From 03c8e7149c9e13dcce591a31f8dc64ed04e01838 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 11 Oct 2024 10:55:55 +0800 Subject: [PATCH 01/32] add DepentdentFilter and SingleColumnValueExcludeFilter --- .../hbase/filter/HBaseFilterUtils.java | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index e58d2590..10c54373 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -45,6 +45,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte toParseableByteArray(byteStream, (CompareFilter) filter); } else if (filter instanceof SingleColumnValueFilter) { toParseableByteArray(byteStream, (SingleColumnValueFilter) filter); + } else if (filter instanceof SingleColumnValueExcludeFilter) { + toParseableByteArray(byteStream, (SingleColumnValueExcludeFilter) filter); } else if (filter instanceof PageFilter) { toParseableByteArray(byteStream, (PageFilter) filter); } else if (filter instanceof ColumnCountGetFilter) { @@ -69,6 +71,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte toParseableByteArray(byteStream, (SkipFilter) filter); } else if (filter instanceof WhileMatchFilter) { toParseableByteArray(byteStream, (WhileMatchFilter) filter); + } else if (filter instanceof DependentColumnFilter) { + toParseableByteArray(byteStream, (DependentColumnFilter) filter); } else { throw new IllegalArgumentException("Invalid filter: " + filter); } @@ -149,6 +153,25 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write(')'); } + // SingleColumnValueExcludeFilter('cf1','col1',=,'binary:123',true,true) + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + SingleColumnValueExcludeFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getFilterIfMissing()).getBytes()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getLatestVersionOnly()).getBytes()); + byteStream.write(')'); + } + // PageFilter(100); private static void toParseableByteArray(ByteArrayOutputStream byteStream, PageFilter filter) throws IOException { @@ -167,6 +190,42 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write(')'); } + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + DependentColumnFilter filter) throws IOException { + boolean dropDependentColumn; + try { + Field field = filter.getClass().getDeclaredField("dropDependentColumn"); + field.setAccessible(true); + dropDependentColumn = (boolean)field.get(filter); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ')' + if (filter.getComparator() == null) { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(dropDependentColumn).getBytes()); + byteStream.write(')'); + } else { // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ',' compare_op ',' comparator ')' + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(dropDependentColumn).getBytes()); + byteStream.write(','); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(')'); + } + } + private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnPaginationFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); From 2aaf7c4d409dbeed6dd6899d1587ce22b7bb63bd Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Wed, 16 Oct 2024 11:54:18 +0800 Subject: [PATCH 02/32] add singleColumnValueExcludeFilter and DependentColumnFilter; add test cases and optimize bufferedMutator test cases --- .../hbase/filter/HBaseFilterUtils.java | 20 +- .../oceanbase/hbase/HTableTestBase.java | 220 ++++++++++++++++++ .../oceanbase/hbase/OHConnectionTest.java | 16 ++ 3 files changed, 242 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index 10c54373..cc512f9e 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -40,13 +40,15 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte throws IOException { if (filter == null) { throw new IllegalArgumentException("Filter is null"); + } else if (filter instanceof DependentColumnFilter) { + toParseableByteArray(byteStream, (DependentColumnFilter) filter); } else if (filter instanceof CompareFilter) { // RowFilter, ValueFilter, QualifierFilter toParseableByteArray(byteStream, (CompareFilter) filter); - } else if (filter instanceof SingleColumnValueFilter) { - toParseableByteArray(byteStream, (SingleColumnValueFilter) filter); } else if (filter instanceof SingleColumnValueExcludeFilter) { toParseableByteArray(byteStream, (SingleColumnValueExcludeFilter) filter); + } else if (filter instanceof SingleColumnValueFilter) { + toParseableByteArray(byteStream, (SingleColumnValueFilter) filter); } else if (filter instanceof PageFilter) { toParseableByteArray(byteStream, (PageFilter) filter); } else if (filter instanceof ColumnCountGetFilter) { @@ -71,8 +73,6 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte toParseableByteArray(byteStream, (SkipFilter) filter); } else if (filter instanceof WhileMatchFilter) { toParseableByteArray(byteStream, (WhileMatchFilter) filter); - } else if (filter instanceof DependentColumnFilter) { - toParseableByteArray(byteStream, (DependentColumnFilter) filter); } else { throw new IllegalArgumentException("Invalid filter: " + filter); } @@ -192,14 +192,6 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, private static void toParseableByteArray(ByteArrayOutputStream byteStream, DependentColumnFilter filter) throws IOException { - boolean dropDependentColumn; - try { - Field field = filter.getClass().getDeclaredField("dropDependentColumn"); - field.setAccessible(true); - dropDependentColumn = (boolean)field.get(filter); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new RuntimeException(e); - } // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ')' if (filter.getComparator() == null) { byteStream.write(filter.getClass().getSimpleName().getBytes()); @@ -208,7 +200,7 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write("','".getBytes()); writeBytesWithEscape(byteStream, filter.getQualifier()); byteStream.write("',".getBytes()); - byteStream.write(Boolean.toString(dropDependentColumn).getBytes()); + byteStream.write(Boolean.toString(filter.getDropDependentColumn()).getBytes()); byteStream.write(')'); } else { // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ',' compare_op ',' comparator ')' byteStream.write(filter.getClass().getSimpleName().getBytes()); @@ -217,7 +209,7 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write("','".getBytes()); writeBytesWithEscape(byteStream, filter.getQualifier()); byteStream.write("',".getBytes()); - byteStream.write(Boolean.toString(dropDependentColumn).getBytes()); + byteStream.write(Boolean.toString(filter.getDropDependentColumn()).getBytes()); byteStream.write(','); byteStream.write(toParseableByteArray(filter.getOperator())); byteStream.write(','); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 4c93efde..a5b891fe 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -509,6 +509,83 @@ public void testFilter() throws Exception { tryPut(hTable, putKey2Column2Value1); tryPut(hTable, putKey2Column2Value2); +// time may be different +// +---------+-----+----------------+--------+ +// | K | Q | T | V | +// +---------+-----+----------------+--------+ +// | getKey1 | abc | -1728834971469 | value1 | +// | getKey1 | abc | -1728834971399 | value2 | +// | getKey1 | abc | -1728834971330 | value1 | +// | getKey1 | def | -1728834971748 | value2 | +// | getKey1 | def | -1728834971679 | value1 | +// | getKey1 | def | -1728834971609 | value2 | +// | getKey1 | def | -1728834971540 | value1 | +// | getKey2 | def | -1728834971887 | value2 | +// | getKey2 | def | -1728834971818 | value1 | +// +---------+-----+----------------+--------+ + + SingleColumnValueFilter singleColumnValueFilter; + singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueFilter); + r = hTable.get(get); + Assert.assertEquals(7, r.raw().length); + + SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; + singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueExcludeFilter); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), true); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + + + filter = new ColumnPrefixFilter(Bytes.toBytes("e")); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -817,6 +894,66 @@ public void testFilter() throws Exception { } Assert.assertEquals(res_count, 12); scanner.close(); + + long timestamp = System.currentTimeMillis(); + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), timestamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timestamp, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + // putKey1Column1Value1 and putKey2Column1Value1 have the same timestamp + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column2Value2); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setStartRow("getKey1".getBytes()); + scan.setStopRow("getKey3".getBytes()); + scan.setMaxVersions(10); + scan.setFilter(dependentColumnFilter); + scanner = hTable.getScanner(scan); + + long prevTimestamp = - 1; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + if (prevTimestamp == -1) { + prevTimestamp = keyValue.getTimestamp(); + } else { + Assert.assertEquals(prevTimestamp, keyValue.getTimestamp()); + } + } + } + scanner.close(); } @Test @@ -1242,6 +1379,15 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(1, r.raw().length); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + // columnCountGetFilter filter 2 get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -1422,6 +1568,67 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(7, r.raw().length); + filterList = new FilterList(); + filterList.addFilter(new SingleColumnValueExcludeFilter(Bytes.toBytes(family), Bytes + .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column1), false)); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), false)); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), true)); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(value2)))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + filterList = new FilterList(); filterList.addFilter(new ColumnCountGetFilter(1)); filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.GREATER, @@ -1547,6 +1754,19 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(7, r.raw().length); + + SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; + singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueExcludeFilter); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index a4a78a61..e8644d7b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -209,6 +209,10 @@ public void testBufferedMutatorWithFlush() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + // only support Put and Delete // for other type of operations, BufferedMutator will not set its type for them Append append = new Append(Bytes.toBytes(key)); @@ -409,6 +413,10 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); @@ -498,6 +506,10 @@ public void testBufferedMutatorWithUserPool() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); @@ -587,6 +599,10 @@ public void testBufferedMutatorConcurrent() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + for (int i = 0; i < 50; ++i) { final int taskId = i; final BufferedMutator thrBufferMutator = ohBufferMutator; From be654df67e8812a255cbbec1babf9dc7abe608b9 Mon Sep 17 00:00:00 2001 From: stuBirdFly <84010733+stuBirdFly@users.noreply.github.com> Date: Tue, 22 Oct 2024 20:35:40 +0800 Subject: [PATCH 03/32] hbase support batch (#84) --- .../com/alipay/oceanbase/hbase/OHTable.java | 380 ++++++++---------- .../oceanbase/hbase/util/BatchError.java | 40 ++ .../oceanbase/hbase/HTableTestBase.java | 18 +- .../hbase/OHTableMultiColumnFamilyTest.java | 109 ++++- 4 files changed, 331 insertions(+), 216 deletions(-) create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 4ecf0aa5..d38eaa46 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -24,6 +24,7 @@ import com.alipay.oceanbase.hbase.result.ClientStreamScanner; import com.alipay.oceanbase.hbase.util.*; import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.mutation.BatchOperation; import com.alipay.oceanbase.rpc.mutation.result.BatchOperationResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; @@ -131,7 +132,7 @@ public class OHTable implements HTableInterface { /** * the buffer of put request */ - private final ArrayList writeBuffer = new ArrayList(); + private final ArrayList writeBuffer = new ArrayList<>(); /** * when the put request reach the write buffer size the do put will * flush commits automatically @@ -336,7 +337,7 @@ public static ThreadPoolExecutor createDefaultThreadPoolExecutor(int coreSize, i System.setProperty(SOFA_THREAD_POOL_LOGGING_CAPABILITY, "false"); } SofaThreadPoolExecutor executor = new SofaThreadPoolExecutor(coreSize, maxThreads, - keepAliveTime, SECONDS, new SynchronousQueue(), "OHTableDefaultExecutePool", + keepAliveTime, SECONDS, new SynchronousQueue<>(), "OHTableDefaultExecutePool", TABLE_HBASE_LOGGER_SPACE); executor.allowCoreThreadTimeOut(true); return executor; @@ -463,13 +464,63 @@ public Boolean[] exists(List gets) throws IOException { } @Override - public void batch(List actions, Object[] results) { - throw new FeatureNotSupportedException("not supported yet."); + public void batch(final List actions, final Object[] results) throws IOException { + BatchError batchError = new BatchError(); + try { + List resultMapSingleOp = new LinkedList<>(); + String realTableName = getTargetTableName(actions); + obTableClient.setRuntimeBatchExecutor(executePool); + BatchOperation batch = buildBatchOperation(realTableName, actions, tableNameString.equals(realTableName), resultMapSingleOp); + BatchOperationResult tmpResults = batch.execute(); + if (results != null) { + if (results.length != actions.size()) { + throw new AssertionError("results.length"); + } + int index = 0; + for (int i = 0; i != results.length; ++i) { + results[i] = tmpResults.getResults().get(index); + index += resultMapSingleOp.get(i); + if (results[i] instanceof ObTableException) { + batchError.add((ObTableException) results[i], actions.get(i), null); + } + } + if (batchError.hasErrors()) { + throw batchError.makeException(); + } + } + } catch (Exception e) { + logger.error(LCD.convert("01-000010"), tableNameString, actions, e); + throw new IOException("batch table " + tableNameString + " error", e); + } + } + + private String getTargetTableName(List actions) { + byte[] family = null; + for (Row action : actions) { + if (action instanceof RowMutations || action instanceof RegionCoprocessorServiceExec) { + throw new FeatureNotSupportedException("not supported yet'"); + } else { + Mutation mutation = (Mutation) action; + if (mutation.getFamilyCellMap().size() != 1) { + return getTargetTableName(tableNameString); + } else { + byte[] nextFamily = mutation.getFamilyCellMap().keySet().iterator().next(); + if (family != null && !Arrays.equals(family, nextFamily)) { + return getTargetTableName(tableNameString); + } else if (family == null) { + family = nextFamily; + } + } + } + } + return getTargetTableName(tableNameString, Bytes.toString(family), configuration); } @Override - public Object[] batch(List actions) { - throw new FeatureNotSupportedException("not supported yet."); + public Object[] batch(List actions) throws IOException { + Object[] results = new Object[actions.size()]; + batch(actions, results); + return results; } @Override @@ -573,7 +624,7 @@ public Result get(final Get get) throws IOException { ServerCallable serverCallable = new ServerCallable(configuration, obTableClient, tableNameString, get.getRow(), get.getRow(), operationTimeout) { public Result call() throws IOException { - List keyValueList = new ArrayList(); + List keyValueList = new ArrayList<>(); byte[] family = new byte[] {}; ObTableClientQueryAsyncStreamResult clientQueryStreamResult; ObTableQueryAsyncRequest request; @@ -860,57 +911,11 @@ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, private void innerDelete(Delete delete) throws IOException { checkArgument(delete.getRow() != null, "row is null"); - List errorCodeList = new ArrayList(); - BatchOperationResult results = null; - try { - checkFamilyViolation(delete.getFamilyMap().keySet(), false); - if (delete.getFamilyMap().isEmpty()) { - // For a Delete operation without any qualifiers, we construct a DeleteFamily request. - // The server then performs the operation on all column families. - KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), - KeyValue.Type.DeleteFamily); - - BatchOperation batch = buildBatchOperation(tableNameString, Arrays.asList(kv), false, null); - results = batch.execute(); - } else if (delete.getFamilyMap().size() > 1) { - // Currently, the Delete Family operation type cannot transmit qualifiers to the server. - // As a result, the server cannot identify which families need to be deleted. - // Therefore, this process is handled sequentially. - boolean has_delete_family = delete.getFamilyMap().entrySet().stream() - .flatMap(entry -> entry.getValue().stream()) - .anyMatch(kv -> KeyValue.Type.codeToType(kv.getType()) == KeyValue.Type.DeleteFamily); - if (!has_delete_family) { - BatchOperation batch = buildBatchOperation(tableNameString, - delete.getFamilyMap(), false, null); - results = batch.execute(); - } else { - for (Map.Entry> entry : delete.getFamilyMap().entrySet()) { - BatchOperation batch = buildBatchOperation( - getTargetTableName(tableNameString, Bytes.toString(entry.getKey()), configuration), - entry.getValue(), false, null); - results = batch.execute(); - } - } - } else { - Map.Entry> entry = delete.getFamilyMap().entrySet().iterator() - .next(); - - BatchOperation batch = buildBatchOperation( - getTargetTableName(tableNameString, Bytes.toString(entry.getKey()), configuration), - entry.getValue(), false, null); - results = batch.execute(); - } - - errorCodeList = results.getErrorCodeList(); - boolean hasError = results.hasError(); - if (hasError) { - throw results.getFirstException(); - } + batch(Collections.singletonList(delete)); } catch (Exception e) { - logger.error(LCD.convert("01-00004"), tableNameString, errorCodeList, e); - throw new IOException("delete table " + tableNameString + " error codes " - + errorCodeList, e); + logger.error(LCD.convert("01-00004"), tableNameString, e); + throw new IOException("delete table " + tableNameString + "error" , e); } } @@ -1187,77 +1192,34 @@ public boolean isAutoFlush() { public void flushCommits() throws IOException { try { + if (writeBuffer.isEmpty()){ + return; + } + Map exceptionRowMap = new LinkedHashMap(); boolean[] resultSuccess = new boolean[writeBuffer.size()]; try { - Map, List>> familyMap = new HashMap, List>>(); - for (int i = 0; i < writeBuffer.size(); i++) { - Put aPut = writeBuffer.get(i); - Map> innerFamilyMap = aPut.getFamilyMap(); - if (innerFamilyMap.size() > 1) { - // Bypass logic: directly construct BatchOperation for puts with family map size > 1 - try { - BatchOperation batch = buildBatchOperation(this.tableNameString, - innerFamilyMap, false, null); - BatchOperationResult results = batch.execute(); - - boolean hasError = results.hasError(); - resultSuccess[i] = !hasError; - if (hasError) { - throw results.getFirstException(); - } - } catch (Exception e) { - logger.error(LCD.convert("01-00008"), tableNameString, null, autoFlush, - writeBuffer.size(), e); - throw new IOException("put table " + tableNameString + " error codes " - + null + "auto flush " + autoFlush - + " current buffer size " + writeBuffer.size(), e); - } - } else { - // Existing logic for puts with family map size = 1 - for (Map.Entry> entry : innerFamilyMap.entrySet()) { - String family = Bytes.toString(entry.getKey()); - Pair, List> keyValueWithIndex = familyMap - .get(family); - if (keyValueWithIndex == null) { - keyValueWithIndex = new Pair, List>( - new ArrayList(), new ArrayList()); - familyMap.put(family, keyValueWithIndex); - } - keyValueWithIndex.getFirst().add(i); - keyValueWithIndex.getSecond().addAll(entry.getValue()); - } - } - } - for (Map.Entry, List>> entry : familyMap - .entrySet()) { - List errorCodeList = new ArrayList(entry.getValue() - .getSecond().size()); - try { - String targetTableName = getTargetTableName(this.tableNameString, - entry.getKey(), configuration); - - BatchOperation batch = buildBatchOperation(targetTableName, entry - .getValue().getSecond(), false, null); - BatchOperationResult results = batch.execute(); - - errorCodeList = results.getErrorCodeList(); - boolean hasError = results.hasError(); - for (Integer index : entry.getValue().getFirst()) { - resultSuccess[index] = !hasError; - } - if (hasError) { - throw results.getFirstException(); + String realTableName = getTargetTableName(writeBuffer); + List resultMapSingleOp = new LinkedList<>(); + BatchOperation batch = buildBatchOperation(realTableName, writeBuffer, tableNameString.equals(realTableName), resultMapSingleOp); + BatchOperationResult results = batch.execute(); + if (results != null) { + int index = 0; + for (int i = 0; i != resultSuccess.length; ++i) { + if (results.getResults().get(index) instanceof ObTableException) { + resultSuccess[i] = false; + exceptionRowMap.put((ObTableException)results.getResults().get(index), writeBuffer.get(i)); + } else { + resultSuccess[i] = true; } - } catch (Exception e) { - logger.error(LCD.convert("01-00008"), tableNameString, errorCodeList, - autoFlush, writeBuffer.size(), e); - throw new IOException("put table " + tableNameString + " error codes " - + errorCodeList + "auto flush " + autoFlush - + " current buffer size " + writeBuffer.size(), e); + index += resultMapSingleOp.get(i); } - } - + } catch (Exception e) { + logger.error(LCD.convert("01-00008"), tableNameString, null, autoFlush, + writeBuffer.size(), e); + throw new IOException("put table " + tableNameString + " error codes " + null + + "auto flush " + autoFlush + " current buffer size " + + writeBuffer.size(), e); } finally { // mutate list so that it is empty for complete success, or contains // only failed records results are returned in the same order as the @@ -1269,6 +1231,12 @@ public void flushCommits() throws IOException { writeBuffer.remove(i); } } + if (!exceptionRowMap.isEmpty()) { + exceptionRowMap.forEach((e, row)->{ + logger.error(LCD.convert("01-00008"), row, tableNameString, autoFlush, + writeBuffer.size(), e); + }); + } } } finally { if (clearBufferOnFail) { @@ -1674,62 +1642,48 @@ public static ObTableBatchOperation buildObTableBatchOperation(List ke return batch; } - private ObTableBatchOperation buildObTableBatchOperation(Map> familyMap, - boolean putToAppend, - List qualifiers) { - ObTableBatchOperation batch = new ObTableBatchOperation(); - for (Map.Entry> entry : familyMap.entrySet()) { - byte[] family = entry.getKey(); - List keyValueList = entry.getValue(); - for (KeyValue kv : keyValueList) { - if (qualifiers != null) { - qualifiers - .add((Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())) - .getBytes()); - } - KeyValue new_kv = modifyQualifier(kv, - (Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())).getBytes()); - batch.addTableOperation(buildObTableOperation(new_kv, putToAppend)); - } - } - batch.setSameType(true); - batch.setSamePropertiesNames(true); - return batch; - } - private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(KeyValue kv, - boolean putToAppend) { + ObTableOperationType operationType, boolean isTableGroup) { KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getType()); - switch (kvType) { - case Put: - ObTableOperationType operationType; - if (putToAppend) { - operationType = APPEND; - } else { - operationType = INSERT_OR_UPDATE; - } + switch (operationType) { + case INSERT_OR_UPDATE: + case APPEND: return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(operationType, ROW_KEY_COLUMNS, new Object[] { kv.getRow(), kv.getQualifier(), kv.getTimestamp() }, V_COLUMNS, new Object[] { kv.getValue() }); - case Delete: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), kv.getQualifier(), kv.getTimestamp() }, null, null); - case DeleteColumn: - return com.alipay.oceanbase.rpc.mutation.Mutation - .getInstance(DEL, ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), kv.getQualifier(), -kv.getTimestamp() }, null, - null); - case DeleteFamily: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), null, -kv.getTimestamp() }, null, null); + case DEL: + switch (kvType) { + case Delete: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), kv.getQualifier(), kv.getTimestamp() }, + null, null); + case Maximum: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), null, -kv.getTimestamp() }, + null, null); + case DeleteColumn: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), kv.getQualifier(), -kv.getTimestamp() }, + null, null); + case DeleteFamily: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), isTableGroup?kv.getQualifier():null, -kv.getTimestamp() }, + null, null); + default: + throw new IllegalArgumentException("illegal mutation type " + kvType); + } default: - throw new IllegalArgumentException("illegal mutation type " + kvType); + throw new IllegalArgumentException("illegal mutation type " + operationType); } } private KeyValue modifyQualifier(KeyValue original, byte[] newQualifier) { - // Extract existing components + // Extract existing components byte[] row = original.getRow(); byte[] family = original.getFamily(); byte[] value = original.getValue(); @@ -1740,36 +1694,65 @@ private KeyValue modifyQualifier(KeyValue original, byte[] newQualifier) { value); } - private BatchOperation buildBatchOperation(String tableName, - Map> familyMap, - boolean putToAppend, List qualifiers) { + private BatchOperation buildBatchOperation(String tableName, List actions, boolean isTableGroup, List resultMapSingleOp) { BatchOperation batch = obTableClient.batchOperation(tableName); - - for (Map.Entry> entry : familyMap.entrySet()) { - byte[] family = entry.getKey(); - List keyValueList = entry.getValue(); - for (KeyValue kv : keyValueList) { - if (qualifiers != null) { - qualifiers.add(kv.getQualifier()); + if (actions != null) { + int posInList = -1; + int singleOpResultNum; + for (Row row : actions) { + singleOpResultNum = 0; + posInList++; + if (!(row instanceof Put) && !(row instanceof Delete)) { + throw new FeatureNotSupportedException( + "not supported other type in batch yet,only support put and delete"); + } else if (row instanceof Put) { + Put put = (Put) row; + if (put.isEmpty()) { + throw new IllegalArgumentException("No columns to insert for #" + + (posInList + 1) + " item"); + } + for (Map.Entry> entry : put.getFamilyMap().entrySet()) { + byte[] family = entry.getKey(); + List keyValueList = entry.getValue(); + for (KeyValue kv : keyValueList) { + singleOpResultNum++; + if(isTableGroup){ + KeyValue new_kv = modifyQualifier(kv,(Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())) + .getBytes()); + batch.addOperation(buildMutation(new_kv, INSERT_OR_UPDATE, isTableGroup)); + } else { + batch.addOperation(buildMutation(kv, INSERT_OR_UPDATE, isTableGroup)); + } + } + } + } else { + Delete delete = (Delete) row; + if (delete.isEmpty()) { + singleOpResultNum++; + KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), + KeyValue.Type.Maximum); + batch.addOperation(buildMutation(kv, DEL, isTableGroup)); + } else { + for (Map.Entry> entry : delete.getFamilyMap() + .entrySet()) { + byte[] family = entry.getKey(); + List keyValueList = entry.getValue(); + List mutations = new LinkedList<>(); + for (KeyValue kv : keyValueList) { + singleOpResultNum++; + if(isTableGroup){ + KeyValue new_kv = modifyQualifier(kv,(Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())) + .getBytes()); + batch.addOperation(buildMutation(new_kv, DEL, true)); + } else { + batch.addOperation(buildMutation(kv, DEL, false)); + } + } + } + } } - KeyValue new_kv = modifyQualifier(kv, - (Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())).getBytes()); - batch.addOperation(buildMutation(new_kv, putToAppend)); - } - } - - batch.setEntityType(ObTableEntityType.HKV); - return batch; - } - - private BatchOperation buildBatchOperation(String tableName, List keyValueList, - boolean putToAppend, List qualifiers) { - BatchOperation batch = obTableClient.batchOperation(tableName); - for (KeyValue kv : keyValueList) { - if (qualifiers != null) { - qualifiers.add(kv.getQualifier()); + resultMapSingleOp.add(singleOpResultNum); } - batch.addOperation(buildMutation(kv, putToAppend)); } batch.setEntityType(ObTableEntityType.HKV); return batch; @@ -1802,15 +1785,6 @@ public static ObTableOperation buildObTableOperation(KeyValue kv, boolean putToA } } - private ObTableQueryRequest buildObTableQueryRequest(ObTableQuery obTableQuery, - String targetTableName) { - ObTableQueryRequest request = new ObTableQueryRequest(); - request.setEntityType(ObTableEntityType.HKV); - request.setTableQuery(obTableQuery); - request.setTableName(targetTableName); - return request; - } - private ObTableQueryAsyncRequest buildObTableQueryAsyncRequest(ObTableQuery obTableQuery, String targetTableName) { ObTableQueryRequest request = new ObTableQueryRequest(); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java b/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java new file mode 100644 index 00000000..640e432d --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java @@ -0,0 +1,40 @@ +package com.alipay.oceanbase.hbase.util; + +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; +import org.apache.hadoop.hbase.client.Row; + +import java.util.ArrayList; +import java.util.List; + +public class BatchError { + private final List throwables = new ArrayList(); + private final List actions = new ArrayList(); + private final List addresses = new ArrayList(); + + public synchronized void add(Throwable ex, Row row, ServerName serverName) { + if (row == null){ + throw new IllegalArgumentException("row cannot be null. location=" + serverName); + } + + throwables.add(ex); + actions.add(row); + addresses.add(serverName != null ? serverName.toString() : "null"); + } + + public boolean hasErrors() { + return !throwables.isEmpty(); + } + + public synchronized RetriesExhaustedWithDetailsException makeException() { + return new RetriesExhaustedWithDetailsException( + new ArrayList(throwables), + new ArrayList(actions), new ArrayList(addresses)); + } + + public synchronized void clear() { + throwables.clear(); + actions.clear(); + addresses.clear(); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 4a1aea84..676dc2a5 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -104,7 +104,7 @@ PRIMARY KEY (`K`, `Q`, `T`) } // test scan with empty family - Scan scan = new Scan(); + Scan scan = new Scan(toBytes(key)); ResultScanner scanner = hTable.getScanner(scan); for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { @@ -3980,7 +3980,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); // reverse scan with MaxVersion @@ -3998,7 +3998,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 3); + Assert.assertEquals(3, res_count); scanner.close(); // reverse scan with pageFilter @@ -4018,7 +4018,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); // reverse scan with not_exist_start_row @@ -4036,7 +4036,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); // reverse scan with abnormal range @@ -4054,7 +4054,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 0); + Assert.assertEquals(0, res_count); scanner.close(); // reverse scan with abnormal range @@ -4072,7 +4072,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 13); + Assert.assertEquals(13, res_count); scanner.close(); hTable.delete(deleteKey1Family); @@ -4421,7 +4421,7 @@ public void testPartitionScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 7); + Assert.assertEquals(7, res_count); scanner.close(); // scan with prefixFilter @@ -4441,7 +4441,7 @@ public void testPartitionScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 2); + Assert.assertEquals(2, res_count); scanner.close(); // scan with singleColumnValueFilter diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java index 1793bb7f..f8d4eed3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java @@ -24,9 +24,7 @@ import org.junit.rules.ExpectedException; import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.*; @@ -48,6 +46,108 @@ public void finish() throws IOException { hTable.close(); } + @Test + public void testMultiColumnFamilyBatch() throws Exception { + byte[] family1 = "family_with_group1".getBytes(); + byte[] family2 = "family_with_group2".getBytes(); + byte[] family3 = "family_with_group3".getBytes(); + + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family1_value = "VVV1".getBytes(); + byte[] family2_value = "VVV2".getBytes(); + byte[] family3_value = "VVV3".getBytes(); + + int rows = 10; + List batchLsit = new LinkedList<>(); + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + Delete delete = new Delete(toBytes("Key" + i)); + batchLsit.add(delete); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + batchLsit.add(put); + } + + // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 + Delete delete = new Delete(toBytes("Key1")); + delete.deleteColumns(family1, family1_column1); + delete.deleteColumns(family2, family2_column1); + batchLsit.add(delete); + hTable.batch(batchLsit); + // f1c2 f1c3 f2c2 f3c1 + Get get = new Get(toBytes("Key1")); + Result result = hTable.get(get); + KeyValue[] keyValues = result.raw(); + assertEquals(4, keyValues.length); + assertFalse(result.containsColumn(family1, family1_column1)); + assertFalse(result.containsColumn(family2, family2_column1)); + + assertTrue(result.containsColumn(family1, family1_column2)); + assertArrayEquals(result.getValue(family1, family1_column2), family1_value); + assertTrue(result.containsColumn(family1, family1_column3)); + assertArrayEquals(result.getValue(family1, family1_column3), family1_value); + assertTrue(result.containsColumn(family2, family2_column2)); + assertArrayEquals(result.getValue(family2, family2_column2), family2_value); + assertTrue(result.containsColumn(family3, family3_column1)); + assertArrayEquals(result.getValue(family3, family3_column1), family3_value); + + // f1c1 f2c1 f2c2 + delete = new Delete(toBytes("Key2")); + delete.deleteColumns(family1, family1_column2); + delete.deleteColumns(family1, family1_column3); + delete.deleteColumns(family3, family3_column1); + batchLsit.add(delete); + // null + hTable.batch(batchLsit); + get = new Get(toBytes("Key2")); + result = hTable.get(get); + keyValues = result.raw(); + assertEquals(3, keyValues.length); + batchLsit.clear(); + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + batchLsit.add(put); + } + + delete = new Delete(toBytes("Key3")); + delete.deleteColumn(family1, family1_column2); + delete.deleteColumn(family2, family2_column1); + batchLsit.add(delete); + hTable.batch(batchLsit); + get = new Get(toBytes("Key3")); + result = hTable.get(get); + keyValues = result.raw(); + assertEquals(6, keyValues.length); + + batchLsit.clear(); + delete = new Delete(toBytes("Key4")); + delete.deleteColumns(family1, family1_column2); + delete.deleteColumns(family2, family2_column1); + delete.deleteFamily(family3); + batchLsit.add(delete); + hTable.batch(batchLsit); + get = new Get(toBytes("Key4")); + get.setMaxVersions(10); + result = hTable.get(get); + keyValues = result.raw(); + assertEquals(6, keyValues.length); + } + @Test public void testMultiColumnFamilyPut() throws Exception { byte[] family1 = "family_with_group1".getBytes(); @@ -487,7 +587,6 @@ public void testMultiColumnFamilyGet() throws Exception { assertEquals(expectedValue, keyValues[i].getValue()); } } - System.out.println(Arrays.toString(result2.raw())); assertEquals(3, keyValues.length); //f2c1 f2c2 @@ -529,6 +628,8 @@ public void testMultiColumnFamilyDelete() throws Exception { for (int i = 0; i < rows; ++i) { Put put = new Put(toBytes("Key" + i)); + Delete delete = new Delete(toBytes("Key" + i)); + hTable.delete(delete); put.add(family1, family1_column1, family1_value); put.add(family1, family1_column2, family1_value); put.add(family1, family1_column3, family1_value); From 697c5d07671b3026cbada259efd297094102e1fe Mon Sep 17 00:00:00 2001 From: Ziyu Shi <57038180+JackShi148@users.noreply.github.com> Date: Wed, 23 Oct 2024 10:49:17 +0800 Subject: [PATCH 04/32] Add DeleteFamilyVersion function and corresponding test cases (#85) * add DepentdentFilter and SingleColumnValueExcludeFilter * add singleColumnValueExcludeFilter and DependentColumnFilter; add test cases and optimize bufferedMutator test cases * add single cf timerange setting in Get * single cf setColumnFamilyTimeRange in Get and Scan * optimize code * add DeleteFamilyVersion and test cases * add DeleteFamilyVersion; optimize test cases * add DeleteFamilyVersion test case and pass * format code * delete useless self-defined table --- .../com/alipay/oceanbase/hbase/OHTable.java | 17 +- .../hbase/filter/HBaseFilterUtils.java | 19 +- .../oceanbase/hbase/util/BatchError.java | 11 +- .../oceanbase/hbase/HTableTestBase.java | 167 ++++----- .../oceanbase/hbase/OHConnectionTest.java | 131 ++++--- .../hbase/OHTableDeleteFamilyVersionTest.java | 348 ++++++++++++++++++ .../hbase/filter/HBaseFilterUtilsTest.java | 51 ++- 7 files changed, 563 insertions(+), 181 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index d38eaa46..6d1cc873 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1656,9 +1656,9 @@ private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(KeyValue kv, switch (kvType) { case Delete: return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), kv.getQualifier(), kv.getTimestamp() }, - null, null); + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), kv.getQualifier(), kv.getTimestamp() }, + null, null); case Maximum: return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, @@ -1671,9 +1671,14 @@ private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(KeyValue kv, null, null); case DeleteFamily: return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), isTableGroup?kv.getQualifier():null, -kv.getTimestamp() }, - null, null); + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), isTableGroup ? kv.getQualifier() : null, -kv.getTimestamp() }, + null, null); + case DeleteFamilyVersion: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), isTableGroup ? kv.getQualifier() : null, kv.getTimestamp() }, + null, null); default: throw new IllegalArgumentException("illegal mutation type " + kvType); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index 85c4f593..ed98d1b3 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -169,7 +169,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, // SingleColumnValueExcludeFilter('cf1','col1',=,'binary:123',true,true) private static void toParseableByteArray(ByteArrayOutputStream byteStream, - SingleColumnValueExcludeFilter filter) throws IOException { + SingleColumnValueExcludeFilter filter) + throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write("('".getBytes()); writeBytesWithEscape(byteStream, filter.getFamily()); @@ -329,12 +330,13 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Times } // MultiRowRangeFilter('a',true,'b',false,'c',true,'d',false); - private static void toParseableByteArray(ByteArrayOutputStream byteStream, MultiRowRangeFilter filter) throws IOException { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + MultiRowRangeFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); List ranges = filter.getRowRanges(); - for (int i = 0; i < ranges.size(); i ++) { + for (int i = 0; i < ranges.size(); i++) { MultiRowRangeFilter.RowRange range = ranges.get(i); byteStream.write("'".getBytes()); byteStream.write(range.getStartRow()); @@ -354,7 +356,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Multi } // InclusiveStopFilter('aaa'); - private static void toParseableByteArray(ByteArrayOutputStream byteStream, InclusiveStopFilter filter) throws IOException { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + InclusiveStopFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); byteStream.write('\''); @@ -364,7 +367,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Inclu } // ColumnRangeFilter('a',true,'b',false); - private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnRangeFilter filter) throws IOException { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + ColumnRangeFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); @@ -382,12 +386,13 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Colum } // MultipleColumnPrefixFilter('a','b','d'); - private static void toParseableByteArray(ByteArrayOutputStream byteStream, MultipleColumnPrefixFilter filter) throws IOException { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + MultipleColumnPrefixFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); byte[][] ranges = filter.getPrefix(); - for (int i = 0; i < ranges.length; i ++) { + for (int i = 0; i < ranges.length; i++) { byte[] range = ranges[i]; byteStream.write("'".getBytes()); byteStream.write(range); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java b/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java index 640e432d..49fe17e9 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java @@ -9,11 +9,11 @@ public class BatchError { private final List throwables = new ArrayList(); - private final List actions = new ArrayList(); - private final List addresses = new ArrayList(); + private final List actions = new ArrayList(); + private final List addresses = new ArrayList(); public synchronized void add(Throwable ex, Row row, ServerName serverName) { - if (row == null){ + if (row == null) { throw new IllegalArgumentException("row cannot be null. location=" + serverName); } @@ -27,9 +27,8 @@ public boolean hasErrors() { } public synchronized RetriesExhaustedWithDetailsException makeException() { - return new RetriesExhaustedWithDetailsException( - new ArrayList(throwables), - new ArrayList(actions), new ArrayList(addresses)); + return new RetriesExhaustedWithDetailsException(new ArrayList(throwables), + new ArrayList(actions), new ArrayList(addresses)); } public synchronized void clear() { diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 676dc2a5..9aa2e1f7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -510,24 +510,24 @@ public void testFilter() throws Exception { tryPut(hTable, putKey2Column2Value1); tryPut(hTable, putKey2Column2Value2); -// time may be different -// +---------+-----+----------------+--------+ -// | K | Q | T | V | -// +---------+-----+----------------+--------+ -// | getKey1 | abc | -1728834971469 | value1 | -// | getKey1 | abc | -1728834971399 | value2 | -// | getKey1 | abc | -1728834971330 | value1 | -// | getKey1 | def | -1728834971748 | value2 | -// | getKey1 | def | -1728834971679 | value1 | -// | getKey1 | def | -1728834971609 | value2 | -// | getKey1 | def | -1728834971540 | value1 | -// | getKey2 | def | -1728834971887 | value2 | -// | getKey2 | def | -1728834971818 | value1 | -// +---------+-----+----------------+--------+ + // time may be different + // +---------+-----+----------------+--------+ + // | K | Q | T | V | + // +---------+-----+----------------+--------+ + // | getKey1 | abc | -1728834971469 | value1 | + // | getKey1 | abc | -1728834971399 | value2 | + // | getKey1 | abc | -1728834971330 | value1 | + // | getKey1 | def | -1728834971748 | value2 | + // | getKey1 | def | -1728834971679 | value1 | + // | getKey1 | def | -1728834971609 | value2 | + // | getKey1 | def | -1728834971540 | value1 | + // | getKey2 | def | -1728834971887 | value2 | + // | getKey2 | def | -1728834971818 | value1 | + // +---------+-----+----------------+--------+ SingleColumnValueFilter singleColumnValueFilter; singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -538,7 +538,7 @@ public void testFilter() throws Exception { SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -547,8 +547,8 @@ public void testFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(4, r.raw().length); - DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), false); + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter( + Bytes.toBytes(family), Bytes.toBytes(column1), false); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -557,7 +557,7 @@ public void testFilter() throws Exception { Assert.assertEquals(3, r.raw().length); dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), true); + Bytes.toBytes(column1), true); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -566,7 +566,7 @@ public void testFilter() throws Exception { Assert.assertEquals(0, r.raw().length); dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); get = new Get(toBytes(key2)); get.setMaxVersions(10); @@ -576,7 +576,7 @@ public void testFilter() throws Exception { Assert.assertEquals(0, r.raw().length); dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); get = new Get(toBytes(key2)); get.setMaxVersions(10); @@ -585,8 +585,6 @@ public void testFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(1, r.raw().length); - - filter = new ColumnPrefixFilter(Bytes.toBytes("e")); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -934,7 +932,7 @@ public void testFilter() throws Exception { tryPut(hTable, putKey2Column2Value2); dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); scan = new Scan(); scan.addFamily(family.getBytes()); @@ -944,7 +942,7 @@ public void testFilter() throws Exception { scan.setFilter(dependentColumnFilter); scanner = hTable.getScanner(scan); - long prevTimestamp = - 1; + long prevTimestamp = -1; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { if (prevTimestamp == -1) { @@ -1280,20 +1278,20 @@ public void testColumnRangeFilter() throws Exception { scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); - ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, Bytes.toBytes("b"), false); + ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, + Bytes.toBytes("b"), false); scan.setFilter(filter); ResultScanner scanner = hTable.getScanner(scan); int res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); res_count += 1; } } @@ -1310,13 +1308,12 @@ public void testColumnRangeFilter() throws Exception { res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); res_count += 1; } } @@ -1333,13 +1330,12 @@ public void testColumnRangeFilter() throws Exception { res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); res_count += 1; } } @@ -1350,11 +1346,7 @@ public void testColumnRangeFilter() throws Exception { scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); - byte[][] range = { - Bytes.toBytes("g"), - Bytes.toBytes("3"), - Bytes.toBytes("d"), - }; + byte[][] range = { Bytes.toBytes("g"), Bytes.toBytes("3"), Bytes.toBytes("d"), }; MultipleColumnPrefixFilter iFilter = new MultipleColumnPrefixFilter(range); scan.setFilter(iFilter); scanner = hTable.getScanner(scan); @@ -1362,13 +1354,12 @@ public void testColumnRangeFilter() throws Exception { res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); res_count += 1; } } @@ -1378,14 +1369,8 @@ public void testColumnRangeFilter() throws Exception { scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); - range = new byte[][]{ - Bytes.toBytes("de"), - Bytes.toBytes("bg"), - Bytes.toBytes("nc"), - Bytes.toBytes("aa"), - Bytes.toBytes("abcd"), - Bytes.toBytes("dea"), - }; + range = new byte[][] { Bytes.toBytes("de"), Bytes.toBytes("bg"), Bytes.toBytes("nc"), + Bytes.toBytes("aa"), Bytes.toBytes("abcd"), Bytes.toBytes("dea"), }; iFilter = new MultipleColumnPrefixFilter(range); scan.setFilter(iFilter); scanner = hTable.getScanner(scan); @@ -1393,13 +1378,12 @@ public void testColumnRangeFilter() throws Exception { res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); res_count += 1; } } @@ -1474,13 +1458,12 @@ public void testFilterNullRange() throws Exception { int res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); res_count += 1; } @@ -2354,8 +2337,8 @@ public void testGetFilter() throws Exception { get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); - DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column1)); + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter( + Bytes.toBytes(family), Bytes.toBytes(column1)); get.setFilter(dependentColumnFilter); r = hTable.get(get); Assert.assertEquals(3, r.raw().length); @@ -2542,7 +2525,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new SingleColumnValueExcludeFilter(Bytes.toBytes(family), Bytes - .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); + .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2552,7 +2535,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column1), false)); + .toBytes(column1), false)); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2562,7 +2545,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2), false)); + .toBytes(column2), false)); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2572,7 +2555,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2))); + .toBytes(column2))); get = new Get(toBytes(key2)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2582,7 +2565,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2), true)); + .toBytes(column2), true)); get = new Get(toBytes(key2)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2592,8 +2575,8 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(value2)))); + .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2)))); get = new Get(toBytes(key2)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2726,10 +2709,9 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(7, r.raw().length); - SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -2738,7 +2720,6 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(4, r.raw().length); - singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index 03617c2c..b40b23ea 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -205,22 +205,25 @@ public void testBufferedMutatorWithFlush() throws Exception { conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); BufferedMutator putBufferMutator = null; BufferedMutator delBufferedMutator = null; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // use defualt params putBufferMutator = connection.getBufferedMutator(tableName); delBufferedMutator = connection.getBufferedMutator(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - Delete delete= new Delete(toBytes(key)); - delete.deleteFamily(toBytes("family_group")); - hTable.delete(delete); + long timestamp = System.currentTimeMillis(); // only support Put and Delete // for other type of operations, BufferedMutator will not set its type for them @@ -234,10 +237,10 @@ public void testBufferedMutatorWithFlush() throws Exception { List mutations = new ArrayList<>(); // test Put Put put1 = new Put(Bytes.toBytes(key)); - put1.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); + put1.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); mutations.add(put1); Put put2 = new Put(Bytes.toBytes(key)); - put2.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); + put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); mutations.add(put2); // test add Mutations with List putBufferMutator.mutate(mutations); @@ -253,7 +256,7 @@ public void testBufferedMutatorWithFlush() throws Exception { Assert.assertThrows(FeatureNotSupportedException.class, () -> { noCfMut.mutate(del); }); - del.deleteFamily(Bytes.toBytes("family_group")); + del.deleteFamily(Bytes.toBytes(family)); // test reuse different type bufferedMutator final BufferedMutator difTypeMut = putBufferMutator; Assert.assertThrows(IllegalArgumentException.class, () -> { @@ -281,8 +284,8 @@ public void testBufferedMutatorWithFlush() throws Exception { if (delBufferedMutator != null) { delBufferedMutator.close(); // test add mutations after closed - Delete delete = new Delete(Bytes.toBytes("putKey")); - delete.deleteFamily(Bytes.toBytes("family_group")); + Delete delete = new Delete(Bytes.toBytes(key)); + delete.deleteFamily(Bytes.toBytes(family)); final BufferedMutator closedMutator = delBufferedMutator; Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); @@ -309,24 +312,29 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); BufferedMutator putBufferMutator = null; BufferedMutator delBufferedMutator = null; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + String family = "family_group"; + long timestamp = System.currentTimeMillis(); try { // use n1 database TableName tableName = TableName.valueOf("n1","test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // use defualt params putBufferMutator = connection.getBufferedMutator(tableName); delBufferedMutator = connection.getBufferedMutator(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - // only support Put and Delete // for other type of operations, BufferedMutator will not set its type for them Append append = new Append(Bytes.toBytes(key)); - append.add("family_group".getBytes(), column1.getBytes(), toBytes("_suffix")); + append.add(toBytes(family), toBytes(column1), toBytes("_suffix")); final BufferedMutator apMut = putBufferMutator; Assert.assertThrows(IllegalArgumentException.class, () -> { apMut.mutate(append); @@ -335,10 +343,10 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { List mutations = new ArrayList<>(); // test Put Put put1 = new Put(Bytes.toBytes(key)); - put1.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); + put1.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); mutations.add(put1); Put put2 = new Put(Bytes.toBytes(key)); - put2.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); + put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); mutations.add(put2); // test add Mutations with List putBufferMutator.mutate(mutations); @@ -354,7 +362,7 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { Assert.assertThrows(FeatureNotSupportedException.class, () -> { noCfMut.mutate(del); }); - del.deleteFamily(Bytes.toBytes("family_group")); + del.deleteFamily(Bytes.toBytes(family)); final BufferedMutator difTypeMut = putBufferMutator; // test reuse different type bufferedMutator Assert.assertThrows(IllegalArgumentException.class, () -> { @@ -410,30 +418,31 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { BufferedMutatorParams params = null; long bufferSize = 45000L; int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); putBufferMutator = connection.getBufferedMutator(params); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - - Delete delete= new Delete(toBytes(key)); - delete.deleteFamily(toBytes("family_group")); - hTable.delete(delete); - List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); for (int j = 0; j < 4; ++j) { Put put = new Put(Bytes.toBytes(key)); - put.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "_" + i + "_" + j), + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "_" + i + "_" + j), timestamp, Bytes.toBytes(value + "_" + i + "_" + j)); mutations.add(put); } @@ -449,14 +458,14 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { } finally { if (putBufferMutator != null) { putBufferMutator.close(); - Get get = new Get(toBytes("putKey")); + Get get = new Get(toBytes(key)); Result r = hTable.get(get); for (KeyValue keyValue : r.raw()) { ++count; } Assert.assertEquals(200, count); - Delete delete = new Delete(toBytes("putKey")); - delete.deleteFamily(toBytes("family_group")); + Delete delete = new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); hTable.delete(delete); r = hTable.get(get); Assert.assertEquals(0, r.raw().length); @@ -496,10 +505,20 @@ public void testBufferedMutatorWithUserPool() throws Exception { BufferedMutatorParams params = null; long bufferSize = 45000L; int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); @@ -512,21 +531,12 @@ public void testBufferedMutatorWithUserPool() throws Exception { ohBufferMutator = connection.getBufferedMutator(params); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - - Delete delete= new Delete(toBytes(key)); - delete.deleteFamily(toBytes("family_group")); - hTable.delete(delete); - List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); for (int j = 0; j < 4; ++j) { Put put = new Put(Bytes.toBytes(key)); - put.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "_" + i + "_" + j), + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "_" + i + "_" + j), timestamp, Bytes.toBytes(value + "_" + i + "_" + j)); mutations.add(put); } @@ -542,14 +552,14 @@ public void testBufferedMutatorWithUserPool() throws Exception { } finally { if (ohBufferMutator != null) { ohBufferMutator.close(); - Get get = new Get(toBytes("putKey")); + Get get = new Get(toBytes(key)); Result r = hTable.get(get); for (KeyValue keyValue : r.raw()) { ++count; } Assert.assertEquals(200, count); - Delete delete = new Delete(toBytes("putKey")); - delete.deleteFamily(toBytes("family_group")); + Delete delete = new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); hTable.delete(delete); r = hTable.get(get); @@ -589,10 +599,20 @@ public void testBufferedMutatorConcurrent() throws Exception { ExecutorService executorService = Executors.newFixedThreadPool(10); long bufferSize = 45000L; int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); @@ -605,15 +625,6 @@ public void testBufferedMutatorConcurrent() throws Exception { ohBufferMutator = connection.getBufferedMutator(params); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - - Delete delete= new Delete(toBytes(key)); - delete.deleteFamily(toBytes("family_group")); - hTable.delete(delete); - for (int i = 0; i < 50; ++i) { final int taskId = i; final BufferedMutator thrBufferMutator = ohBufferMutator; @@ -626,7 +637,7 @@ public void testBufferedMutatorConcurrent() throws Exception { long thrTimestamp = timestamp; Put put = new Put(Bytes.toBytes(thrKey)); - put.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(thrColumn), + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(thrColumn), thrTimestamp, Bytes.toBytes(thrValue)); mutations.add(put); } @@ -661,14 +672,14 @@ public void testBufferedMutatorConcurrent() throws Exception { } if (ohBufferMutator != null) { ohBufferMutator.close(); - Get get = new Get(toBytes("putKey")); + Get get = new Get(toBytes(key)); Result r = hTable.get(get); for (KeyValue keyValue : r.raw()) { ++count; } Assert.assertEquals(200, count); - Delete delete = new Delete(toBytes("putKey")); - delete.deleteFamily(toBytes("family_group")); + Delete delete = new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); hTable.delete(delete); r = hTable.get(get); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java new file mode 100644 index 00000000..2ed4c190 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java @@ -0,0 +1,348 @@ +package com.alipay.oceanbase.hbase; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; +import org.junit.rules.ExpectedException; + +import java.io.IOException; + +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + +public class OHTableDeleteFamilyVersionTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + protected HTableInterface hTable; + private static final String key1 = "scanKey1x"; + private static final String key2 = "scanKey2x"; + private static final String key3 = "scanKey3x"; + private static final String column1 = "column1"; + private static final String column2 = "column2"; + private static final String column3 = "column3"; + private static final String value1 = "value1"; + private static final String value2 = "value2"; + private static final String value3 = "value3"; + private static final String family1 = "family_with_group1"; + private static final String family2 = "family_with_group2"; + + @Before + public void before() throws Exception { + hTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); + ((OHTableClient) hTable).init(); + } + + @After + public void finish() throws IOException { + hTable.close(); + } + + public void tryPut(Table hTable, Put put) throws Exception { + hTable.put(put); + Thread.sleep(1); + } + + @Test + public void testDeleteFamilyVerison() throws Exception { + // delete previous data + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family1)); + deleteKey1Family.deleteFamily(toBytes(family2)); + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family1)); + deleteKey2Family.deleteFamily(toBytes(family2)); + Delete deleteKey3Family = new Delete(toBytes(key3)); + deleteKey3Family.deleteFamily(toBytes(family1)); + deleteKey3Family.deleteFamily(toBytes(family2)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); + putKey1Fam1Column1MinTs.add(toBytes(family1), toBytes(column1), minTimeStamp, + toBytes(value1)); + + Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp1, toBytes(value2)); + + Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); + putKey1Fam1Column2MinTs.add(toBytes(family1), toBytes(column2), minTimeStamp, + toBytes(value1)); + + Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts6.add(toBytes(family1), toBytes(column2), timeStamp6, toBytes(value1)); + + Put putKey2Fam1Column3Ts6 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp3, toBytes(value1)); + + tryPut(hTable, putKey1Fam1Column1MinTs); + tryPut(hTable, putKey3Fam1Column1Ts1); + tryPut(hTable, putKey1Fam1Column2MinTs); + tryPut(hTable, putKey1Fam1Column2Ts3); + tryPut(hTable, putKey2Fam1Column2Ts3); + tryPut(hTable, putKey2Fam1Column3Ts1); + tryPut(hTable, putKey3Fam1Column3Ts1); + tryPut(hTable, putKey3Fam1Column2Ts6); + tryPut(hTable, putKey2Fam1Column3Ts6); + + // test DeleteFamilyVersion single cf + Get get = new Get(toBytes(key1)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + Result r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + Delete delKey1MinTs = new Delete(toBytes(key1)); + delKey1MinTs.deleteFamilyVersion(toBytes(family1), minTimeStamp); + hTable.delete(delKey1MinTs); + + get = new Get(toBytes(key1)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts1 = new Delete(toBytes(key3)); + delKey3Ts1.deleteFamilyVersion(toBytes(family1), timeStamp1); + hTable.delete(delKey3Ts1); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey2Ts3 = new Delete(toBytes(key2)); + delKey2Ts3.deleteFamilyVersion(family1.getBytes(), timeStamp3); + hTable.delete(delKey2Ts3); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Scan scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(toBytes(family1)); + scan.setMaxVersions(10); + ResultScanner scanner = hTable.getScanner(scan); + int key1Cnt = 0, key2Cnt = 0, key3Cnt = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (key1.equals(Bytes.toString(kv.getRow()))) { + ++key1Cnt; + } else if (key2.equals(Bytes.toString(kv.getRow()))) { + ++key2Cnt; + } else { + ++key3Cnt; + } + } + } + Assert.assertEquals(1, key1Cnt); + Assert.assertEquals(1, key2Cnt); + Assert.assertEquals(1, key3Cnt); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + + // test DeleteFamilyVersion multiple cf + Put putKey1Fam1Column3Ts6 = new Put(toBytes(key1)); + putKey1Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp6, toBytes(value3)); + + Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column2Ts2.add(toBytes(family2), toBytes(column2), timeStamp2, toBytes(value1)); + + Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column3Ts2.add(toBytes(family2), toBytes(column3), timeStamp2, toBytes(value1)); + + Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts1.add(toBytes(family1), toBytes(column2), timeStamp1, toBytes(value2)); + + Put putKey2Fam1Column2Ts8 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts8.add(toBytes(family1), toBytes(column2), timeStamp8, toBytes(value2)); + + Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam2Column3Ts1.add(toBytes(family2), toBytes(column3), timeStamp3, toBytes(value3)); + + Put putKey2Fam1Column1Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp8, toBytes(value1)); + + Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); + putKey2Fam2Column1Ts3.add(toBytes(family2), toBytes(column1), timeStamp3, toBytes(value2)); + + Put putKey3Fam1Column2Ts9 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts9.add(toBytes(family1), toBytes(column2), timeStamp9, toBytes(value2)); + + Put putKey3Fam2Column3Ts10 = new Put(toBytes(key3)); + putKey3Fam2Column3Ts10 + .add(toBytes(family2), toBytes(column3), timeStamp10, toBytes(value1)); + + Put putKey3Fam2Column1Ts10 = new Put(toBytes(key3)); + putKey3Fam2Column1Ts10 + .add(toBytes(family2), toBytes(column1), timeStamp10, toBytes(value2)); + + Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts2.add(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); + + tryPut(hTable, putKey1Fam1Column3Ts6); + tryPut(hTable, putKey1Fam2Column2Ts2); + tryPut(hTable, putKey1Fam2Column3Ts2); + tryPut(hTable, putKey1Fam1Column2Ts1); + tryPut(hTable, putKey2Fam1Column2Ts8); + tryPut(hTable, putKey2Fam2Column3Ts1); + tryPut(hTable, putKey2Fam1Column1Ts1); + tryPut(hTable, putKey2Fam2Column1Ts3); + tryPut(hTable, putKey3Fam1Column2Ts9); + tryPut(hTable, putKey3Fam2Column3Ts10); + tryPut(hTable, putKey3Fam2Column1Ts10); + tryPut(hTable, putKey3Fam1Column2Ts2); + + Get getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = hTable.get(getKey1); + Assert.assertEquals(4, r.raw().length); + + Get getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = hTable.get(getKey2); + Assert.assertEquals(4, r.raw().length); + + Get getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setMaxVersions(10); + r = hTable.get(getKey3); + Assert.assertEquals(4, r.raw().length); + + Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); + delKey1Ts_6_2.deleteFamilyVersion(toBytes(family1), timeStamp6); + delKey1Ts_6_2.deleteFamilyVersion(toBytes(family2), timeStamp2); + hTable.delete(delKey1Ts_6_2); + + getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = hTable.get(getKey1); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp1, kv.getTimestamp()); + } + + Delete delKey2Ts_8_3 = new Delete(toBytes(key2)); + delKey2Ts_8_3.deleteFamilyVersion(toBytes(family1), timeStamp8); + delKey2Ts_8_3.deleteFamilyVersion(toBytes(family2), timeStamp3); + hTable.delete(delKey2Ts_8_3); + + getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = hTable.get(getKey2); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts_2_10 = new Delete(toBytes(key3)); + delKey3Ts_2_10.deleteFamilyVersion(toBytes(family1), timeStamp2); + delKey3Ts_2_10.deleteFamilyVersion(toBytes(family2), timeStamp10); + hTable.delete(delKey3Ts_2_10); + + getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setMaxVersions(10); + r = hTable.get(getKey3); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp9, kv.getTimestamp()); + } + + scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(toBytes(family1)); + scan.addFamily(toBytes(family2)); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + int ts1Cnt = 0, ts9Cnt = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (kv.getTimestamp() == timeStamp1) { + ++ts1Cnt; + } else if (kv.getTimestamp() == timeStamp9) { + ++ts9Cnt; + } + } + } + Assert.assertEquals(1, ts1Cnt); + Assert.assertEquals(1, ts9Cnt); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java index f0e66604..4fac8402 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java @@ -138,6 +138,39 @@ public void testSingleColumnValueFilter() throws IOException { } } + @Test + public void testSingleColumnValueExcludeFilter() throws IOException { + for (int i = 0; i < ops.length; i++) { + String expect = String + .format( + "SingleColumnValueExcludeFilter('family','qualifier',%s,'binary:value',false,true)", + opFlags[i]); + SingleColumnValueExcludeFilter filter = new SingleColumnValueExcludeFilter( + "family".getBytes(), "qualifier".getBytes(), ops[i], "value".getBytes()); + Assert.assertArrayEquals(expect.getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + } + + @Test + public void testDependentColumnFilter() throws IOException { + DependentColumnFilter filter = new DependentColumnFilter("family".getBytes(), + "qualifier".getBytes()); + String expect = "DependentColumnFilter('family','qualifier',false)"; + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + filter = new DependentColumnFilter("family".getBytes(), "qualifier".getBytes(), true); + expect = "DependentColumnFilter('family','qualifier',true)"; + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + for (int i = 0; i < ops.length; ++i) { + filter = new DependentColumnFilter("family".getBytes(), "qualifier".getBytes(), false, + ops[i], new BinaryComparator("value".getBytes())); + expect = String.format( + "DependentColumnFilter('family','qualifier',false,%s,'binary:value')", opFlags[i]); + Assert.assertArrayEquals(expect.getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + } + @Test public void testPageFilter() throws IOException { PageFilter filter = new PageFilter(128); @@ -188,24 +221,24 @@ public void testMultiRowRangeFilter() throws IOException { @Test public void testInclusiveStopFilter() throws IOException { InclusiveStopFilter filter = new InclusiveStopFilter(Bytes.toBytes("aaa")); - Assert.assertArrayEquals("InclusiveStopFilter('aaa')".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + Assert.assertArrayEquals("InclusiveStopFilter('aaa')".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test public void testColumnRangeFilter() throws IOException { - ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, Bytes.toBytes("b"), false); - Assert.assertArrayEquals("ColumnRangeFilter('a',true,'b',false)".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, + Bytes.toBytes("b"), false); + Assert.assertArrayEquals("ColumnRangeFilter('a',true,'b',false)".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test public void testMultipleColumnPrefixFilter() throws IOException { - byte[][] prefix = { - Bytes.toBytes("a"), - Bytes.toBytes("b"), - Bytes.toBytes("d"), - }; + byte[][] prefix = { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("d"), }; MultipleColumnPrefixFilter filter = new MultipleColumnPrefixFilter(prefix); - Assert.assertArrayEquals("MultipleColumnPrefixFilter('a','b','d')".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + Assert.assertArrayEquals("MultipleColumnPrefixFilter('a','b','d')".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test From 7733e4a54e2b384b1653dd92ff451b6e94a95ef3 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Wed, 23 Oct 2024 11:38:34 +0800 Subject: [PATCH 05/32] remove DeleteFamilyVersion file and move all cases to MultiColumnFamilyTest --- .../hbase/OHTableDeleteFamilyVersionTest.java | 348 ------------------ .../hbase/OHTableMultiColumnFamilyTest.java | 309 ++++++++++++++++ 2 files changed, 309 insertions(+), 348 deletions(-) delete mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java deleted file mode 100644 index 2ed4c190..00000000 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java +++ /dev/null @@ -1,348 +0,0 @@ -package com.alipay.oceanbase.hbase; - -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.*; -import org.junit.rules.ExpectedException; - -import java.io.IOException; - -import static org.apache.hadoop.hbase.util.Bytes.toBytes; - -public class OHTableDeleteFamilyVersionTest { - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - protected HTableInterface hTable; - private static final String key1 = "scanKey1x"; - private static final String key2 = "scanKey2x"; - private static final String key3 = "scanKey3x"; - private static final String column1 = "column1"; - private static final String column2 = "column2"; - private static final String column3 = "column3"; - private static final String value1 = "value1"; - private static final String value2 = "value2"; - private static final String value3 = "value3"; - private static final String family1 = "family_with_group1"; - private static final String family2 = "family_with_group2"; - - @Before - public void before() throws Exception { - hTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); - ((OHTableClient) hTable).init(); - } - - @After - public void finish() throws IOException { - hTable.close(); - } - - public void tryPut(Table hTable, Put put) throws Exception { - hTable.put(put); - Thread.sleep(1); - } - - @Test - public void testDeleteFamilyVerison() throws Exception { - // delete previous data - Delete deleteKey1Family = new Delete(toBytes(key1)); - deleteKey1Family.deleteFamily(toBytes(family1)); - deleteKey1Family.deleteFamily(toBytes(family2)); - Delete deleteKey2Family = new Delete(toBytes(key2)); - deleteKey2Family.deleteFamily(toBytes(family1)); - deleteKey2Family.deleteFamily(toBytes(family2)); - Delete deleteKey3Family = new Delete(toBytes(key3)); - deleteKey3Family.deleteFamily(toBytes(family1)); - deleteKey3Family.deleteFamily(toBytes(family2)); - - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - - long minTimeStamp = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp1 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp2 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp3 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp4 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp5 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp6 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp7 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp8 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp9 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp10 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp11 = System.currentTimeMillis(); - Thread.sleep(5); - long maxTimeStamp = System.currentTimeMillis(); - - Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); - putKey1Fam1Column1MinTs.add(toBytes(family1), toBytes(column1), minTimeStamp, - toBytes(value1)); - - Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); - putKey3Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp1, toBytes(value2)); - - Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); - putKey1Fam1Column2MinTs.add(toBytes(family1), toBytes(column2), minTimeStamp, - toBytes(value1)); - - Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); - putKey1Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); - - Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); - putKey2Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); - - Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); - putKey2Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); - - Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); - putKey3Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); - - Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts6.add(toBytes(family1), toBytes(column2), timeStamp6, toBytes(value1)); - - Put putKey2Fam1Column3Ts6 = new Put(toBytes(key2)); - putKey2Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp3, toBytes(value1)); - - tryPut(hTable, putKey1Fam1Column1MinTs); - tryPut(hTable, putKey3Fam1Column1Ts1); - tryPut(hTable, putKey1Fam1Column2MinTs); - tryPut(hTable, putKey1Fam1Column2Ts3); - tryPut(hTable, putKey2Fam1Column2Ts3); - tryPut(hTable, putKey2Fam1Column3Ts1); - tryPut(hTable, putKey3Fam1Column3Ts1); - tryPut(hTable, putKey3Fam1Column2Ts6); - tryPut(hTable, putKey2Fam1Column3Ts6); - - // test DeleteFamilyVersion single cf - Get get = new Get(toBytes(key1)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(minTimeStamp); - get.setMaxVersions(10); - Result r = hTable.get(get); - Assert.assertEquals(2, r.raw().length); - - get = new Get(toBytes(key3)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp1); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(2, r.raw().length); - - get = new Get(toBytes(key2)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp3); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(2, r.raw().length); - - Delete delKey1MinTs = new Delete(toBytes(key1)); - delKey1MinTs.deleteFamilyVersion(toBytes(family1), minTimeStamp); - hTable.delete(delKey1MinTs); - - get = new Get(toBytes(key1)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(minTimeStamp); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Delete delKey3Ts1 = new Delete(toBytes(key3)); - delKey3Ts1.deleteFamilyVersion(toBytes(family1), timeStamp1); - hTable.delete(delKey3Ts1); - - get = new Get(toBytes(key3)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp1); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Delete delKey2Ts3 = new Delete(toBytes(key2)); - delKey2Ts3.deleteFamilyVersion(family1.getBytes(), timeStamp3); - hTable.delete(delKey2Ts3); - - get = new Get(toBytes(key2)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp3); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Scan scan = new Scan(); - scan.setStartRow(toBytes(key1)); - scan.setStopRow("scanKey4x".getBytes()); - scan.addFamily(toBytes(family1)); - scan.setMaxVersions(10); - ResultScanner scanner = hTable.getScanner(scan); - int key1Cnt = 0, key2Cnt = 0, key3Cnt = 0; - for (Result result : scanner) { - for (KeyValue kv : result.raw()) { - if (key1.equals(Bytes.toString(kv.getRow()))) { - ++key1Cnt; - } else if (key2.equals(Bytes.toString(kv.getRow()))) { - ++key2Cnt; - } else { - ++key3Cnt; - } - } - } - Assert.assertEquals(1, key1Cnt); - Assert.assertEquals(1, key2Cnt); - Assert.assertEquals(1, key3Cnt); - - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - - // test DeleteFamilyVersion multiple cf - Put putKey1Fam1Column3Ts6 = new Put(toBytes(key1)); - putKey1Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp6, toBytes(value3)); - - Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); - putKey1Fam2Column2Ts2.add(toBytes(family2), toBytes(column2), timeStamp2, toBytes(value1)); - - Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); - putKey1Fam2Column3Ts2.add(toBytes(family2), toBytes(column3), timeStamp2, toBytes(value1)); - - Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); - putKey1Fam1Column2Ts1.add(toBytes(family1), toBytes(column2), timeStamp1, toBytes(value2)); - - Put putKey2Fam1Column2Ts8 = new Put(toBytes(key2)); - putKey2Fam1Column2Ts8.add(toBytes(family1), toBytes(column2), timeStamp8, toBytes(value2)); - - Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); - putKey2Fam2Column3Ts1.add(toBytes(family2), toBytes(column3), timeStamp3, toBytes(value3)); - - Put putKey2Fam1Column1Ts1 = new Put(toBytes(key2)); - putKey2Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp8, toBytes(value1)); - - Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); - putKey2Fam2Column1Ts3.add(toBytes(family2), toBytes(column1), timeStamp3, toBytes(value2)); - - Put putKey3Fam1Column2Ts9 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts9.add(toBytes(family1), toBytes(column2), timeStamp9, toBytes(value2)); - - Put putKey3Fam2Column3Ts10 = new Put(toBytes(key3)); - putKey3Fam2Column3Ts10 - .add(toBytes(family2), toBytes(column3), timeStamp10, toBytes(value1)); - - Put putKey3Fam2Column1Ts10 = new Put(toBytes(key3)); - putKey3Fam2Column1Ts10 - .add(toBytes(family2), toBytes(column1), timeStamp10, toBytes(value2)); - - Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts2.add(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); - - tryPut(hTable, putKey1Fam1Column3Ts6); - tryPut(hTable, putKey1Fam2Column2Ts2); - tryPut(hTable, putKey1Fam2Column3Ts2); - tryPut(hTable, putKey1Fam1Column2Ts1); - tryPut(hTable, putKey2Fam1Column2Ts8); - tryPut(hTable, putKey2Fam2Column3Ts1); - tryPut(hTable, putKey2Fam1Column1Ts1); - tryPut(hTable, putKey2Fam2Column1Ts3); - tryPut(hTable, putKey3Fam1Column2Ts9); - tryPut(hTable, putKey3Fam2Column3Ts10); - tryPut(hTable, putKey3Fam2Column1Ts10); - tryPut(hTable, putKey3Fam1Column2Ts2); - - Get getKey1 = new Get(toBytes(key1)); - getKey1.addFamily(toBytes(family1)); - getKey1.addFamily(toBytes(family2)); - getKey1.setMaxVersions(10); - r = hTable.get(getKey1); - Assert.assertEquals(4, r.raw().length); - - Get getKey2 = new Get(toBytes(key2)); - getKey2.addFamily(toBytes(family1)); - getKey2.addFamily(toBytes(family2)); - getKey2.setMaxVersions(10); - r = hTable.get(getKey2); - Assert.assertEquals(4, r.raw().length); - - Get getKey3 = new Get(toBytes(key3)); - getKey3.addFamily(toBytes(family1)); - getKey3.addFamily(toBytes(family2)); - getKey3.setMaxVersions(10); - r = hTable.get(getKey3); - Assert.assertEquals(4, r.raw().length); - - Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); - delKey1Ts_6_2.deleteFamilyVersion(toBytes(family1), timeStamp6); - delKey1Ts_6_2.deleteFamilyVersion(toBytes(family2), timeStamp2); - hTable.delete(delKey1Ts_6_2); - - getKey1 = new Get(toBytes(key1)); - getKey1.addFamily(toBytes(family1)); - getKey1.addFamily(toBytes(family2)); - getKey1.setMaxVersions(10); - r = hTable.get(getKey1); - Assert.assertEquals(1, r.raw().length); - for (KeyValue kv : r.raw()) { - Assert.assertEquals(timeStamp1, kv.getTimestamp()); - } - - Delete delKey2Ts_8_3 = new Delete(toBytes(key2)); - delKey2Ts_8_3.deleteFamilyVersion(toBytes(family1), timeStamp8); - delKey2Ts_8_3.deleteFamilyVersion(toBytes(family2), timeStamp3); - hTable.delete(delKey2Ts_8_3); - - getKey2 = new Get(toBytes(key2)); - getKey2.addFamily(toBytes(family1)); - getKey2.addFamily(toBytes(family2)); - getKey2.setMaxVersions(10); - r = hTable.get(getKey2); - Assert.assertEquals(0, r.raw().length); - - Delete delKey3Ts_2_10 = new Delete(toBytes(key3)); - delKey3Ts_2_10.deleteFamilyVersion(toBytes(family1), timeStamp2); - delKey3Ts_2_10.deleteFamilyVersion(toBytes(family2), timeStamp10); - hTable.delete(delKey3Ts_2_10); - - getKey3 = new Get(toBytes(key3)); - getKey3.addFamily(toBytes(family1)); - getKey3.addFamily(toBytes(family2)); - getKey3.setMaxVersions(10); - r = hTable.get(getKey3); - Assert.assertEquals(1, r.raw().length); - for (KeyValue kv : r.raw()) { - Assert.assertEquals(timeStamp9, kv.getTimestamp()); - } - - scan = new Scan(); - scan.setStartRow(toBytes(key1)); - scan.setStopRow("scanKey4x".getBytes()); - scan.addFamily(toBytes(family1)); - scan.addFamily(toBytes(family2)); - scan.setMaxVersions(10); - scanner = hTable.getScanner(scan); - int ts1Cnt = 0, ts9Cnt = 0; - for (Result result : scanner) { - for (KeyValue kv : result.raw()) { - if (kv.getTimestamp() == timeStamp1) { - ++ts1Cnt; - } else if (kv.getTimestamp() == timeStamp9) { - ++ts9Cnt; - } - } - } - Assert.assertEquals(1, ts1Cnt); - Assert.assertEquals(1, ts9Cnt); - - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - } -} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java index f8d4eed3..0bf4b93f 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.util.Bytes; import org.junit.*; import org.junit.rules.ExpectedException; @@ -610,6 +611,13 @@ public void testMultiColumnFamilyGet() throws Exception { @Test public void testMultiColumnFamilyDelete() throws Exception { + String key1 = "scanKey1x"; + String key2 = "scanKey2x"; + String key3 = "scanKey3x"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + byte[] family1 = "family_with_group1".getBytes(); byte[] family2 = "family_with_group2".getBytes(); byte[] family3 = "family_with_group3".getBytes(); @@ -619,6 +627,7 @@ public void testMultiColumnFamilyDelete() throws Exception { byte[] family1_column3 = "family1_column3".getBytes(); byte[] family2_column1 = "family2_column1".getBytes(); byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family2_column3 = "family2_column3".getBytes(); byte[] family3_column1 = "family3_column1".getBytes(); byte[] family1_value = "VVV1".getBytes(); byte[] family2_value = "VVV2".getBytes(); @@ -736,5 +745,305 @@ public void testMultiColumnFamilyDelete() throws Exception { assertEquals(oldTimestamp, result.getColumnCells(family2, family2_column1).get(0) .getTimestamp()); assertTrue(lastTimestamp > oldTimestamp); + + // delete previous data + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(family1); + deleteKey1Family.deleteFamily(family2); + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(family1); + deleteKey2Family.deleteFamily(family2); + Delete deleteKey3Family = new Delete(toBytes(key3)); + deleteKey3Family.deleteFamily(family1); + deleteKey3Family.deleteFamily(family2); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); + putKey1Fam1Column1MinTs.add(family1, family1_column1, minTimeStamp, + toBytes(value1)); + + Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column1Ts1.add(family1, family1_column1, timeStamp1, toBytes(value2)); + + Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); + putKey1Fam1Column2MinTs.add(family1, family1_column2, minTimeStamp, + toBytes(value1)); + + Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts3.add(family1, family1_column2, timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts3.add(family1, family1_column2, timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts1.add(family1, family1_column3, timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column3Ts1.add(family1, family1_column3, timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts6.add(family1, family1_column2, timeStamp6, toBytes(value1)); + + Put putKey2Fam1Column3Ts6 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts6.add(family1, family1_column3, timeStamp3, toBytes(value1)); + + hTable.put(putKey1Fam1Column1MinTs); + hTable.put(putKey3Fam1Column1Ts1); + hTable.put(putKey1Fam1Column2MinTs); + hTable.put(putKey1Fam1Column2Ts3); + hTable.put(putKey2Fam1Column2Ts3); + hTable.put(putKey2Fam1Column3Ts1); + hTable.put(putKey3Fam1Column3Ts1); + hTable.put(putKey3Fam1Column2Ts6); + hTable.put(putKey2Fam1Column3Ts6); + + // test DeleteFamilyVersion single cf + get = new Get(toBytes(key1)); + get.addFamily(family1); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + Result r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key3)); + get.addFamily(family1); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key2)); + get.addFamily(family1); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + Delete delKey1MinTs = new Delete(toBytes(key1)); + delKey1MinTs.deleteFamilyVersion(family1, minTimeStamp); + hTable.delete(delKey1MinTs); + + get = new Get(toBytes(key1)); + get.addFamily(family1); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts1 = new Delete(toBytes(key3)); + delKey3Ts1.deleteFamilyVersion(family1, timeStamp1); + hTable.delete(delKey3Ts1); + + get = new Get(toBytes(key3)); + get.addFamily(family1); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey2Ts3 = new Delete(toBytes(key2)); + delKey2Ts3.deleteFamilyVersion(family1, timeStamp3); + hTable.delete(delKey2Ts3); + + get = new Get(toBytes(key2)); + get.addFamily(family1); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Scan scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(family1); + scan.setMaxVersions(10); + ResultScanner scanner = hTable.getScanner(scan); + int key1Cnt = 0, key2Cnt = 0, key3Cnt = 0; + for (Result res : scanner) { + for (KeyValue kv : res.raw()) { + if (key1.equals(Bytes.toString(kv.getRow()))) { + ++key1Cnt; + } else if (key2.equals(Bytes.toString(kv.getRow()))) { + ++key2Cnt; + } else { + ++key3Cnt; + } + } + } + Assert.assertEquals(1, key1Cnt); + Assert.assertEquals(1, key2Cnt); + Assert.assertEquals(1, key3Cnt); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + + // test DeleteFamilyVersion multiple cf + Put putKey1Fam1Column3Ts6 = new Put(toBytes(key1)); + putKey1Fam1Column3Ts6.add(family1, family1_column3, timeStamp6, toBytes(value3)); + + Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column2Ts2.add(family2, family2_column2, timeStamp2, toBytes(value1)); + + Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column3Ts2.add(family2, family2_column3, timeStamp2, toBytes(value1)); + + Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts1.add(family1, family1_column2, timeStamp1, toBytes(value2)); + + Put putKey2Fam1Column2Ts8 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts8.add(family1, family1_column2, timeStamp8, toBytes(value2)); + + Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam2Column3Ts1.add(family2, family2_column3, timeStamp3, toBytes(value3)); + + Put putKey2Fam1Column1Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column1Ts1.add(family1, family1_column1, timeStamp8, toBytes(value1)); + + Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); + putKey2Fam2Column1Ts3.add(family2, family2_column1, timeStamp3, toBytes(value2)); + + Put putKey3Fam1Column2Ts9 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts9.add(family1, family1_column2, timeStamp9, toBytes(value2)); + + Put putKey3Fam2Column3Ts10 = new Put(toBytes(key3)); + putKey3Fam2Column3Ts10 + .add(family2, family2_column3, timeStamp10, toBytes(value1)); + + Put putKey3Fam2Column1Ts10 = new Put(toBytes(key3)); + putKey3Fam2Column1Ts10 + .add(family2, family2_column1, timeStamp10, toBytes(value2)); + + Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts2.add(family1, family1_column2, timeStamp2, toBytes(value1)); + + hTable.put(putKey1Fam1Column3Ts6); + hTable.put(putKey1Fam2Column2Ts2); + hTable.put(putKey1Fam2Column3Ts2); + hTable.put(putKey1Fam1Column2Ts1); + hTable.put(putKey2Fam1Column2Ts8); + hTable.put(putKey2Fam2Column3Ts1); + hTable.put(putKey2Fam1Column1Ts1); + hTable.put(putKey2Fam2Column1Ts3); + hTable.put(putKey3Fam1Column2Ts9); + hTable.put(putKey3Fam2Column3Ts10); + hTable.put(putKey3Fam2Column1Ts10); + hTable.put(putKey3Fam1Column2Ts2); + + Get getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(family1); + getKey1.addFamily(family2); + getKey1.setMaxVersions(10); + r = hTable.get(getKey1); + Assert.assertEquals(4, r.raw().length); + + Get getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(family1); + getKey2.addFamily(family2); + getKey2.setMaxVersions(10); + r = hTable.get(getKey2); + Assert.assertEquals(4, r.raw().length); + + Get getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(family1); + getKey3.addFamily(family2); + getKey3.setMaxVersions(10); + r = hTable.get(getKey3); + Assert.assertEquals(4, r.raw().length); + + Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); + delKey1Ts_6_2.deleteFamilyVersion(family1, timeStamp6); + delKey1Ts_6_2.deleteFamilyVersion(family2, timeStamp2); + hTable.delete(delKey1Ts_6_2); + + getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(family1); + getKey1.addFamily(family2); + getKey1.setMaxVersions(10); + r = hTable.get(getKey1); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp1, kv.getTimestamp()); + } + + Delete delKey2Ts_8_3 = new Delete(toBytes(key2)); + delKey2Ts_8_3.deleteFamilyVersion(family1, timeStamp8); + delKey2Ts_8_3.deleteFamilyVersion(family2, timeStamp3); + hTable.delete(delKey2Ts_8_3); + + getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(family1); + getKey2.addFamily(family2); + getKey2.setMaxVersions(10); + r = hTable.get(getKey2); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts_2_10 = new Delete(toBytes(key3)); + delKey3Ts_2_10.deleteFamilyVersion(family1, timeStamp2); + delKey3Ts_2_10.deleteFamilyVersion(family2, timeStamp10); + hTable.delete(delKey3Ts_2_10); + + getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(family1); + getKey3.addFamily(family2); + getKey3.setMaxVersions(10); + r = hTable.get(getKey3); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp9, kv.getTimestamp()); + } + + scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(family1); + scan.addFamily(family2); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + int ts1Cnt = 0, ts9Cnt = 0; + for (Result res : scanner) { + for (KeyValue kv : res.raw()) { + if (kv.getTimestamp() == timeStamp1) { + ++ts1Cnt; + } else if (kv.getTimestamp() == timeStamp9) { + ++ts9Cnt; + } + } + } + Assert.assertEquals(1, ts1Cnt); + Assert.assertEquals(1, ts9Cnt); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); } } From 34114912efc1876b7a8b9cf81de061f08792e061 Mon Sep 17 00:00:00 2001 From: stuBirdFly <84010733+stuBirdFly@users.noreply.github.com> Date: Wed, 23 Oct 2024 11:42:52 +0800 Subject: [PATCH 06/32] hbase support batchCallBack (#86) --- .../com/alipay/oceanbase/hbase/OHTable.java | 23 +++++++++++---- .../hbase/OHTableMultiColumnFamilyTest.java | 28 +++++++++++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 6d1cc873..4712527b 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -526,15 +526,26 @@ public Object[] batch(List actions) throws IOException { @Override public void batchCallback(List actions, Object[] results, Batch.Callback callback) throws IOException, - InterruptedException { - throw new FeatureNotSupportedException("not supported yet'"); + InterruptedException { + try { + batch(actions, results); + } finally { + if (results != null) { + for (int i = 0; i < results.length; i++) { + if (!(results[i] instanceof ObTableException)) { + callback.update(null, actions.get(i).getRow(), (R) results[i]); + } + } + } + } } @Override - public Object[] batchCallback(List actions, Batch.Callback callback) - throws IOException, - InterruptedException { - throw new FeatureNotSupportedException("not supported yet'"); + public Object[] batchCallback( + final List actions, final Batch.Callback callback) throws IOException, InterruptedException { + Object[] results = new Object[actions.size()]; + batchCallback(actions, results, callback); + return results; } public static int compareByteArray(byte[] bt1, byte[] bt2) { diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java index f8d4eed3..a649e58e 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java @@ -17,8 +17,10 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.filter.PrefixFilter; import org.junit.*; import org.junit.rules.ExpectedException; @@ -146,6 +148,32 @@ public void testMultiColumnFamilyBatch() throws Exception { result = hTable.get(get); keyValues = result.raw(); assertEquals(6, keyValues.length); + + batchLsit.clear(); + final long[] updateCounter = new long[] { 0L }; + delete = new Delete(toBytes("Key5")); + delete.deleteColumns(family1, family1_column2); + delete.deleteColumns(family2, family2_column1); + delete.deleteFamily(family3); + batchLsit.add(delete); + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + batchLsit.add(put); + } + hTable.batchCallback(batchLsit, new Batch.Callback() { + @Override + public void update(byte[] region, byte[] row, MutationResult result) { + updateCounter[0]++; + } + }); + assertEquals(11, updateCounter[0]); + } @Test From 9c6be3f6cf7686b611fa2501b4d44212faaa382d Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Wed, 23 Oct 2024 21:36:37 +0800 Subject: [PATCH 07/32] adjust bufferdMutatorImpl 1.x to new batch --- .../com/alipay/oceanbase/hbase/OHTable.java | 18 ++ .../hbase/util/OHBufferedMutatorImpl.java | 184 ++++++--------- .../oceanbase/hbase/OHConnectionTest.java | 198 +++++++++------- .../hbase/OHTableMultiColumnFamilyTest.java | 217 ++++++++++++++++-- 4 files changed, 409 insertions(+), 208 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 6d1cc873..88ba3dc4 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -198,8 +198,13 @@ public OHTable(Configuration configuration, String tableName) throws IOException DEFAULT_HBASE_HTABLE_THREAD_KEEP_ALIVE_TIME); this.executePool = createDefaultThreadPoolExecutor(1, maxThreads, keepAliveTime); OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); + int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, ohConnectionConf)); + this.obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); + this.obTableClient.setRuntimeRetryTimes(numRetries); + setOperationTimeout(ohConnectionConf.getOperationTimeout()); finishSetUp(); } @@ -246,8 +251,13 @@ public OHTable(Configuration configuration, final byte[] tableName, this.executePool = executePool; this.cleanupPoolOnClose = false; OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); + int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, ohConnectionConf)); + this.obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); + this.obTableClient.setRuntimeRetryTimes(numRetries); + setOperationTimeout(ohConnectionConf.getOperationTimeout()); finishSetUp(); } @@ -301,6 +311,7 @@ public OHTable(TableName tableName, Connection connection, } else { this.cleanupPoolOnClose = false; } + this.rpcTimeout = connectionConfig.getRpcTimeout(); this.operationTimeout = connectionConfig.getOperationTimeout(); this.operationExecuteInPool = this.configuration.getBoolean( @@ -311,8 +322,15 @@ public OHTable(TableName tableName, Connection connection, DEFAULT_HBASE_HTABLE_PUT_WRITE_BUFFER_CHECK); this.writeBufferSize = connectionConfig.getWriteBufferSize(); this.tableName = tableName.getName(); + int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, connectionConfig)); + this.obTableClient.setRpcExecuteTimeout(rpcTimeout); + this.obTableClient.setRuntimeRetryTimes(numRetries); + setOperationTimeout(operationTimeout); + + finishSetUp(); } /** diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index 0cef7e62..30a18170 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -18,14 +18,11 @@ package com.alipay.oceanbase.hbase.util; import com.alipay.oceanbase.hbase.OHTable; -import com.alipay.oceanbase.rpc.ObTableClient; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.*; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger; import java.io.IOException; @@ -34,7 +31,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.LCD; @InterfaceAudience.Private @@ -44,44 +40,40 @@ public class OHBufferedMutatorImpl implements BufferedMutator { private final ExceptionListener listener; - protected final ObTableClient obTableClient; + private final OHTable ohTable; private final TableName tableName; private volatile Configuration conf; private final OHConnectionConfiguration connectionConfig; + @VisibleForTesting final ConcurrentLinkedQueue asyncWriteBuffer = new ConcurrentLinkedQueue(); + @VisibleForTesting AtomicLong currentAsyncBufferSize = new AtomicLong(0); - private AtomicReference> type = new AtomicReference<>(null); private final long writeBufferSize; private final int maxKeyValueSize; private boolean closed = false; private final ExecutorService pool; - private final int rpcTimeout; public OHBufferedMutatorImpl(OHConnectionImpl ohConnection, BufferedMutatorParams params) throws IOException { if (ohConnection == null || ohConnection.isClosed()) { throw new IllegalArgumentException("Connection is null or closed."); } - // create a ObTableClient to do rpc operations - this.obTableClient = ObTableClientManager.getOrCreateObTableClient(ohConnection - .getOHConnectionConfiguration()); - // init params in OHBufferedMutatorImpl this.tableName = params.getTableName(); this.conf = ohConnection.getConfiguration(); this.connectionConfig = ohConnection.getOHConnectionConfiguration(); this.listener = params.getListener(); this.pool = params.getPool(); - this.obTableClient.setRuntimeBatchExecutor(pool); this.writeBufferSize = params.getWriteBufferSize() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params .getWriteBufferSize() : connectionConfig.getWriteBufferSize(); this.maxKeyValueSize = params.getMaxKeyValueSize() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params .getMaxKeyValueSize() : connectionConfig.getMaxKeyValueSize(); - this.rpcTimeout = connectionConfig.getRpcTimeout(); - this.obTableClient.setRpcExecuteTimeout(rpcTimeout); + + // create an OHTable object to do batch work + this.ohTable = new OHTable(tableName, ohConnection, connectionConfig, pool); } @Override @@ -119,26 +111,18 @@ public void mutate(List mutations) throws IOException { } long toAddSize = 0; - // check if every mutation's family is the same - // check if mutations are the same type for (Mutation m : mutations) { - OHTable.checkFamilyViolation(m.getFamilyMap().keySet(), true); validateInsUpAndDelete(m); - Class curType = m.getClass(); - // set the type of this BufferedMutator - if (type.get() == null) { - type.compareAndSet(null, mutations.get(0).getClass()); - } - if (!type.get().equals(curType)) { - throw new IllegalArgumentException("Not support different type in one batch."); - } toAddSize += m.heapSize(); } currentAsyncBufferSize.addAndGet(toAddSize); asyncWriteBuffer.addAll(mutations); - asyncExecute(false); + if (currentAsyncBufferSize.get() > writeBufferSize) { + execute(false); + } + } /** @@ -150,7 +134,11 @@ private void validateInsUpAndDelete(Mutation mt) throws IllegalArgumentException throw new IllegalArgumentException("Only support for Put and Delete for now."); } if (mt instanceof Put) { + // family empty check is in validatePut HTable.validatePut((Put) mt, maxKeyValueSize); + OHTable.checkFamilyViolation(mt.getFamilyMap().keySet(), true); + } else { + OHTable.checkFamilyViolation(mt.getFamilyMap().keySet(), false); } } @@ -161,91 +149,65 @@ private void validateInsUpAndDelete(Mutation mt) throws IllegalArgumentException * @param flushAll - if true, sends all the writes and wait for all of them to finish before * returning. */ - private void asyncExecute(boolean flushAll) throws IOException { + private void execute(boolean flushAll) throws IOException { LinkedList execBuffer = new LinkedList<>(); - ObTableBatchOperationRequest request = null; - // namespace n1, n1:table_name - // namespace default, table_name - String tableNameString = tableName.getNameAsString(); + long dequeuedSize = 0L; try { - while (true) { - try{ - if (!flushAll || asyncWriteBuffer.isEmpty()) { - if (currentAsyncBufferSize.get() <= writeBufferSize) { - break; - } - } - Mutation m; - while ((m = asyncWriteBuffer.poll()) != null) { - execBuffer.add(m); - long size = m.heapSize(); - currentAsyncBufferSize.addAndGet(-size); - } - // in concurrent situation, asyncWriteBuffer may be empty here - // for other threads flush all buffer - if (execBuffer.isEmpty()) { - break; - } - // for now, operations' family is the same - byte[] family = execBuffer.getFirst().getFamilyMap().firstKey(); - ObTableBatchOperation batch = buildObTableBatchOperation(execBuffer); - // table_name$cf_name - String targetTableName = OHTable.getTargetTableName(tableNameString, Bytes.toString(family), conf); - request = OHTable.buildObTableBatchOperationRequest(batch, targetTableName); - } catch (Exception ex) { - LOGGER.error("Errors occur before mutation operation", ex); - throw new IllegalArgumentException("Errors occur before mutation operation", ex); - } - try { - ObTableBatchOperationResult result = (ObTableBatchOperationResult) obTableClient.execute(request); - } catch (Exception ex) { - LOGGER.debug("Errors occur during mutation operation", ex); - Mutation m = null; - try { - // retry every single operation - while (!execBuffer.isEmpty()) { - // poll elements from execBuffer to recollect remaining operations - m = execBuffer.poll(); - byte[] family = m.getFamilyMap().firstKey(); - ObTableBatchOperation batch = buildObTableBatchOperation(Collections.singletonList(m)); - String targetTableName = OHTable.getTargetTableName(tableNameString, Bytes.toString(family), conf); - request = OHTable.buildObTableBatchOperationRequest(batch, targetTableName); - ObTableBatchOperationResult result = (ObTableBatchOperationResult) obTableClient.execute(request); - } - } catch (Exception newEx) { - if (m != null) { - execBuffer.addFirst(m); - } - // if retry fails, only recollect remaining operations - while(!execBuffer.isEmpty()) { - m = execBuffer.poll(); - long size = m.heapSize(); - asyncWriteBuffer.add(m); - currentAsyncBufferSize.addAndGet(size); - } - throw newEx; + Mutation m; + while ((writeBufferSize <= 0 || dequeuedSize < (writeBufferSize * 2) || flushAll) + && (m = asyncWriteBuffer.poll()) != null) { + execBuffer.add(m); + long size = m.heapSize(); + currentAsyncBufferSize.addAndGet(-size); + dequeuedSize += size; + } + + if (execBuffer.isEmpty()) { + return; + } + try { + ohTable.batch(execBuffer); + } catch (Exception ex) { + if (ex.getCause() instanceof RetriesExhaustedWithDetailsException) { + // retry one time + LOGGER.debug(tableName + ": One or more of the operations have failed -" + + " waiting for all operations to complete (successfully or not) and retries"); + RetriesExhaustedWithDetailsException retryException = (RetriesExhaustedWithDetailsException) ex.getCause(); + execBuffer.clear(); + for (int i = 0; i < retryException.getNumExceptions(); ++i) { + execBuffer.add((Mutation) retryException.getRow(i)); } + ohTable.batch(execBuffer); + } else { + LOGGER.error("Errors unrelated to operations occur during mutation operation", ex); + throw ex; } } } catch (Exception ex) { LOGGER.error(LCD.convert("01-00026"), ex); - // if the cause is illegal argument, directly throw to user - if (ex instanceof IllegalArgumentException) { - throw (IllegalArgumentException) ex; - } - // TODO: need to collect error information and actions during batch operations - // TODO: maybe keep in ObTableBatchOperationResult - List throwables = new ArrayList(); - List actions = new ArrayList(); - List addresses = new ArrayList(); - throwables.add(ex); - RetriesExhaustedWithDetailsException error = new RetriesExhaustedWithDetailsException( - new ArrayList(throwables), - new ArrayList(actions), new ArrayList(addresses)); - if (listener == null) { - throw error; + if (ex.getCause() instanceof RetriesExhaustedWithDetailsException) { + // retry one time + LOGGER.error(tableName + ": One or more of the operations have failed after retry."); + RetriesExhaustedWithDetailsException retryException = (RetriesExhaustedWithDetailsException) ex.getCause(); + // recollect mutations + execBuffer.clear(); + for (int i = 0; i < retryException.getNumExceptions(); ++i) { + execBuffer.add((Mutation) retryException.getRow(i)); + } + if (listener != null) { + listener.onException(retryException, this); + } else { + throw retryException; + } } else { - listener.onException(error, this); + LOGGER.error("Errors unrelated to operations occur during mutation operation", ex); + throw ex; + } + } finally { + for (Mutation mutation : execBuffer) { + long size = mutation.heapSize(); + currentAsyncBufferSize.addAndGet(size); + asyncWriteBuffer.add(mutation); } } } @@ -256,7 +218,7 @@ public void close() throws IOException { return; } try { - asyncExecute(true); + execute(true); } finally { // the pool in ObTableClient will be shut down too this.pool.shutdown(); @@ -279,21 +241,11 @@ public void close() throws IOException { */ @Override public void flush() throws IOException { - asyncExecute(true); + execute(true); } @Override public long getWriteBufferSize() { return this.writeBufferSize; } - - private ObTableBatchOperation buildObTableBatchOperation(List execBuffer) { - List keyValueList = new LinkedList<>(); - for (Mutation mutation : execBuffer) { - for (Map.Entry> entry : mutation.getFamilyMap().entrySet()) { - keyValueList.addAll(entry.getValue()); - } - } - return OHTable.buildObTableBatchOperation(keyValueList, false, null); - } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index b40b23ea..d3b3fd93 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -28,6 +28,7 @@ import org.junit.Assert; import org.junit.Test; +import java.security.Key; import java.util.ArrayList; import java.util.List; import java.util.concurrent.*; @@ -203,8 +204,7 @@ public void testBufferedMutatorWithFlush() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); conf.set("rs.list.acquire.read.timeout", "10000"); conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); - BufferedMutator putBufferMutator = null; - BufferedMutator delBufferedMutator = null; + BufferedMutator bufferMutator = null; String key = "putKey"; String column1 = "putColumn1"; String value = "value333444"; @@ -219,17 +219,15 @@ public void testBufferedMutatorWithFlush() throws Exception { hTable.delete(delete); // use defualt params - putBufferMutator = connection.getBufferedMutator(tableName); - delBufferedMutator = connection.getBufferedMutator(tableName); + bufferMutator = connection.getBufferedMutator(tableName); long timestamp = System.currentTimeMillis(); // only support Put and Delete - // for other type of operations, BufferedMutator will not set its type for them Append append = new Append(Bytes.toBytes(key)); append.add("family_group".getBytes(), column1.getBytes(), toBytes("_suffix")); - final BufferedMutator apMut = putBufferMutator; + final BufferedMutator apMut = bufferMutator; Assert.assertThrows(IllegalArgumentException.class, () -> { apMut.mutate(append); }); @@ -243,31 +241,46 @@ public void testBufferedMutatorWithFlush() throws Exception { put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); mutations.add(put2); // test add Mutations with List - putBufferMutator.mutate(mutations); - putBufferMutator.flush(); + bufferMutator.mutate(mutations); + bufferMutator.flush(); Get get = new Get(toBytes(key)); Result r = hTable.get(get); Assert.assertEquals(2, r.raw().length); - Delete del = new Delete(Bytes.toBytes(key)); - final BufferedMutator noCfMut = putBufferMutator; - // test mutation without setting family - Assert.assertThrows(FeatureNotSupportedException.class, () -> { - noCfMut.mutate(del); - }); - del.deleteFamily(Bytes.toBytes(family)); - // test reuse different type bufferedMutator - final BufferedMutator difTypeMut = putBufferMutator; + Put put3 = new Put(Bytes.toBytes(key)); + final BufferedMutator noCfMut = bufferMutator; + // test Put without setting family Assert.assertThrows(IllegalArgumentException.class, () -> { - difTypeMut.mutate(del); + noCfMut.mutate(put3); }); + put3.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "2"), timestamp, Bytes.toBytes(value)); // test add Mutation directly - delBufferedMutator.mutate(del); - delBufferedMutator.flush(); + bufferMutator.mutate(put3); + bufferMutator.flush(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + // test Delete + Delete del = new Delete(toBytes(key)); + // test without setting family, delete all + bufferMutator.mutate(del); + bufferMutator.flush(); r = hTable.get(get); Assert.assertEquals(0, r.raw().length); + + // test hybrid mutations + mutations.clear(); + mutations.add(put1); + mutations.add(put2); + mutations.add(del); + mutations.add(put3); + bufferMutator.mutate(mutations); + bufferMutator.flush(); + + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); } catch (Exception ex) { if (ex instanceof RetriesExhaustedWithDetailsException) { ((RetriesExhaustedWithDetailsException) ex).getCauses().get(0).printStackTrace(); @@ -276,17 +289,14 @@ public void testBufferedMutatorWithFlush() throws Exception { } Assert.assertTrue(false); } finally { - if (putBufferMutator != null ) { - putBufferMutator.close(); + if (bufferMutator != null ) { + bufferMutator.close(); // test flush after closed - putBufferMutator.flush(); - } - if (delBufferedMutator != null) { - delBufferedMutator.close(); + bufferMutator.flush(); // test add mutations after closed Delete delete = new Delete(Bytes.toBytes(key)); delete.deleteFamily(Bytes.toBytes(family)); - final BufferedMutator closedMutator = delBufferedMutator; + final BufferedMutator closedMutator = bufferMutator; Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); }); @@ -310,32 +320,31 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); conf.set("rs.list.acquire.read.timeout", "10000"); conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); - BufferedMutator putBufferMutator = null; - BufferedMutator delBufferedMutator = null; + BufferedMutator bufferMutator = null; String key = "putKey"; String column1 = "putColumn1"; String value = "value333444"; String family = "family_group"; - long timestamp = System.currentTimeMillis(); try { // use n1 database TableName tableName = TableName.valueOf("n1","test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + Assert.assertEquals("n1:test", Bytes.toString(((OHTable) hTable).getTableName())); Delete delete= new Delete(toBytes(key)); delete.deleteFamily(toBytes(family)); hTable.delete(delete); // use defualt params - putBufferMutator = connection.getBufferedMutator(tableName); - delBufferedMutator = connection.getBufferedMutator(tableName); + bufferMutator = connection.getBufferedMutator(tableName); + + long timestamp = System.currentTimeMillis(); // only support Put and Delete - // for other type of operations, BufferedMutator will not set its type for them Append append = new Append(Bytes.toBytes(key)); - append.add(toBytes(family), toBytes(column1), toBytes("_suffix")); - final BufferedMutator apMut = putBufferMutator; + append.add("family_group".getBytes(), column1.getBytes(), toBytes("_suffix")); + final BufferedMutator apMut = bufferMutator; Assert.assertThrows(IllegalArgumentException.class, () -> { apMut.mutate(append); }); @@ -349,31 +358,46 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); mutations.add(put2); // test add Mutations with List - putBufferMutator.mutate(mutations); - putBufferMutator.flush(); + bufferMutator.mutate(mutations); + bufferMutator.flush(); Get get = new Get(toBytes(key)); Result r = hTable.get(get); Assert.assertEquals(2, r.raw().length); - Delete del = new Delete(Bytes.toBytes(key)); - final BufferedMutator noCfMut = putBufferMutator; - // test mutation without setting family - Assert.assertThrows(FeatureNotSupportedException.class, () -> { - noCfMut.mutate(del); - }); - del.deleteFamily(Bytes.toBytes(family)); - final BufferedMutator difTypeMut = putBufferMutator; - // test reuse different type bufferedMutator + Put put3 = new Put(Bytes.toBytes(key)); + final BufferedMutator noCfMut = bufferMutator; + // test Put without setting family Assert.assertThrows(IllegalArgumentException.class, () -> { - difTypeMut.mutate(del); + noCfMut.mutate(put3); }); + put3.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "2"), timestamp, Bytes.toBytes(value)); // test add Mutation directly - delBufferedMutator.mutate(del); - delBufferedMutator.flush(); + bufferMutator.mutate(put3); + bufferMutator.flush(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + // test Delete + Delete del = new Delete(toBytes(key)); + // test without setting family, delete all + bufferMutator.mutate(del); + bufferMutator.flush(); r = hTable.get(get); Assert.assertEquals(0, r.raw().length); + + // test hybrid mutations + mutations.clear(); + mutations.add(put1); + mutations.add(put2); + mutations.add(del); + mutations.add(put3); + bufferMutator.mutate(mutations); + bufferMutator.flush(); + + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); } catch (Exception ex) { if (ex instanceof RetriesExhaustedWithDetailsException) { ((RetriesExhaustedWithDetailsException) ex).getCauses().get(0).printStackTrace(); @@ -382,17 +406,14 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { } Assert.assertTrue(false); } finally { - if (putBufferMutator != null ) { - putBufferMutator.close(); + if (bufferMutator != null ) { + bufferMutator.close(); // test flush after closed - putBufferMutator.flush(); - } - if (delBufferedMutator != null) { - delBufferedMutator.close(); + bufferMutator.flush(); // test add mutations after closed - Delete delete = new Delete(Bytes.toBytes("putKey")); - delete.deleteFamily(Bytes.toBytes("family_group")); - final BufferedMutator closedMutator = delBufferedMutator; + Delete delete = new Delete(Bytes.toBytes(key)); + delete.deleteFamily(Bytes.toBytes(family)); + final BufferedMutator closedMutator = bufferMutator; Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); }); @@ -414,7 +435,7 @@ PRIMARY KEY (`K`, `Q`, `T`) public void testBufferedMutatorWithAutoFlush() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); conf.set("rs.list.acquire.read.timeout", "10000"); - BufferedMutator putBufferMutator = null; + BufferedMutator bufferMutator = null; BufferedMutatorParams params = null; long bufferSize = 45000L; int count = 0; @@ -435,7 +456,7 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); - putBufferMutator = connection.getBufferedMutator(params); + bufferMutator = connection.getBufferedMutator(params); List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { @@ -446,8 +467,20 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { timestamp, Bytes.toBytes(value + "_" + i + "_" + j)); mutations.add(put); } - putBufferMutator.mutate(mutations); + if (i % 10 == 0) { // 0, 10, 20, 30, 40 + for(int j = 0; j < 4; ++j) { + Delete del = new Delete(Bytes.toBytes(key)); + del.addColumns(toBytes(family), toBytes(column1 + "_" + i + "_" + j)); + mutations.add(del); + } + } + bufferMutator.mutate(mutations); } + Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); + Result r = hTable.get(get); + count = r.raw().length; + Assert.assertTrue(count > 0); } catch (Exception ex) { if (ex instanceof RetriesExhaustedWithDetailsException) { ((RetriesExhaustedWithDetailsException) ex).getCauses().get(0).printStackTrace(); @@ -456,14 +489,13 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { } Assert.assertTrue(false); } finally { - if (putBufferMutator != null) { - putBufferMutator.close(); + if (bufferMutator != null) { + bufferMutator.close(); Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); Result r = hTable.get(get); - for (KeyValue keyValue : r.raw()) { - ++count; - } - Assert.assertEquals(200, count); + count = r.raw().length; + Assert.assertEquals(180, count); Delete delete = new Delete(toBytes(key)); delete.deleteFamily(toBytes(family)); hTable.delete(delete); @@ -471,12 +503,12 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { Assert.assertEquals(0, r.raw().length); // test add mutations after closed - final BufferedMutator closedMutator = putBufferMutator; + final BufferedMutator closedMutator = bufferMutator; Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); }); // test flush after closed - putBufferMutator.flush(); + bufferMutator.flush(); } if (params != null) { @@ -540,8 +572,20 @@ public void testBufferedMutatorWithUserPool() throws Exception { timestamp, Bytes.toBytes(value + "_" + i + "_" + j)); mutations.add(put); } + if (i % 10 == 0) { // 0, 10, 20, 30, 40 + for(int j = 0; j < 4; ++j) { + Delete del = new Delete(Bytes.toBytes(key)); + del.addColumns(toBytes(family), toBytes(column1 + "_" + i + "_" + j)); + mutations.add(del); + } + } ohBufferMutator.mutate(mutations); } + Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); + Result r = hTable.get(get); + count = r.raw().length; + Assert.assertTrue(count > 0); } catch (Exception ex) { if (ex instanceof RetriesExhaustedWithDetailsException) { ((RetriesExhaustedWithDetailsException) ex).getCauses().get(0).printStackTrace(); @@ -553,11 +597,10 @@ public void testBufferedMutatorWithUserPool() throws Exception { if (ohBufferMutator != null) { ohBufferMutator.close(); Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); Result r = hTable.get(get); - for (KeyValue keyValue : r.raw()) { - ++count; - } - Assert.assertEquals(200, count); + count = r.raw().length; + Assert.assertEquals(180, count); Delete delete = new Delete(toBytes(key)); delete.deleteFamily(toBytes(family)); hTable.delete(delete); @@ -625,18 +668,18 @@ public void testBufferedMutatorConcurrent() throws Exception { ohBufferMutator = connection.getBufferedMutator(params); + // BufferedMutator is not concurrently safe for (int i = 0; i < 50; ++i) { final int taskId = i; final BufferedMutator thrBufferMutator = ohBufferMutator; executorService.submit(() -> { List mutations = new ArrayList<>(); for (int j = 0; j < 4; ++j) { - String thrKey = key; String thrColumn = column1 + "_" + taskId + "_" + j; String thrValue = value + "_" + taskId + "_" + j; long thrTimestamp = timestamp; - Put put = new Put(Bytes.toBytes(thrKey)); + Put put = new Put(Bytes.toBytes(key)); put.addColumn(Bytes.toBytes(family), Bytes.toBytes(thrColumn), thrTimestamp, Bytes.toBytes(thrValue)); mutations.add(put); @@ -673,10 +716,9 @@ public void testBufferedMutatorConcurrent() throws Exception { if (ohBufferMutator != null) { ohBufferMutator.close(); Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); Result r = hTable.get(get); - for (KeyValue keyValue : r.raw()) { - ++count; - } + count = r.raw().length; Assert.assertEquals(200, count); Delete delete = new Delete(toBytes(key)); delete.deleteFamily(toBytes(family)); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java index 0bf4b93f..d9e15a08 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java @@ -17,7 +17,9 @@ package com.alipay.oceanbase.hbase; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.util.Bytes; @@ -47,6 +49,197 @@ public void finish() throws IOException { hTable.close(); } + @Test + public void testMulfiColumnFamilyBufferedMutator() throws Exception { + byte[] family1 = "family_with_group1".getBytes(); + byte[] family2 = "family_with_group2".getBytes(); + byte[] family3 = "family_with_group3".getBytes(); + + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family3_column2 = "family3_column2".getBytes(); + byte[] family1_value = "VVV1".getBytes(); + byte[] family2_value = "VVV2".getBytes(); + byte[] family3_value = "VVV3".getBytes(); + + Configuration conf = ObHTableTestUtil.newConfiguration(); + TableName tableName = TableName.valueOf("test_multi_cf"); + Connection connection = ConnectionFactory.createConnection(conf); + BufferedMutator mutator = connection.getBufferedMutator(tableName); + + int rows = 10; + List keys = new ArrayList<>(); + List mutations = new ArrayList<>(); + for (int i = 0; i < rows; ++i) { + String key = "Key" + i; + keys.add(key); + Delete delete = new Delete(toBytes(key)); + mutations.add(delete); + Put put = new Put(toBytes(key)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + mutations.add(put); + } + mutator.mutate(mutations); + + // test force flush + mutator.flush(); + Get get = new Get(toBytes("Key2")); + get.addFamily(family1); + get.addFamily(family2); + Result result = hTable.get(get); + Assert.assertEquals(5, result.raw().length); + + mutations.clear(); + for (int i = 0; i < rows; ++i) { + if (i % 5 == 0) { // 0, 5 + Delete delete = new Delete(toBytes("Key" + i)); + delete.deleteFamily(family2); + delete.deleteFamily(family3); + mutations.add(delete); + } + } + mutator.mutate(mutations); + mutator.flush(); + + get = new Get(toBytes("Key0")); + result = hTable.get(get); + Assert.assertEquals(3, result.raw().length); + Assert.assertFalse(result.containsColumn(family2, family2_column1)); + Assert.assertFalse(result.containsColumn(family2, family2_column2)); + Assert.assertFalse(result.containsColumn(family3, family3_column1)); + + get = new Get(toBytes("Key5")); + result = hTable.get(get); + Assert.assertEquals(3, result.raw().length); + Assert.assertFalse(result.containsColumn(family2, family2_column1)); + Assert.assertFalse(result.containsColumn(family2, family2_column2)); + Assert.assertFalse(result.containsColumn(family3, family3_column1)); + + mutations.clear(); + for (String key : keys) { + Delete delete = new Delete(toBytes(key)); + mutations.add(delete); + } + mutator.mutate(mutations); + mutator.flush(); + + Scan scan = new Scan(); + scan.setStartRow(toBytes("Key0")); + scan.setStopRow(toBytes("Key10")); + scan.addFamily(family1); + scan.addFamily(family2); + scan.addFamily(family3); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result r : scanner) { + count += r.raw().length; + } + Assert.assertEquals(0, count); + + // test auto flush + long bufferSize = 45000L; + BufferedMutatorParams params = new BufferedMutatorParams(tableName); + params.writeBufferSize(bufferSize); + mutator = connection.getBufferedMutator(params); + + while (true) { + for (int i = 0; i < rows; ++i) { + mutations.clear(); + Put put = new Put(toBytes(keys.get(i))); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family3, family3_column1, family2_value); + put.add(family3, family3_column2, family3_value); + mutations.add(put); + if (i % 3 == 0) { // 0, 3, 6, 9 + Delete delete = new Delete(toBytes(keys.get(i))); + delete.deleteFamily(family1); + delete.deleteFamily(family2); + mutations.add(delete); + } + mutator.mutate(mutations); + } + + get = new Get(toBytes("Key0")); + result = hTable.get(get); + if (!result.isEmpty()) { + break; + } + } + get = new Get(toBytes("Key2")); + result = hTable.get(get); + Assert.assertEquals(6 , result.raw().length); + Assert.assertTrue(result.containsColumn(family1, family1_column1)); + Assert.assertTrue(result.containsColumn(family1, family1_column2)); + Assert.assertTrue(result.containsColumn(family1, family1_column3)); + Assert.assertTrue(result.containsColumn(family2, family2_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column2)); + + get = new Get(toBytes("Key3")); + result = hTable.get(get); + if (result.containsColumn(family1, family1_column1) || result.containsColumn(family2, family2_column1)) { + mutator.flush(); + } + get = new Get(toBytes("Key3")); + result = hTable.get(get); + Assert.assertEquals(2, result.raw().length); + Assert.assertFalse(result.containsColumn(family1, family1_column1)); + Assert.assertFalse(result.containsColumn(family1, family1_column2)); + Assert.assertFalse(result.containsColumn(family1, family1_column3)); + Assert.assertFalse(result.containsColumn(family2, family2_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column2)); + + get = new Get(toBytes("Key9")); + result = hTable.get(get); + if (result.containsColumn(family1, family1_column1) || result.containsColumn(family2, family2_column1)) { + mutator.flush(); + } + get = new Get(toBytes("Key9")); + result = hTable.get(get); + Assert.assertEquals(2, result.raw().length); + Assert.assertFalse(result.containsColumn(family1, family1_column1)); + Assert.assertFalse(result.containsColumn(family1, family1_column2)); + Assert.assertFalse(result.containsColumn(family1, family1_column3)); + Assert.assertFalse(result.containsColumn(family2, family2_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column2)); + + // clean data + mutations.clear(); + for (String key : keys) { + Delete delete = new Delete(toBytes(key)); + mutations.add(delete); + } + mutator.mutate(mutations); + mutator.flush(); + + scan = new Scan(); + scan.setStartRow(toBytes("Key0")); + scan.setStopRow(toBytes("Key10")); + scan.addFamily(family1); + scan.addFamily(family2); + scan.addFamily(family3); + scanner = hTable.getScanner(scan); + count = 0; + for (Result r : scanner) { + count += r.raw().length; + } + Assert.assertEquals(0, count); + } + @Test public void testMultiColumnFamilyBatch() throws Exception { byte[] family1 = "family_with_group1".getBytes(); @@ -611,12 +804,12 @@ public void testMultiColumnFamilyGet() throws Exception { @Test public void testMultiColumnFamilyDelete() throws Exception { - String key1 = "scanKey1x"; - String key2 = "scanKey2x"; - String key3 = "scanKey3x"; - String value1 = "value1"; - String value2 = "value2"; - String value3 = "value3"; + String key1 = "scanKey1x"; + String key2 = "scanKey2x"; + String key3 = "scanKey3x"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; byte[] family1 = "family_with_group1".getBytes(); byte[] family2 = "family_with_group2".getBytes(); @@ -788,15 +981,13 @@ public void testMultiColumnFamilyDelete() throws Exception { long maxTimeStamp = System.currentTimeMillis(); Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); - putKey1Fam1Column1MinTs.add(family1, family1_column1, minTimeStamp, - toBytes(value1)); + putKey1Fam1Column1MinTs.add(family1, family1_column1, minTimeStamp, toBytes(value1)); Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); putKey3Fam1Column1Ts1.add(family1, family1_column1, timeStamp1, toBytes(value2)); Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); - putKey1Fam1Column2MinTs.add(family1, family1_column2, minTimeStamp, - toBytes(value1)); + putKey1Fam1Column2MinTs.add(family1, family1_column2, minTimeStamp, toBytes(value1)); Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); putKey1Fam1Column2Ts3.add(family1, family1_column2, timeStamp3, toBytes(value2)); @@ -936,12 +1127,10 @@ public void testMultiColumnFamilyDelete() throws Exception { putKey3Fam1Column2Ts9.add(family1, family1_column2, timeStamp9, toBytes(value2)); Put putKey3Fam2Column3Ts10 = new Put(toBytes(key3)); - putKey3Fam2Column3Ts10 - .add(family2, family2_column3, timeStamp10, toBytes(value1)); + putKey3Fam2Column3Ts10.add(family2, family2_column3, timeStamp10, toBytes(value1)); Put putKey3Fam2Column1Ts10 = new Put(toBytes(key3)); - putKey3Fam2Column1Ts10 - .add(family2, family2_column1, timeStamp10, toBytes(value2)); + putKey3Fam2Column1Ts10.add(family2, family2_column1, timeStamp10, toBytes(value2)); Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); putKey3Fam1Column2Ts2.add(family1, family1_column2, timeStamp2, toBytes(value1)); From 5347033602fc36b5609872a89327aa8021690d29 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Wed, 23 Oct 2024 21:45:16 +0800 Subject: [PATCH 08/32] bufferedMutator do not retry, batch retry in table client --- .../hbase/util/OHBufferedMutatorImpl.java | 22 ++----------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index 30a18170..53cd04a2 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -165,29 +165,11 @@ private void execute(boolean flushAll) throws IOException { if (execBuffer.isEmpty()) { return; } - try { - ohTable.batch(execBuffer); - } catch (Exception ex) { - if (ex.getCause() instanceof RetriesExhaustedWithDetailsException) { - // retry one time - LOGGER.debug(tableName + ": One or more of the operations have failed -" - + " waiting for all operations to complete (successfully or not) and retries"); - RetriesExhaustedWithDetailsException retryException = (RetriesExhaustedWithDetailsException) ex.getCause(); - execBuffer.clear(); - for (int i = 0; i < retryException.getNumExceptions(); ++i) { - execBuffer.add((Mutation) retryException.getRow(i)); - } - ohTable.batch(execBuffer); - } else { - LOGGER.error("Errors unrelated to operations occur during mutation operation", ex); - throw ex; - } - } + ohTable.batch(execBuffer); } catch (Exception ex) { LOGGER.error(LCD.convert("01-00026"), ex); if (ex.getCause() instanceof RetriesExhaustedWithDetailsException) { - // retry one time - LOGGER.error(tableName + ": One or more of the operations have failed after retry."); + LOGGER.error(tableName + ": One or more of the operations have failed after retries."); RetriesExhaustedWithDetailsException retryException = (RetriesExhaustedWithDetailsException) ex.getCause(); // recollect mutations execBuffer.clear(); From d1d5b52d79d85c2df27c0e73ae478e7ddce5f77b Mon Sep 17 00:00:00 2001 From: stuBirdFly <84010733+stuBirdFly@users.noreply.github.com> Date: Tue, 22 Oct 2024 20:35:40 +0800 Subject: [PATCH 09/32] hbase support batch (#84) --- .../com/alipay/oceanbase/hbase/OHTable.java | 384 ++++++++---------- .../oceanbase/hbase/util/BatchError.java | 40 ++ .../oceanbase/hbase/HTableTestBase.java | 16 +- .../hbase/OHTableMultiColumnFamilyTest.java | 113 +++++- 4 files changed, 336 insertions(+), 217 deletions(-) create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 8cb29cb6..a2daecfa 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -24,6 +24,7 @@ import com.alipay.oceanbase.hbase.result.ClientStreamScanner; import com.alipay.oceanbase.hbase.util.*; import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.mutation.BatchOperation; import com.alipay.oceanbase.rpc.mutation.result.BatchOperationResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; @@ -131,7 +132,7 @@ public class OHTable implements Table { /** * the buffer of put request */ - private final ArrayList writeBuffer = new ArrayList(); + private final ArrayList writeBuffer = new ArrayList<>(); /** * when the put request reach the write buffer size the do put will * flush commits automatically @@ -336,7 +337,7 @@ public static ThreadPoolExecutor createDefaultThreadPoolExecutor(int coreSize, i System.setProperty(SOFA_THREAD_POOL_LOGGING_CAPABILITY, "false"); } SofaThreadPoolExecutor executor = new SofaThreadPoolExecutor(coreSize, maxThreads, - keepAliveTime, SECONDS, new SynchronousQueue(), "OHTableDefaultExecutePool", + keepAliveTime, SECONDS, new SynchronousQueue<>(), "OHTableDefaultExecutePool", TABLE_HBASE_LOGGER_SPACE); executor.allowCoreThreadTimeOut(true); return executor; @@ -463,8 +464,56 @@ public boolean[] exists(List gets) throws IOException { } @Override - public void batch(List actions, Object[] results) { - throw new FeatureNotSupportedException("not supported yet."); + public void batch(final List actions, final Object[] results) throws IOException { + BatchError batchError = new BatchError(); + try { + List resultMapSingleOp = new LinkedList<>(); + String realTableName = getTargetTableName(actions); + obTableClient.setRuntimeBatchExecutor(executePool); + BatchOperation batch = buildBatchOperation(realTableName, actions, tableNameString.equals(realTableName), resultMapSingleOp); + BatchOperationResult tmpResults = batch.execute(); + if (results != null) { + if (results.length != actions.size()) { + throw new AssertionError("results.length"); + } + int index = 0; + for (int i = 0; i != results.length; ++i) { + results[i] = tmpResults.getResults().get(index); + index += resultMapSingleOp.get(i); + if (results[i] instanceof ObTableException) { + batchError.add((ObTableException) results[i], actions.get(i), null); + } + } + if (batchError.hasErrors()) { + throw batchError.makeException(); + } + } + } catch (Exception e) { + logger.error(LCD.convert("01-000010"), tableNameString, actions, e); + throw new IOException("batch table " + tableNameString + " error", e); + } + } + + private String getTargetTableName(List actions) { + byte[] family = null; + for (Row action : actions) { + if (action instanceof RowMutations || action instanceof RegionCoprocessorServiceExec) { + throw new FeatureNotSupportedException("not supported yet'"); + } else { + Mutation mutation = (Mutation) action; + if (mutation.getFamilyCellMap().size() != 1) { + return getTargetTableName(tableNameString); + } else { + byte[] nextFamily = mutation.getFamilyCellMap().keySet().iterator().next(); + if (family != null && !Arrays.equals(family, nextFamily)) { + return getTargetTableName(tableNameString); + } else if (family == null) { + family = nextFamily; + } + } + } + } + return getTargetTableName(tableNameString, Bytes.toString(family), configuration); } @Override @@ -561,7 +610,7 @@ public Result get(final Get get) throws IOException { ServerCallable serverCallable = new ServerCallable(configuration, obTableClient, tableNameString, get.getRow(), get.getRow(), operationTimeout) { public Result call() throws IOException { - List keyValueList = new ArrayList(); + List keyValueList = new ArrayList<>(); byte[] family = new byte[] {}; ObTableClientQueryAsyncStreamResult clientQueryStreamResult; ObTableQueryAsyncRequest request; @@ -837,57 +886,13 @@ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, private void innerDelete(Delete delete) throws IOException { checkArgument(delete.getRow() != null, "row is null"); - List errorCodeList = new ArrayList(); - BatchOperationResult results = null; - try { - checkFamilyViolation(delete.getFamilyCellMap().keySet(), false); - if (delete.getFamilyCellMap().isEmpty()) { - // For a Delete operation without any qualifiers, we construct a DeleteFamily request. - // The server then performs the operation on all column families. - KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), - KeyValue.Type.DeleteFamily); - - BatchOperation batch = buildBatchOperation(tableNameString, Arrays.asList(kv), false, null); - results = batch.execute(); - } else if (delete.getFamilyCellMap().size() > 1) { - // Currently, the Delete Family operation type cannot transmit qualifiers to the server. - // As a result, the server cannot identify which families need to be deleted. - // Therefore, this process is handled sequentially. - boolean has_delete_family = delete.getFamilyCellMap().entrySet().stream() - .flatMap(entry -> entry.getValue().stream()) - .anyMatch(kv -> kv.getType() == Cell.Type.DeleteFamily); - if (!has_delete_family) { - BatchOperation batch = buildBatchOperation(tableNameString, - delete.getFamilyCellMap(), false, null); - results = batch.execute(); - } else { - for (Map.Entry> entry : delete.getFamilyCellMap().entrySet()) { - BatchOperation batch = buildBatchOperation( - getTargetTableName(tableNameString, Bytes.toString(entry.getKey()), configuration), - entry.getValue(), false, null); - results = batch.execute(); - } - } - } else { - Map.Entry> entry = delete.getFamilyCellMap().entrySet().iterator() - .next(); - - BatchOperation batch = buildBatchOperation( - getTargetTableName(tableNameString, Bytes.toString(entry.getKey()), configuration), - entry.getValue(), false, null); - results = batch.execute(); - } - - errorCodeList = results.getErrorCodeList(); - boolean hasError = results.hasError(); - if (hasError) { - throw results.getFirstException(); - } + List actions = Collections.singletonList(delete); + Object[] results = new Object[actions.size()]; + batch(actions, results); } catch (Exception e) { - logger.error(LCD.convert("01-00004"), tableNameString, errorCodeList, e); - throw new IOException("delete table " + tableNameString + " error codes " - + errorCodeList, e); + logger.error(LCD.convert("01-00004"), tableNameString, e); + throw new IOException("delete table " + tableNameString + "error" , e); } } @@ -1151,77 +1156,34 @@ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, lo public void flushCommits() throws IOException { try { + if (writeBuffer.isEmpty()){ + return; + } + Map exceptionRowMap = new LinkedHashMap(); boolean[] resultSuccess = new boolean[writeBuffer.size()]; try { - Map, List>> familyMap = new HashMap, List>>(); - for (int i = 0; i < writeBuffer.size(); i++) { - Put aPut = writeBuffer.get(i); - Map> innerFamilyMap = aPut.getFamilyCellMap(); - if (innerFamilyMap.size() > 1) { - // Bypass logic: directly construct BatchOperation for puts with family map size > 1 - try { - BatchOperation batch = buildBatchOperation(this.tableNameString, - innerFamilyMap, false, null); - BatchOperationResult results = batch.execute(); - - boolean hasError = results.hasError(); - resultSuccess[i] = !hasError; - if (hasError) { - throw results.getFirstException(); - } - } catch (Exception e) { - logger.error(LCD.convert("01-00008"), tableNameString, null, autoFlush, - writeBuffer.size(), e); - throw new IOException("put table " + tableNameString + " error codes " - + null + "auto flush " + autoFlush - + " current buffer size " + writeBuffer.size(), e); - } - } else { - // Existing logic for puts with family map size = 1 - for (Map.Entry> entry : innerFamilyMap.entrySet()) { - String family = Bytes.toString(entry.getKey()); - Pair, List> keyValueWithIndex = familyMap - .get(family); - if (keyValueWithIndex == null) { - keyValueWithIndex = new Pair, List>( - new ArrayList(), new ArrayList()); - familyMap.put(family, keyValueWithIndex); - } - keyValueWithIndex.getFirst().add(i); - keyValueWithIndex.getSecond().addAll(entry.getValue()); - } - } - } - for (Map.Entry, List>> entry : familyMap - .entrySet()) { - List errorCodeList = new ArrayList(entry.getValue() - .getSecond().size()); - try { - String targetTableName = getTargetTableName(this.tableNameString, - entry.getKey(), configuration); - - BatchOperation batch = buildBatchOperation(targetTableName, entry - .getValue().getSecond(), false, null); - BatchOperationResult results = batch.execute(); - - errorCodeList = results.getErrorCodeList(); - boolean hasError = results.hasError(); - for (Integer index : entry.getValue().getFirst()) { - resultSuccess[index] = !hasError; - } - if (hasError) { - throw results.getFirstException(); + String realTableName = getTargetTableName(writeBuffer); + List resultMapSingleOp = new LinkedList<>(); + BatchOperation batch = buildBatchOperation(realTableName, writeBuffer, tableNameString.equals(realTableName), resultMapSingleOp); + BatchOperationResult results = batch.execute(); + if (results != null) { + int index = 0; + for (int i = 0; i != resultSuccess.length; ++i) { + if (results.getResults().get(index) instanceof ObTableException) { + resultSuccess[i] = false; + exceptionRowMap.put((ObTableException)results.getResults().get(index), writeBuffer.get(i)); + } else { + resultSuccess[i] = true; } - } catch (Exception e) { - logger.error(LCD.convert("01-00008"), tableNameString, errorCodeList, - autoFlush, writeBuffer.size(), e); - throw new IOException("put table " + tableNameString + " error codes " - + errorCodeList + "auto flush " + autoFlush - + " current buffer size " + writeBuffer.size(), e); + index += resultMapSingleOp.get(i); } - } - + } catch (Exception e) { + logger.error(LCD.convert("01-00008"), tableNameString, null, autoFlush, + writeBuffer.size(), e); + throw new IOException("put table " + tableNameString + " error codes " + null + + "auto flush " + autoFlush + " current buffer size " + + writeBuffer.size(), e); } finally { // mutate list so that it is empty for complete success, or contains // only failed records results are returned in the same order as the @@ -1233,6 +1195,12 @@ public void flushCommits() throws IOException { writeBuffer.remove(i); } } + if (!exceptionRowMap.isEmpty()) { + exceptionRowMap.forEach((e, row)->{ + logger.error(LCD.convert("01-00008"), row, tableNameString, autoFlush, + writeBuffer.size(), e); + }); + } } } finally { if (clearBufferOnFail) { @@ -1562,65 +1530,51 @@ public static ObTableBatchOperation buildObTableBatchOperation(List keyVal return batch; } - private ObTableBatchOperation buildObTableBatchOperation(Map> familyMap, - boolean putToAppend, - List qualifiers) { - ObTableBatchOperation batch = new ObTableBatchOperation(); - for (Map.Entry> entry : familyMap.entrySet()) { - byte[] family = entry.getKey(); - List keyValueList = entry.getValue(); - for (Cell kv : keyValueList) { - if (qualifiers != null) { - qualifiers - .add((Bytes.toString(family) + "." + Bytes.toString(CellUtil.cloneQualifier(kv))) - .getBytes()); - } - KeyValue new_kv = modifyQualifier(kv, - (Bytes.toString(family) + "." + Bytes.toString(CellUtil.cloneQualifier(kv))).getBytes()); - batch.addTableOperation(buildObTableOperation(new_kv, putToAppend)); - } - } - batch.setSameType(true); - batch.setSamePropertiesNames(true); - return batch; - } - private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(Cell kv, - boolean putToAppend) { - Cell.Type kvType = kv.getType(); - switch (kvType) { - case Put: - ObTableOperationType operationType; - if (putToAppend) { - operationType = APPEND; - } else { - operationType = INSERT_OR_UPDATE; - } + ObTableOperationType operationType, boolean isTableGroup) { + KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getType().getCode()); + switch (operationType) { + case INSERT_OR_UPDATE: + case APPEND: return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(operationType, ROW_KEY_COLUMNS, new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, V_COLUMNS, new Object[] { CellUtil.cloneValue(kv) }); - case Delete: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, null, null); - case DeleteColumn: - return com.alipay.oceanbase.rpc.mutation.Mutation - .getInstance(DEL, ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), -kv.getTimestamp() }, null, - null); - case DeleteFamily: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), null, -kv.getTimestamp() }, null, null); + case DEL: + switch (kvType) { + case Delete: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, + null, null); + case Maximum: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), null, -kv.getTimestamp() }, + null, null); + case DeleteColumn: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), -kv.getTimestamp() }, + null, null); + case DeleteFamily: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), isTableGroup?CellUtil.cloneQualifier(kv):null, -kv.getTimestamp() }, + null, null); + default: + throw new IllegalArgumentException("illegal mutation type " + kvType); + } default: - throw new IllegalArgumentException("illegal mutation type " + kvType); + throw new IllegalArgumentException("illegal mutation type " + operationType); } } private KeyValue modifyQualifier(Cell original, byte[] newQualifier) { - // Extract existing components - byte[] row = original.getRowArray(); - byte[] family = original.getFamilyArray(); - byte[] value = original.getValueArray(); + // Extract existing components + byte[] row = CellUtil.cloneRow(original); + byte[] family = CellUtil.cloneFamily(original); + byte[] value = CellUtil.cloneValue(original); long timestamp = original.getTimestamp(); KeyValue.Type type = KeyValue.Type.codeToType(original.getType().getCode()); // Create a new KeyValue with the modified qualifier @@ -1628,37 +1582,66 @@ private KeyValue modifyQualifier(Cell original, byte[] newQualifier) { value); } - private BatchOperation buildBatchOperation(String tableName, - Map> familyMap, - boolean putToAppend, List qualifiers) { + private BatchOperation buildBatchOperation(String tableName, List actions, boolean isTableGroup, List resultMapSingleOp) { BatchOperation batch = obTableClient.batchOperation(tableName); - - for (Map.Entry> entry : familyMap.entrySet()) { - byte[] family = entry.getKey(); - List keyValueList = entry.getValue(); - for (Cell kv : keyValueList) { - if (qualifiers != null) { - qualifiers.add(CellUtil.cloneQualifier(kv)); + if (actions != null) { + int posInList = -1; + int singleOpResultNum; + for (Row row : actions) { + singleOpResultNum = 0; + posInList++; + if (!(row instanceof Put) && !(row instanceof Delete)) { + throw new FeatureNotSupportedException( + "not supported other type in batch yet,only support put and delete"); + } else if (row instanceof Put) { + Put put = (Put) row; + if (put.isEmpty()) { + throw new IllegalArgumentException("No columns to insert for #" + + (posInList + 1) + " item"); + } + for (Map.Entry> entry : put.getFamilyCellMap().entrySet()) { + byte[] family = entry.getKey(); + List keyValueList = entry.getValue(); + for (Cell kv : keyValueList) { + singleOpResultNum++; + if(isTableGroup){ + KeyValue new_kv = modifyQualifier(kv,(Bytes.toString(family) + "." + Bytes.toString(CellUtil.cloneQualifier(kv))) + .getBytes()); + batch.addOperation(buildMutation(new_kv, INSERT_OR_UPDATE, isTableGroup)); + } else { + batch.addOperation(buildMutation(kv, INSERT_OR_UPDATE, isTableGroup)); + } + } + } + } else { + Delete delete = (Delete) row; + if (delete.isEmpty()) { + singleOpResultNum++; + KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), + KeyValue.Type.Maximum); + batch.addOperation(buildMutation(kv, DEL, isTableGroup)); + } else { + for (Map.Entry> entry : delete.getFamilyCellMap() + .entrySet()) { + byte[] family = entry.getKey(); + List keyValueList = entry.getValue(); + List mutations = new LinkedList<>(); + for (Cell kv : keyValueList) { + singleOpResultNum++; + if(isTableGroup){ + KeyValue new_kv = modifyQualifier(kv,(Bytes.toString(family) + "." + Bytes.toString(CellUtil.cloneQualifier(kv))) + .getBytes()); + batch.addOperation(buildMutation(new_kv, DEL, true)); + } else { + batch.addOperation(buildMutation(kv, DEL, false)); + } + } + } + } } - KeyValue new_kv = modifyQualifier(kv, - (Bytes.toString(family) + "." + Bytes.toString(CellUtil.cloneQualifier(kv))).getBytes()); - batch.addOperation(buildMutation(new_kv, putToAppend)); + resultMapSingleOp.add(singleOpResultNum); } } - - batch.setEntityType(ObTableEntityType.HKV); - return batch; - } - - private BatchOperation buildBatchOperation(String tableName, List keyValueList, - boolean putToAppend, List qualifiers) { - BatchOperation batch = obTableClient.batchOperation(tableName); - for (Cell kv : keyValueList) { - if (qualifiers != null) { - qualifiers.add(CellUtil.cloneQualifier(kv)); - } - batch.addOperation(buildMutation(kv, putToAppend)); - } batch.setEntityType(ObTableEntityType.HKV); return batch; } @@ -1690,15 +1673,6 @@ public static ObTableOperation buildObTableOperation(Cell kv, boolean putToAppen } } - private ObTableQueryRequest buildObTableQueryRequest(ObTableQuery obTableQuery, - String targetTableName) { - ObTableQueryRequest request = new ObTableQueryRequest(); - request.setEntityType(ObTableEntityType.HKV); - request.setTableQuery(obTableQuery); - request.setTableName(targetTableName); - return request; - } - private ObTableQueryAsyncRequest buildObTableQueryAsyncRequest(ObTableQuery obTableQuery, String targetTableName) { ObTableQueryRequest request = new ObTableQueryRequest(); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java b/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java new file mode 100644 index 00000000..640e432d --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java @@ -0,0 +1,40 @@ +package com.alipay.oceanbase.hbase.util; + +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; +import org.apache.hadoop.hbase.client.Row; + +import java.util.ArrayList; +import java.util.List; + +public class BatchError { + private final List throwables = new ArrayList(); + private final List actions = new ArrayList(); + private final List addresses = new ArrayList(); + + public synchronized void add(Throwable ex, Row row, ServerName serverName) { + if (row == null){ + throw new IllegalArgumentException("row cannot be null. location=" + serverName); + } + + throwables.add(ex); + actions.add(row); + addresses.add(serverName != null ? serverName.toString() : "null"); + } + + public boolean hasErrors() { + return !throwables.isEmpty(); + } + + public synchronized RetriesExhaustedWithDetailsException makeException() { + return new RetriesExhaustedWithDetailsException( + new ArrayList(throwables), + new ArrayList(actions), new ArrayList(addresses)); + } + + public synchronized void clear() { + throwables.clear(); + actions.clear(); + addresses.clear(); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 2148302d..5413c6d0 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -3986,7 +3986,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); // reverse scan with MaxVersion @@ -4004,7 +4004,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 3); + Assert.assertEquals(3, res_count); scanner.close(); // reverse scan with pageFilter @@ -4024,7 +4024,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); // reverse scan with not_exist_start_row @@ -4042,7 +4042,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); // reverse scan with abnormal range @@ -4060,7 +4060,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 0); + Assert.assertEquals(0, res_count); scanner.close(); // reverse scan with abnormal range @@ -4078,7 +4078,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 13); + Assert.assertEquals(13, res_count); scanner.close(); hTable.delete(deleteKey1Family); @@ -4427,7 +4427,7 @@ public void testPartitionScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 7); + Assert.assertEquals(7, res_count); scanner.close(); // scan with prefixFilter @@ -4447,7 +4447,7 @@ public void testPartitionScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 2); + Assert.assertEquals(2, res_count); scanner.close(); // scan with singleColumnValueFilter diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java index 3af773bf..5a2e1c15 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java @@ -26,9 +26,7 @@ import org.junit.rules.ExpectedException; import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.*; @@ -50,6 +48,112 @@ public void finish() throws IOException { hTable.close(); } + @Test + public void testMultiColumnFamilyBatch() throws Exception { + byte[] family1 = "family_with_group1".getBytes(); + byte[] family2 = "family_with_group2".getBytes(); + byte[] family3 = "family_with_group3".getBytes(); + + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family1_value = "VVV1".getBytes(); + byte[] family2_value = "VVV2".getBytes(); + byte[] family3_value = "VVV3".getBytes(); + + int rows = 10; + List batchLsit = new LinkedList<>(); + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + Delete delete = new Delete(toBytes("Key" + i)); + batchLsit.add(delete); + put.addColumn(family1, family1_column1, family1_value); + put.addColumn(family1, family1_column2, family1_value); + put.addColumn(family1, family1_column3, family1_value); + put.addColumn(family2, family2_column1, family2_value); + put.addColumn(family2, family2_column2, family2_value); + put.addColumn(family3, family3_column1, family3_value); + batchLsit.add(put); + } + + // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 + Delete delete = new Delete(toBytes("Key1")); + delete.addColumns(family1, family1_column1); + delete.addColumns(family2, family2_column1); + batchLsit.add(delete); + Object[] results = new Object[batchLsit.size()]; + hTable.batch(batchLsit, results); + // f1c2 f1c3 f2c2 f3c1 + Get get = new Get(toBytes("Key1")); + Result result = hTable.get(get); + Cell[] keyValues = result.rawCells(); + assertEquals(4, keyValues.length); + assertFalse(result.containsColumn(family1, family1_column1)); + assertFalse(result.containsColumn(family2, family2_column1)); + + assertTrue(result.containsColumn(family1, family1_column2)); + assertArrayEquals(result.getValue(family1, family1_column2), family1_value); + assertTrue(result.containsColumn(family1, family1_column3)); + assertArrayEquals(result.getValue(family1, family1_column3), family1_value); + assertTrue(result.containsColumn(family2, family2_column2)); + assertArrayEquals(result.getValue(family2, family2_column2), family2_value); + assertTrue(result.containsColumn(family3, family3_column1)); + assertArrayEquals(result.getValue(family3, family3_column1), family3_value); + + // f1c1 f2c1 f2c2 + delete = new Delete(toBytes("Key2")); + delete.addColumns(family1, family1_column2); + delete.addColumns(family1, family1_column3); + delete.addColumns(family3, family3_column1); + batchLsit.add(delete); + // null + results = new Object[batchLsit.size()]; + hTable.batch(batchLsit, results); + get = new Get(toBytes("Key2")); + result = hTable.get(get); + keyValues = result.rawCells(); + assertEquals(3, keyValues.length); + batchLsit.clear(); + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.addColumn(family1, family1_column1, family1_value); + put.addColumn(family1, family1_column2, family1_value); + put.addColumn(family1, family1_column3, family1_value); + put.addColumn(family2, family2_column1, family2_value); + put.addColumn(family2, family2_column2, family2_value); + put.addColumn(family3, family3_column1, family3_value); + batchLsit.add(put); + } + + delete = new Delete(toBytes("Key3")); + delete.addColumn(family1, family1_column2); + delete.addColumn(family2, family2_column1); + batchLsit.add(delete); + results = new Object[batchLsit.size()]; + hTable.batch(batchLsit, results); + get = new Get(toBytes("Key3")); + result = hTable.get(get); + keyValues = result.rawCells(); + assertEquals(6, keyValues.length); + + batchLsit.clear(); + delete = new Delete(toBytes("Key4")); + delete.addColumns(family1, family1_column2); + delete.addColumns(family2, family2_column1); + delete.addFamily(family3); + batchLsit.add(delete); + results = new Object[batchLsit.size()]; + hTable.batch(batchLsit, results); + get = new Get(toBytes("Key4")); + get.setMaxVersions(10); + result = hTable.get(get); + keyValues = result.rawCells(); + assertEquals(6, keyValues.length); + } + @Test public void testMultiColumnFamilyPut() throws Exception { byte[] family1 = "family_with_group1".getBytes(); @@ -486,7 +590,6 @@ public void testMultiColumnFamilyGet() throws Exception { assertEquals(expectedValue, CellUtil.cloneValue(keyValues[i])); } } - System.out.println(Arrays.toString(result2.rawCells())); assertEquals(3, keyValues.length); //f2c1 f2c2 @@ -528,6 +631,8 @@ public void testMultiColumnFamilyDelete() throws Exception { for (int i = 0; i < rows; ++i) { Put put = new Put(toBytes("Key" + i)); + Delete delete = new Delete(toBytes("Key" + i)); + hTable.delete(delete); put.addColumn(family1, family1_column1, family1_value); put.addColumn(family1, family1_column2, family1_value); put.addColumn(family1, family1_column3, family1_value); From 60814007c2da114126dae1f849a3067746fec324 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Thu, 24 Oct 2024 10:49:25 +0800 Subject: [PATCH 10/32] add rpcTimeout and operationTimetout setting in bufferedMutator --- .../hbase/util/OHBufferedMutatorImpl.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index 53cd04a2..22338f91 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -54,6 +54,8 @@ public class OHBufferedMutatorImpl implements BufferedMutator { private final int maxKeyValueSize; private boolean closed = false; private final ExecutorService pool; + private int rpcTimeout; + private int operationTimeout; public OHBufferedMutatorImpl(OHConnectionImpl ohConnection, BufferedMutatorParams params) throws IOException { @@ -66,6 +68,8 @@ public OHBufferedMutatorImpl(OHConnectionImpl ohConnection, BufferedMutatorParam this.connectionConfig = ohConnection.getOHConnectionConfiguration(); this.listener = params.getListener(); this.pool = params.getPool(); + this.rpcTimeout = connectionConfig.getRpcTimeout(); + this.operationTimeout = connectionConfig.getOperationTimeout(); this.writeBufferSize = params.getWriteBufferSize() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params .getWriteBufferSize() : connectionConfig.getWriteBufferSize(); @@ -230,4 +234,14 @@ public void flush() throws IOException { public long getWriteBufferSize() { return this.writeBufferSize; } + + public void setRpcTimeout(int rpcTimeout) { + this.rpcTimeout = rpcTimeout; + this.ohTable.setRpcTimeout(rpcTimeout); + } + + public void setOperationTimeout(int operationTimeout) { + this.operationTimeout = operationTimeout; + this.ohTable.setOperationTimeout(operationTimeout); + } } From 3884e08e8769cc968a9fdd1576cc0022b79fd07d Mon Sep 17 00:00:00 2001 From: miyuan-ljr Date: Thu, 24 Oct 2024 15:05:06 +0800 Subject: [PATCH 11/32] fix test --- .../com/alipay/oceanbase/hbase/OHTable.java | 89 +++++++++++-------- 1 file changed, 54 insertions(+), 35 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index a2daecfa..8c8c204a 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1531,40 +1531,38 @@ public static ObTableBatchOperation buildObTableBatchOperation(List keyVal } private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(Cell kv, - ObTableOperationType operationType, boolean isTableGroup) { - KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getType().getCode()); + OHOpType operationType, boolean isTableGroup) { switch (operationType) { case INSERT_OR_UPDATE: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(INSERT_OR_UPDATE, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, V_COLUMNS, + new Object[] { CellUtil.cloneValue(kv) }); case APPEND: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(operationType, + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(APPEND, ROW_KEY_COLUMNS, new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, V_COLUMNS, new Object[] { CellUtil.cloneValue(kv) }); - case DEL: - switch (kvType) { - case Delete: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, - null, null); - case Maximum: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), null, -kv.getTimestamp() }, - null, null); - case DeleteColumn: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), -kv.getTimestamp() }, - null, null); - case DeleteFamily: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), isTableGroup?CellUtil.cloneQualifier(kv):null, -kv.getTimestamp() }, - null, null); - default: - throw new IllegalArgumentException("illegal mutation type " + kvType); - } + case Delete: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, + null, null); + case DeleteAll: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), null, -kv.getTimestamp() }, + null, null); + case DeleteColumn: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), -kv.getTimestamp() }, + null, null); + case DeleteFamily: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), isTableGroup?CellUtil.cloneQualifier(kv):null, -kv.getTimestamp() }, + null, null); default: throw new IllegalArgumentException("illegal mutation type " + operationType); } @@ -1607,9 +1605,9 @@ private BatchOperation buildBatchOperation(String tableName, List if(isTableGroup){ KeyValue new_kv = modifyQualifier(kv,(Bytes.toString(family) + "." + Bytes.toString(CellUtil.cloneQualifier(kv))) .getBytes()); - batch.addOperation(buildMutation(new_kv, INSERT_OR_UPDATE, isTableGroup)); + batch.addOperation(buildMutation(new_kv, OHOpType.INSERT_OR_UPDATE, isTableGroup)); } else { - batch.addOperation(buildMutation(kv, INSERT_OR_UPDATE, isTableGroup)); + batch.addOperation(buildMutation(kv, OHOpType.INSERT_OR_UPDATE, isTableGroup)); } } } @@ -1617,9 +1615,8 @@ private BatchOperation buildBatchOperation(String tableName, List Delete delete = (Delete) row; if (delete.isEmpty()) { singleOpResultNum++; - KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), - KeyValue.Type.Maximum); - batch.addOperation(buildMutation(kv, DEL, isTableGroup)); + KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp()); + batch.addOperation(buildMutation(kv, OHOpType.DeleteAll, isTableGroup)); } else { for (Map.Entry> entry : delete.getFamilyCellMap() .entrySet()) { @@ -1631,9 +1628,9 @@ private BatchOperation buildBatchOperation(String tableName, List if(isTableGroup){ KeyValue new_kv = modifyQualifier(kv,(Bytes.toString(family) + "." + Bytes.toString(CellUtil.cloneQualifier(kv))) .getBytes()); - batch.addOperation(buildMutation(new_kv, DEL, true)); + batch.addOperation(buildMutation(new_kv, getDeleteType(new_kv.getType()), true)); } else { - batch.addOperation(buildMutation(kv, DEL, false)); + batch.addOperation(buildMutation(kv, getDeleteType(kv.getType()), false)); } } } @@ -1780,4 +1777,26 @@ public byte[][] getEndKeys() throws IOException { public Pair getStartEndKeys() throws IOException { return new Pair<>(getStartKeys(), getEndKeys()); } + + public static enum OHOpType { + INSERT_OR_UPDATE, + APPEND, + Delete, + DeleteAll, + DeleteColumn, + DeleteFamily, + } + + public static OHOpType getDeleteType(Cell.Type type) { + switch (type) { + case Delete: + return OHOpType.Delete; + case DeleteColumn: + return OHOpType.DeleteColumn; + case DeleteFamily: + return OHOpType.DeleteFamily; + default: + throw new IllegalArgumentException("illegal mutation type " + type); + } + } } \ No newline at end of file From 676ec51dfee1570eb809d528f828da20f9fa3fd4 Mon Sep 17 00:00:00 2001 From: Ziyu Shi <57038180+JackShi148@users.noreply.github.com> Date: Wed, 23 Oct 2024 10:49:17 +0800 Subject: [PATCH 12/32] Add DeleteFamilyVersion function and corresponding test cases (#85) * add DepentdentFilter and SingleColumnValueExcludeFilter * add singleColumnValueExcludeFilter and DependentColumnFilter; add test cases and optimize bufferedMutator test cases * add single cf timerange setting in Get * single cf setColumnFamilyTimeRange in Get and Scan * optimize code * add DeleteFamilyVersion and test cases * add DeleteFamilyVersion; optimize test cases * add DeleteFamilyVersion test case and pass * format code * delete useless self-defined table --- .../com/alipay/oceanbase/hbase/OHTable.java | 6 + .../hbase/filter/HBaseFilterUtils.java | 19 +- .../oceanbase/hbase/util/BatchError.java | 11 +- .../oceanbase/hbase/HTableTestBase.java | 89 ++--- .../oceanbase/hbase/OHConnectionTest.java | 131 ++++--- .../hbase/OHTableDeleteFamilyVersionTest.java | 350 ++++++++++++++++++ .../hbase/filter/HBaseFilterUtilsTest.java | 51 ++- 7 files changed, 524 insertions(+), 133 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 8c8c204a..e0df60b8 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1563,6 +1563,11 @@ private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(Cell kv, ROW_KEY_COLUMNS, new Object[] { CellUtil.cloneRow(kv), isTableGroup?CellUtil.cloneQualifier(kv):null, -kv.getTimestamp() }, null, null); + case DeleteFamilyVersion: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), isTableGroup ? CellUtil.cloneQualifier(kv) : null, kv.getTimestamp() }, + null, null); default: throw new IllegalArgumentException("illegal mutation type " + operationType); } @@ -1785,6 +1790,7 @@ public static enum OHOpType { DeleteAll, DeleteColumn, DeleteFamily, + DeleteFamilyVersion, } public static OHOpType getDeleteType(Cell.Type type) { diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index 85c4f593..ed98d1b3 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -169,7 +169,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, // SingleColumnValueExcludeFilter('cf1','col1',=,'binary:123',true,true) private static void toParseableByteArray(ByteArrayOutputStream byteStream, - SingleColumnValueExcludeFilter filter) throws IOException { + SingleColumnValueExcludeFilter filter) + throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write("('".getBytes()); writeBytesWithEscape(byteStream, filter.getFamily()); @@ -329,12 +330,13 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Times } // MultiRowRangeFilter('a',true,'b',false,'c',true,'d',false); - private static void toParseableByteArray(ByteArrayOutputStream byteStream, MultiRowRangeFilter filter) throws IOException { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + MultiRowRangeFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); List ranges = filter.getRowRanges(); - for (int i = 0; i < ranges.size(); i ++) { + for (int i = 0; i < ranges.size(); i++) { MultiRowRangeFilter.RowRange range = ranges.get(i); byteStream.write("'".getBytes()); byteStream.write(range.getStartRow()); @@ -354,7 +356,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Multi } // InclusiveStopFilter('aaa'); - private static void toParseableByteArray(ByteArrayOutputStream byteStream, InclusiveStopFilter filter) throws IOException { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + InclusiveStopFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); byteStream.write('\''); @@ -364,7 +367,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Inclu } // ColumnRangeFilter('a',true,'b',false); - private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnRangeFilter filter) throws IOException { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + ColumnRangeFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); @@ -382,12 +386,13 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Colum } // MultipleColumnPrefixFilter('a','b','d'); - private static void toParseableByteArray(ByteArrayOutputStream byteStream, MultipleColumnPrefixFilter filter) throws IOException { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + MultipleColumnPrefixFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); byte[][] ranges = filter.getPrefix(); - for (int i = 0; i < ranges.length; i ++) { + for (int i = 0; i < ranges.length; i++) { byte[] range = ranges[i]; byteStream.write("'".getBytes()); byteStream.write(range); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java b/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java index 640e432d..49fe17e9 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java @@ -9,11 +9,11 @@ public class BatchError { private final List throwables = new ArrayList(); - private final List actions = new ArrayList(); - private final List addresses = new ArrayList(); + private final List actions = new ArrayList(); + private final List addresses = new ArrayList(); public synchronized void add(Throwable ex, Row row, ServerName serverName) { - if (row == null){ + if (row == null) { throw new IllegalArgumentException("row cannot be null. location=" + serverName); } @@ -27,9 +27,8 @@ public boolean hasErrors() { } public synchronized RetriesExhaustedWithDetailsException makeException() { - return new RetriesExhaustedWithDetailsException( - new ArrayList(throwables), - new ArrayList(actions), new ArrayList(addresses)); + return new RetriesExhaustedWithDetailsException(new ArrayList(throwables), + new ArrayList(actions), new ArrayList(addresses)); } public synchronized void clear() { diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 5413c6d0..3a350d04 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -523,24 +523,24 @@ public void testFilter() throws Exception { tryPut(hTable, putKey2Column2Value1); tryPut(hTable, putKey2Column2Value2); -// time may be different -// +---------+-----+----------------+--------+ -// | K | Q | T | V | -// +---------+-----+----------------+--------+ -// | getKey1 | abc | -1728834971469 | value1 | -// | getKey1 | abc | -1728834971399 | value2 | -// | getKey1 | abc | -1728834971330 | value1 | -// | getKey1 | def | -1728834971748 | value2 | -// | getKey1 | def | -1728834971679 | value1 | -// | getKey1 | def | -1728834971609 | value2 | -// | getKey1 | def | -1728834971540 | value1 | -// | getKey2 | def | -1728834971887 | value2 | -// | getKey2 | def | -1728834971818 | value1 | -// +---------+-----+----------------+--------+ + // time may be different + // +---------+-----+----------------+--------+ + // | K | Q | T | V | + // +---------+-----+----------------+--------+ + // | getKey1 | abc | -1728834971469 | value1 | + // | getKey1 | abc | -1728834971399 | value2 | + // | getKey1 | abc | -1728834971330 | value1 | + // | getKey1 | def | -1728834971748 | value2 | + // | getKey1 | def | -1728834971679 | value1 | + // | getKey1 | def | -1728834971609 | value2 | + // | getKey1 | def | -1728834971540 | value1 | + // | getKey2 | def | -1728834971887 | value2 | + // | getKey2 | def | -1728834971818 | value1 | + // +---------+-----+----------------+--------+ SingleColumnValueFilter singleColumnValueFilter; singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -551,7 +551,7 @@ public void testFilter() throws Exception { SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -560,8 +560,8 @@ public void testFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(4, r.rawCells().length); - DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), false); + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter( + Bytes.toBytes(family), Bytes.toBytes(column1), false); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -570,7 +570,7 @@ public void testFilter() throws Exception { Assert.assertEquals(3, r.rawCells().length); dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), true); + Bytes.toBytes(column1), true); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -579,7 +579,7 @@ public void testFilter() throws Exception { Assert.assertEquals(0, r.rawCells().length); dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); get = new Get(toBytes(key2)); get.setMaxVersions(10); @@ -589,7 +589,7 @@ public void testFilter() throws Exception { Assert.assertEquals(0, r.rawCells().length); dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); get = new Get(toBytes(key2)); get.setMaxVersions(10); @@ -598,8 +598,6 @@ public void testFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(1, r.rawCells().length); - - filter = new ColumnPrefixFilter(Bytes.toBytes("e")); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -947,7 +945,7 @@ public void testFilter() throws Exception { tryPut(hTable, putKey2Column2Value2); dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); scan = new Scan(); scan.addFamily(family.getBytes()); @@ -957,7 +955,7 @@ public void testFilter() throws Exception { scan.setFilter(dependentColumnFilter); scanner = hTable.getScanner(scan); - long prevTimestamp = - 1; + long prevTimestamp = -1; for (Result result : scanner) { for (Cell keyValue : result.rawCells()) { if (prevTimestamp == -1) { @@ -1293,7 +1291,8 @@ public void testColumnRangeFilter() throws Exception { scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); - ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, Bytes.toBytes("b"), false); + ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, + Bytes.toBytes("b"), false); scan.setFilter(filter); ResultScanner scanner = hTable.getScanner(scan); @@ -1363,11 +1362,7 @@ public void testColumnRangeFilter() throws Exception { scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); - byte[][] range = { - Bytes.toBytes("g"), - Bytes.toBytes("3"), - Bytes.toBytes("d"), - }; + byte[][] range = { Bytes.toBytes("g"), Bytes.toBytes("3"), Bytes.toBytes("d"), }; MultipleColumnPrefixFilter iFilter = new MultipleColumnPrefixFilter(range); scan.setFilter(iFilter); scanner = hTable.getScanner(scan); @@ -1391,14 +1386,8 @@ public void testColumnRangeFilter() throws Exception { scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); - range = new byte[][]{ - Bytes.toBytes("de"), - Bytes.toBytes("bg"), - Bytes.toBytes("nc"), - Bytes.toBytes("aa"), - Bytes.toBytes("abcd"), - Bytes.toBytes("dea"), - }; + range = new byte[][] { Bytes.toBytes("de"), Bytes.toBytes("bg"), Bytes.toBytes("nc"), + Bytes.toBytes("aa"), Bytes.toBytes("abcd"), Bytes.toBytes("dea"), }; iFilter = new MultipleColumnPrefixFilter(range); scan.setFilter(iFilter); scanner = hTable.getScanner(scan); @@ -2367,8 +2356,8 @@ public void testGetFilter() throws Exception { get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); - DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column1)); + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter( + Bytes.toBytes(family), Bytes.toBytes(column1)); get.setFilter(dependentColumnFilter); r = hTable.get(get); Assert.assertEquals(3, r.rawCells().length); @@ -2555,7 +2544,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new SingleColumnValueExcludeFilter(Bytes.toBytes(family), Bytes - .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); + .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2565,7 +2554,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column1), false)); + .toBytes(column1), false)); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2575,7 +2564,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2), false)); + .toBytes(column2), false)); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2585,7 +2574,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2))); + .toBytes(column2))); get = new Get(toBytes(key2)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2595,7 +2584,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2), true)); + .toBytes(column2), true)); get = new Get(toBytes(key2)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2605,8 +2594,8 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(value2)))); + .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2)))); get = new Get(toBytes(key2)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2739,10 +2728,9 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(7, r.rawCells().length); - SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -2751,7 +2739,6 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(4, r.rawCells().length); - singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index 1a6875b1..2909bcb6 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -207,22 +207,25 @@ public void testBufferedMutatorWithFlush() throws Exception { conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); BufferedMutator putBufferMutator = null; BufferedMutator delBufferedMutator = null; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.addFamily(toBytes(family)); + hTable.delete(delete); + // use defualt params putBufferMutator = connection.getBufferedMutator(tableName); delBufferedMutator = connection.getBufferedMutator(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - Delete delete= new Delete(toBytes(key)); - delete.addFamily(toBytes("family_group")); - hTable.delete(delete); + long timestamp = System.currentTimeMillis(); // only support Put and Delete // for other type of operations, BufferedMutator will not set its type for them @@ -236,10 +239,10 @@ public void testBufferedMutatorWithFlush() throws Exception { List mutations = new ArrayList<>(); // test Put Put put1 = new Put(Bytes.toBytes(key)); - put1.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); + put1.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); mutations.add(put1); Put put2 = new Put(Bytes.toBytes(key)); - put2.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); + put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); mutations.add(put2); // test add Mutations with List putBufferMutator.mutate(mutations); @@ -255,7 +258,7 @@ public void testBufferedMutatorWithFlush() throws Exception { Assert.assertThrows(FeatureNotSupportedException.class, () -> { noCfMut.mutate(del); }); - del.addFamily(Bytes.toBytes("family_group")); + del.addFamily(Bytes.toBytes(family)); // test reuse different type bufferedMutator final BufferedMutator difTypeMut = putBufferMutator; Assert.assertThrows(IllegalArgumentException.class, () -> { @@ -283,8 +286,8 @@ public void testBufferedMutatorWithFlush() throws Exception { if (delBufferedMutator != null) { delBufferedMutator.close(); // test add mutations after closed - Delete delete = new Delete(Bytes.toBytes("putKey")); - delete.addFamily(Bytes.toBytes("family_group")); + Delete delete = new Delete(Bytes.toBytes(key)); + delete.addFamily(Bytes.toBytes(family)); final BufferedMutator closedMutator = delBufferedMutator; Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); @@ -311,24 +314,29 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); BufferedMutator putBufferMutator = null; BufferedMutator delBufferedMutator = null; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + String family = "family_group"; + long timestamp = System.currentTimeMillis(); try { // use n1 database TableName tableName = TableName.valueOf("n1","test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.addFamily(toBytes(family)); + hTable.delete(delete); + // use defualt params putBufferMutator = connection.getBufferedMutator(tableName); delBufferedMutator = connection.getBufferedMutator(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - // only support Put and Delete // for other type of operations, BufferedMutator will not set its type for them Append append = new Append(Bytes.toBytes(key)); - append.add("family_group".getBytes(), column1.getBytes(), toBytes("_suffix")); + append.add(toBytes(family), toBytes(column1), toBytes("_suffix")); final BufferedMutator apMut = putBufferMutator; Assert.assertThrows(IllegalArgumentException.class, () -> { apMut.mutate(append); @@ -337,10 +345,10 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { List mutations = new ArrayList<>(); // test Put Put put1 = new Put(Bytes.toBytes(key)); - put1.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); + put1.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); mutations.add(put1); Put put2 = new Put(Bytes.toBytes(key)); - put2.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); + put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); mutations.add(put2); // test add Mutations with List putBufferMutator.mutate(mutations); @@ -356,7 +364,7 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { Assert.assertThrows(FeatureNotSupportedException.class, () -> { noCfMut.mutate(del); }); - del.addFamily(Bytes.toBytes("family_group")); + del.addFamily(Bytes.toBytes(family)); final BufferedMutator difTypeMut = putBufferMutator; // test reuse different type bufferedMutator Assert.assertThrows(IllegalArgumentException.class, () -> { @@ -412,30 +420,31 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { BufferedMutatorParams params = null; long bufferSize = 45000L; int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.addFamily(toBytes(family)); + hTable.delete(delete); + // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); putBufferMutator = connection.getBufferedMutator(params); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - - Delete delete= new Delete(toBytes(key)); - delete.addFamily(toBytes("family_group")); - hTable.delete(delete); - List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); for (int j = 0; j < 4; ++j) { Put put = new Put(Bytes.toBytes(key)); - put.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "_" + i + "_" + j), + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "_" + i + "_" + j), timestamp, Bytes.toBytes(value + "_" + i + "_" + j)); mutations.add(put); } @@ -451,14 +460,14 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { } finally { if (putBufferMutator != null) { putBufferMutator.close(); - Get get = new Get(toBytes("putKey")); + Get get = new Get(toBytes(key)); Result r = hTable.get(get); for (Cell keyValue : r.rawCells()) { ++count; } Assert.assertEquals(200, count); - Delete delete = new Delete(toBytes("putKey")); - delete.addFamily(toBytes("family_group")); + Delete delete = new Delete(toBytes(key)); + delete.addFamily(toBytes(family)); hTable.delete(delete); r = hTable.get(get); Assert.assertEquals(0, r.rawCells().length); @@ -498,10 +507,20 @@ public void testBufferedMutatorWithUserPool() throws Exception { BufferedMutatorParams params = null; long bufferSize = 45000L; int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.addFamily(toBytes(family)); + hTable.delete(delete); + // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); @@ -514,21 +533,12 @@ public void testBufferedMutatorWithUserPool() throws Exception { ohBufferMutator = connection.getBufferedMutator(params); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - - Delete delete= new Delete(toBytes(key)); - delete.addFamily(toBytes("family_group")); - hTable.delete(delete); - List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); for (int j = 0; j < 4; ++j) { Put put = new Put(Bytes.toBytes(key)); - put.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "_" + i + "_" + j), + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "_" + i + "_" + j), timestamp, Bytes.toBytes(value + "_" + i + "_" + j)); mutations.add(put); } @@ -544,14 +554,14 @@ public void testBufferedMutatorWithUserPool() throws Exception { } finally { if (ohBufferMutator != null) { ohBufferMutator.close(); - Get get = new Get(toBytes("putKey")); + Get get = new Get(toBytes(key)); Result r = hTable.get(get); for (Cell keyValue : r.rawCells()) { ++count; } Assert.assertEquals(200, count); - Delete delete = new Delete(toBytes("putKey")); - delete.addFamily(toBytes("family_group")); + Delete delete = new Delete(toBytes(key)); + delete.addFamily(toBytes(family)); hTable.delete(delete); r = hTable.get(get); @@ -591,10 +601,20 @@ public void testBufferedMutatorConcurrent() throws Exception { ExecutorService executorService = Executors.newFixedThreadPool(10); long bufferSize = 45000L; int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.addFamily(toBytes(family)); + hTable.delete(delete); + // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); @@ -607,15 +627,6 @@ public void testBufferedMutatorConcurrent() throws Exception { ohBufferMutator = connection.getBufferedMutator(params); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - - Delete delete= new Delete(toBytes(key)); - delete.addFamily(toBytes("family_group")); - hTable.delete(delete); - for (int i = 0; i < 50; ++i) { final int taskId = i; final BufferedMutator thrBufferMutator = ohBufferMutator; @@ -628,7 +639,7 @@ public void testBufferedMutatorConcurrent() throws Exception { long thrTimestamp = timestamp; Put put = new Put(Bytes.toBytes(thrKey)); - put.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(thrColumn), + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(thrColumn), thrTimestamp, Bytes.toBytes(thrValue)); mutations.add(put); } @@ -663,14 +674,14 @@ public void testBufferedMutatorConcurrent() throws Exception { } if (ohBufferMutator != null) { ohBufferMutator.close(); - Get get = new Get(toBytes("putKey")); + Get get = new Get(toBytes(key)); Result r = hTable.get(get); for (Cell keyValue : r.rawCells()) { ++count; } Assert.assertEquals(200, count); - Delete delete = new Delete(toBytes("putKey")); - delete.addFamily(toBytes("family_group")); + Delete delete = new Delete(toBytes(key)); + delete.addFamily(toBytes(family)); hTable.delete(delete); r = hTable.get(get); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java new file mode 100644 index 00000000..26ee365a --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java @@ -0,0 +1,350 @@ +package com.alipay.oceanbase.hbase; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; +import org.junit.rules.ExpectedException; + +import java.io.IOException; + +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + +public class OHTableDeleteFamilyVersionTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + protected Table hTable; + private static final String key1 = "scanKey1x"; + private static final String key2 = "scanKey2x"; + private static final String key3 = "scanKey3x"; + private static final String column1 = "column1"; + private static final String column2 = "column2"; + private static final String column3 = "column3"; + private static final String value1 = "value1"; + private static final String value2 = "value2"; + private static final String value3 = "value3"; + private static final String family1 = "family_with_group1"; + private static final String family2 = "family_with_group2"; + + @Before + public void before() throws Exception { + hTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); + ((OHTableClient) hTable).init(); + } + + @After + public void finish() throws IOException { + hTable.close(); + } + + public void tryPut(Table hTable, Put put) throws Exception { + hTable.put(put); + Thread.sleep(1); + } + + @Test + public void testDeleteFamilyVerison() throws Exception { + // delete previous data + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.addFamily(toBytes(family1)); + deleteKey1Family.addFamily(toBytes(family2)); + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.addFamily(toBytes(family1)); + deleteKey2Family.addFamily(toBytes(family2)); + Delete deleteKey3Family = new Delete(toBytes(key3)); + deleteKey3Family.addFamily(toBytes(family1)); + deleteKey3Family.addFamily(toBytes(family2)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); + putKey1Fam1Column1MinTs.addColumn(toBytes(family1), toBytes(column1), minTimeStamp, + toBytes(value1)); + + Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column1Ts1.addColumn(toBytes(family1), toBytes(column1), timeStamp1, toBytes(value2)); + + Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); + putKey1Fam1Column2MinTs.addColumn(toBytes(family1), toBytes(column2), minTimeStamp, + toBytes(value1)); + + Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts3.addColumn(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts3.addColumn(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts1.addColumn(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column3Ts1.addColumn(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts6.addColumn(toBytes(family1), toBytes(column2), timeStamp6, toBytes(value1)); + + Put putKey2Fam1Column3Ts6 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts6.addColumn(toBytes(family1), toBytes(column3), timeStamp3, toBytes(value1)); + + tryPut(hTable, putKey1Fam1Column1MinTs); + tryPut(hTable, putKey3Fam1Column1Ts1); + tryPut(hTable, putKey1Fam1Column2MinTs); + tryPut(hTable, putKey1Fam1Column2Ts3); + tryPut(hTable, putKey2Fam1Column2Ts3); + tryPut(hTable, putKey2Fam1Column3Ts1); + tryPut(hTable, putKey3Fam1Column3Ts1); + tryPut(hTable, putKey3Fam1Column2Ts6); + tryPut(hTable, putKey2Fam1Column3Ts6); + + // test DeleteFamilyVersion single cf + Get get = new Get(toBytes(key1)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + Result r = hTable.get(get); + Assert.assertEquals(2, r.rawCells().length); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(2, r.rawCells().length); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(2, r.rawCells().length); + + Delete delKey1MinTs = new Delete(toBytes(key1)); + delKey1MinTs.addFamilyVersion(toBytes(family1), minTimeStamp); + hTable.delete(delKey1MinTs); + + get = new Get(toBytes(key1)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.rawCells().length); + + Delete delKey3Ts1 = new Delete(toBytes(key3)); + delKey3Ts1.addFamilyVersion(toBytes(family1), timeStamp1); + hTable.delete(delKey3Ts1); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.rawCells().length); + + Delete delKey2Ts3 = new Delete(toBytes(key2)); + delKey2Ts3.addFamilyVersion(family1.getBytes(), timeStamp3); + hTable.delete(delKey2Ts3); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.rawCells().length); + + Scan scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(toBytes(family1)); + scan.setMaxVersions(10); + ResultScanner scanner = hTable.getScanner(scan); + int key1Cnt = 0, key2Cnt = 0, key3Cnt = 0; + for (Result result : scanner) { + for (Cell kv : result.rawCells()) { + if (key1.equals(Bytes.toString(CellUtil.cloneRow(kv)))) { + ++key1Cnt; + } else if (key2.equals(Bytes.toString(CellUtil.cloneRow(kv)))) { + ++key2Cnt; + } else { + ++key3Cnt; + } + } + } + Assert.assertEquals(1, key1Cnt); + Assert.assertEquals(1, key2Cnt); + Assert.assertEquals(1, key3Cnt); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + + // test DeleteFamilyVersion multiple cf + Put putKey1Fam1Column3Ts6 = new Put(toBytes(key1)); + putKey1Fam1Column3Ts6.addColumn(toBytes(family1), toBytes(column3), timeStamp6, toBytes(value3)); + + Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column2Ts2.addColumn(toBytes(family2), toBytes(column2), timeStamp2, toBytes(value1)); + + Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column3Ts2.addColumn(toBytes(family2), toBytes(column3), timeStamp2, toBytes(value1)); + + Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts1.addColumn(toBytes(family1), toBytes(column2), timeStamp1, toBytes(value2)); + + Put putKey2Fam1Column2Ts8 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts8.addColumn(toBytes(family1), toBytes(column2), timeStamp8, toBytes(value2)); + + Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam2Column3Ts1.addColumn(toBytes(family2), toBytes(column3), timeStamp3, toBytes(value3)); + + Put putKey2Fam1Column1Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column1Ts1.addColumn(toBytes(family1), toBytes(column1), timeStamp8, toBytes(value1)); + + Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); + putKey2Fam2Column1Ts3.addColumn(toBytes(family2), toBytes(column1), timeStamp3, toBytes(value2)); + + Put putKey3Fam1Column2Ts9 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts9.addColumn(toBytes(family1), toBytes(column2), timeStamp9, toBytes(value2)); + + Put putKey3Fam2Column3Ts10 = new Put(toBytes(key3)); + putKey3Fam2Column3Ts10 + .addColumn(toBytes(family2), toBytes(column3), timeStamp10, toBytes(value1)); + + Put putKey3Fam2Column1Ts10 = new Put(toBytes(key3)); + putKey3Fam2Column1Ts10 + .addColumn(toBytes(family2), toBytes(column1), timeStamp10, toBytes(value2)); + + Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts2.addColumn(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); + + tryPut(hTable, putKey1Fam1Column3Ts6); + tryPut(hTable, putKey1Fam2Column2Ts2); + tryPut(hTable, putKey1Fam2Column3Ts2); + tryPut(hTable, putKey1Fam1Column2Ts1); + tryPut(hTable, putKey2Fam1Column2Ts8); + tryPut(hTable, putKey2Fam2Column3Ts1); + tryPut(hTable, putKey2Fam1Column1Ts1); + tryPut(hTable, putKey2Fam2Column1Ts3); + tryPut(hTable, putKey3Fam1Column2Ts9); + tryPut(hTable, putKey3Fam2Column3Ts10); + tryPut(hTable, putKey3Fam2Column1Ts10); + tryPut(hTable, putKey3Fam1Column2Ts2); + + Get getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = hTable.get(getKey1); + Assert.assertEquals(4, r.rawCells().length); + + Get getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = hTable.get(getKey2); + Assert.assertEquals(4, r.rawCells().length); + + Get getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setMaxVersions(10); + r = hTable.get(getKey3); + Assert.assertEquals(4, r.rawCells().length); + + Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); + delKey1Ts_6_2.addFamilyVersion(toBytes(family1), timeStamp6); + delKey1Ts_6_2.addFamilyVersion(toBytes(family2), timeStamp2); + hTable.delete(delKey1Ts_6_2); + + getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = hTable.get(getKey1); + Assert.assertEquals(1, r.rawCells().length); + for (Cell kv : r.rawCells()) { + Assert.assertEquals(timeStamp1, kv.getTimestamp()); + } + + Delete delKey2Ts_8_3 = new Delete(toBytes(key2)); + delKey2Ts_8_3.addFamilyVersion(toBytes(family1), timeStamp8); + delKey2Ts_8_3.addFamilyVersion(toBytes(family2), timeStamp3); + hTable.delete(delKey2Ts_8_3); + + getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = hTable.get(getKey2); + Assert.assertEquals(0, r.rawCells().length); + + Delete delKey3Ts_2_10 = new Delete(toBytes(key3)); + delKey3Ts_2_10.addFamilyVersion(toBytes(family1), timeStamp2); + delKey3Ts_2_10.addFamilyVersion(toBytes(family2), timeStamp10); + hTable.delete(delKey3Ts_2_10); + + getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setMaxVersions(10); + r = hTable.get(getKey3); + Assert.assertEquals(1, r.rawCells().length); + for (Cell kv : r.rawCells()) { + Assert.assertEquals(timeStamp9, kv.getTimestamp()); + } + + scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(toBytes(family1)); + scan.addFamily(toBytes(family2)); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + int ts1Cnt = 0, ts9Cnt = 0; + for (Result result : scanner) { + for (Cell kv : result.rawCells()) { + if (kv.getTimestamp() == timeStamp1) { + ++ts1Cnt; + } else if (kv.getTimestamp() == timeStamp9) { + ++ts9Cnt; + } + } + } + Assert.assertEquals(1, ts1Cnt); + Assert.assertEquals(1, ts9Cnt); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java index f0e66604..4fac8402 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java @@ -138,6 +138,39 @@ public void testSingleColumnValueFilter() throws IOException { } } + @Test + public void testSingleColumnValueExcludeFilter() throws IOException { + for (int i = 0; i < ops.length; i++) { + String expect = String + .format( + "SingleColumnValueExcludeFilter('family','qualifier',%s,'binary:value',false,true)", + opFlags[i]); + SingleColumnValueExcludeFilter filter = new SingleColumnValueExcludeFilter( + "family".getBytes(), "qualifier".getBytes(), ops[i], "value".getBytes()); + Assert.assertArrayEquals(expect.getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + } + + @Test + public void testDependentColumnFilter() throws IOException { + DependentColumnFilter filter = new DependentColumnFilter("family".getBytes(), + "qualifier".getBytes()); + String expect = "DependentColumnFilter('family','qualifier',false)"; + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + filter = new DependentColumnFilter("family".getBytes(), "qualifier".getBytes(), true); + expect = "DependentColumnFilter('family','qualifier',true)"; + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + for (int i = 0; i < ops.length; ++i) { + filter = new DependentColumnFilter("family".getBytes(), "qualifier".getBytes(), false, + ops[i], new BinaryComparator("value".getBytes())); + expect = String.format( + "DependentColumnFilter('family','qualifier',false,%s,'binary:value')", opFlags[i]); + Assert.assertArrayEquals(expect.getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + } + @Test public void testPageFilter() throws IOException { PageFilter filter = new PageFilter(128); @@ -188,24 +221,24 @@ public void testMultiRowRangeFilter() throws IOException { @Test public void testInclusiveStopFilter() throws IOException { InclusiveStopFilter filter = new InclusiveStopFilter(Bytes.toBytes("aaa")); - Assert.assertArrayEquals("InclusiveStopFilter('aaa')".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + Assert.assertArrayEquals("InclusiveStopFilter('aaa')".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test public void testColumnRangeFilter() throws IOException { - ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, Bytes.toBytes("b"), false); - Assert.assertArrayEquals("ColumnRangeFilter('a',true,'b',false)".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, + Bytes.toBytes("b"), false); + Assert.assertArrayEquals("ColumnRangeFilter('a',true,'b',false)".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test public void testMultipleColumnPrefixFilter() throws IOException { - byte[][] prefix = { - Bytes.toBytes("a"), - Bytes.toBytes("b"), - Bytes.toBytes("d"), - }; + byte[][] prefix = { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("d"), }; MultipleColumnPrefixFilter filter = new MultipleColumnPrefixFilter(prefix); - Assert.assertArrayEquals("MultipleColumnPrefixFilter('a','b','d')".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + Assert.assertArrayEquals("MultipleColumnPrefixFilter('a','b','d')".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test From 5d43b53857c3e563b871ce790f20743309e34d62 Mon Sep 17 00:00:00 2001 From: stuBirdFly <84010733+stuBirdFly@users.noreply.github.com> Date: Wed, 23 Oct 2024 11:42:52 +0800 Subject: [PATCH 13/32] hbase support batchCallBack (#86) --- .../com/alipay/oceanbase/hbase/OHTable.java | 14 +++++++-- .../hbase/OHTableMultiColumnFamilyTest.java | 29 ++++++++++++++++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index e0df60b8..60117770 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -519,8 +519,18 @@ private String getTargetTableName(List actions) { @Override public void batchCallback(List actions, Object[] results, Batch.Callback callback) throws IOException, - InterruptedException { - throw new FeatureNotSupportedException("not supported yet'"); + InterruptedException { + try { + batch(actions, results); + } finally { + if (results != null) { + for (int i = 0; i < results.length; i++) { + if (!(results[i] instanceof ObTableException)) { + callback.update(null, actions.get(i).getRow(), (R) results[i]); + } + } + } + } } public static int compareByteArray(byte[] bt1, byte[] bt2) { diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java index 5a2e1c15..7e1cfe08 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java @@ -17,10 +17,11 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.filter.PrefixFilter; import org.junit.*; import org.junit.rules.ExpectedException; @@ -152,6 +153,32 @@ public void testMultiColumnFamilyBatch() throws Exception { result = hTable.get(get); keyValues = result.rawCells(); assertEquals(6, keyValues.length); + + batchLsit.clear(); + final long[] updateCounter = new long[] { 0L }; + delete = new Delete(toBytes("Key5")); + delete.deleteColumns(family1, family1_column2); + delete.deleteColumns(family2, family2_column1); + delete.deleteFamily(family3); + batchLsit.add(delete); + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + batchLsit.add(put); + } + hTable.batchCallback(batchLsit, new Batch.Callback() { + @Override + public void update(byte[] region, byte[] row, MutationResult result) { + updateCounter[0]++; + } + }); + assertEquals(11, updateCounter[0]); + } @Test From f801cfe0a899925791aa3b19dddd364bdab58700 Mon Sep 17 00:00:00 2001 From: miyuan-ljr Date: Thu, 24 Oct 2024 15:27:01 +0800 Subject: [PATCH 14/32] fix test --- .../hbase/OHTableMultiColumnFamilyTest.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java index 7e1cfe08..8f252c5f 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java @@ -157,21 +157,22 @@ public void testMultiColumnFamilyBatch() throws Exception { batchLsit.clear(); final long[] updateCounter = new long[] { 0L }; delete = new Delete(toBytes("Key5")); - delete.deleteColumns(family1, family1_column2); - delete.deleteColumns(family2, family2_column1); - delete.deleteFamily(family3); + delete.addColumns(family1, family1_column2); + delete.addColumns(family2, family2_column1); + delete.addFamily(family3); batchLsit.add(delete); for (int i = 0; i < rows; ++i) { Put put = new Put(toBytes("Key" + i)); - put.add(family1, family1_column1, family1_value); - put.add(family1, family1_column2, family1_value); - put.add(family1, family1_column3, family1_value); - put.add(family2, family2_column1, family2_value); - put.add(family2, family2_column2, family2_value); - put.add(family3, family3_column1, family3_value); + put.addColumn(family1, family1_column1, family1_value); + put.addColumn(family1, family1_column2, family1_value); + put.addColumn(family1, family1_column3, family1_value); + put.addColumn(family2, family2_column1, family2_value); + put.addColumn(family2, family2_column2, family2_value); + put.addColumn(family3, family3_column1, family3_value); batchLsit.add(put); } - hTable.batchCallback(batchLsit, new Batch.Callback() { + results = new Object[batchLsit.size()]; + hTable.batchCallback(batchLsit, results, new Batch.Callback() { @Override public void update(byte[] region, byte[] row, MutationResult result) { updateCounter[0]++; From 4c96d00543fb9b211bb31209cdc685869ac1510e Mon Sep 17 00:00:00 2001 From: miyuan-ljr Date: Fri, 25 Oct 2024 11:27:10 +0800 Subject: [PATCH 15/32] fix test --- src/main/java/com/alipay/oceanbase/hbase/OHTable.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 60117770..44ea1df7 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1811,6 +1811,8 @@ public static OHOpType getDeleteType(Cell.Type type) { return OHOpType.DeleteColumn; case DeleteFamily: return OHOpType.DeleteFamily; + case DeleteFamilyVersion: + return OHOpType.DeleteFamilyVersion; default: throw new IllegalArgumentException("illegal mutation type " + type); } From f8568b62d3a2f0b054e371d399cfa4e788b92913 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 25 Oct 2024 14:11:36 +0800 Subject: [PATCH 16/32] init hbase_2.0 bufferedMutatorImpl --- .../com/alipay/oceanbase/hbase/OHTable.java | 105 +++++---- .../hbase/constants/OHConstants.java | 2 +- .../hbase/util/OHBufferedMutatorImpl.java | 200 ++++++++++++++---- .../hbase/util/OHConnectionConfiguration.java | 27 ++- .../hbase/util/OHConnectionImpl.java | 3 - .../oceanbase/hbase/OHConnectionTest.java | 65 ++++-- 6 files changed, 296 insertions(+), 106 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 7b15cbd9..d3a1d669 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -36,7 +36,6 @@ import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.*; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncRequest; import com.alipay.oceanbase.rpc.stream.ObTableClientQueryAsyncStreamResult; -import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult; import com.alipay.oceanbase.rpc.table.ObHBaseParams; import com.alipay.oceanbase.rpc.table.ObKVParams; import com.alipay.sofa.common.thread.SofaThreadPoolExecutor; @@ -1593,45 +1592,43 @@ private ObTableBatchOperation buildObTableBatchOperation(Map> } private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(Cell kv, - ObTableOperationType operationType, boolean isTableGroup) { - Cell.Type kvType = kv.getType(); + OHOpType operationType, boolean isTableGroup) { switch (operationType) { case INSERT_OR_UPDATE: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(INSERT_OR_UPDATE, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, V_COLUMNS, + new Object[] { CellUtil.cloneValue(kv) }); case APPEND: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(operationType, - ROW_KEY_COLUMNS, + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(APPEND, + ROW_KEY_COLUMNS, new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, V_COLUMNS, new Object[] { CellUtil.cloneValue(kv) }); - case DEL: - switch (kvType) { - case Delete: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, - null, null); -// case Maximum: -// return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, -// ROW_KEY_COLUMNS, -// new Object[] { CellUtil.cloneRow(kv), null, -kv.getTimestamp() }, -// null, null); - case DeleteColumn: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), -kv.getTimestamp() }, - null, null); - case DeleteFamily: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), isTableGroup ? CellUtil.cloneQualifier(kv) : null, -kv.getTimestamp() }, - null, null); - case DeleteFamilyVersion: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), isTableGroup ? CellUtil.cloneQualifier(kv) : null, kv.getTimestamp() }, - null, null); - default: - throw new IllegalArgumentException("illegal mutation type " + kvType); - } + case Delete: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, + null, null); + case DeleteAll: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), null, -kv.getTimestamp() }, + null, null); + case DeleteColumn: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), -kv.getTimestamp() }, + null, null); + case DeleteFamily: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), isTableGroup?CellUtil.cloneQualifier(kv):null, -kv.getTimestamp() }, + null, null); + case DeleteFamilyVersion: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), isTableGroup ? CellUtil.cloneQualifier(kv) : null, kv.getTimestamp() }, + null, null); default: throw new IllegalArgumentException("illegal mutation type " + operationType); } @@ -1674,9 +1671,9 @@ private BatchOperation buildBatchOperation(String tableName, List if(isTableGroup){ KeyValue new_kv = modifyQualifier(kv,(Bytes.toString(family) + "." + Bytes.toString(CellUtil.cloneQualifier(kv))) .getBytes()); - batch.addOperation(buildMutation(new_kv, INSERT_OR_UPDATE, isTableGroup)); + batch.addOperation(buildMutation(new_kv, OHOpType.INSERT_OR_UPDATE, isTableGroup)); } else { - batch.addOperation(buildMutation(kv, INSERT_OR_UPDATE, isTableGroup)); + batch.addOperation(buildMutation(kv, OHOpType.INSERT_OR_UPDATE, isTableGroup)); } } } @@ -1684,9 +1681,8 @@ private BatchOperation buildBatchOperation(String tableName, List Delete delete = (Delete) row; if (delete.isEmpty()) { singleOpResultNum++; - KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), - KeyValue.Type.Maximum); - batch.addOperation(buildMutation(kv, DEL, isTableGroup)); + KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp()); + batch.addOperation(buildMutation(kv, OHOpType.DeleteAll, isTableGroup)); } else { for (Map.Entry> entry : delete.getFamilyCellMap() .entrySet()) { @@ -1698,9 +1694,9 @@ private BatchOperation buildBatchOperation(String tableName, List if(isTableGroup){ KeyValue new_kv = modifyQualifier(kv,(Bytes.toString(family) + "." + Bytes.toString(CellUtil.cloneQualifier(kv))) .getBytes()); - batch.addOperation(buildMutation(new_kv, DEL, true)); + batch.addOperation(buildMutation(new_kv, getDeleteType(new_kv.getType()), true)); } else { - batch.addOperation(buildMutation(kv, DEL, false)); + batch.addOperation(buildMutation(kv, getDeleteType(kv.getType()), false)); } } } @@ -1847,4 +1843,29 @@ public byte[][] getEndKeys() throws IOException { public Pair getStartEndKeys() throws IOException { return new Pair<>(getStartKeys(), getEndKeys()); } + + public static enum OHOpType { + INSERT_OR_UPDATE, + APPEND, + Delete, + DeleteAll, + DeleteColumn, + DeleteFamily, + DeleteFamilyVersion, + } + + public static OHOpType getDeleteType(Cell.Type type) { + switch (type) { + case Delete: + return OHOpType.Delete; + case DeleteColumn: + return OHOpType.DeleteColumn; + case DeleteFamily: + return OHOpType.DeleteFamily; + case DeleteFamilyVersion: + return OHOpType.DeleteFamilyVersion; + default: + throw new IllegalArgumentException("illegal mutation type " + type); + } + } } \ No newline at end of file diff --git a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java index e10ef0b1..b4e70677 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java +++ b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java @@ -157,7 +157,7 @@ public final class OHConstants { public static final long WRITE_BUFFER_SIZE_DEFAULT = 2097152L; - public static final int MAX_KEYVALUE_SIZE_DEFAULT = -1; + public static final int MAX_KEYVALUE_SIZE_DEFAULT = 10485760; public static final String SOCKET_TIMEOUT = "ipc.socket.timeout"; diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index cdbe9b95..e5006a8c 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -18,16 +18,12 @@ package com.alipay.oceanbase.hbase.util; import com.alipay.oceanbase.hbase.OHTable; -import com.google.common.annotations.VisibleForTesting; -import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.*; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; +import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import java.io.IOException; @@ -35,6 +31,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.LCD; @@ -50,17 +47,25 @@ public class OHBufferedMutatorImpl implements BufferedMutator { private volatile Configuration conf; private final OHConnectionConfiguration connectionConfig; - @VisibleForTesting - final ConcurrentLinkedQueue asyncWriteBuffer = new ConcurrentLinkedQueue(); - @VisibleForTesting - AtomicLong currentAsyncBufferSize = new AtomicLong(0); + private final ConcurrentLinkedQueue asyncWriteBuffer = new ConcurrentLinkedQueue(); + private final AtomicLong currentAsyncBufferSize = new AtomicLong(0); + + private final AtomicLong firstRecordInBufferTimestamp = new AtomicLong(0); + private final AtomicLong executedWriteBufferPeriodicFlushes = new AtomicLong(0); + + private final AtomicLong writeBufferPeriodicFlushTimeoutMs = new AtomicLong(0); + private final AtomicLong writeBufferPeriodicFlushTimerTickMs = + new AtomicLong(MIN_WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS); + private Timer writeBufferPeriodicFlushTimer = null; private final long writeBufferSize; private final int maxKeyValueSize; - private boolean closed = false; private final ExecutorService pool; - private int rpcTimeout; - private int operationTimeout; + private final AtomicInteger undealtMutationCount = new AtomicInteger(0); + private final AtomicInteger rpcTimeout; + private final AtomicInteger operationTimeout; + private final boolean cleanipPoolOnClose; + private volatile boolean closed = false; public OHBufferedMutatorImpl(OHConnectionImpl ohConnection, BufferedMutatorParams params) throws IOException { @@ -72,9 +77,29 @@ public OHBufferedMutatorImpl(OHConnectionImpl ohConnection, BufferedMutatorParam this.conf = ohConnection.getConfiguration(); this.connectionConfig = ohConnection.getOHConnectionConfiguration(); this.listener = params.getListener(); - this.pool = params.getPool(); - this.rpcTimeout = connectionConfig.getRpcTimeout(); - this.operationTimeout = connectionConfig.getOperationTimeout(); + if (params.getPool() == null) { // need to verify necessity + this.pool = HTable.getDefaultExecutor(conf); + this.cleanipPoolOnClose = true; + } else { + this.pool = params.getPool(); + this.cleanipPoolOnClose = false; + } + this.rpcTimeout = new AtomicInteger( + params.getRpcTimeout() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? + params.getRpcTimeout() : connectionConfig.getRpcTimeout() + ); + this.operationTimeout = new AtomicInteger( + params.getOperationTimeout() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? + params.getOperationTimeout() : connectionConfig.getOperationTimeout() + ); + + long newPeriodicFlushTimeoutMs = + params.getWriteBufferPeriodicFlushTimeoutMs() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? + params.getWriteBufferPeriodicFlushTimeoutMs() : connectionConfig.getWriteBufferPeriodicFlushTimeoutMs(); + long newPeriodicFlushTimeIntervalMs = + params.getWriteBufferPeriodicFlushTimerTickMs() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? + params.getWriteBufferPeriodicFlushTimerTickMs() : connectionConfig.getWriteBufferPeriodicFlushTimerTickMs(); + this.setWriteBufferPeriodicFlush(newPeriodicFlushTimeoutMs, newPeriodicFlushTimeIntervalMs); this.writeBufferSize = params.getWriteBufferSize() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params .getWriteBufferSize() : connectionConfig.getWriteBufferSize(); @@ -112,26 +137,33 @@ public void mutate(Mutation mutation) throws IOException { */ @Override public void mutate(List mutations) throws IOException { - if (closed) { - throw new IllegalStateException("Cannot put when the BufferedMutator is closed."); - } + checkClose(); if (mutations.isEmpty()) { return; } long toAddSize = 0; + int toAddCount = 0; for (Mutation m : mutations) { validateInsUpAndDelete(m); toAddSize += m.heapSize(); + ++toAddCount; } + if (currentAsyncBufferSize.get() == 0) { + firstRecordInBufferTimestamp.set(System.currentTimeMillis()); + } + undealtMutationCount.addAndGet(toAddCount); currentAsyncBufferSize.addAndGet(toAddSize); asyncWriteBuffer.addAll(mutations); - if (currentAsyncBufferSize.get() > writeBufferSize) { - execute(false); - } + execute(false); + } + private void checkClose() { + if (closed) { + throw new IllegalStateException("Cannot put when the BufferedMutator is closed."); + } } /** @@ -151,36 +183,97 @@ private void validateInsUpAndDelete(Mutation mt) throws IllegalArgumentException } } + public void timeTriggerForWriteBufferPeriodicFlush() { + if (currentAsyncBufferSize.get() == 0) { + return; + } + long now = System.currentTimeMillis(); + if (firstRecordInBufferTimestamp.get() + writeBufferPeriodicFlushTimeoutMs.get() > now) { + // too soon to execute + return; + } + try { + executedWriteBufferPeriodicFlushes.incrementAndGet(); + flush(); + } catch (Exception e) { + LOGGER.error("Errors occur during timeTriggerForWriteBufferPeriodicFlush: { " + e.getMessage() + " }"); + } + } + + @Override + public synchronized void setWriteBufferPeriodicFlush(long timeoutMs, long timerTickMs) { + long originalTimeoutMs = this.writeBufferPeriodicFlushTimeoutMs.get(); + long originalTimeTickMs = this.writeBufferPeriodicFlushTimerTickMs.get(); + + writeBufferPeriodicFlushTimeoutMs.set(Math.max(0, timeoutMs)); + writeBufferPeriodicFlushTimerTickMs.set( + Math.max(MIN_WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS, timerTickMs)); + + // if time parameters are updated, stop the old timer + if (writeBufferPeriodicFlushTimeoutMs.get() != originalTimeoutMs || + writeBufferPeriodicFlushTimerTickMs.get() != originalTimeTickMs) { + if (writeBufferPeriodicFlushTimer != null) { + writeBufferPeriodicFlushTimer.cancel(); + writeBufferPeriodicFlushTimer = null; + } + } + + if (writeBufferPeriodicFlushTimer == null && + writeBufferPeriodicFlushTimeoutMs.get() > 0) { + writeBufferPeriodicFlushTimer = new Timer(true); + writeBufferPeriodicFlushTimer.schedule(new TimerTask() { + @Override + public void run() { + OHBufferedMutatorImpl.this.timeTriggerForWriteBufferPeriodicFlush(); + } + }, this.writeBufferPeriodicFlushTimerTickMs.get(), + this.writeBufferPeriodicFlushTimerTickMs.get()); + } + } + /** * Send the operations in the buffer to the servers. Does not wait for the server's answer. If * there is an error, either throw the error, or use the listener to deal with the error. * - * @param flushAll - if true, sends all the writes and wait for all of them to finish before - * returning. + * @param flushAll - if true, force to commit all mutations in asyncWriteBuffer; else to commit only if + * larger than writeBufferSize */ private void execute(boolean flushAll) throws IOException { LinkedList execBuffer = new LinkedList<>(); - long dequeuedSize = 0L; try { - Mutation m; - while ((writeBufferSize <= 0 || dequeuedSize < (writeBufferSize * 2) || flushAll) - && (m = asyncWriteBuffer.poll()) != null) { - execBuffer.add(m); - long size = m.heapSize(); - currentAsyncBufferSize.addAndGet(-size); - dequeuedSize += size; + if (flushAll || currentAsyncBufferSize.get() > writeBufferSize) { + Mutation m; + int dealtCount = 0; + while ((m = asyncWriteBuffer.poll())!= null) { + execBuffer.add(m); + long size = m.heapSize(); + currentAsyncBufferSize.addAndGet(-size); + ++dealtCount; + } + if (currentAsyncBufferSize.get() > 0) { + while (!execBuffer.isEmpty()) { + m = execBuffer.getFirst(); + long size = m.heapSize(); + currentAsyncBufferSize.addAndGet(size); + asyncWriteBuffer.add(m); + } + throw new IllegalStateException("Fetch error null value during execute"); + } + undealtMutationCount.addAndGet(-dealtCount); } if (execBuffer.isEmpty()) { return; } ohTable.batch(execBuffer); + // if commit all successfully, clean execBuffer + execBuffer.clear(); } catch (Exception ex) { LOGGER.error(LCD.convert("01-00026"), ex); if (ex.getCause() instanceof RetriesExhaustedWithDetailsException) { LOGGER.error(tableName + ": One or more of the operations have failed after retries."); RetriesExhaustedWithDetailsException retryException = (RetriesExhaustedWithDetailsException) ex.getCause(); - // recollect mutations + // recollect failed mutations execBuffer.clear(); for (int i = 0; i < retryException.getNumExceptions(); ++i) { execBuffer.add((Mutation) retryException.getRow(i)); @@ -199,28 +292,38 @@ private void execute(boolean flushAll) throws IOException { long size = mutation.heapSize(); currentAsyncBufferSize.addAndGet(size); asyncWriteBuffer.add(mutation); + undealtMutationCount.incrementAndGet(); } } } + @Override + public void disableWriteBufferPeriodicFlush() { + setWriteBufferPeriodicFlush(0, MIN_WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS); + } + @Override public void close() throws IOException { if (closed) { return; } + // reset timeout, timeInterval and Timer + disableWriteBufferPeriodicFlush(); try { execute(true); } finally { - // the pool in ObTableClient will be shut down too - this.pool.shutdown(); - try { - if (!pool.awaitTermination(600, TimeUnit.SECONDS)) { - LOGGER - .warn("close() failed to terminate pool after 10 minutes. Abandoning pool."); + if (cleanipPoolOnClose) { + // the pool in ObTableClient will be shut down too + this.pool.shutdown(); + try { + if (!pool.awaitTermination(600, TimeUnit.SECONDS)) { + LOGGER + .warn("close() failed to terminate pool after 10 minutes. Abandoning pool."); + } + } catch (InterruptedException e) { + LOGGER.warn("waitForTermination interrupted"); + Thread.currentThread().interrupt(); } - } catch (InterruptedException e) { - LOGGER.warn("waitForTermination interrupted"); - Thread.currentThread().interrupt(); } closed = true; } @@ -232,21 +335,30 @@ public void close() throws IOException { */ @Override public void flush() throws IOException { + checkClose(); execute(true); } + /** + * Count the mutations which haven't been processed. + */ + @VisibleForTesting + int size() { + return undealtMutationCount.get(); + } + @Override public long getWriteBufferSize() { return this.writeBufferSize; } public void setRpcTimeout(int rpcTimeout) { - this.rpcTimeout = rpcTimeout; + this.rpcTimeout.set(rpcTimeout); this.ohTable.setRpcTimeout(rpcTimeout); } public void setOperationTimeout(int operationTimeout) { - this.operationTimeout = operationTimeout; + this.operationTimeout.set(operationTimeout); this.ohTable.setOperationTimeout(operationTimeout); } } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java index f602fdda..8ff8c677 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java @@ -25,6 +25,10 @@ import java.util.Properties; import static com.alipay.oceanbase.hbase.constants.OHConstants.*; +import static org.apache.hadoop.hbase.client.ConnectionConfiguration.WRITE_BUFFER_PERIODIC_FLUSH_TIMEOUT_MS; +import static org.apache.hadoop.hbase.client.ConnectionConfiguration.WRITE_BUFFER_PERIODIC_FLUSH_TIMEOUT_MS_DEFAULT; +import static org.apache.hadoop.hbase.client.ConnectionConfiguration.WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS; +import static org.apache.hadoop.hbase.client.ConnectionConfiguration.WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS_DEFAULT; import static org.apache.commons.lang.StringUtils.isBlank; import static org.apache.hadoop.hbase.ipc.RpcClient.DEFAULT_SOCKET_TIMEOUT_CONNECT; import static org.apache.hadoop.hbase.ipc.RpcClient.SOCKET_TIMEOUT_CONNECT; @@ -48,6 +52,8 @@ public class OHConnectionConfiguration { private final int maxKeyValueSize; private final int rpcTimeout; private final int rpcConnectTimeout; + private final long writeBufferPeriodicFlushTimeoutMs; + private final long writeBufferPeriodicFlushTimerTickMs; public OHConnectionConfiguration(Configuration conf) { this.paramUrl = conf.get(HBASE_OCEANBASE_PARAM_URL); @@ -64,9 +70,16 @@ public OHConnectionConfiguration(Configuration conf) { } this.database = database; this.writeBufferSize = conf.getLong(WRITE_BUFFER_SIZE_KEY, WRITE_BUFFER_SIZE_DEFAULT); - this.operationTimeout = conf.getInt("hbase.client.operation.timeout", 1200000); + this.operationTimeout = conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, + HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT); this.rpcTimeout = conf.getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT); + this.writeBufferPeriodicFlushTimeoutMs = conf.getLong( + WRITE_BUFFER_PERIODIC_FLUSH_TIMEOUT_MS, + WRITE_BUFFER_PERIODIC_FLUSH_TIMEOUT_MS_DEFAULT); + this.writeBufferPeriodicFlushTimerTickMs = conf.getLong( + WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS, + WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS_DEFAULT); int rpcConnectTimeout = -1; if (conf.get(SOCKET_TIMEOUT_CONNECT) != null) { rpcConnectTimeout = conf.getInt(SOCKET_TIMEOUT_CONNECT, DEFAULT_SOCKET_TIMEOUT_CONNECT); @@ -79,8 +92,8 @@ public OHConnectionConfiguration(Configuration conf) { } } this.rpcConnectTimeout = rpcConnectTimeout; - this.scannerCaching = conf.getInt("hbase.client.scanner.caching", Integer.MAX_VALUE); - this.scannerMaxResultSize = conf.getLong("hbase.client.scanner.max.result.size", + this.scannerCaching = conf.getInt(HConstants.HBASE_CLIENT_SCANNER_CACHING, Integer.MAX_VALUE); + this.scannerMaxResultSize = conf.getLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, WRITE_BUFFER_SIZE_DEFAULT); this.maxKeyValueSize = conf.getInt(MAX_KEYVALUE_SIZE_KEY, MAX_KEYVALUE_SIZE_DEFAULT); properties = new Properties(); @@ -167,4 +180,12 @@ public boolean isOdpMode() { public String getDatabase() { return this.database; } + + public long getWriteBufferPeriodicFlushTimeoutMs() { + return this.writeBufferPeriodicFlushTimeoutMs; + } + + public long getWriteBufferPeriodicFlushTimerTickMs() { + return this.writeBufferPeriodicFlushTimerTickMs; + } } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java index ad0219f7..939a88f5 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java @@ -129,9 +129,6 @@ public BufferedMutator getBufferedMutator(BufferedMutatorParams params) throws I if (params.getTableName() == null) { throw new IllegalArgumentException("TableName cannot be null."); } - if (params.getPool() == null) { - params.pool(HTable.getDefaultExecutor(getConfiguration())); - } if (params.getWriteBufferSize() == BUFFERED_PARAM_UNSET) { params.writeBufferSize(connectionConfig.getWriteBufferSize()); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index 5513a426..f9827ef2 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -265,6 +265,7 @@ public void testBufferedMutatorWithFlush() throws Exception { // test Delete Delete del = new Delete(toBytes(key)); + del.addFamily(toBytes(family)); // test without setting family, delete all bufferMutator.mutate(del); bufferMutator.flush(); @@ -294,11 +295,13 @@ public void testBufferedMutatorWithFlush() throws Exception { if (bufferMutator != null ) { bufferMutator.close(); // test flush after closed - bufferMutator.flush(); + final BufferedMutator closedMutator = bufferMutator; + Assert.assertThrows(IllegalStateException.class, () -> { + closedMutator.flush(); + }); // test add mutations after closed Delete delete = new Delete(Bytes.toBytes(key)); delete.addFamily(Bytes.toBytes(family)); - final BufferedMutator closedMutator = bufferMutator; Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); }); @@ -411,11 +414,13 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { if (bufferMutator != null ) { bufferMutator.close(); // test flush after closed - bufferMutator.flush(); + final BufferedMutator closedMutator = bufferMutator; + Assert.assertThrows(IllegalStateException.class, () -> { + closedMutator.flush(); + }); // test add mutations after closed Delete delete = new Delete(Bytes.toBytes(key)); delete.addFamily(Bytes.toBytes(family)); - final BufferedMutator closedMutator = bufferMutator; Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); }); @@ -510,7 +515,9 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { closedMutator.mutate(delete); }); // test flush after closed - bufferMutator.flush(); + Assert.assertThrows(IllegalStateException.class, () -> { + closedMutator.flush(); + }); } if (params != null) { @@ -537,6 +544,7 @@ public void testBufferedMutatorWithUserPool() throws Exception { conf.set("rs.list.acquire.read.timeout", "10000"); BufferedMutator ohBufferMutator = null; BufferedMutatorParams params = null; + ThreadPoolExecutor pool = null; long bufferSize = 45000L; int count = 0; String key = "putKey"; @@ -559,7 +567,7 @@ public void testBufferedMutatorWithUserPool() throws Exception { // set thread pool long keepAliveTime = conf.getLong("hbase.hconnection.threads.keepalivetime", 60); - ThreadPoolExecutor pool = new ThreadPoolExecutor(10, 256, keepAliveTime, TimeUnit.SECONDS, new SynchronousQueue(), Threads.newDaemonThreadFactory("htable")); + pool = new ThreadPoolExecutor(10, 256, keepAliveTime, TimeUnit.SECONDS, new SynchronousQueue(), Threads.newDaemonThreadFactory("htable")); pool.allowCoreThreadTimeOut(true); params.pool(pool); @@ -615,10 +623,23 @@ public void testBufferedMutatorWithUserPool() throws Exception { closedMutator.mutate(delete); }); // test flush after closed - ohBufferMutator.flush(); + Assert.assertThrows(IllegalStateException.class, () -> { + closedMutator.flush(); + }); } if (params != null) { - if (params.getPool() != null) { + if (pool != null) { + // self-defined pool must be shutdown by users + Assert.assertFalse(params.getPool().isShutdown()); + pool.shutdown(); + try { + if (!pool.awaitTermination(600, TimeUnit.SECONDS)) { + System.out.println("close() failed to terminate pool after 10 minutes. Abandoning pool."); + } + } catch (InterruptedException e) { + System.out.println("waitForTermination interrupted"); + Thread.currentThread().interrupt(); + } Assert.assertTrue(params.getPool().isShutdown()); } } @@ -641,6 +662,7 @@ public void testBufferedMutatorConcurrent() throws Exception { conf.set("rs.list.acquire.read.timeout", "10000"); BufferedMutator ohBufferMutator = null; BufferedMutatorParams params = null; + ThreadPoolExecutor pool = null; ExecutorService executorService = Executors.newFixedThreadPool(10); long bufferSize = 45000L; int count = 0; @@ -664,7 +686,7 @@ public void testBufferedMutatorConcurrent() throws Exception { // set thread pool long keepAliveTime = conf.getLong("hbase.hconnection.threads.keepalivetime", 60); - ThreadPoolExecutor pool = new ThreadPoolExecutor(10, 256, keepAliveTime, TimeUnit.SECONDS, new SynchronousQueue(), Threads.newDaemonThreadFactory("htable")); + pool = new ThreadPoolExecutor(10, 256, keepAliveTime, TimeUnit.SECONDS, new SynchronousQueue(), Threads.newDaemonThreadFactory("htable")); pool.allowCoreThreadTimeOut(true); params.pool(pool); @@ -706,9 +728,10 @@ public void testBufferedMutatorConcurrent() throws Exception { } Assert.assertTrue(false); } finally { + System.out.println(1); executorService.shutdown(); try { - if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) { + if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { executorService.shutdownNow(); } } catch (InterruptedException e) { @@ -716,16 +739,18 @@ public void testBufferedMutatorConcurrent() throws Exception { Thread.currentThread().interrupt(); } if (ohBufferMutator != null) { + System.out.println(3); ohBufferMutator.close(); Get get = new Get(toBytes(key)); get.addFamily(toBytes(family)); Result r = hTable.get(get); + System.out.println(4); count = r.rawCells().length; Assert.assertEquals(200, count); Delete delete = new Delete(toBytes(key)); delete.addFamily(toBytes(family)); hTable.delete(delete); - + System.out.println(5); r = hTable.get(get); Assert.assertEquals(0, r.rawCells().length); // test add mutations after closed @@ -734,10 +759,24 @@ public void testBufferedMutatorConcurrent() throws Exception { closedMutator.mutate(delete); }); // test flush after closed - ohBufferMutator.flush(); + Assert.assertThrows(IllegalStateException.class, () -> { + closedMutator.flush(); + }); } + System.out.println(2); if (params != null) { - if (params.getPool() != null) { + if (pool != null) { + // self-defined pool must be shutdown by users + Assert.assertFalse(params.getPool().isShutdown()); + pool.shutdown(); + try { + if (!pool.awaitTermination(600, TimeUnit.SECONDS)) { + System.out.println("close() failed to terminate pool after 10 minutes. Abandoning pool."); + } + } catch (InterruptedException e) { + System.out.println("waitForTermination interrupted"); + Thread.currentThread().interrupt(); + } Assert.assertTrue(params.getPool().isShutdown()); } } From 56f838fa74249ef5aebceeb37eb91f16a06637c6 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 25 Oct 2024 15:33:29 +0800 Subject: [PATCH 17/32] pass single bufferedMutator test --- .../hbase/util/OHBufferedMutatorImpl.java | 21 ++- .../hbase/util/OHConnectionImpl.java | 1 + .../oceanbase/hbase/OHConnectionTest.java | 172 +++++++++++++++--- .../hbase/OHTableMultiColumnFamilyTest.java | 65 +++++++ 4 files changed, 231 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index e5006a8c..34d03bfb 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -167,7 +167,7 @@ private void checkClose() { } /** - * Check whether the mutation is Put or Delete in 1.x + * Check whether the mutation is Put or Delete in 2.x * @param mt - mutation operation */ private void validateInsUpAndDelete(Mutation mt) throws IllegalArgumentException { @@ -183,6 +183,9 @@ private void validateInsUpAndDelete(Mutation mt) throws IllegalArgumentException } } + /** + * triggered to do periodic flush if reach the time limit + * */ public void timeTriggerForWriteBufferPeriodicFlush() { if (currentAsyncBufferSize.get() == 0) { return; @@ -200,6 +203,11 @@ public void timeTriggerForWriteBufferPeriodicFlush() { } } + /** + * set time for periodic flush timer + * @param timeoutMs control when to flush from collecting first mutation + * @param timerTickMs control time interval to trigger the timer + * */ @Override public synchronized void setWriteBufferPeriodicFlush(long timeoutMs, long timerTickMs) { long originalTimeoutMs = this.writeBufferPeriodicFlushTimeoutMs.get(); @@ -297,6 +305,9 @@ private void execute(boolean flushAll) throws IOException { } } + /** + * reset the time parameters and cancel the timer (if exists) + * */ @Override public void disableWriteBufferPeriodicFlush() { setWriteBufferPeriodicFlush(0, MIN_WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS); @@ -331,7 +342,6 @@ public void close() throws IOException { /** * Force to commit all operations - * do not care whether the pool is shut down or this BufferedMutator is closed */ @Override public void flush() throws IOException { @@ -343,10 +353,15 @@ public void flush() throws IOException { * Count the mutations which haven't been processed. */ @VisibleForTesting - int size() { + public int size() { return undealtMutationCount.get(); } + @VisibleForTesting + public ExecutorService getPool() { + return pool; + } + @Override public long getWriteBufferSize() { return this.writeBufferSize; diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java index 939a88f5..b3ba4744 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java @@ -129,6 +129,7 @@ public BufferedMutator getBufferedMutator(BufferedMutatorParams params) throws I if (params.getTableName() == null) { throw new IllegalArgumentException("TableName cannot be null."); } + // do not check whether the pool is null, cause now the bufferedMutator will take over the control if (params.getWriteBufferSize() == BUFFERED_PARAM_UNSET) { params.writeBufferSize(connectionConfig.getWriteBufferSize()); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index f9827ef2..e6689ab1 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -261,6 +261,12 @@ public void testBufferedMutatorWithFlush() throws Exception { bufferMutator.mutate(put3); bufferMutator.flush(); r = hTable.get(get); +// for (Cell c : r.rawCells()) { +// System.out.println("2. Key: " + Bytes.toString(CellUtil.cloneRow(c)) + +// ", Qualifier: " + Bytes.toString(CellUtil.cloneQualifier(c)) + +// ", Time: " + c.getTimestamp() + +// ", Value: " + Bytes.toString(CellUtil.cloneValue(c))); +// } Assert.assertEquals(3, r.rawCells().length); // test Delete @@ -305,6 +311,9 @@ public void testBufferedMutatorWithFlush() throws Exception { Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); }); + if (((OHBufferedMutatorImpl) bufferMutator).getPool() != null) { + Assert.assertTrue(((OHBufferedMutatorImpl) bufferMutator).getPool().isShutdown()); + } } } } @@ -424,6 +433,9 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); }); + if (((OHBufferedMutatorImpl) bufferMutator).getPool() != null) { + Assert.assertTrue(((OHBufferedMutatorImpl) bufferMutator).getPool().isShutdown()); + } } } } @@ -518,11 +530,8 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.flush(); }); - } - - if (params != null) { - if (params.getPool() != null) { - Assert.assertTrue(params.getPool().isShutdown()); + if (((OHBufferedMutatorImpl) bufferMutator).getPool() != null) { + Assert.assertTrue(((OHBufferedMutatorImpl) bufferMutator).getPool().isShutdown()); } } } @@ -626,21 +635,20 @@ public void testBufferedMutatorWithUserPool() throws Exception { Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.flush(); }); - } - if (params != null) { - if (pool != null) { + ExecutorService bufferPool = ((OHBufferedMutatorImpl) ohBufferMutator).getPool(); + if (bufferPool != null) { // self-defined pool must be shutdown by users - Assert.assertFalse(params.getPool().isShutdown()); - pool.shutdown(); + Assert.assertFalse(bufferPool.isShutdown()); + bufferPool.shutdown(); try { - if (!pool.awaitTermination(600, TimeUnit.SECONDS)) { + if (!bufferPool.awaitTermination(600, TimeUnit.SECONDS)) { System.out.println("close() failed to terminate pool after 10 minutes. Abandoning pool."); } } catch (InterruptedException e) { System.out.println("waitForTermination interrupted"); Thread.currentThread().interrupt(); } - Assert.assertTrue(params.getPool().isShutdown()); + Assert.assertTrue(bufferPool.isShutdown()); } } } @@ -693,7 +701,7 @@ public void testBufferedMutatorConcurrent() throws Exception { ohBufferMutator = connection.getBufferedMutator(params); // BufferedMutator is not concurrently safe - for (int i = 0; i < 50; ++i) { + for (int i = 0; i < 10; ++i) { final int taskId = i; final BufferedMutator thrBufferMutator = ohBufferMutator; executorService.submit(() -> { @@ -728,7 +736,6 @@ public void testBufferedMutatorConcurrent() throws Exception { } Assert.assertTrue(false); } finally { - System.out.println(1); executorService.shutdown(); try { if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { @@ -739,18 +746,15 @@ public void testBufferedMutatorConcurrent() throws Exception { Thread.currentThread().interrupt(); } if (ohBufferMutator != null) { - System.out.println(3); ohBufferMutator.close(); Get get = new Get(toBytes(key)); get.addFamily(toBytes(family)); Result r = hTable.get(get); - System.out.println(4); count = r.rawCells().length; - Assert.assertEquals(200, count); + Assert.assertEquals(40, count); Delete delete = new Delete(toBytes(key)); delete.addFamily(toBytes(family)); hTable.delete(delete); - System.out.println(5); r = hTable.get(get); Assert.assertEquals(0, r.rawCells().length); // test add mutations after closed @@ -762,22 +766,140 @@ public void testBufferedMutatorConcurrent() throws Exception { Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.flush(); }); + ExecutorService bufferPool = ((OHBufferedMutatorImpl) ohBufferMutator).getPool(); + if (bufferPool != null) { + // self-defined pool must be shutdown by users + Assert.assertFalse(bufferPool.isShutdown()); + bufferPool.shutdown(); + try { + if (!bufferPool.awaitTermination(600, TimeUnit.SECONDS)) { + System.out.println("close() failed to terminate pool after 10 minutes. Abandoning pool."); + } + } catch (InterruptedException e) { + System.out.println("waitForTermination interrupted"); + Thread.currentThread().interrupt(); + } + Assert.assertTrue(bufferPool.isShutdown()); + } + } + } + } + + @Test + public void testBufferedMutatorPeriodicFlush() throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + conf.set("rs.list.acquire.read.timeout", "10000"); + BufferedMutator ohBufferMutator = null; + BufferedMutatorParams params = null; + ThreadPoolExecutor pool = null; + long bufferSize = 45000L; + int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; + try { + TableName tableName = TableName.valueOf("test"); + connection = ConnectionFactory.createConnection(conf); + hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.addFamily(toBytes(family)); + hTable.delete(delete); + + // set params + params = new BufferedMutatorParams(tableName); + params.writeBufferSize(bufferSize); + // set periodic flush timeout to enable Timer + params.setWriteBufferPeriodicFlushTimeoutMs(100); + + // set thread pool + long keepAliveTime = conf.getLong("hbase.hconnection.threads.keepalivetime", 60); + pool = new ThreadPoolExecutor(10, 256, keepAliveTime, TimeUnit.SECONDS, new SynchronousQueue(), Threads.newDaemonThreadFactory("htable")); + pool.allowCoreThreadTimeOut(true); + params.pool(pool); + + ohBufferMutator = connection.getBufferedMutator(params); + + List mutations = new ArrayList<>(); + for (int i = 0; i < 50; ++i) { + mutations.clear(); + for (int j = 0; j < 4; ++j) { + Put put = new Put(Bytes.toBytes(key)); + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "_" + i + "_" + j), + timestamp, Bytes.toBytes(value + "_" + i + "_" + j)); + mutations.add(put); + } + if (i % 10 == 0) { // 0, 10, 20, 30, 40 + for(int j = 0; j < 4; ++j) { + Delete del = new Delete(Bytes.toBytes(key)); + del.addColumns(toBytes(family), toBytes(column1 + "_" + i + "_" + j)); + mutations.add(del); + } + } + ohBufferMutator.mutate(mutations); + } + + // test auto flush + Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); + Result r = hTable.get(get); + count = r.rawCells().length; + Assert.assertTrue(count > 0); + + // test timer periodic flush + int lastUndealtCount = ((OHBufferedMutatorImpl) ohBufferMutator).size(); + Thread.sleep(1000); + int currentUndealtCount = ((OHBufferedMutatorImpl) ohBufferMutator).size(); + Assert.assertNotEquals(lastUndealtCount, currentUndealtCount); + r = hTable.get(get); + int newCount = r.rawCells().length; + Assert.assertNotEquals(count, newCount); + } catch (Exception ex) { + if (ex instanceof RetriesExhaustedWithDetailsException) { + ((RetriesExhaustedWithDetailsException) ex).getCauses().get(0).printStackTrace(); + } else { + ex.printStackTrace(); } - System.out.println(2); - if (params != null) { - if (pool != null) { + Assert.assertTrue(false); + } finally { + if (ohBufferMutator != null) { + ohBufferMutator.close(); + Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); + Result r = hTable.get(get); + count = r.rawCells().length; + Assert.assertEquals(180, count); + Delete delete = new Delete(toBytes(key)); + delete.addFamily(toBytes(family)); + hTable.delete(delete); + + r = hTable.get(get); + Assert.assertEquals(0, r.rawCells().length); + // test add mutations after closed + final BufferedMutator closedMutator = ohBufferMutator; + Assert.assertThrows(IllegalStateException.class, () -> { + closedMutator.mutate(delete); + }); + // test flush after closed + Assert.assertThrows(IllegalStateException.class, () -> { + closedMutator.flush(); + }); + ExecutorService bufferPool = ((OHBufferedMutatorImpl) ohBufferMutator).getPool(); + if (bufferPool != null) { // self-defined pool must be shutdown by users - Assert.assertFalse(params.getPool().isShutdown()); - pool.shutdown(); + Assert.assertFalse(bufferPool.isShutdown()); + bufferPool.shutdown(); try { - if (!pool.awaitTermination(600, TimeUnit.SECONDS)) { + if (!bufferPool.awaitTermination(600, TimeUnit.SECONDS)) { System.out.println("close() failed to terminate pool after 10 minutes. Abandoning pool."); } } catch (InterruptedException e) { System.out.println("waitForTermination interrupted"); Thread.currentThread().interrupt(); } - Assert.assertTrue(params.getPool().isShutdown()); + Assert.assertTrue(bufferPool.isShutdown()); } } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java index 830ee869..2c7c8f0c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java @@ -17,6 +17,7 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.OHBufferedMutatorImpl; import org.apache.hadoop.conf.Configuration; import com.alipay.oceanbase.rpc.mutation.result.MutationResult; import org.apache.hadoop.hbase.Cell; @@ -220,6 +221,69 @@ public void testMulfiColumnFamilyBufferedMutator() throws Exception { Assert.assertTrue(result.containsColumn(family3, family3_column1)); Assert.assertTrue(result.containsColumn(family3, family3_column2)); + mutations.clear(); + for (String key : keys) { + Delete delete = new Delete(toBytes(key)); + mutations.add(delete); + } + mutator.mutate(mutations); + mutator.flush(); + + scan = new Scan(); + scan.setStartRow(toBytes("Key0")); + scan.setStopRow(toBytes("Key10")); + scan.addFamily(family1); + scan.addFamily(family2); + scan.addFamily(family3); + scanner = hTable.getScanner(scan); + count = 0; + for (Result r : scanner) { + count += r.rawCells().length; + } + Assert.assertEquals(0, count); + + // test periodic flush + params.setWriteBufferPeriodicFlushTimeoutMs(100); + mutator = connection.getBufferedMutator(params); + while (true) { + for (int i = 0; i < rows; ++i) { + mutations.clear(); + Put put = new Put(toBytes(keys.get(i))); + put.addColumn(family1, family1_column1, family1_value); + put.addColumn(family1, family1_column2, family1_value); + put.addColumn(family1, family1_column3, family1_value); + put.addColumn(family2, family2_column1, family2_value); + put.addColumn(family3, family3_column1, family2_value); + put.addColumn(family3, family3_column2, family3_value); + mutations.add(put); + if (i % 3 == 0) { // 0, 3, 6, 9 + Delete delete = new Delete(toBytes(keys.get(i))); + delete.addFamily(family1); + delete.addFamily(family2); + mutations.add(delete); + } + mutator.mutate(mutations); + } + + get = new Get(toBytes("Key0")); + result = hTable.get(get); + if (!result.isEmpty()) { + break; + } + } + get = new Get(toBytes("Key2")); + result = hTable.get(get); + count = result.rawCells().length; + Assert.assertTrue(count > 0); + // test timer periodic flush + int lastUndealtCount = ((OHBufferedMutatorImpl) mutator).size(); + Thread.sleep(1000); + int currentUndealtCount = ((OHBufferedMutatorImpl) mutator).size(); + Assert.assertNotEquals(lastUndealtCount, currentUndealtCount); + result = hTable.get(get); + int newCount = result.rawCells().length; + Assert.assertNotEquals(count, newCount); + // clean data mutations.clear(); for (String key : keys) { @@ -228,6 +292,7 @@ public void testMulfiColumnFamilyBufferedMutator() throws Exception { } mutator.mutate(mutations); mutator.flush(); + mutator.close(); scan = new Scan(); scan.setStartRow(toBytes("Key0")); From 24ae6d7b8e355318dd766fc5989f34c015fef302 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 25 Oct 2024 15:35:59 +0800 Subject: [PATCH 18/32] remove useless comments --- .../java/com/alipay/oceanbase/hbase/OHConnectionTest.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index e6689ab1..c67c1a35 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -261,12 +261,6 @@ public void testBufferedMutatorWithFlush() throws Exception { bufferMutator.mutate(put3); bufferMutator.flush(); r = hTable.get(get); -// for (Cell c : r.rawCells()) { -// System.out.println("2. Key: " + Bytes.toString(CellUtil.cloneRow(c)) + -// ", Qualifier: " + Bytes.toString(CellUtil.cloneQualifier(c)) + -// ", Time: " + c.getTimestamp() + -// ", Value: " + Bytes.toString(CellUtil.cloneValue(c))); -// } Assert.assertEquals(3, r.rawCells().length); // test Delete From 20a689ab958088badc57be8cfbf2b8606b4acbe9 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 25 Oct 2024 17:01:10 +0800 Subject: [PATCH 19/32] format code --- .../com/alipay/oceanbase/hbase/OHTable.java | 112 ++++++------ .../alipay/oceanbase/hbase/OHTablePool.java | 16 +- .../hbase/util/OHBufferedMutatorImpl.java | 107 ++++++------ .../hbase/util/OHConnectionConfiguration.java | 20 +-- .../hbase/util/OHConnectionImpl.java | 3 +- .../oceanbase/hbase/HTableTestBase.java | 163 +++++++++++------- .../oceanbase/hbase/OHConnectionTest.java | 20 ++- .../hbase/OHTableDeleteFamilyVersionTest.java | 61 ++++--- .../hbase/OHTableMultiColumnFamilyTest.java | 6 +- 9 files changed, 286 insertions(+), 222 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 6f839a9f..8e13c036 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -198,7 +198,7 @@ public OHTable(Configuration configuration, String tableName) throws IOException this.executePool = createDefaultThreadPoolExecutor(1, maxThreads, keepAliveTime); OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, ohConnectionConf)); this.obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); @@ -251,7 +251,7 @@ public OHTable(Configuration configuration, final byte[] tableName, this.cleanupPoolOnClose = false; OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, ohConnectionConf)); this.obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); @@ -321,7 +321,7 @@ public OHTable(TableName tableName, Connection connection, this.writeBufferSize = connectionConfig.getWriteBufferSize(); this.tableName = tableName.getName(); int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, connectionConfig)); this.obTableClient.setRpcExecuteTimeout(rpcTimeout); @@ -535,7 +535,7 @@ private String getTargetTableName(List actions) { @Override public void batchCallback(List actions, Object[] results, Batch.Callback callback) throws IOException, - InterruptedException { + InterruptedException { try { batch(actions, results); } finally { @@ -560,8 +560,8 @@ public static int compareByteArray(byte[] bt1, byte[] bt2) { } private void getMaxRowFromResult(AbstractQueryStreamResult clientQueryStreamResult, - List keyValueList, boolean isTableGroup, - byte[] family) throws Exception { + List keyValueList, boolean isTableGroup, byte[] family) + throws Exception { byte[][] familyAndQualifier = new byte[2][]; KeyValue kv = null; while (clientQueryStreamResult.next()) { @@ -849,7 +849,8 @@ private void validatePut(Put put) { throw new IllegalArgumentException("family is empty"); } for (Cell kv : entry.getValue()) { - if (kv.getRowLength() + kv.getValueLength() + kv.getQualifierLength() + Bytes.toBytes(kv.getTimestamp()).length + kv.getFamilyLength() > maxKeyValueSize) { + if (kv.getRowLength() + kv.getValueLength() + kv.getQualifierLength() + + Bytes.toBytes(kv.getTimestamp()).length + kv.getFamilyLength() > maxKeyValueSize) { throw new IllegalArgumentException("KeyValue size too large"); } } @@ -918,7 +919,7 @@ private void innerDelete(Delete delete) throws IOException { batch(actions, results); } catch (Exception e) { logger.error(LCD.convert("01-00004"), tableNameString, e); - throw new IOException("delete table " + tableNameString + "error" , e); + throw new IOException("delete table " + tableNameString + "error", e); } } @@ -1557,43 +1558,53 @@ public static ObTableBatchOperation buildObTableBatchOperation(List keyVal } private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(Cell kv, - OHOpType operationType, boolean isTableGroup) { + OHOpType operationType, + boolean isTableGroup) { switch (operationType) { case INSERT_OR_UPDATE: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(INSERT_OR_UPDATE, - ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, V_COLUMNS, - new Object[] { CellUtil.cloneValue(kv) }); + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance( + INSERT_OR_UPDATE, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), + kv.getTimestamp() }, V_COLUMNS, + new Object[] { CellUtil.cloneValue(kv) }); case APPEND: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(APPEND, + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance( + APPEND, ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, V_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), + kv.getTimestamp() }, V_COLUMNS, new Object[] { CellUtil.cloneValue(kv) }); case Delete: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, - null, null); + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance( + DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), + kv.getTimestamp() }, null, null); case DeleteAll: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), null, -kv.getTimestamp() }, - null, null); + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), null, -kv.getTimestamp() }, null, null); case DeleteColumn: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), -kv.getTimestamp() }, - null, null); + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance( + DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), + -kv.getTimestamp() }, null, null); case DeleteFamily: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + return com.alipay.oceanbase.rpc.mutation.Mutation + .getInstance( + DEL, ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), isTableGroup?CellUtil.cloneQualifier(kv):null, -kv.getTimestamp() }, - null, null); + new Object[] { CellUtil.cloneRow(kv), + isTableGroup ? CellUtil.cloneQualifier(kv) : null, + -kv.getTimestamp() }, null, null); case DeleteFamilyVersion: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), isTableGroup ? CellUtil.cloneQualifier(kv) : null, kv.getTimestamp() }, - null, null); + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance( + DEL, + ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), + isTableGroup ? CellUtil.cloneQualifier(kv) : null, kv.getTimestamp() }, + null, null); default: throw new IllegalArgumentException("illegal mutation type " + operationType); } @@ -1607,8 +1618,7 @@ private KeyValue modifyQualifier(Cell original, byte[] newQualifier) { long timestamp = original.getTimestamp(); KeyValue.Type type = KeyValue.Type.codeToType(original.getType().getCode()); // Create a new KeyValue with the modified qualifier - return new KeyValue(row, family, newQualifier, timestamp, type, - value); + return new KeyValue(row, family, newQualifier, timestamp, type, value); } private BatchOperation buildBatchOperation(String tableName, List actions, boolean isTableGroup, List resultMapSingleOp) { @@ -1684,18 +1694,24 @@ public static ObTableOperation buildObTableOperation(Cell kv, boolean putToAppen } else { operationType = INSERT_OR_UPDATE; } - return getInstance(operationType, - new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, V_COLUMNS, + return getInstance( + operationType, + new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), + kv.getTimestamp() }, V_COLUMNS, new Object[] { CellUtil.cloneValue(kv) }); case Delete: - return getInstance(DEL, - new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), kv.getTimestamp() }, null, null); + return getInstance( + DEL, + new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), + kv.getTimestamp() }, null, null); case DeleteColumn: - return getInstance(DEL, - new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), -kv.getTimestamp() }, null, null); + return getInstance( + DEL, + new Object[] { CellUtil.cloneRow(kv), CellUtil.cloneQualifier(kv), + -kv.getTimestamp() }, null, null); case DeleteFamily: - return getInstance(DEL, new Object[] { CellUtil.cloneRow(kv), null, -kv.getTimestamp() }, - null, null); + return getInstance(DEL, + new Object[] { CellUtil.cloneRow(kv), null, -kv.getTimestamp() }, null, null); default: throw new IllegalArgumentException("illegal mutation type " + kvType); } @@ -1810,13 +1826,7 @@ public Pair getStartEndKeys() throws IOException { } public static enum OHOpType { - INSERT_OR_UPDATE, - APPEND, - Delete, - DeleteAll, - DeleteColumn, - DeleteFamily, - DeleteFamilyVersion, + INSERT_OR_UPDATE, APPEND, Delete, DeleteAll, DeleteColumn, DeleteFamily, DeleteFamilyVersion, } public static OHOpType getDeleteType(Cell.Type type) { diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java b/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java index 67183aa6..87ec2113 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java @@ -49,19 +49,19 @@ public class OHTablePool implements Closeable { - private String originTabelName = null; - private final PoolMap tables; - private final int maxSize; - private final PoolMap.PoolType poolType; - private final Configuration config; - private final OHTableFactory tableFactory; + private String originTabelName = null; + private final PoolMap tables; + private final int maxSize; + private final PoolMap.PoolType poolType; + private final Configuration config; + private final OHTableFactory tableFactory; // A map of table attributes used for the table created by this pool. The map // key is composed of Table_Name + SEPARATOR + Attribute_Name, and the value // is byte value of attribute. - private ConcurrentHashMap tableAttributes; + private ConcurrentHashMap tableAttributes; - private ConcurrentHashMap tableExtendAttributes; + private ConcurrentHashMap tableExtendAttributes; /** * Default Constructor. Default HBaseConfiguration and no limit on pool size. diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index 6fdef097..55fa8d48 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -37,35 +37,40 @@ @InterfaceAudience.Private public class OHBufferedMutatorImpl implements BufferedMutator { - private static final Logger LOGGER = TableHBaseLoggerFactory - .getLogger(OHBufferedMutatorImpl.class); - - private final ExceptionListener listener; - - private final OHTable ohTable; - private final TableName tableName; - private volatile Configuration conf; - private final OHConnectionConfiguration connectionConfig; - - private final ConcurrentLinkedQueue asyncWriteBuffer = new ConcurrentLinkedQueue(); - private final AtomicLong currentAsyncBufferSize = new AtomicLong(0); - - private final AtomicLong firstRecordInBufferTimestamp = new AtomicLong(0); - private final AtomicLong executedWriteBufferPeriodicFlushes = new AtomicLong(0); - - private final AtomicLong writeBufferPeriodicFlushTimeoutMs = new AtomicLong(0); - private final AtomicLong writeBufferPeriodicFlushTimerTickMs = - new AtomicLong(MIN_WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS); - private Timer writeBufferPeriodicFlushTimer = null; - - private final long writeBufferSize; - private final int maxKeyValueSize; - private final ExecutorService pool; - private final AtomicInteger undealtMutationCount = new AtomicInteger(0); - private final AtomicInteger rpcTimeout; - private final AtomicInteger operationTimeout; - private final boolean cleanipPoolOnClose; - private volatile boolean closed = false; + private static final Logger LOGGER = TableHBaseLoggerFactory + .getLogger(OHBufferedMutatorImpl.class); + + private final ExceptionListener listener; + + private final OHTable ohTable; + private final TableName tableName; + private volatile Configuration conf; + private final OHConnectionConfiguration connectionConfig; + + private final ConcurrentLinkedQueue asyncWriteBuffer = new ConcurrentLinkedQueue(); + private final AtomicLong currentAsyncBufferSize = new AtomicLong( + 0); + + private final AtomicLong firstRecordInBufferTimestamp = new AtomicLong( + 0); + private final AtomicLong executedWriteBufferPeriodicFlushes = new AtomicLong( + 0); + + private final AtomicLong writeBufferPeriodicFlushTimeoutMs = new AtomicLong( + 0); + private final AtomicLong writeBufferPeriodicFlushTimerTickMs = new AtomicLong( + MIN_WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS); + private Timer writeBufferPeriodicFlushTimer = null; + + private final long writeBufferSize; + private final int maxKeyValueSize; + private final ExecutorService pool; + private final AtomicInteger undealtMutationCount = new AtomicInteger( + 0); + private final AtomicInteger rpcTimeout; + private final AtomicInteger operationTimeout; + private final boolean cleanipPoolOnClose; + private volatile boolean closed = false; public OHBufferedMutatorImpl(OHConnectionImpl ohConnection, BufferedMutatorParams params) throws IOException { @@ -85,20 +90,18 @@ public OHBufferedMutatorImpl(OHConnectionImpl ohConnection, BufferedMutatorParam this.cleanipPoolOnClose = false; } this.rpcTimeout = new AtomicInteger( - params.getRpcTimeout() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? - params.getRpcTimeout() : connectionConfig.getRpcTimeout() - ); + params.getRpcTimeout() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params + .getRpcTimeout() : connectionConfig.getRpcTimeout()); this.operationTimeout = new AtomicInteger( - params.getOperationTimeout() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? - params.getOperationTimeout() : connectionConfig.getOperationTimeout() - ); - - long newPeriodicFlushTimeoutMs = - params.getWriteBufferPeriodicFlushTimeoutMs() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? - params.getWriteBufferPeriodicFlushTimeoutMs() : connectionConfig.getWriteBufferPeriodicFlushTimeoutMs(); - long newPeriodicFlushTimeIntervalMs = - params.getWriteBufferPeriodicFlushTimerTickMs() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? - params.getWriteBufferPeriodicFlushTimerTickMs() : connectionConfig.getWriteBufferPeriodicFlushTimerTickMs(); + params.getOperationTimeout() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params + .getOperationTimeout() : connectionConfig.getOperationTimeout()); + + long newPeriodicFlushTimeoutMs = params.getWriteBufferPeriodicFlushTimeoutMs() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params + .getWriteBufferPeriodicFlushTimeoutMs() : connectionConfig + .getWriteBufferPeriodicFlushTimeoutMs(); + long newPeriodicFlushTimeIntervalMs = params.getWriteBufferPeriodicFlushTimerTickMs() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params + .getWriteBufferPeriodicFlushTimerTickMs() : connectionConfig + .getWriteBufferPeriodicFlushTimerTickMs(); this.setWriteBufferPeriodicFlush(newPeriodicFlushTimeoutMs, newPeriodicFlushTimeIntervalMs); this.writeBufferSize = params.getWriteBufferSize() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params @@ -199,7 +202,8 @@ public void timeTriggerForWriteBufferPeriodicFlush() { executedWriteBufferPeriodicFlushes.incrementAndGet(); flush(); } catch (Exception e) { - LOGGER.error("Errors occur during timeTriggerForWriteBufferPeriodicFlush: { " + e.getMessage() + " }"); + LOGGER.error("Errors occur during timeTriggerForWriteBufferPeriodicFlush: { " + + e.getMessage() + " }"); } } @@ -214,20 +218,19 @@ public synchronized void setWriteBufferPeriodicFlush(long timeoutMs, long timerT long originalTimeTickMs = this.writeBufferPeriodicFlushTimerTickMs.get(); writeBufferPeriodicFlushTimeoutMs.set(Math.max(0, timeoutMs)); - writeBufferPeriodicFlushTimerTickMs.set( - Math.max(MIN_WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS, timerTickMs)); + writeBufferPeriodicFlushTimerTickMs.set(Math.max( + MIN_WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS, timerTickMs)); // if time parameters are updated, stop the old timer - if (writeBufferPeriodicFlushTimeoutMs.get() != originalTimeoutMs || - writeBufferPeriodicFlushTimerTickMs.get() != originalTimeTickMs) { + if (writeBufferPeriodicFlushTimeoutMs.get() != originalTimeoutMs + || writeBufferPeriodicFlushTimerTickMs.get() != originalTimeTickMs) { if (writeBufferPeriodicFlushTimer != null) { writeBufferPeriodicFlushTimer.cancel(); writeBufferPeriodicFlushTimer = null; } } - if (writeBufferPeriodicFlushTimer == null && - writeBufferPeriodicFlushTimeoutMs.get() > 0) { + if (writeBufferPeriodicFlushTimer == null && writeBufferPeriodicFlushTimeoutMs.get() > 0) { writeBufferPeriodicFlushTimer = new Timer(true); writeBufferPeriodicFlushTimer.schedule(new TimerTask() { @Override @@ -235,7 +238,7 @@ public void run() { OHBufferedMutatorImpl.this.timeTriggerForWriteBufferPeriodicFlush(); } }, this.writeBufferPeriodicFlushTimerTickMs.get(), - this.writeBufferPeriodicFlushTimerTickMs.get()); + this.writeBufferPeriodicFlushTimerTickMs.get()); } } @@ -319,7 +322,7 @@ public void close() throws IOException { if (closed) { return; } - // reset timeout, timeInterval and Timer + // reset timeout, timeTick and Timer disableWriteBufferPeriodicFlush(); try { execute(true); @@ -330,7 +333,7 @@ public void close() throws IOException { try { if (!pool.awaitTermination(600, TimeUnit.SECONDS)) { LOGGER - .warn("close() failed to terminate pool after 10 minutes. Abandoning pool."); + .warn("close() failed to terminate pool after 10 minutes. Abandoning pool."); } } catch (InterruptedException e) { LOGGER.warn("waitForTermination interrupted"); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java index 8ff8c677..fd032267 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java @@ -52,8 +52,8 @@ public class OHConnectionConfiguration { private final int maxKeyValueSize; private final int rpcTimeout; private final int rpcConnectTimeout; - private final long writeBufferPeriodicFlushTimeoutMs; - private final long writeBufferPeriodicFlushTimerTickMs; + private final long writeBufferPeriodicFlushTimeoutMs; + private final long writeBufferPeriodicFlushTimerTickMs; public OHConnectionConfiguration(Configuration conf) { this.paramUrl = conf.get(HBASE_OCEANBASE_PARAM_URL); @@ -71,15 +71,14 @@ public OHConnectionConfiguration(Configuration conf) { this.database = database; this.writeBufferSize = conf.getLong(WRITE_BUFFER_SIZE_KEY, WRITE_BUFFER_SIZE_DEFAULT); this.operationTimeout = conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, - HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT); + HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT); this.rpcTimeout = conf.getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT); this.writeBufferPeriodicFlushTimeoutMs = conf.getLong( - WRITE_BUFFER_PERIODIC_FLUSH_TIMEOUT_MS, - WRITE_BUFFER_PERIODIC_FLUSH_TIMEOUT_MS_DEFAULT); + WRITE_BUFFER_PERIODIC_FLUSH_TIMEOUT_MS, WRITE_BUFFER_PERIODIC_FLUSH_TIMEOUT_MS_DEFAULT); this.writeBufferPeriodicFlushTimerTickMs = conf.getLong( - WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS, - WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS_DEFAULT); + WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS, + WRITE_BUFFER_PERIODIC_FLUSH_TIMERTICK_MS_DEFAULT); int rpcConnectTimeout = -1; if (conf.get(SOCKET_TIMEOUT_CONNECT) != null) { rpcConnectTimeout = conf.getInt(SOCKET_TIMEOUT_CONNECT, DEFAULT_SOCKET_TIMEOUT_CONNECT); @@ -92,9 +91,10 @@ public OHConnectionConfiguration(Configuration conf) { } } this.rpcConnectTimeout = rpcConnectTimeout; - this.scannerCaching = conf.getInt(HConstants.HBASE_CLIENT_SCANNER_CACHING, Integer.MAX_VALUE); - this.scannerMaxResultSize = conf.getLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, - WRITE_BUFFER_SIZE_DEFAULT); + this.scannerCaching = conf.getInt(HConstants.HBASE_CLIENT_SCANNER_CACHING, + Integer.MAX_VALUE); + this.scannerMaxResultSize = conf.getLong( + HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, WRITE_BUFFER_SIZE_DEFAULT); this.maxKeyValueSize = conf.getInt(MAX_KEYVALUE_SIZE_KEY, MAX_KEYVALUE_SIZE_DEFAULT); properties = new Properties(); for (Property property : Property.values()) { diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java index b3ba4744..3964f52f 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java @@ -56,8 +56,7 @@ public class OHConnectionImpl implements Connection { private final OHConnectionConfiguration connectionConfig; - OHConnectionImpl(Configuration conf, ExecutorService pool, - final User user) throws IOException { + OHConnectionImpl(Configuration conf, ExecutorService pool, final User user) throws IOException { this.conf = conf; this.batchPool = pool; this.connectionConfig = new OHConnectionConfiguration(conf); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 3a350d04..d6368678 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -68,7 +68,8 @@ PRIMARY KEY (`K`, `Q`, `T`) long timestamp = System.currentTimeMillis(); // put data Put put = new Put(toBytes(key)); - KeyValue kv = new KeyValue(toBytes(key), "family_group".getBytes(), column1.getBytes(), timestamp, toBytes(value + "1")); + KeyValue kv = new KeyValue(toBytes(key), "family_group".getBytes(), column1.getBytes(), + timestamp, toBytes(value + "1")); put.add(kv); hTable.put(put); // test get with empty family @@ -113,8 +114,9 @@ PRIMARY KEY (`K`, `Q`, `T`) Assert.assertEquals(column1, Bytes.toString(CellUtil.cloneQualifier(keyValue))); Assert.assertEquals(timestamp, keyValue.getTimestamp()); Assert.assertEquals(value + "1", Bytes.toString(CellUtil.cloneValue(keyValue))); - System.out.println("rowKey: " + new String(CellUtil.cloneRow(keyValue)) + " family :" - + new String(CellUtil.cloneFamily(keyValue)) + " columnQualifier:" + System.out.println("rowKey: " + new String(CellUtil.cloneRow(keyValue)) + + " family :" + new String(CellUtil.cloneFamily(keyValue)) + + " columnQualifier:" + new String(CellUtil.cloneQualifier(keyValue)) + " timestamp:" + keyValue.getTimestamp() + " value:" + new String(CellUtil.cloneValue(keyValue))); @@ -138,7 +140,8 @@ private void testBasic(String family) throws Exception { hTable.delete(delete); Put put = new Put(toBytes(key)); - KeyValue kv = new KeyValue(toBytes(key), family.getBytes(), column1.getBytes(), timestamp, toBytes(value)); + KeyValue kv = new KeyValue(toBytes(key), family.getBytes(), column1.getBytes(), timestamp, + toBytes(value)); put.add(kv); hTable.put(put); Get get = new Get(toBytes(key)); @@ -147,7 +150,8 @@ private void testBasic(String family) throws Exception { Assert.assertEquals(1, r.rawCells().length); for (Cell keyValue : r.rawCells()) { - System.out.println("rowKey: " + new String(CellUtil.cloneRow(keyValue)) + " columnQualifier:" + System.out.println("rowKey: " + new String(CellUtil.cloneRow(keyValue)) + + " columnQualifier:" + new String(CellUtil.cloneQualifier(keyValue)) + " timestamp:" + keyValue.getTimestamp() + " value:" + new String(CellUtil.cloneValue(keyValue))); @@ -158,7 +162,8 @@ private void testBasic(String family) throws Exception { } put = new Put(toBytes(key)); - kv = new KeyValue(toBytes(key), family.getBytes(), column1.getBytes(), timestamp + 1, toBytes(value)); + kv = new KeyValue(toBytes(key), family.getBytes(), column1.getBytes(), timestamp + 1, + toBytes(value)); put.add(kv); hTable.put(put); get = new Get(toBytes(key)); @@ -176,7 +181,8 @@ private void testBasic(String family) throws Exception { hTable.delete(delete); for (Cell keyValue : r.rawCells()) { - System.out.println("rowKey: " + new String(CellUtil.cloneRow(keyValue)) + " columnQualifier:" + System.out.println("rowKey: " + new String(CellUtil.cloneRow(keyValue)) + + " columnQualifier:" + new String(CellUtil.cloneQualifier(keyValue)) + " timestamp:" + keyValue.getTimestamp() + " value:" + new String(CellUtil.cloneValue(keyValue))); @@ -207,12 +213,15 @@ private void testBasic(String family) throws Exception { boolean countAdd = true; for (Cell keyValue : result.rawCells()) { System.out.println("rowKey: " + new String(CellUtil.cloneRow(keyValue)) - + " columnQualifier:" + new String(CellUtil.cloneQualifier(keyValue)) + + " columnQualifier:" + + new String(CellUtil.cloneQualifier(keyValue)) + " timestamp:" + keyValue.getTimestamp() + " value:" + new String(CellUtil.cloneValue(keyValue))); Assert.assertEquals(key + "_" + i, Bytes.toString(CellUtil.cloneRow(keyValue))); - Assert.assertTrue(column1.equals(Bytes.toString(CellUtil.cloneQualifier(keyValue))) - || column2.equals(Bytes.toString(CellUtil.cloneQualifier(keyValue)))); + Assert.assertTrue(column1.equals(Bytes.toString(CellUtil + .cloneQualifier(keyValue))) + || column2.equals(Bytes.toString(CellUtil + .cloneQualifier(keyValue)))); Assert.assertEquals(timestamp + 2, keyValue.getTimestamp()); Assert.assertEquals(value, Bytes.toString(CellUtil.cloneValue(keyValue))); if (countAdd) { @@ -235,12 +244,15 @@ private void testBasic(String family) throws Exception { boolean countAdd = true; for (Cell keyValue : result.rawCells()) { System.out.println("rowKey: " + new String(CellUtil.cloneRow(keyValue)) - + " columnQualifier:" + new String(CellUtil.cloneQualifier(keyValue)) + + " columnQualifier:" + + new String(CellUtil.cloneQualifier(keyValue)) + " timestamp:" + keyValue.getTimestamp() + " value:" + new String(CellUtil.cloneValue(keyValue))); Assert.assertEquals(key + "_" + i, Bytes.toString(CellUtil.cloneRow(keyValue))); - Assert.assertTrue(column1.equals(Bytes.toString(CellUtil.cloneQualifier(keyValue))) - || column2.equals(Bytes.toString(CellUtil.cloneQualifier(keyValue)))); + Assert.assertTrue(column1.equals(Bytes.toString(CellUtil + .cloneQualifier(keyValue))) + || column2.equals(Bytes.toString(CellUtil + .cloneQualifier(keyValue)))); Assert.assertEquals(value, Bytes.toString(CellUtil.cloneValue(keyValue))); if (countAdd) { countAdd = false; @@ -279,7 +291,8 @@ public void testHugeData() throws IOException { System.arraycopy(keyBytes, 0, rowkey, 0, keyBytes.length); System.arraycopy(currentBytes, 0, rowkey, keyBytes.length, currentBytes.length); Put put = new Put(rowkey); - KeyValue kv = new KeyValue(rowkey, "family1".getBytes(), column1.getBytes(), toBytes(value)); + KeyValue kv = new KeyValue(rowkey, "family1".getBytes(), column1.getBytes(), + toBytes(value)); put.add(kv); kv = new KeyValue(rowkey, "family1".getBytes(), column2.getBytes(), toBytes(value)); put.add(kv); @@ -909,7 +922,8 @@ public void testFilter() throws Exception { long timestamp = System.currentTimeMillis(); putKey1Column1Value1 = new Put(toBytes(key1)); - putKey1Column1Value1.addColumn(toBytes(family), toBytes(column1), timestamp, toBytes(value1)); + putKey1Column1Value1.addColumn(toBytes(family), toBytes(column1), timestamp, + toBytes(value1)); putKey1Column1Value2 = new Put(toBytes(key1)); putKey1Column1Value2.addColumn(toBytes(family), toBytes(column1), toBytes(value2)); @@ -921,7 +935,8 @@ public void testFilter() throws Exception { putKey1Column2Value1.addColumn(toBytes(family), toBytes(column2), toBytes(value1)); putKey2Column1Value1 = new Put(toBytes(key2)); - putKey2Column1Value1.addColumn(toBytes(family), toBytes(column1), timestamp, toBytes(value1)); + putKey2Column1Value1.addColumn(toBytes(family), toBytes(column1), timestamp, + toBytes(value1)); putKey2Column1Value2 = new Put(toBytes(key2)); putKey2Column1Value2.addColumn(toBytes(family), toBytes(column1), toBytes(value2)); @@ -1299,13 +1314,13 @@ public void testColumnRangeFilter() throws Exception { int res_count = 0; for (Result result : scanner) { for (Cell keyValue : result.rawCells()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", Bytes.toString(result.getRow()), Bytes.toString(CellUtil.cloneFamily(keyValue)), - Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), - Bytes.toString(CellUtil.cloneValue(keyValue)) - ); + Bytes.toString(CellUtil.cloneQualifier(keyValue)), keyValue.getTimestamp(), + Bytes.toString(CellUtil.cloneValue(keyValue))); res_count += 1; } } @@ -1322,13 +1337,13 @@ public void testColumnRangeFilter() throws Exception { res_count = 0; for (Result result : scanner) { for (Cell keyValue : result.rawCells()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", Bytes.toString(result.getRow()), Bytes.toString(CellUtil.cloneFamily(keyValue)), - Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), - Bytes.toString(CellUtil.cloneValue(keyValue)) - ); + Bytes.toString(CellUtil.cloneQualifier(keyValue)), keyValue.getTimestamp(), + Bytes.toString(CellUtil.cloneValue(keyValue))); res_count += 1; } } @@ -1345,13 +1360,13 @@ public void testColumnRangeFilter() throws Exception { res_count = 0; for (Result result : scanner) { for (Cell keyValue : result.rawCells()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", Bytes.toString(result.getRow()), Bytes.toString(CellUtil.cloneFamily(keyValue)), - Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), - Bytes.toString(CellUtil.cloneValue(keyValue)) - ); + Bytes.toString(CellUtil.cloneQualifier(keyValue)), keyValue.getTimestamp(), + Bytes.toString(CellUtil.cloneValue(keyValue))); res_count += 1; } } @@ -1370,13 +1385,13 @@ public void testColumnRangeFilter() throws Exception { res_count = 0; for (Result result : scanner) { for (Cell keyValue : result.rawCells()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", Bytes.toString(result.getRow()), Bytes.toString(CellUtil.cloneFamily(keyValue)), - Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), - Bytes.toString(CellUtil.cloneValue(keyValue)) - ); + Bytes.toString(CellUtil.cloneQualifier(keyValue)), keyValue.getTimestamp(), + Bytes.toString(CellUtil.cloneValue(keyValue))); res_count += 1; } } @@ -1395,13 +1410,13 @@ public void testColumnRangeFilter() throws Exception { res_count = 0; for (Result result : scanner) { for (Cell keyValue : result.rawCells()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", Bytes.toString(result.getRow()), Bytes.toString(CellUtil.cloneFamily(keyValue)), - Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), - Bytes.toString(CellUtil.cloneValue(keyValue)) - ); + Bytes.toString(CellUtil.cloneQualifier(keyValue)), keyValue.getTimestamp(), + Bytes.toString(CellUtil.cloneValue(keyValue))); res_count += 1; } } @@ -1476,13 +1491,13 @@ public void testFilterNullRange() throws Exception { int res_count = 0; for (Result result : scanner) { for (Cell keyValue : result.rawCells()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", Bytes.toString(result.getRow()), Bytes.toString(CellUtil.cloneFamily(keyValue)), - Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), - Bytes.toString(CellUtil.cloneValue(keyValue)) - ); + Bytes.toString(CellUtil.cloneQualifier(keyValue)), keyValue.getTimestamp(), + Bytes.toString(CellUtil.cloneValue(keyValue))); Assert.assertArrayEquals(key2.getBytes(), CellUtil.cloneRow(keyValue)); res_count += 1; } @@ -2644,7 +2659,8 @@ public void testGetFilter() throws Exception { get.setFilter(filterList); r = hTable.get(get); Assert.assertEquals(2, r.rawCells().length); - Assert.assertFalse(Bytes.equals(CellUtil.cloneQualifier(r.rawCells()[0]), CellUtil.cloneQualifier(r.rawCells()[1]))); + Assert.assertFalse(Bytes.equals(CellUtil.cloneQualifier(r.rawCells()[0]), + CellUtil.cloneQualifier(r.rawCells()[1]))); filterList = new FilterList(); filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, @@ -5077,8 +5093,9 @@ public void testQualifyNull() throws Exception { r = hTable.get(get); for (Cell kv : r.rawCells()) { System.out.println("K = [" + Bytes.toString(CellUtil.cloneRow(kv)) + "] Q =[" - + Bytes.toString(CellUtil.cloneQualifier(kv)) + "] T = [" + kv.getTimestamp() - + "] V = [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]"); + + Bytes.toString(CellUtil.cloneQualifier(kv)) + "] T = [" + + kv.getTimestamp() + "] V = [" + + Bytes.toString(CellUtil.cloneValue(kv)) + "]"); } Assert.assertEquals(1, r.rawCells().length); Assert.assertEquals(key, Bytes.toString(CellUtil.cloneRow(r.rawCells()[0]))); @@ -5306,8 +5323,8 @@ public void testColumnQualifier() throws Exception { Result result = hTable.get(get); Assert.assertEquals("", 1, result.rawCells().length); Assert.assertEquals("", 0, result.rawCells()[0].getQualifierLength()); - Assert - .assertArrayEquals("", "value1_qualifier_null".getBytes(), CellUtil.cloneValue(result.rawCells()[0])); + Assert.assertArrayEquals("", "value1_qualifier_null".getBytes(), + CellUtil.cloneValue(result.rawCells()[0])); put = new Put(("key_c_q_empty").getBytes()); put.addColumn("family1".getBytes(), "".getBytes(), "value1_qualifier_empty".getBytes()); @@ -5360,14 +5377,20 @@ public void testTTLColumnLevel() throws Exception { get.setMaxVersions(1); Result result = hTable.get(get); Assert.assertEquals("", 2, result.rawCells().length); - Assert.assertEquals("", 1, result - .getColumnCells("family_ttl".getBytes(), ("column1").getBytes()).size()); - Assert.assertEquals("", 1, result - .getColumnCells("family_ttl".getBytes(), ("column2").getBytes()).size()); - Assert.assertArrayEquals("", "column1_value1_ttl_column".getBytes(), - CellUtil.cloneValue(result.getColumnCells("family_ttl".getBytes(), ("column1").getBytes()).get(0))); - Assert.assertArrayEquals("", "column2_value1_ttl_column".getBytes(), - CellUtil.cloneValue(result.getColumnCells("family_ttl".getBytes(), ("column2").getBytes()).get(0))); + Assert.assertEquals("", 1, + result.getColumnCells("family_ttl".getBytes(), ("column1").getBytes()).size()); + Assert.assertEquals("", 1, + result.getColumnCells("family_ttl".getBytes(), ("column2").getBytes()).size()); + Assert.assertArrayEquals( + "", + "column1_value1_ttl_column".getBytes(), + CellUtil.cloneValue(result.getColumnCells("family_ttl".getBytes(), + ("column1").getBytes()).get(0))); + Assert.assertArrayEquals( + "", + "column2_value1_ttl_column".getBytes(), + CellUtil.cloneValue(result.getColumnCells("family_ttl".getBytes(), + ("column2").getBytes()).get(0))); //过期之后不能再查出数据 Thread.sleep(4 * 1000L); @@ -5388,14 +5411,20 @@ public void testTTLFamilyLevel() throws Exception { get.setMaxVersions(1); Result result = hTable.get(get); Assert.assertEquals("", 2, result.rawCells().length); - Assert.assertEquals("", 1, result - .getColumnCells("family_ttl".getBytes(), ("column1").getBytes()).size()); - Assert.assertEquals("", 1, result - .getColumnCells("family_ttl".getBytes(), ("column2").getBytes()).size()); - Assert.assertArrayEquals("", "column1_value_ttl_family".getBytes(), - CellUtil.cloneValue(result.getColumnCells("family_ttl".getBytes(), ("column1").getBytes()).get(0))); - Assert.assertArrayEquals("", "column2_value_ttl_family".getBytes(), - CellUtil.cloneValue(result.getColumnCells("family_ttl".getBytes(), ("column2").getBytes()).get(0))); + Assert.assertEquals("", 1, + result.getColumnCells("family_ttl".getBytes(), ("column1").getBytes()).size()); + Assert.assertEquals("", 1, + result.getColumnCells("family_ttl".getBytes(), ("column2").getBytes()).size()); + Assert.assertArrayEquals( + "", + "column1_value_ttl_family".getBytes(), + CellUtil.cloneValue(result.getColumnCells("family_ttl".getBytes(), + ("column1").getBytes()).get(0))); + Assert.assertArrayEquals( + "", + "column2_value_ttl_family".getBytes(), + CellUtil.cloneValue(result.getColumnCells("family_ttl".getBytes(), + ("column2").getBytes()).get(0))); //过期之后不能再查出数据 Thread.sleep(4 * 1000L); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index b81e3707..aac67652 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -17,12 +17,10 @@ package com.alipay.oceanbase.hbase; -import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; import com.alipay.oceanbase.hbase.util.OHBufferedMutatorImpl; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; @@ -30,7 +28,6 @@ import org.junit.Assert; import org.junit.Test; -import java.security.Key; import java.util.ArrayList; import java.util.List; import java.util.concurrent.*; @@ -115,7 +112,8 @@ private void testBasic() throws Exception { hTable.delete(delete); for (Cell keyValue : r.rawCells()) { - System.out.println("rowKey: " + new String(CellUtil.cloneRow(keyValue)) + " columnQualifier:" + System.out.println("rowKey: " + new String(CellUtil.cloneRow(keyValue)) + + " columnQualifier:" + new String(CellUtil.cloneQualifier(keyValue)) + " timestamp:" + keyValue.getTimestamp() + " value:" + new String(CellUtil.cloneValue(keyValue))); @@ -146,8 +144,10 @@ private void testBasic() throws Exception { boolean countAdd = true; for (Cell keyValue : result.rawCells()) { Assert.assertEquals(key + "_" + i, Bytes.toString(CellUtil.cloneRow(keyValue))); - Assert.assertTrue(column1.equals(Bytes.toString(CellUtil.cloneQualifier(keyValue))) - || column2.equals(Bytes.toString(CellUtil.cloneQualifier(keyValue)))); + Assert.assertTrue(column1.equals(Bytes.toString(CellUtil + .cloneQualifier(keyValue))) + || column2.equals(Bytes.toString(CellUtil + .cloneQualifier(keyValue)))); Assert.assertEquals(timestamp + 2, keyValue.getTimestamp()); Assert.assertEquals(value, Bytes.toString(CellUtil.cloneValue(keyValue))); if (countAdd) { @@ -169,8 +169,10 @@ private void testBasic() throws Exception { boolean countAdd = true; for (Cell keyValue : result.rawCells()) { Assert.assertEquals(key + "_" + i, Bytes.toString(CellUtil.cloneRow(keyValue))); - Assert.assertTrue(column1.equals(Bytes.toString(CellUtil.cloneQualifier(keyValue))) - || column2.equals(Bytes.toString(CellUtil.cloneQualifier(keyValue)))); + Assert.assertTrue(column1.equals(Bytes.toString(CellUtil + .cloneQualifier(keyValue))) + || column2.equals(Bytes.toString(CellUtil + .cloneQualifier(keyValue)))); Assert.assertEquals(value, Bytes.toString(CellUtil.cloneValue(keyValue))); if (countAdd) { countAdd = false; @@ -848,6 +850,8 @@ public void testBufferedMutatorPeriodicFlush() throws Exception { Thread.sleep(1000); int currentUndealtCount = ((OHBufferedMutatorImpl) ohBufferMutator).size(); Assert.assertNotEquals(lastUndealtCount, currentUndealtCount); + // after periodic flush, all mutations will be committed + Assert.assertEquals(0, currentUndealtCount); r = hTable.get(get); int newCount = r.rawCells().length; Assert.assertNotEquals(count, newCount); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java index 26ee365a..2296df18 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java @@ -16,7 +16,7 @@ public class OHTableDeleteFamilyVersionTest { @Rule public ExpectedException expectedException = ExpectedException.none(); - protected Table hTable; + protected Table hTable; private static final String key1 = "scanKey1x"; private static final String key2 = "scanKey2x"; private static final String key3 = "scanKey3x"; @@ -93,29 +93,36 @@ public void testDeleteFamilyVerison() throws Exception { toBytes(value1)); Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); - putKey3Fam1Column1Ts1.addColumn(toBytes(family1), toBytes(column1), timeStamp1, toBytes(value2)); + putKey3Fam1Column1Ts1.addColumn(toBytes(family1), toBytes(column1), timeStamp1, + toBytes(value2)); Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); putKey1Fam1Column2MinTs.addColumn(toBytes(family1), toBytes(column2), minTimeStamp, toBytes(value1)); Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); - putKey1Fam1Column2Ts3.addColumn(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + putKey1Fam1Column2Ts3.addColumn(toBytes(family1), toBytes(column2), timeStamp3, + toBytes(value2)); Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); - putKey2Fam1Column2Ts3.addColumn(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + putKey2Fam1Column2Ts3.addColumn(toBytes(family1), toBytes(column2), timeStamp3, + toBytes(value2)); Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); - putKey2Fam1Column3Ts1.addColumn(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + putKey2Fam1Column3Ts1.addColumn(toBytes(family1), toBytes(column3), timeStamp1, + toBytes(value2)); Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); - putKey3Fam1Column3Ts1.addColumn(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + putKey3Fam1Column3Ts1.addColumn(toBytes(family1), toBytes(column3), timeStamp1, + toBytes(value2)); Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts6.addColumn(toBytes(family1), toBytes(column2), timeStamp6, toBytes(value1)); + putKey3Fam1Column2Ts6.addColumn(toBytes(family1), toBytes(column2), timeStamp6, + toBytes(value1)); Put putKey2Fam1Column3Ts6 = new Put(toBytes(key2)); - putKey2Fam1Column3Ts6.addColumn(toBytes(family1), toBytes(column3), timeStamp3, toBytes(value1)); + putKey2Fam1Column3Ts6.addColumn(toBytes(family1), toBytes(column3), timeStamp3, + toBytes(value1)); tryPut(hTable, putKey1Fam1Column1MinTs); tryPut(hTable, putKey3Fam1Column1Ts1); @@ -210,42 +217,52 @@ public void testDeleteFamilyVerison() throws Exception { // test DeleteFamilyVersion multiple cf Put putKey1Fam1Column3Ts6 = new Put(toBytes(key1)); - putKey1Fam1Column3Ts6.addColumn(toBytes(family1), toBytes(column3), timeStamp6, toBytes(value3)); + putKey1Fam1Column3Ts6.addColumn(toBytes(family1), toBytes(column3), timeStamp6, + toBytes(value3)); Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); - putKey1Fam2Column2Ts2.addColumn(toBytes(family2), toBytes(column2), timeStamp2, toBytes(value1)); + putKey1Fam2Column2Ts2.addColumn(toBytes(family2), toBytes(column2), timeStamp2, + toBytes(value1)); Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); - putKey1Fam2Column3Ts2.addColumn(toBytes(family2), toBytes(column3), timeStamp2, toBytes(value1)); + putKey1Fam2Column3Ts2.addColumn(toBytes(family2), toBytes(column3), timeStamp2, + toBytes(value1)); Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); - putKey1Fam1Column2Ts1.addColumn(toBytes(family1), toBytes(column2), timeStamp1, toBytes(value2)); + putKey1Fam1Column2Ts1.addColumn(toBytes(family1), toBytes(column2), timeStamp1, + toBytes(value2)); Put putKey2Fam1Column2Ts8 = new Put(toBytes(key2)); - putKey2Fam1Column2Ts8.addColumn(toBytes(family1), toBytes(column2), timeStamp8, toBytes(value2)); + putKey2Fam1Column2Ts8.addColumn(toBytes(family1), toBytes(column2), timeStamp8, + toBytes(value2)); Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); - putKey2Fam2Column3Ts1.addColumn(toBytes(family2), toBytes(column3), timeStamp3, toBytes(value3)); + putKey2Fam2Column3Ts1.addColumn(toBytes(family2), toBytes(column3), timeStamp3, + toBytes(value3)); Put putKey2Fam1Column1Ts1 = new Put(toBytes(key2)); - putKey2Fam1Column1Ts1.addColumn(toBytes(family1), toBytes(column1), timeStamp8, toBytes(value1)); + putKey2Fam1Column1Ts1.addColumn(toBytes(family1), toBytes(column1), timeStamp8, + toBytes(value1)); Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); - putKey2Fam2Column1Ts3.addColumn(toBytes(family2), toBytes(column1), timeStamp3, toBytes(value2)); + putKey2Fam2Column1Ts3.addColumn(toBytes(family2), toBytes(column1), timeStamp3, + toBytes(value2)); Put putKey3Fam1Column2Ts9 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts9.addColumn(toBytes(family1), toBytes(column2), timeStamp9, toBytes(value2)); + putKey3Fam1Column2Ts9.addColumn(toBytes(family1), toBytes(column2), timeStamp9, + toBytes(value2)); Put putKey3Fam2Column3Ts10 = new Put(toBytes(key3)); - putKey3Fam2Column3Ts10 - .addColumn(toBytes(family2), toBytes(column3), timeStamp10, toBytes(value1)); + putKey3Fam2Column3Ts10.addColumn(toBytes(family2), toBytes(column3), timeStamp10, + toBytes(value1)); Put putKey3Fam2Column1Ts10 = new Put(toBytes(key3)); - putKey3Fam2Column1Ts10 - .addColumn(toBytes(family2), toBytes(column1), timeStamp10, toBytes(value2)); + putKey3Fam2Column1Ts10.addColumn(toBytes(family2), toBytes(column1), timeStamp10, + toBytes(value2)); Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts2.addColumn(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); + putKey3Fam1Column2Ts2.addColumn(toBytes(family1), toBytes(column2), timeStamp2, + toBytes(value1)); tryPut(hTable, putKey1Fam1Column3Ts6); tryPut(hTable, putKey1Fam2Column2Ts2); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java index a7472e52..a3646e96 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java @@ -38,9 +38,9 @@ public class OHTableMultiColumnFamilyTest { @Rule - public ExpectedException expectedException = ExpectedException.none(); + public ExpectedException expectedException = ExpectedException.none(); - protected Table hTable; + protected Table hTable; @Before public void before() throws Exception { @@ -281,6 +281,8 @@ public void testMulfiColumnFamilyBufferedMutator() throws Exception { Thread.sleep(1000); int currentUndealtCount = ((OHBufferedMutatorImpl) mutator).size(); Assert.assertNotEquals(lastUndealtCount, currentUndealtCount); + // after periodic flush, all mutations will be committed + Assert.assertEquals(0, currentUndealtCount); result = hTable.get(get); int newCount = result.rawCells().length; Assert.assertNotEquals(count, newCount); From 6759d9603c65f5e698a4d91e42aace1f5a1c668b Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 25 Oct 2024 17:57:37 +0800 Subject: [PATCH 20/32] add inherited interface in bufferedMutator; fix concurrent bug in bufferedMuator execution --- .../hbase/util/OHBufferedMutatorImpl.java | 53 +++++++++++-------- .../oceanbase/hbase/OHConnectionTest.java | 4 +- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index 55fa8d48..1f0af10d 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -148,7 +148,7 @@ public void mutate(List mutations) throws IOException { long toAddSize = 0; int toAddCount = 0; for (Mutation m : mutations) { - validateInsUpAndDelete(m); + validateOperation(m); toAddSize += m.heapSize(); ++toAddCount; } @@ -170,10 +170,13 @@ private void checkClose() { } /** - * Check whether the mutation is Put or Delete in 2.x + * Check mutations in 2.x * @param mt - mutation operation */ - private void validateInsUpAndDelete(Mutation mt) throws IllegalArgumentException { + private void validateOperation(Mutation mt) throws IllegalArgumentException { + if (mt == null) { + throw new IllegalArgumentException("Mutation operation cannot be null."); + } if (!(mt instanceof Put) && !(mt instanceof Delete)) { throw new IllegalArgumentException("Only support for Put and Delete for now."); } @@ -261,15 +264,6 @@ private void execute(boolean flushAll) throws IOException { currentAsyncBufferSize.addAndGet(-size); ++dealtCount; } - if (currentAsyncBufferSize.get() > 0) { - while (!execBuffer.isEmpty()) { - m = execBuffer.getFirst(); - long size = m.heapSize(); - currentAsyncBufferSize.addAndGet(size); - asyncWriteBuffer.add(m); - } - throw new IllegalStateException("Fetch error null value during execute"); - } undealtMutationCount.addAndGet(-dealtCount); } @@ -353,17 +347,14 @@ public void flush() throws IOException { execute(true); } - /** - * Count the mutations which haven't been processed. - */ - @VisibleForTesting - public int size() { - return undealtMutationCount.get(); + @Override + public long getWriteBufferPeriodicFlushTimeoutMs() { + return writeBufferPeriodicFlushTimeoutMs.get(); } - @VisibleForTesting - public ExecutorService getPool() { - return pool; + @Override + public long getWriteBufferPeriodicFlushTimerTickMs() { + return writeBufferPeriodicFlushTimerTickMs.get(); } @Override @@ -371,13 +362,33 @@ public long getWriteBufferSize() { return this.writeBufferSize; } + @Override public void setRpcTimeout(int rpcTimeout) { this.rpcTimeout.set(rpcTimeout); this.ohTable.setRpcTimeout(rpcTimeout); } + @Override public void setOperationTimeout(int operationTimeout) { this.operationTimeout.set(operationTimeout); this.ohTable.setOperationTimeout(operationTimeout); } + + /** + * Count the mutations which haven't been processed. + */ + @VisibleForTesting + public int size() { + return undealtMutationCount.get(); + } + + @VisibleForTesting + public ExecutorService getPool() { + return pool; + } + + @VisibleForTesting + protected long getExecutedWriteBufferPeriodicFlushes() { + return executedWriteBufferPeriodicFlushes.get(); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index aac67652..cf26ba6a 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -697,7 +697,7 @@ public void testBufferedMutatorConcurrent() throws Exception { ohBufferMutator = connection.getBufferedMutator(params); // BufferedMutator is not concurrently safe - for (int i = 0; i < 10; ++i) { + for (int i = 0; i < 50; ++i) { final int taskId = i; final BufferedMutator thrBufferMutator = ohBufferMutator; executorService.submit(() -> { @@ -747,7 +747,7 @@ public void testBufferedMutatorConcurrent() throws Exception { get.addFamily(toBytes(family)); Result r = hTable.get(get); count = r.rawCells().length; - Assert.assertEquals(40, count); + Assert.assertEquals(200, count); Delete delete = new Delete(toBytes(key)); delete.addFamily(toBytes(family)); hTable.delete(delete); From d83966ac591c4a72b9f5400cd9b72d6f582d2d0b Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Mon, 28 Oct 2024 14:08:42 +0800 Subject: [PATCH 21/32] develop tableBuilder --- .../com/alipay/oceanbase/hbase/OHTable.java | 60 ++++++++++++++- .../hbase/util/OHConnectionConfiguration.java | 21 +++++ .../hbase/util/OHConnectionImpl.java | 14 +++- .../hbase/util/ObTableBuilderBase.java | 74 ++++++++++++++++++ .../oceanbase/hbase/OHConnectionTest.java | 50 ++++++++++-- .../hbase/OHTableMultiColumnFamilyTest.java | 77 ++++++++++++++++++- 6 files changed, 285 insertions(+), 11 deletions(-) create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/ObTableBuilderBase.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 8e13c036..79eb372b 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -104,6 +104,16 @@ public class OHTable implements Table { */ private int rpcTimeout; + /** + * timeout for each read rpc request + */ + private int readRpcTimeout; + + /** + * timeout for each write rpc request + */ + private int writeRpcTimeout; + /** * if the Get executing pool is specified by user cleanupPoolOnClose will be false , * which means that user is responsible for the pool @@ -292,7 +302,7 @@ public OHTable(final byte[] tableName, final ObTableClient obTableClient, public OHTable(TableName tableName, Connection connection, OHConnectionConfiguration connectionConfig, ExecutorService executePool) throws IOException { - checkArgument(connection.getConfiguration() != null, "configuration is null."); + checkArgument(connection != null, "connection is null."); checkArgument(tableName != null, "tableName is null."); checkArgument(connection.getConfiguration() != null, "configuration is null."); checkArgument(tableName.getName() != null, "tableNameString is null."); @@ -311,6 +321,8 @@ public OHTable(TableName tableName, Connection connection, this.cleanupPoolOnClose = false; } this.rpcTimeout = connectionConfig.getRpcTimeout(); + this.readRpcTimeout = connectionConfig.getReadRpcTimeout(); + this.writeRpcTimeout = connectionConfig.getWriteRpcTimeout(); this.operationTimeout = connectionConfig.getOperationTimeout(); this.operationExecuteInPool = this.configuration.getBoolean( HBASE_CLIENT_OPERATION_EXECUTE_IN_POOL, @@ -331,6 +343,50 @@ public OHTable(TableName tableName, Connection connection, finishSetUp(); } + public OHTable(Connection connection, ObTableBuilderBase builder, + OHConnectionConfiguration connectionConfig, ExecutorService executePool) + throws IOException { + checkArgument(connection != null, "connection is null."); + checkArgument(connection.getConfiguration() != null, "configuration is null."); + checkArgument(builder != null, "builder is null"); + checkArgument(connectionConfig != null, "connectionConfig is null."); + TableName builderTableName = builder.getTableName(); + this.tableName = builderTableName.getName(); + this.tableNameString = builderTableName.getNameAsString(); + this.configuration = connection.getConfiguration(); + this.executePool = executePool; + if (executePool == null) { + int maxThreads = configuration.getInt(HBASE_HTABLE_PRIVATE_THREADS_MAX, + DEFAULT_HBASE_HTABLE_PRIVATE_THREADS_MAX); + long keepAliveTime = configuration.getLong(HBASE_HTABLE_THREAD_KEEP_ALIVE_TIME, + DEFAULT_HBASE_HTABLE_THREAD_KEEP_ALIVE_TIME); + this.executePool = createDefaultThreadPoolExecutor(1, maxThreads, keepAliveTime); + this.cleanupPoolOnClose = true; + } else { + this.cleanupPoolOnClose = false; + } + this.rpcTimeout = builder.getRpcTimeout(); + this.readRpcTimeout = builder.getReadRpcTimeout(); + this.writeRpcTimeout = builder.getWriteRpcTimeout(); + this.operationTimeout = builder.getOperationTimeout(); + this.operationExecuteInPool = this.configuration.getBoolean( + HBASE_CLIENT_OPERATION_EXECUTE_IN_POOL, + (this.operationTimeout != HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT)); + this.maxKeyValueSize = connectionConfig.getMaxKeyValueSize(); + this.putWriteBufferCheck = this.configuration.getInt(HBASE_HTABLE_PUT_WRITE_BUFFER_CHECK, + DEFAULT_HBASE_HTABLE_PUT_WRITE_BUFFER_CHECK); + this.writeBufferSize = connectionConfig.getWriteBufferSize(); + int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( + this.tableNameString, connectionConfig)); + this.obTableClient.setRpcExecuteTimeout(rpcTimeout); + this.obTableClient.setRuntimeRetryTimes(numRetries); + setOperationTimeout(operationTimeout); + + finishSetUp(); + } + /** * 创建默认的线程池 * Using the "direct handoff" approach, new threads will only be created @@ -388,7 +444,7 @@ private void finishSetUp() { private OHConnectionConfiguration setUserDefinedNamespace(String tableNameString, OHConnectionConfiguration ohConnectionConf) - throws IOException { + throws IllegalArgumentException { if (tableNameString.indexOf(':') != -1) { String[] params = tableNameString.split(":"); if (params.length != 2) { diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java index fd032267..9b98dd0b 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java @@ -47,10 +47,13 @@ public class OHConnectionConfiguration { private final boolean odpMode; private final long writeBufferSize; private final int operationTimeout; + private final int metaOperationTimeout; private final int scannerCaching; private final long scannerMaxResultSize; private final int maxKeyValueSize; private final int rpcTimeout; + private final int readRpcTimeout; + private final int writeRpcTimeout; private final int rpcConnectTimeout; private final long writeBufferPeriodicFlushTimeoutMs; private final long writeBufferPeriodicFlushTimerTickMs; @@ -70,10 +73,16 @@ public OHConnectionConfiguration(Configuration conf) { } this.database = database; this.writeBufferSize = conf.getLong(WRITE_BUFFER_SIZE_KEY, WRITE_BUFFER_SIZE_DEFAULT); + this.metaOperationTimeout = conf.getInt(HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT, + HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT); this.operationTimeout = conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT); this.rpcTimeout = conf.getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT); + this.readRpcTimeout = conf.getInt(HConstants.HBASE_RPC_READ_TIMEOUT_KEY, + HConstants.DEFAULT_HBASE_RPC_TIMEOUT); + this.writeRpcTimeout = conf.getInt(HConstants.HBASE_RPC_WRITE_TIMEOUT_KEY, + HConstants.DEFAULT_HBASE_RPC_TIMEOUT); this.writeBufferPeriodicFlushTimeoutMs = conf.getLong( WRITE_BUFFER_PERIODIC_FLUSH_TIMEOUT_MS, WRITE_BUFFER_PERIODIC_FLUSH_TIMEOUT_MS_DEFAULT); this.writeBufferPeriodicFlushTimerTickMs = conf.getLong( @@ -117,6 +126,10 @@ public long getWriteBufferSize() { return this.writeBufferSize; } + public int getMetaOperationTimeout() { + return this.metaOperationTimeout; + } + public int getOperationTimeout() { return this.operationTimeout; } @@ -133,6 +146,14 @@ public int getRpcTimeout() { return this.rpcTimeout; } + public int getReadRpcTimeout() { + return this.readRpcTimeout; + } + + public int getWriteRpcTimeout() { + return this.writeRpcTimeout; + } + public int getRpcConnectTimeout() { return this.rpcConnectTimeout; } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java index 3964f52f..44df648c 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java @@ -176,8 +176,18 @@ public boolean isClosed() { } @Override - public TableBuilder getTableBuilder(TableName tableName, ExecutorService executorService) { - throw new FeatureNotSupportedException("not supported yet'"); + public TableBuilder getTableBuilder(TableName tableName, ExecutorService pool) { + return new ObTableBuilderBase(tableName, connectionConfig) { + @Override + public Table build() { + try { + return new OHTable(OHConnectionImpl.this, this, connectionConfig, pool); + } catch (Exception e) { + LOGGER.error("Fail to build new OHTable", e); + throw new RuntimeException(e); + } + } + }; } @Override diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableBuilderBase.java b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableBuilderBase.java new file mode 100644 index 00000000..48fad23b --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableBuilderBase.java @@ -0,0 +1,74 @@ +package com.alipay.oceanbase.hbase.util; + +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.TableBuilder; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +abstract public class ObTableBuilderBase implements TableBuilder { + protected TableName tableName; + + protected int operationTimeout; + + protected int rpcTimeout; + + protected int readRpcTimeout; + + protected int writeRpcTimeout; + + ObTableBuilderBase(TableName tableName, OHConnectionConfiguration ohConnConf) { + if (tableName == null) { + throw new IllegalArgumentException("The provided tableName is null"); + } + this.tableName = tableName; + this.operationTimeout = tableName.isSystemTable() ? ohConnConf.getMetaOperationTimeout() + : ohConnConf.getOperationTimeout(); + this.rpcTimeout = ohConnConf.getRpcTimeout(); + this.readRpcTimeout = ohConnConf.getReadRpcTimeout(); + this.writeRpcTimeout = ohConnConf.getWriteRpcTimeout(); + } + + @Override + public ObTableBuilderBase setOperationTimeout(int timeout) { + this.operationTimeout = timeout; + return this; + } + + @Override + public ObTableBuilderBase setRpcTimeout(int timeout) { + this.rpcTimeout = timeout; + return this; + } + + @Override + public ObTableBuilderBase setReadRpcTimeout(int timeout) { + this.readRpcTimeout = timeout; + return this; + } + + @Override + public ObTableBuilderBase setWriteRpcTimeout(int timeout) { + this.writeRpcTimeout = timeout; + return this; + } + + public TableName getTableName() { + return this.tableName; + } + + public int getOperationTimeout() { + return this.operationTimeout; + } + + public int getRpcTimeout() { + return this.rpcTimeout; + } + + public int getReadRpcTimeout() { + return this.readRpcTimeout; + } + + public int getWriteRpcTimeout() { + return this.writeRpcTimeout; + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index cf26ba6a..7e11c031 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -68,6 +68,44 @@ public void testConnectionByXml() throws Exception { testBasic(); } + @Test + public void testGetTableByTableBuilder() throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + conf.set("rs.list.acquire.read.timeout", "10000"); + conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); + connection = ConnectionFactory.createConnection(conf); + TableName tableName = TableName.valueOf("test"); + TableBuilder builder = connection.getTableBuilder(tableName, null); + // build a OHTable with default params + hTable = builder.build(); + testBasic(); + + // set params for TableBuilder + long keepAliveTime = conf.getLong("hbase.hconnection.threads.keepalivetime", 60); + ThreadPoolExecutor pool = new ThreadPoolExecutor(10, 256, keepAliveTime, TimeUnit.SECONDS, + new SynchronousQueue(), Threads.newDaemonThreadFactory("htable")); + pool.allowCoreThreadTimeOut(true); + builder = connection.getTableBuilder(tableName, pool); + builder.setOperationTimeout(1500000); + builder.setRpcTimeout(40000); + hTable = builder.build(); + testBasic(); + + hTable.close(); + Assert.assertFalse(pool.isShutdown()); + pool.shutdown(); + try { + if (!pool.awaitTermination(10, TimeUnit.SECONDS)) { + System.out + .println("close() failed to terminate pool after 10 seconds. Abandoning pool."); + } + } catch (InterruptedException e) { + System.out.println("waitForTermination interrupted"); + Thread.currentThread().interrupt(); + } + Assert.assertTrue(pool.isShutdown()); + } + private void testBasic() throws Exception { String key = "putKey"; String column1 = "putColumn1"; @@ -637,8 +675,8 @@ public void testBufferedMutatorWithUserPool() throws Exception { Assert.assertFalse(bufferPool.isShutdown()); bufferPool.shutdown(); try { - if (!bufferPool.awaitTermination(600, TimeUnit.SECONDS)) { - System.out.println("close() failed to terminate pool after 10 minutes. Abandoning pool."); + if (!bufferPool.awaitTermination(10, TimeUnit.SECONDS)) { + System.out.println("close() failed to terminate pool after 10 seconds. Abandoning pool."); } } catch (InterruptedException e) { System.out.println("waitForTermination interrupted"); @@ -768,8 +806,8 @@ public void testBufferedMutatorConcurrent() throws Exception { Assert.assertFalse(bufferPool.isShutdown()); bufferPool.shutdown(); try { - if (!bufferPool.awaitTermination(600, TimeUnit.SECONDS)) { - System.out.println("close() failed to terminate pool after 10 minutes. Abandoning pool."); + if (!bufferPool.awaitTermination(10, TimeUnit.SECONDS)) { + System.out.println("close() failed to terminate pool after 10 seconds. Abandoning pool."); } } catch (InterruptedException e) { System.out.println("waitForTermination interrupted"); @@ -891,8 +929,8 @@ public void testBufferedMutatorPeriodicFlush() throws Exception { Assert.assertFalse(bufferPool.isShutdown()); bufferPool.shutdown(); try { - if (!bufferPool.awaitTermination(600, TimeUnit.SECONDS)) { - System.out.println("close() failed to terminate pool after 10 minutes. Abandoning pool."); + if (!bufferPool.awaitTermination(10, TimeUnit.SECONDS)) { + System.out.println("close() failed to terminate pool after 10 seconds. Abandoning pool."); } } catch (InterruptedException e) { System.out.println("waitForTermination interrupted"); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java index a3646e96..0441a071 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.util.*; +import static org.apache.hadoop.hbase.ipc.RpcClient.SOCKET_TIMEOUT_CONNECT; import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.*; @@ -54,7 +55,81 @@ public void finish() throws IOException { } @Test - public void testMulfiColumnFamilyBufferedMutator() throws Exception { + public void testMultiColumnFamilyTableBuilder() throws Exception { + byte[] family1 = "family_with_group1".getBytes(); + byte[] family2 = "family_with_group2".getBytes(); + byte[] family3 = "family_with_group3".getBytes(); + + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family1_value = "VVV1".getBytes(); + byte[] family2_value = "VVV2".getBytes(); + byte[] family3_value = "VVV3".getBytes(); + + Configuration conf = ObHTableTestUtil.newConfiguration(); + conf.set("rs.list.acquire.read.timeout", "10000"); + conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); + Connection connection = ConnectionFactory.createConnection(conf); + TableName tableName = TableName.valueOf("test_multi_cf"); + TableBuilder builder = connection.getTableBuilder(tableName, null); + // build a OHTable with default params + hTable = builder.build(); + + Delete delete = new Delete(toBytes("Key0")); + delete.addFamily(family1); + delete.addFamily(family2); + delete.addFamily(family3); + hTable.delete(delete); + + Put put = new Put(toBytes("Key0")); + put.addColumn(family1, family1_column1, family1_value); + put.addColumn(family1, family1_column2, family1_value); + put.addColumn(family1, family1_column3, family1_value); + put.addColumn(family2, family2_column1, family2_value); + put.addColumn(family2, family2_column2, family2_value); + put.addColumn(family3, family3_column1, family3_value); + hTable.put(put); + + int count = 0; + Get get = new Get(toBytes("Key0")); + get.setMaxVersions(); + Result r = hTable.get(get); + Assert.assertEquals(6, r.rawCells().length); + + delete = new Delete(toBytes("Key0")); + delete.addFamily(family1); + delete.addFamily(family2); + delete.addFamily(family3); + hTable.delete(delete); + r = hTable.get(get); + Assert.assertEquals(0, r.rawCells().length); + + // set params for TableBuilder + builder.setOperationTimeout(1500000); + builder.setRpcTimeout(40000); + hTable = builder.build(); + + put = new Put(toBytes("Key0")); + put.addColumn(family1, family1_column1, family1_value); + put.addColumn(family1, family1_column2, family1_value); + put.addColumn(family2, family2_column1, family2_value); + put.addColumn(family3, family3_column1, family3_value); + hTable.put(put); + + r = hTable.get(get); + Assert.assertEquals(4, r.rawCells().length); + + hTable.delete(delete); + r = hTable.get(get); + Assert.assertEquals(0, r.rawCells().length); + } + + @Test + public void testMultiColumnFamilyBufferedMutator() throws Exception { byte[] family1 = "family_with_group1".getBytes(); byte[] family2 = "family_with_group2".getBytes(); byte[] family3 = "family_with_group3".getBytes(); From 3a44d73ffc0012df588c286f54c21cbf40e93ff3 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Mon, 28 Oct 2024 15:07:21 +0800 Subject: [PATCH 22/32] fix typo --- .../oceanbase/hbase/util/OHBufferedMutatorImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index 1f0af10d..54ce1b9e 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -69,7 +69,7 @@ public class OHBufferedMutatorImpl implements BufferedMutator { 0); private final AtomicInteger rpcTimeout; private final AtomicInteger operationTimeout; - private final boolean cleanipPoolOnClose; + private final boolean cleanupPoolOnClose; private volatile boolean closed = false; public OHBufferedMutatorImpl(OHConnectionImpl ohConnection, BufferedMutatorParams params) @@ -84,10 +84,10 @@ public OHBufferedMutatorImpl(OHConnectionImpl ohConnection, BufferedMutatorParam this.listener = params.getListener(); if (params.getPool() == null) { // need to verify necessity this.pool = HTable.getDefaultExecutor(conf); - this.cleanipPoolOnClose = true; + this.cleanupPoolOnClose = true; } else { this.pool = params.getPool(); - this.cleanipPoolOnClose = false; + this.cleanupPoolOnClose = false; } this.rpcTimeout = new AtomicInteger( params.getRpcTimeout() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params @@ -321,7 +321,7 @@ public void close() throws IOException { try { execute(true); } finally { - if (cleanipPoolOnClose) { + if (cleanupPoolOnClose) { // the pool in ObTableClient will be shut down too this.pool.shutdown(); try { From 21c6a0ff72f1c8b2b8133ae79345365777b6b106 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Mon, 28 Oct 2024 15:10:12 +0800 Subject: [PATCH 23/32] fix exception erros message --- .../com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index 54ce1b9e..d6e63d40 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -165,7 +165,7 @@ public void mutate(List mutations) throws IOException { private void checkClose() { if (closed) { - throw new IllegalStateException("Cannot put when the BufferedMutator is closed."); + throw new IllegalStateException("The BufferedMutator is closed."); } } From 14fc1461d5496b55cb2b4d051cb6c90d803edf02 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Mon, 28 Oct 2024 15:33:57 +0800 Subject: [PATCH 24/32] update time to wait pool to shutdown --- .../com/alipay/oceanbase/hbase/OHConnectionTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index cf26ba6a..719c1de4 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -637,8 +637,8 @@ public void testBufferedMutatorWithUserPool() throws Exception { Assert.assertFalse(bufferPool.isShutdown()); bufferPool.shutdown(); try { - if (!bufferPool.awaitTermination(600, TimeUnit.SECONDS)) { - System.out.println("close() failed to terminate pool after 10 minutes. Abandoning pool."); + if (!bufferPool.awaitTermination(10, TimeUnit.SECONDS)) { + System.out.println("close() failed to terminate pool after 10 seconds. Abandoning pool."); } } catch (InterruptedException e) { System.out.println("waitForTermination interrupted"); @@ -768,8 +768,8 @@ public void testBufferedMutatorConcurrent() throws Exception { Assert.assertFalse(bufferPool.isShutdown()); bufferPool.shutdown(); try { - if (!bufferPool.awaitTermination(600, TimeUnit.SECONDS)) { - System.out.println("close() failed to terminate pool after 10 minutes. Abandoning pool."); + if (!bufferPool.awaitTermination(10, TimeUnit.SECONDS)) { + System.out.println("close() failed to terminate pool after 10 seconds. Abandoning pool."); } } catch (InterruptedException e) { System.out.println("waitForTermination interrupted"); @@ -891,8 +891,8 @@ public void testBufferedMutatorPeriodicFlush() throws Exception { Assert.assertFalse(bufferPool.isShutdown()); bufferPool.shutdown(); try { - if (!bufferPool.awaitTermination(600, TimeUnit.SECONDS)) { - System.out.println("close() failed to terminate pool after 10 minutes. Abandoning pool."); + if (!bufferPool.awaitTermination(10, TimeUnit.SECONDS)) { + System.out.println("close() failed to terminate pool after 10 seconds. Abandoning pool."); } } catch (InterruptedException e) { System.out.println("waitForTermination interrupted"); From f31ae96f2b1a9b8a46726749bd8213c2966fa679 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Mon, 28 Oct 2024 17:03:28 +0800 Subject: [PATCH 25/32] optimize rpcTimeout and operationTimeout setting in OHTable finishSetup --- .../com/alipay/oceanbase/hbase/OHTable.java | 22 +++++++++++++++---- .../hbase/util/OHConnectionImpl.java | 3 ++- .../oceanbase/hbase/OHConnectionTest.java | 8 ++++++- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 4e853aa8..3ff2c48c 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -427,11 +427,15 @@ private void finishSetUp() { HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD); - this.rpcTimeout = configuration.getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, - HConstants.DEFAULT_HBASE_RPC_TIMEOUT); - this.operationTimeout = this.configuration.getInt( + this.rpcTimeout = this.rpcTimeout <= 0 ? configuration.getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, + HConstants.DEFAULT_HBASE_RPC_TIMEOUT) : this.rpcTimeout; + this.readRpcTimeout = this.readRpcTimeout <= 0 ? configuration.getInt(HConstants.HBASE_RPC_READ_TIMEOUT_KEY, + HConstants.DEFAULT_HBASE_RPC_TIMEOUT) : this.readRpcTimeout; + this.writeRpcTimeout = this.writeRpcTimeout <= 0 ? configuration.getInt(HConstants.HBASE_RPC_WRITE_TIMEOUT_KEY, + HConstants.DEFAULT_HBASE_RPC_TIMEOUT) : this.writeRpcTimeout; + this.operationTimeout = this.operationTimeout <= 0 ? this.configuration.getInt( HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, - HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT); + HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT) : this.operationTimeout; this.operationExecuteInPool = this.configuration.getBoolean( HBASE_CLIENT_OPERATION_EXECUTE_IN_POOL, (this.operationTimeout != HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT)); @@ -1377,6 +1381,16 @@ public int getRpcTimeout() { return this.rpcTimeout; } + @Override + public int getReadRpcTimeout() { + return this.readRpcTimeout; + } + + @Override + public int getWriteRpcTimeout() { + return this.writeRpcTimeout; + } + public void setRuntimeBatchExecutor(ExecutorService runtimeBatchExecutor) { this.obTableClient.setRuntimeBatchExecutor(runtimeBatchExecutor); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java index 44df648c..41a0690a 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java @@ -181,7 +181,8 @@ public TableBuilder getTableBuilder(TableName tableName, ExecutorService pool) { @Override public Table build() { try { - return new OHTable(OHConnectionImpl.this, this, connectionConfig, pool); + return new OHTable(OHConnectionImpl.this, this, + OHConnectionImpl.this.connectionConfig, pool); } catch (Exception e) { LOGGER.error("Fail to build new OHTable", e); throw new RuntimeException(e); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index 7e11c031..7fcae9d0 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -86,9 +86,15 @@ public void testGetTableByTableBuilder() throws Exception { new SynchronousQueue(), Threads.newDaemonThreadFactory("htable")); pool.allowCoreThreadTimeOut(true); builder = connection.getTableBuilder(tableName, pool); - builder.setOperationTimeout(1500000); builder.setRpcTimeout(40000); + builder.setReadRpcTimeout(30000); + builder.setWriteRpcTimeout(50000); + builder.setOperationTimeout(1500000); hTable = builder.build(); + Assert.assertEquals(40000, hTable.getRpcTimeout()); + Assert.assertEquals(30000, hTable.getReadRpcTimeout()); + Assert.assertEquals(50000, hTable.getWriteRpcTimeout()); + Assert.assertEquals(1500000, hTable.getOperationTimeout()); testBasic(); hTable.close(); From d932b908acf160c642eb2e69fe5c9efad9bd515f Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Mon, 28 Oct 2024 21:13:09 +0800 Subject: [PATCH 26/32] develop ObCheckAndMutateBuilder --- .../com/alipay/oceanbase/hbase/OHTable.java | 127 ++++++- .../alipay/oceanbase/hbase/OHTableClient.java | 6 + .../alipay/oceanbase/hbase/OHTablePool.java | 5 + .../oceanbase/hbase/HTableTestBase.java | 326 +++++++++++++++++- 4 files changed, 452 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 3ff2c48c..546fdc98 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.slf4j.Logger; +import org.tukaani.xz.check.Check; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -64,6 +65,7 @@ import static com.alipay.oceanbase.hbase.constants.OHConstants.*; import static com.alipay.oceanbase.hbase.util.Preconditions.checkArgument; +import static com.alipay.oceanbase.hbase.util.Preconditions.checkNotNull; import static com.alipay.oceanbase.hbase.util.TableHBaseLoggerFactory.LCD; import static com.alipay.oceanbase.hbase.util.TableHBaseLoggerFactory.TABLE_HBASE_LOGGER_SPACE; import static com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperation.getInstance; @@ -964,7 +966,7 @@ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, RowMutations rowMutations = new RowMutations(row); rowMutations.add(put); try { - return checkAndMutation(row, family, qualifier, compareOp, value, rowMutations); + return checkAndMutation(row, family, qualifier, compareOp, value, null, rowMutations); } catch (Exception e) { logger.error(LCD.convert("01-00005"), put, tableNameString, e); throw new IOException("checkAndPut type table:" + tableNameString + " e.msg:" @@ -1020,7 +1022,7 @@ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, RowMutations rowMutations = new RowMutations(row); rowMutations.add(delete); try { - return checkAndMutation(row, family, qualifier, compareOp, value, rowMutations); + return checkAndMutation(row, family, qualifier, compareOp, value, null, rowMutations); } catch (Exception e) { logger.error(LCD.convert("01-00005"), delete, tableNameString, e); throw new IOException("checkAndDelete type table:" + tableNameString + " e.msg:" @@ -1033,7 +1035,7 @@ public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, CompareFilter.CompareOp compareOp, byte[] value, RowMutations rowMutations) throws IOException { try { - return checkAndMutation(row, family, qualifier, compareOp, value, rowMutations); + return checkAndMutation(row, family, qualifier, compareOp, value, null, rowMutations); } catch (Exception e) { logger.error(LCD.convert("01-00005"), rowMutations, tableNameString, e); throw new IOException("checkAndMutate type table:" + tableNameString + " e.msg:" @@ -1041,18 +1043,21 @@ public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, } } + @Override + public CheckAndMutateBuilder checkAndMutate(byte[]row , byte[] family) { + return new ObCheckAndMutateBuilderImpl(row, family); + } + private boolean checkAndMutation(byte[] row, byte[] family, byte[] qualifier, CompareFilter.CompareOp compareOp, byte[] value, - RowMutations rowMutations) throws Exception { + TimeRange timeRange, RowMutations rowMutations) throws Exception { checkArgument(row != null, "row is null"); checkArgument(isNotBlank(Bytes.toString(family)), "family is blank"); checkArgument(Bytes.equals(row, rowMutations.getRow()), - "mutation row is not equal check row"); - + "mutation row is not equal check row"); checkArgument(!rowMutations.getMutations().isEmpty(), "mutation is empty"); - byte[] filterString = buildCheckAndMutateFilterString(family, qualifier, compareOp, value); - ObHTableFilter filter = buildObHTableFilter(filterString, null, 1, qualifier); + ObHTableFilter filter = buildObHTableFilter(filterString, timeRange, 1, qualifier); List mutations = rowMutations.getMutations(); List keyValueList = new LinkedList<>(); // only one family operation is allowed @@ -1905,4 +1910,110 @@ public static OHOpType getDeleteType(Cell.Type type) { throw new IllegalArgumentException("illegal mutation type " + type); } } + + private class ObCheckAndMutateBuilderImpl implements CheckAndMutateBuilder { + private final byte[] row; + private final byte[] family; + private byte[] qualifier; + private byte[] value; + private TimeRange timeRange; + private CompareOperator cmpOp; + + ObCheckAndMutateBuilderImpl(byte[] row, byte[] family) { + this.row = checkNotNull(row, "The provided row is null."); + this.family = checkNotNull(family, "The provided family is null."); + } + + @Override + public CheckAndMutateBuilder qualifier(byte[] qualifier) { + this.qualifier = checkNotNull(qualifier, "The provided qualifier is null. You could" + + " use an empty byte array, or do not call this method if you want a null qualifier."); + return this; + } + + @Override + public CheckAndMutateBuilder timeRange(TimeRange timeRange) { + this.timeRange = timeRange; + return this; + } + + @Override + public CheckAndMutateBuilder ifNotExists() { + this.cmpOp = CompareOperator.EQUAL; + this.value = null; + return this; + } + + @Override + public CheckAndMutateBuilder ifMatches(CompareOperator cmpOp, byte[] value) { + this.cmpOp = checkNotNull(cmpOp, "The provided cmpOp is null."); + this.value = checkNotNull(value , "The provided value is null."); + return this; + } + + @Override + public boolean thenPut(Put put) throws IOException { + checkCmpOp(); + RowMutations rowMutations = new RowMutations(row); + rowMutations.add(put); + try { + return checkAndMutation(row, family, qualifier, getCompareOp(cmpOp), value, timeRange, rowMutations); + } catch(Exception e) { + logger.error(LCD.convert("01-00005"), rowMutations, tableNameString, e); + throw new IOException("checkAndMutate type table:" + tableNameString + " e.msg:" + + e.getMessage() + " error.", e); + } + } + + @Override + public boolean thenDelete(Delete delete) throws IOException { + checkCmpOp(); + RowMutations rowMutations = new RowMutations(row); + rowMutations.add(delete); + try { + return checkAndMutation(row, family, qualifier, getCompareOp(cmpOp), value, timeRange, rowMutations); + } catch(Exception e) { + logger.error(LCD.convert("01-00005"), rowMutations, tableNameString, e); + throw new IOException("checkAndMutate type table:" + tableNameString + " e.msg:" + + e.getMessage() + " error.", e); + } + } + + @Override + public boolean thenMutate(RowMutations mutation) throws IOException { + checkCmpOp(); + try { + return checkAndMutation(row, family, qualifier, getCompareOp(cmpOp), value, timeRange, mutation); + } catch(Exception e) { + logger.error(LCD.convert("01-00005"), mutation, tableNameString, e); + throw new IOException("checkAndMutate type table:" + tableNameString + " e.msg:" + + e.getMessage() + " error.", e); + } + } + + + private void checkCmpOp() { + checkNotNull(this.cmpOp, "The compare condition is null. Please use" + + " ifNotExists/ifEquals/ifMatches before executing the request"); + } + + private CompareFilter.CompareOp getCompareOp(CompareOperator cmpOp) { + switch (cmpOp) { + case LESS: + return CompareFilter.CompareOp.LESS; + case LESS_OR_EQUAL: + return CompareFilter.CompareOp.LESS_OR_EQUAL; + case EQUAL: + return CompareFilter.CompareOp.EQUAL; + case NOT_EQUAL: + return CompareFilter.CompareOp.NOT_EQUAL; + case GREATER_OR_EQUAL: + return CompareFilter.CompareOp.GREATER_OR_EQUAL; + case GREATER: + return CompareFilter.CompareOp.GREATER; + default: + return CompareFilter.CompareOp.NO_OP; + } + } + } } \ No newline at end of file diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java b/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java index 1894e8c5..eec11a60 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java @@ -173,6 +173,12 @@ public boolean checkAndMutate(byte[] row, byte[] family, byte[] qualifier, return ohTable.checkAndMutate(row, family, qualifier, compareOp, value, mutations); } + @Override + public CheckAndMutateBuilder checkAndMutate(byte[] row, byte[] family) { + checkStatus(); + return ohTable.checkAndMutate(row, family); + } + @Override public void setOperationTimeout(int i) { checkStatus(); diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java b/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java index 87ec2113..d392840c 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java @@ -826,6 +826,11 @@ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, return table.checkAndDelete(row, family, qualifier, compareOp, value, delete); } + @Override + public CheckAndMutateBuilder checkAndMutate(byte[] row, byte[] family) { + return table.checkAndMutate(row, family); + } + @Override public Result increment(Increment increment) throws IOException { return table.increment(increment); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index d6368678..c7ee74e2 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -18,12 +18,10 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.*; +import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.junit.Assert; @@ -4623,6 +4621,69 @@ public void testCheckAndPut() throws IOException, InterruptedException { r = hTable.get(get); Assert.assertEquals(3, r.rawCells().length); Assert.assertEquals("value1", Bytes.toString(CellUtil.cloneValue(r.rawCells()[0]))); + + // test CheckAndMutateBuilder + delete.addFamily(family.getBytes()); + hTable.delete(delete); + r = hTable.get(get); + Assert.assertEquals(0, r.rawCells().length); + + put = new Put(key.getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), value.getBytes()); + hTable.put(put); + get = new Get(key.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + get.addColumn(family.getBytes(), column.getBytes()); + r = hTable.get(get); + Assert.assertEquals(1, r.rawCells().length); + + Table.CheckAndMutateBuilder builder = hTable.checkAndMutate(toBytes(key), toBytes(family)); + put = new Put(key.getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), "value1".getBytes()); + ret = builder.qualifier(toBytes(column)).ifEquals(toBytes(value)).thenPut(put); + Assert.assertTrue(ret); + + ret = builder.qualifier(toBytes(column)) + .ifMatches(CompareOperator.GREATER, toBytes("value1")).thenPut(put); + Assert.assertFalse(ret); + ret = builder.qualifier(toBytes(column)) + .ifMatches(CompareOperator.GREATER_OR_EQUAL, toBytes("value1")).thenPut(put); + Assert.assertTrue(ret); + ret = builder.qualifier(toBytes(column)).ifMatches(CompareOperator.LESS, toBytes("")) + .thenPut(put); + Assert.assertFalse(ret); + ret = builder.qualifier(toBytes(column)) + .ifMatches(CompareOperator.LESS_OR_EQUAL, toBytes("")).thenPut(put); + Assert.assertFalse(ret); + + get = new Get(key.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + get.addColumn(family.getBytes(), column.getBytes()); + r = hTable.get(get); + Assert.assertEquals(3, r.rawCells().length); + Assert.assertEquals("value1", Bytes.toString(CellUtil.cloneValue(r.rawCells()[0]))); + + // test TimeRange + long t = System.currentTimeMillis(); + put = new Put(key.getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), t, value.getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), t + 3, "value1".getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), t + 5, "value2".getBytes()); + hTable.put(put); + put = new Put(toBytes(key)); + put.addColumn(toBytes(family), toBytes(column), toBytes(value)); + TimeRange timeRange = new TimeRange(t + 1, t + 3); + ret = builder.qualifier(toBytes(column)).timeRange(timeRange).ifEquals(toBytes("value1")) + .thenPut(put); + Assert.assertFalse(ret); + timeRange = new TimeRange(t, t + 2); + ret = builder.qualifier(toBytes(column)).timeRange(timeRange).ifEquals(toBytes(value)) + .thenPut(put); + Assert.assertTrue(ret); + + hTable.delete(delete); + r = hTable.get(get); + Assert.assertEquals(0, r.rawCells().length); } @Test @@ -4720,6 +4781,104 @@ public void testCheckAndDelete() throws IOException { r = hTable.get(get); Assert.assertEquals(0, r.rawCells().length); + // test CheckAndMutateBuilder + delete = new Delete(key.getBytes()); + delete.addFamily(family.getBytes()); + hTable.delete(delete); + put = new Put(key.getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), value.getBytes()); + hTable.put(put); + + // check delete column + delete = new Delete(key.getBytes()); + delete.addColumn(family.getBytes(), column.getBytes()); + Table.CheckAndMutateBuilder builder = hTable.checkAndMutate(toBytes(key), toBytes(family)); + ret = builder.qualifier(toBytes(column)).ifEquals(toBytes(value)).thenDelete(delete); + Assert.assertTrue(ret); + put = new Put(key.getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), "value6".getBytes()); + hTable.put(put); + ret = builder.qualifier(toBytes(column)) + .ifMatches(CompareOperator.GREATER, toBytes("value5")).thenDelete(delete); + Assert.assertTrue(ret); + put = new Put(key.getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), "value5".getBytes()); + hTable.put(put); + ret = builder.qualifier(toBytes(column)) + .ifMatches(CompareOperator.GREATER_OR_EQUAL, toBytes("value5")).thenDelete(delete); + Assert.assertTrue(ret); + put = new Put(key.getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), "value1".getBytes()); + hTable.put(put); + ret = builder.qualifier(toBytes(column)).ifMatches(CompareOperator.LESS, toBytes("value1")) + .thenDelete(delete); + Assert.assertFalse(ret); + ret = builder.qualifier(toBytes(column)) + .ifMatches(CompareOperator.LESS_OR_EQUAL, toBytes("value1")).thenDelete(delete); + Assert.assertTrue(ret); + + get = new Get(key.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + get.addColumn(family.getBytes(), column.getBytes()); + r = hTable.get(get); + Assert.assertEquals(0, r.rawCells().length); + + // check delete columns + t = System.currentTimeMillis(); + put = new Put(key.getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), t, value.getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), t + 1, value.getBytes()); + hTable.put(put); + get = new Get(key.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + get.addColumn(family.getBytes(), column.getBytes()); + r = hTable.get(get); + Assert.assertEquals(2, r.rawCells().length); + delete = new Delete(key.getBytes()); + delete.addColumns(family.getBytes(), column.getBytes()); + ret = builder.qualifier(toBytes(column)).ifEquals(toBytes(value)).thenDelete(delete); + Assert.assertTrue(ret); + get = new Get(key.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + get.addColumn(family.getBytes(), column.getBytes()); + r = hTable.get(get); + Assert.assertEquals(0, r.rawCells().length); + + // check delete family + t = System.currentTimeMillis(); + put = new Put(key.getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), t, value.getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), t + 1, value.getBytes()); + put.addColumn(family.getBytes(), column2.getBytes(), t, value.getBytes()); + put.addColumn(family.getBytes(), column2.getBytes(), t + 1, value.getBytes()); + hTable.put(put); + get = new Get(key.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + get.addFamily(family.getBytes()); + r = hTable.get(get); + Assert.assertEquals(4, r.rawCells().length); + + // test TimeRange + put = new Put(key.getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), value.getBytes()); + TimeRange timeRange = new TimeRange(t + 2, t + 4); + ret = builder.qualifier(toBytes(column)).timeRange(timeRange).ifEquals(toBytes(value)) + .thenPut(put); + Assert.assertFalse(ret); + timeRange = new TimeRange(t, t + 2); + ret = builder.qualifier(toBytes(column)).timeRange(timeRange).ifEquals(toBytes(value)) + .thenPut(put); + Assert.assertTrue(ret); + + delete = new Delete(key.getBytes()); + delete.addFamily(family.getBytes()); + ret = builder.qualifier(toBytes(column)).ifEquals(toBytes(value)).thenDelete(delete); + Assert.assertTrue(ret); + get = new Get(key.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + get.addFamily(family.getBytes()); + r = hTable.get(get); + Assert.assertEquals(0, r.rawCells().length); } @Test @@ -4867,6 +5026,165 @@ public void testCheckAndMutate() throws IOException { get.setMaxVersions(Integer.MAX_VALUE); r = hTable.get(get); Assert.assertEquals(10, r.rawCells().length); + + // test CheckAndMutateBuilder + delete = new Delete(key.getBytes()); + delete.addFamily(family.getBytes()); + hTable.delete(delete); + + t = System.currentTimeMillis(); + // put + put1 = new Put(key.getBytes()); + put1.addColumn(family.getBytes(), column1.getBytes(), t, value1.getBytes()); + put1.addColumn(family.getBytes(), column2.getBytes(), t, value2.getBytes()); + + put2 = new Put(key.getBytes()); + put2.addColumn(family.getBytes(), column1.getBytes(), t + 3, value2.getBytes()); + put2.addColumn(family.getBytes(), column2.getBytes(), t + 3, value1.getBytes()); + + put3 = new Put(key.getBytes()); + put3.addColumn(family.getBytes(), column1.getBytes(), t + 5, value1.getBytes()); + put3.addColumn(family.getBytes(), column2.getBytes(), t + 5, value2.getBytes()); + + rowMutations = new RowMutations(key.getBytes()); + rowMutations.add(put1); + rowMutations.add(put2); + rowMutations.add(put3); + + //put data + Table.CheckAndMutateBuilder builder = hTable.checkAndMutate(toBytes(key), + family.getBytes(StandardCharsets.UTF_8)); + ret = builder.qualifier(toBytes(column1)).ifNotExists().thenMutate(rowMutations); + Assert.assertTrue(ret); + + get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + r = hTable.get(get); + Assert.assertEquals(6, r.rawCells().length); + + t = System.currentTimeMillis() + 7; + // put + put1 = new Put(key.getBytes()); + put1.addColumn(family.getBytes(), column1.getBytes(), t, value1.getBytes()); + put1.addColumn(family.getBytes(), column2.getBytes(), t, value2.getBytes()); + + put2 = new Put(key.getBytes()); + put2.addColumn(family.getBytes(), column1.getBytes(), t + 3, value2.getBytes()); + put2.addColumn(family.getBytes(), column2.getBytes(), t + 3, value1.getBytes()); + + put3 = new Put(key.getBytes()); + put3.addColumn(family.getBytes(), column1.getBytes(), t + 5, value1.getBytes()); + put3.addColumn(family.getBytes(), column2.getBytes(), t + 5, value2.getBytes()); + rowMutations = new RowMutations(key.getBytes()); + rowMutations.add(put1); + rowMutations.add(put2); + rowMutations.add(put3); + // test greater op + ret = builder.qualifier(toBytes(column1)).ifMatches(CompareOperator.LESS, toBytes(value1)) + .thenMutate(rowMutations); + Assert.assertFalse(ret); + get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + r = hTable.get(get); + Assert.assertEquals(6, r.rawCells().length); + + // test less op + ret = builder.qualifier(toBytes(column1)).ifMatches(CompareOperator.LESS, toBytes(value1)) + .thenMutate(rowMutations); + ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column1.getBytes(), + CompareFilter.CompareOp.LESS, value2.getBytes(), rowMutations); + Assert.assertTrue(ret); + get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + r = hTable.get(get); + Assert.assertEquals(12, r.rawCells().length); + + t = System.currentTimeMillis() + 14; + // put + put1 = new Put(key.getBytes()); + put1.addColumn(family.getBytes(), column1.getBytes(), t, value1.getBytes()); + put1.addColumn(family.getBytes(), column2.getBytes(), t, value2.getBytes()); + + put2 = new Put(key.getBytes()); + put2.addColumn(family.getBytes(), column1.getBytes(), t + 3, value2.getBytes()); + put2.addColumn(family.getBytes(), column2.getBytes(), t + 3, value1.getBytes()); + + put3 = new Put(key.getBytes()); + put3.addColumn(family.getBytes(), column1.getBytes(), t + 5, value1.getBytes()); + put3.addColumn(family.getBytes(), column2.getBytes(), t + 5, value2.getBytes()); + rowMutations = new RowMutations(key.getBytes()); + rowMutations.add(put1); + rowMutations.add(put2); + rowMutations.add(put3); + // test NO_OP + try { + builder.qualifier(toBytes(column1)).ifMatches(CompareOperator.NO_OP, toBytes(value1)) + .thenMutate(rowMutations); + fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("checkAndMutate")); + } + + // test equal op + ret = builder.qualifier(toBytes(column1)).ifEquals(toBytes(value1)) + .thenMutate(rowMutations); + Assert.assertTrue(ret); + get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + r = hTable.get(get); + Assert.assertEquals(18, r.rawCells().length); + + t = System.currentTimeMillis() + 21; + // put + put1 = new Put(key.getBytes()); + put1.addColumn(family.getBytes(), column1.getBytes(), t, value1.getBytes()); + put1.addColumn(family.getBytes(), column2.getBytes(), t, value2.getBytes()); + + // delete + delete1 = new Delete(key.getBytes()); + delete1.addColumns(family.getBytes(), column1.getBytes()); + + // check delete and put + rowMutations = new RowMutations(key.getBytes()); + rowMutations.add(delete1); + rowMutations.add(put1); + ret = builder.qualifier(toBytes(column1)).ifEquals(toBytes(value1)) + .thenMutate(rowMutations); + Assert.assertTrue(ret); + get = new Get(key.getBytes()); + get.addColumn(family.getBytes(), column1.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + r = hTable.get(get); + Assert.assertEquals(1, r.rawCells().length); + + get = new Get(key.getBytes()); + get.addColumn(family.getBytes(), column2.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + r = hTable.get(get); + Assert.assertEquals(10, r.rawCells().length); + + // check TimeRange + put1 = new Put(toBytes(key)); + put1.addColumn(toBytes(family), toBytes(column1), toBytes(value1)); + rowMutations = new RowMutations(key.getBytes()); + rowMutations.add(put1); + rowMutations.add(delete1); + TimeRange timeRange = new TimeRange(t + 1, t + 3); + ret = builder.qualifier(toBytes(column1)).timeRange(timeRange).ifEquals(toBytes(value1)) + .thenMutate(rowMutations); + Assert.assertFalse(ret); + timeRange = new TimeRange(t, t + 2); + ret = builder.qualifier(toBytes(column1)).timeRange(timeRange).ifEquals(toBytes(value1)) + .thenMutate(rowMutations); + Assert.assertTrue(ret); + + delete = new Delete(key.getBytes()); + delete.addFamily(family.getBytes()); + hTable.delete(delete); } @Test From 46671217b5c57b458dcb5877863dcb6d240abc3b Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Tue, 29 Oct 2024 11:06:22 +0800 Subject: [PATCH 27/32] add test cases --- src/main/java/com/alipay/oceanbase/hbase/OHTable.java | 7 +++---- .../java/com/alipay/oceanbase/hbase/HTableTestBase.java | 9 ++++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 546fdc98..2a72e3a2 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1960,7 +1960,7 @@ public boolean thenPut(Put put) throws IOException { return checkAndMutation(row, family, qualifier, getCompareOp(cmpOp), value, timeRange, rowMutations); } catch(Exception e) { logger.error(LCD.convert("01-00005"), rowMutations, tableNameString, e); - throw new IOException("checkAndMutate type table:" + tableNameString + " e.msg:" + throw new IOException("checkAndMutate type table: " + tableNameString + " e.msg: " + e.getMessage() + " error.", e); } } @@ -1974,7 +1974,7 @@ public boolean thenDelete(Delete delete) throws IOException { return checkAndMutation(row, family, qualifier, getCompareOp(cmpOp), value, timeRange, rowMutations); } catch(Exception e) { logger.error(LCD.convert("01-00005"), rowMutations, tableNameString, e); - throw new IOException("checkAndMutate type table:" + tableNameString + " e.msg:" + throw new IOException("checkAndMutate type table: " + tableNameString + " e.msg: " + e.getMessage() + " error.", e); } } @@ -1986,12 +1986,11 @@ public boolean thenMutate(RowMutations mutation) throws IOException { return checkAndMutation(row, family, qualifier, getCompareOp(cmpOp), value, timeRange, mutation); } catch(Exception e) { logger.error(LCD.convert("01-00005"), mutation, tableNameString, e); - throw new IOException("checkAndMutate type table:" + tableNameString + " e.msg:" + throw new IOException("checkAndMutate type table: " + tableNameString + " e.msg: " + e.getMessage() + " error.", e); } } - private void checkCmpOp() { checkNotNull(this.cmpOp, "The compare condition is null. Please use" + " ifNotExists/ifEquals/ifMatches before executing the request"); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index c7ee74e2..092186aa 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -4642,6 +4642,11 @@ public void testCheckAndPut() throws IOException, InterruptedException { put.addColumn(family.getBytes(), column.getBytes(), "value1".getBytes()); ret = builder.qualifier(toBytes(column)).ifEquals(toBytes(value)).thenPut(put); Assert.assertTrue(ret); + Put difFamPut = new Put(key.getBytes()); + difFamPut.addColumn("family_group".getBytes(), column.getBytes(), "value1".getBytes()); + Assert.assertThrows(IOException.class, () -> { + builder.qualifier(toBytes(column)).ifEquals(toBytes(value)).thenPut(difFamPut); + }); ret = builder.qualifier(toBytes(column)) .ifMatches(CompareOperator.GREATER, toBytes("value1")).thenPut(put); @@ -5091,10 +5096,8 @@ public void testCheckAndMutate() throws IOException { Assert.assertEquals(6, r.rawCells().length); // test less op - ret = builder.qualifier(toBytes(column1)).ifMatches(CompareOperator.LESS, toBytes(value1)) + ret = builder.qualifier(toBytes(column1)).ifMatches(CompareOperator.LESS, toBytes(value2)) .thenMutate(rowMutations); - ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column1.getBytes(), - CompareFilter.CompareOp.LESS, value2.getBytes(), rowMutations); Assert.assertTrue(ret); get = new Get(key.getBytes()); get.addFamily(family.getBytes()); From e51e6be3b8370b954e3a2854155d1e00351ef44e Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Tue, 29 Oct 2024 12:16:56 +0800 Subject: [PATCH 28/32] add import --- .../java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java index 9895a10a..88e34922 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java @@ -18,6 +18,7 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.util.OHBufferedMutatorImpl; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.conf.Configuration; import com.alipay.oceanbase.rpc.mutation.result.MutationResult; import org.apache.hadoop.hbase.Cell; From 9cde5ba35a22ec6c4edc088420fde4cbd851c126 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Tue, 29 Oct 2024 18:14:18 +0800 Subject: [PATCH 29/32] pass test cases after merge --- .../hbase/filter/HBaseFilterUtils.java | 3 +- .../hbase/HTableMultiCFTestBase.java | 81 ++++++++++++------- .../oceanbase/hbase/HTableTestBase.java | 55 +++++++------ .../oceanbase/hbase/OHConnectionTest.java | 24 +++--- .../hbase/filter/HBaseFilterUtilsTest.java | 5 +- src/test/java/unit_test_db.sql | 6 +- 6 files changed, 100 insertions(+), 74 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index 4a14021a..3b8dec3c 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -355,7 +355,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Times } // ColumnValueFilter('cf','q') - private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnValueFilter filter) throws IOException { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + ColumnValueFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); byteStream.write("'".getBytes()); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java index 88e34922..0eaef049 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java @@ -97,29 +97,36 @@ public void testDeleteFamilyVerison() throws Exception { toBytes(value1)); Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); - putKey3Fam1Column1Ts1.addColumn(toBytes(family1), toBytes(column1), timeStamp1, toBytes(value2)); + putKey3Fam1Column1Ts1.addColumn(toBytes(family1), toBytes(column1), timeStamp1, + toBytes(value2)); Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); putKey1Fam1Column2MinTs.addColumn(toBytes(family1), toBytes(column2), minTimeStamp, toBytes(value1)); Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); - putKey1Fam1Column2Ts3.addColumn(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + putKey1Fam1Column2Ts3.addColumn(toBytes(family1), toBytes(column2), timeStamp3, + toBytes(value2)); Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); - putKey2Fam1Column2Ts3.addColumn(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + putKey2Fam1Column2Ts3.addColumn(toBytes(family1), toBytes(column2), timeStamp3, + toBytes(value2)); Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); - putKey2Fam1Column3Ts1.addColumn(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + putKey2Fam1Column3Ts1.addColumn(toBytes(family1), toBytes(column3), timeStamp1, + toBytes(value2)); Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); - putKey3Fam1Column3Ts1.addColumn(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + putKey3Fam1Column3Ts1.addColumn(toBytes(family1), toBytes(column3), timeStamp1, + toBytes(value2)); Put putKey3Fam1Column2Ts4 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts4.addColumn(toBytes(family1), toBytes(column2), timeStamp4, toBytes(value1)); + putKey3Fam1Column2Ts4.addColumn(toBytes(family1), toBytes(column2), timeStamp4, + toBytes(value1)); Put putKey2Fam1Column3Ts3 = new Put(toBytes(key2)); - putKey2Fam1Column3Ts3.addColumn(toBytes(family1), toBytes(column3), timeStamp3, toBytes(value1)); + putKey2Fam1Column3Ts3.addColumn(toBytes(family1), toBytes(column3), timeStamp3, + toBytes(value1)); tryPut(multiCfHTable, putKey1Fam1Column1MinTs); tryPut(multiCfHTable, putKey3Fam1Column1Ts1); @@ -137,21 +144,21 @@ public void testDeleteFamilyVerison() throws Exception { get.setTimeStamp(minTimeStamp); get.setMaxVersions(10); Result r = multiCfHTable.get(get); - Assert.assertEquals(2, r.getRow().length); + Assert.assertEquals(2, r.rawCells().length); get = new Get(toBytes(key3)); get.addFamily(toBytes(family1)); get.setTimeStamp(timeStamp1); get.setMaxVersions(10); r = multiCfHTable.get(get); - Assert.assertEquals(2, r.getRow().length); + Assert.assertEquals(2, r.rawCells().length); get = new Get(toBytes(key2)); get.addFamily(toBytes(family1)); get.setTimeStamp(timeStamp3); get.setMaxVersions(10); r = multiCfHTable.get(get); - Assert.assertEquals(2, r.getRow().length); + Assert.assertEquals(2, r.rawCells().length); Delete delKey1MinTs = new Delete(toBytes(key1)); delKey1MinTs.addFamilyVersion(toBytes(family1), minTimeStamp); @@ -162,7 +169,7 @@ public void testDeleteFamilyVerison() throws Exception { get.setTimeStamp(minTimeStamp); get.setMaxVersions(10); r = multiCfHTable.get(get); - Assert.assertEquals(0, r.getRow().length); + Assert.assertEquals(0, r.rawCells().length); Delete delKey3Ts1 = new Delete(toBytes(key3)); delKey3Ts1.addFamilyVersion(toBytes(family1), timeStamp1); @@ -173,7 +180,7 @@ public void testDeleteFamilyVerison() throws Exception { get.setTimeStamp(timeStamp1); get.setMaxVersions(10); r = multiCfHTable.get(get); - Assert.assertEquals(0, r.getRow().length); + Assert.assertEquals(0, r.rawCells().length); Delete delKey2Ts3 = new Delete(toBytes(key2)); delKey2Ts3.addFamilyVersion(family1.getBytes(), timeStamp3); @@ -184,7 +191,7 @@ public void testDeleteFamilyVerison() throws Exception { get.setTimeStamp(timeStamp3); get.setMaxVersions(10); r = multiCfHTable.get(get); - Assert.assertEquals(0, r.getRow().length); + Assert.assertEquals(0, r.rawCells().length); Scan scan = new Scan(); scan.setStartRow(toBytes(key1)); @@ -214,40 +221,52 @@ public void testDeleteFamilyVerison() throws Exception { // test DeleteFamilyVersion multiple cf Put putKey1Fam1Column3Ts4 = new Put(toBytes(key1)); - putKey1Fam1Column3Ts4.addColumn(toBytes(family1), toBytes(column3), timeStamp4, toBytes(value3)); + putKey1Fam1Column3Ts4.addColumn(toBytes(family1), toBytes(column3), timeStamp4, + toBytes(value3)); Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); - putKey1Fam2Column2Ts2.addColumn(toBytes(family2), toBytes(column2), timeStamp2, toBytes(value1)); + putKey1Fam2Column2Ts2.addColumn(toBytes(family2), toBytes(column2), timeStamp2, + toBytes(value1)); Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); - putKey1Fam2Column3Ts2.addColumn(toBytes(family2), toBytes(column3), timeStamp2, toBytes(value1)); + putKey1Fam2Column3Ts2.addColumn(toBytes(family2), toBytes(column3), timeStamp2, + toBytes(value1)); Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); - putKey1Fam1Column2Ts1.addColumn(toBytes(family1), toBytes(column2), timeStamp1, toBytes(value2)); + putKey1Fam1Column2Ts1.addColumn(toBytes(family1), toBytes(column2), timeStamp1, + toBytes(value2)); Put putKey2Fam1Column2Ts5 = new Put(toBytes(key2)); - putKey2Fam1Column2Ts5.addColumn(toBytes(family1), toBytes(column2), timeStamp5, toBytes(value2)); + putKey2Fam1Column2Ts5.addColumn(toBytes(family1), toBytes(column2), timeStamp5, + toBytes(value2)); Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); - putKey2Fam2Column3Ts1.addColumn(toBytes(family2), toBytes(column3), timeStamp3, toBytes(value3)); + putKey2Fam2Column3Ts1.addColumn(toBytes(family2), toBytes(column3), timeStamp3, + toBytes(value3)); Put putKey2Fam1Column1Ts5 = new Put(toBytes(key2)); - putKey2Fam1Column1Ts5.addColumn(toBytes(family1), toBytes(column1), timeStamp5, toBytes(value1)); + putKey2Fam1Column1Ts5.addColumn(toBytes(family1), toBytes(column1), timeStamp5, + toBytes(value1)); Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); - putKey2Fam2Column1Ts3.addColumn(toBytes(family2), toBytes(column1), timeStamp3, toBytes(value2)); + putKey2Fam2Column1Ts3.addColumn(toBytes(family2), toBytes(column1), timeStamp3, + toBytes(value2)); Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts6.addColumn(toBytes(family1), toBytes(column2), timeStamp6, toBytes(value2)); + putKey3Fam1Column2Ts6.addColumn(toBytes(family1), toBytes(column2), timeStamp6, + toBytes(value2)); Put putKey3Fam2Column3Ts7 = new Put(toBytes(key3)); - putKey3Fam2Column3Ts7.addColumn(toBytes(family2), toBytes(column3), timeStamp7, toBytes(value1)); + putKey3Fam2Column3Ts7.addColumn(toBytes(family2), toBytes(column3), timeStamp7, + toBytes(value1)); Put putKey3Fam2Column1Ts7 = new Put(toBytes(key3)); - putKey3Fam2Column1Ts7.addColumn(toBytes(family2), toBytes(column1), timeStamp7, toBytes(value2)); + putKey3Fam2Column1Ts7.addColumn(toBytes(family2), toBytes(column1), timeStamp7, + toBytes(value2)); Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts2.addColumn(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); + putKey3Fam1Column2Ts2.addColumn(toBytes(family1), toBytes(column2), timeStamp2, + toBytes(value1)); tryPut(multiCfHTable, putKey1Fam1Column3Ts4); tryPut(multiCfHTable, putKey1Fam2Column2Ts2); @@ -267,21 +286,21 @@ public void testDeleteFamilyVerison() throws Exception { getKey1.addFamily(toBytes(family2)); getKey1.setMaxVersions(10); r = multiCfHTable.get(getKey1); - Assert.assertEquals(4, r.getRow().length); + Assert.assertEquals(4, r.rawCells().length); Get getKey2 = new Get(toBytes(key2)); getKey2.addFamily(toBytes(family1)); getKey2.addFamily(toBytes(family2)); getKey2.setMaxVersions(10); r = multiCfHTable.get(getKey2); - Assert.assertEquals(4, r.getRow().length); + Assert.assertEquals(4, r.rawCells().length); Get getKey3 = new Get(toBytes(key3)); getKey3.addFamily(toBytes(family1)); getKey3.addFamily(toBytes(family2)); getKey3.setMaxVersions(10); r = multiCfHTable.get(getKey3); - Assert.assertEquals(4, r.getRow().length); + Assert.assertEquals(4, r.rawCells().length); Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); delKey1Ts_6_2.addFamilyVersion(toBytes(family1), timeStamp4); @@ -293,7 +312,7 @@ public void testDeleteFamilyVerison() throws Exception { getKey1.addFamily(toBytes(family2)); getKey1.setMaxVersions(10); r = multiCfHTable.get(getKey1); - Assert.assertEquals(1, r.getRow().length); + Assert.assertEquals(1, r.rawCells().length); for (Cell cell : r.rawCells()) { Assert.assertEquals(timeStamp1, cell.getTimestamp()); } @@ -308,7 +327,7 @@ public void testDeleteFamilyVerison() throws Exception { getKey2.addFamily(toBytes(family2)); getKey2.setMaxVersions(10); r = multiCfHTable.get(getKey2); - Assert.assertEquals(0, r.getRow().length); + Assert.assertEquals(0, r.rawCells().length); Delete delKey3Ts_2_7 = new Delete(toBytes(key3)); delKey3Ts_2_7.addFamilyVersion(toBytes(family1), timeStamp2); @@ -320,7 +339,7 @@ public void testDeleteFamilyVerison() throws Exception { getKey3.addFamily(toBytes(family2)); getKey3.setMaxVersions(10); r = multiCfHTable.get(getKey3); - Assert.assertEquals(1, r.getRow().length); + Assert.assertEquals(1, r.rawCells().length); for (Cell cell : r.rawCells()) { Assert.assertEquals(timeStamp6, cell.getTimestamp()); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index fdef443e..a7c402fa 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -1860,20 +1860,21 @@ public void testColumnValueFilter() throws Exception { scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); - ColumnValueFilter filter = new ColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column2), CompareOperator.EQUAL, Bytes.toBytes(value1)); + ColumnValueFilter filter = new ColumnValueFilter(Bytes.toBytes(family), + Bytes.toBytes(column2), CompareOperator.EQUAL, Bytes.toBytes(value1)); scan.setFilter(filter); ResultScanner scanner = hTable.getScanner(scan); int res_count = 0; for (Result result : scanner) { for (Cell keyValue : result.rawCells()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", Bytes.toString(result.getRow()), Bytes.toString(CellUtil.cloneFamily(keyValue)), Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), - Bytes.toString(CellUtil.cloneValue(keyValue)) - ); + keyValue.getTimestamp(), Bytes.toString(CellUtil.cloneValue(keyValue))); res_count += 1; } } @@ -1883,20 +1884,21 @@ public void testColumnValueFilter() throws Exception { scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); - filter = new ColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column2), CompareOperator.EQUAL, new BinaryPrefixComparator(Bytes.toBytes("value"))); + filter = new ColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column2), + CompareOperator.EQUAL, new BinaryPrefixComparator(Bytes.toBytes("value"))); scan.setFilter(filter); scanner = hTable.getScanner(scan); res_count = 0; for (Result result : scanner) { for (Cell keyValue : result.rawCells()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", Bytes.toString(result.getRow()), Bytes.toString(CellUtil.cloneFamily(keyValue)), Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), - Bytes.toString(CellUtil.cloneValue(keyValue)) - ); + keyValue.getTimestamp(), Bytes.toString(CellUtil.cloneValue(keyValue))); res_count += 1; } } @@ -1906,20 +1908,21 @@ public void testColumnValueFilter() throws Exception { scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); - filter = new ColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column2), CompareOperator.GREATER, Bytes.toBytes(value1)); + filter = new ColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column2), + CompareOperator.GREATER, Bytes.toBytes(value1)); scan.setFilter(filter); scanner = hTable.getScanner(scan); res_count = 0; for (Result result : scanner) { for (Cell keyValue : result.rawCells()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", Bytes.toString(result.getRow()), Bytes.toString(CellUtil.cloneFamily(keyValue)), Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), - Bytes.toString(CellUtil.cloneValue(keyValue)) - ); + keyValue.getTimestamp(), Bytes.toString(CellUtil.cloneValue(keyValue))); res_count += 1; } } @@ -1929,20 +1932,21 @@ public void testColumnValueFilter() throws Exception { scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); - filter = new ColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column1), CompareOperator.NOT_EQUAL, Bytes.toBytes(value1)); + filter = new ColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column1), + CompareOperator.NOT_EQUAL, Bytes.toBytes(value1)); scan.setFilter(filter); scanner = hTable.getScanner(scan); res_count = 0; for (Result result : scanner) { for (Cell keyValue : result.rawCells()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", Bytes.toString(result.getRow()), Bytes.toString(CellUtil.cloneFamily(keyValue)), Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), - Bytes.toString(CellUtil.cloneValue(keyValue)) - ); + keyValue.getTimestamp(), Bytes.toString(CellUtil.cloneValue(keyValue))); res_count += 1; } } @@ -1952,20 +1956,21 @@ public void testColumnValueFilter() throws Exception { scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); - filter = new ColumnValueFilter(Bytes.toBytes("ff"), Bytes.toBytes(column1), CompareOperator.NOT_EQUAL, Bytes.toBytes(value1)); + filter = new ColumnValueFilter(Bytes.toBytes("ff"), Bytes.toBytes(column1), + CompareOperator.NOT_EQUAL, Bytes.toBytes(value1)); scan.setFilter(filter); scanner = hTable.getScanner(scan); res_count = 0; for (Result result : scanner) { for (Cell keyValue : result.rawCells()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", Bytes.toString(result.getRow()), Bytes.toString(CellUtil.cloneFamily(keyValue)), Bytes.toString(CellUtil.cloneQualifier(keyValue)), - keyValue.getTimestamp(), - Bytes.toString(CellUtil.cloneValue(keyValue)) - ); + keyValue.getTimestamp(), Bytes.toString(CellUtil.cloneValue(keyValue))); res_count += 1; } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index 23479f9a..6ac69b6b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -72,13 +72,6 @@ public void testConnectionByXml() throws Exception { hTable.close(); } - @BeforeClass - public static void before() throws Exception { - // use self-defined namespace "n1" - hTable = ObHTableTestUtil.newOHTableClient("n1:test"); - ((OHTableClient) hTable).init(); - } - @AfterClass public static void finish() throws IOException { hTable.close(); @@ -86,6 +79,8 @@ public static void finish() throws IOException { @Test public void testRefreshTableEntry() throws Exception { + hTable = ObHTableTestUtil.newOHTableClient("n1:test"); + ((OHTableClient) hTable).init(); ((OHTableClient) hTable).refreshTableEntry("family1", false); ((OHTableClient) hTable).refreshTableEntry("family1", true); } @@ -139,7 +134,8 @@ private void testBasic() throws Exception { hTable.delete(delete); for (Cell keyValue : r.rawCells()) { - System.out.println("rowKey: " + new String(CellUtil.cloneRow(keyValue)) + " columnQualifier:" + System.out.println("rowKey: " + new String(CellUtil.cloneRow(keyValue)) + + " columnQualifier:" + new String(CellUtil.cloneQualifier(keyValue)) + " timestamp:" + keyValue.getTimestamp() + " value:" + new String(CellUtil.cloneValue(keyValue))); @@ -170,8 +166,10 @@ private void testBasic() throws Exception { boolean countAdd = true; for (Cell keyValue : result.rawCells()) { Assert.assertEquals(key + "_" + i, Bytes.toString(CellUtil.cloneRow(keyValue))); - Assert.assertTrue(column1.equals(Bytes.toString(CellUtil.cloneQualifier(keyValue))) - || column2.equals(Bytes.toString(CellUtil.cloneQualifier(keyValue)))); + Assert.assertTrue(column1.equals(Bytes.toString(CellUtil + .cloneQualifier(keyValue))) + || column2.equals(Bytes.toString(CellUtil + .cloneQualifier(keyValue)))); Assert.assertEquals(timestamp + 2, keyValue.getTimestamp()); Assert.assertEquals(value, Bytes.toString(CellUtil.cloneValue(keyValue))); if (countAdd) { @@ -193,8 +191,10 @@ private void testBasic() throws Exception { boolean countAdd = true; for (Cell keyValue : result.rawCells()) { Assert.assertEquals(key + "_" + i, Bytes.toString(CellUtil.cloneRow(keyValue))); - Assert.assertTrue(column1.equals(Bytes.toString(CellUtil.cloneQualifier(keyValue))) - || column2.equals(Bytes.toString(CellUtil.cloneQualifier(keyValue)))); + Assert.assertTrue(column1.equals(Bytes.toString(CellUtil + .cloneQualifier(keyValue))) + || column2.equals(Bytes.toString(CellUtil + .cloneQualifier(keyValue)))); Assert.assertEquals(value, Bytes.toString(CellUtil.cloneValue(keyValue))); if (countAdd) { countAdd = false; diff --git a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java index b379295d..30a06fbe 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java @@ -210,10 +210,11 @@ public void testFuzzyRowFilter() throws IOException { @Test public void testColumnValueFilter() throws IOException { - ColumnValueFilter filter = new ColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("q"), CompareOperator.EQUAL, Bytes.toBytes("v")); + ColumnValueFilter filter = new ColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("q"), + CompareOperator.EQUAL, Bytes.toBytes("v")); System.out.println(Bytes.toString(HBaseFilterUtils.toParseableByteArray(filter))); Assert.assertArrayEquals("ColumnValueFilter('cf','q',=,'binary:v')".getBytes(), - HBaseFilterUtils.toParseableByteArray(filter)); + HBaseFilterUtils.toParseableByteArray(filter)); } @Test diff --git a/src/test/java/unit_test_db.sql b/src/test/java/unit_test_db.sql index c5681bd7..8fe8a9b3 100644 --- a/src/test/java/unit_test_db.sql +++ b/src/test/java/unit_test_db.sql @@ -168,7 +168,7 @@ CREATE TABLE `test$family'1` ( `T` bigint(20) NOT NULL, `V` varbinary(1024) DEFAULT NULL, PRIMARY KEY (`K`, `Q`, `T`) -) TABLEGROUP = test; +); CREATE TABLE `test_t$family'1` ( `K` varbinary(1024) NOT NULL, @@ -176,7 +176,7 @@ CREATE TABLE `test_t$family'1` ( `T` bigint(20) NOT NULL, `V` varbinary(1024) DEFAULT NULL, PRIMARY KEY (`K`, `Q`, `T`) -) TABLEGROUP = test_t; +); CREATE TABLEGROUP test_multi_cf SHARDING = 'ADAPTIVE'; @@ -228,7 +228,7 @@ CREATE TABLE `n1:test$family'1` ( `T` bigint(20) NOT NULL, `V` varbinary(1024) DEFAULT NULL, PRIMARY KEY (`K`, `Q`, `T`) -) TABLEGROUP = `n1:test`; +); CREATE TABLE `n1:test$family_with_local_index` ( `K` varbinary(1024) NOT NULL, From ce5b1e94117ff12aeaf65ef311d9a152f764470a Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Tue, 29 Oct 2024 19:25:47 +0800 Subject: [PATCH 30/32] pass all test cases after merge --- .../java/com/alipay/oceanbase/hbase/OHConnectionTest.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index a8fd4229..805de10e 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -36,7 +36,6 @@ import static com.alipay.oceanbase.hbase.constants.OHConstants.SOCKET_TIMEOUT; import static org.apache.hadoop.hbase.ipc.RpcClient.SOCKET_TIMEOUT_CONNECT; import static org.apache.hadoop.hbase.util.Bytes.toBytes; -import static org.junit.Assert.*; public class OHConnectionTest { protected static Table hTable; @@ -72,11 +71,6 @@ public void testConnectionByXml() throws Exception { hTable.close(); } - @AfterClass - public static void finish() throws IOException { - hTable.close(); - } - @Test public void testRefreshTableEntry() throws Exception { hTable = ObHTableTestUtil.newOHTableClient("n1:test"); From 9e0cb531e2c2b1accdb4591bcd2c5c60df1d3a3c Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Wed, 30 Oct 2024 10:26:08 +0800 Subject: [PATCH 31/32] remove useless import --- src/main/java/com/alipay/oceanbase/hbase/OHTable.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 2974f61d..c68b6f43 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -36,7 +36,6 @@ import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.*; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncRequest; import com.alipay.oceanbase.rpc.stream.ObTableClientQueryAsyncStreamResult; -import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult; import com.alipay.oceanbase.rpc.table.ObHBaseParams; import com.alipay.oceanbase.rpc.table.ObKVParams; import com.alipay.sofa.common.thread.SofaThreadPoolExecutor; @@ -56,7 +55,6 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.slf4j.Logger; -import org.tukaani.xz.check.Check; import java.io.ByteArrayOutputStream; import java.io.IOException; From f08cd045a97ecee3f08413229b600b45b0566cdd Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Wed, 30 Oct 2024 13:13:51 +0800 Subject: [PATCH 32/32] use temporary multiCfHTable so that does not influence other cases --- .../hbase/HTableMultiCFTestBase.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java index 6de98730..b4f63a4d 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java @@ -388,13 +388,13 @@ public void testMultiColumnFamilyTableBuilder() throws Exception { TableName tableName = TableName.valueOf("test_multi_cf"); TableBuilder builder = connection.getTableBuilder(tableName, null); // build a OHTable with default params - multiCfHTable = builder.build(); + Table tmpMultiCfHTable = builder.build(); Delete delete = new Delete(toBytes("Key0")); delete.addFamily(family1); delete.addFamily(family2); delete.addFamily(family3); - multiCfHTable.delete(delete); + tmpMultiCfHTable.delete(delete); Put put = new Put(toBytes("Key0")); put.addColumn(family1, family1_column1, family1_value); @@ -403,39 +403,39 @@ public void testMultiColumnFamilyTableBuilder() throws Exception { put.addColumn(family2, family2_column1, family2_value); put.addColumn(family2, family2_column2, family2_value); put.addColumn(family3, family3_column1, family3_value); - multiCfHTable.put(put); + tmpMultiCfHTable.put(put); int count = 0; Get get = new Get(toBytes("Key0")); get.setMaxVersions(); - Result r = multiCfHTable.get(get); + Result r = tmpMultiCfHTable.get(get); Assert.assertEquals(6, r.rawCells().length); delete = new Delete(toBytes("Key0")); delete.addFamily(family1); delete.addFamily(family2); delete.addFamily(family3); - multiCfHTable.delete(delete); - r = multiCfHTable.get(get); + tmpMultiCfHTable.delete(delete); + r = tmpMultiCfHTable.get(get); Assert.assertEquals(0, r.rawCells().length); // set params for TableBuilder builder.setOperationTimeout(1500000); builder.setRpcTimeout(40000); - multiCfHTable = builder.build(); + tmpMultiCfHTable = builder.build(); put = new Put(toBytes("Key0")); put.addColumn(family1, family1_column1, family1_value); put.addColumn(family1, family1_column2, family1_value); put.addColumn(family2, family2_column1, family2_value); put.addColumn(family3, family3_column1, family3_value); - multiCfHTable.put(put); + tmpMultiCfHTable.put(put); - r = multiCfHTable.get(get); + r = tmpMultiCfHTable.get(get); Assert.assertEquals(4, r.rawCells().length); - multiCfHTable.delete(delete); - r = multiCfHTable.get(get); + tmpMultiCfHTable.delete(delete); + r = tmpMultiCfHTable.get(get); Assert.assertEquals(0, r.rawCells().length); }