From 03c8e7149c9e13dcce591a31f8dc64ed04e01838 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 11 Oct 2024 10:55:55 +0800 Subject: [PATCH 01/10] add DepentdentFilter and SingleColumnValueExcludeFilter --- .../hbase/filter/HBaseFilterUtils.java | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index e58d2590..10c54373 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -45,6 +45,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte toParseableByteArray(byteStream, (CompareFilter) filter); } else if (filter instanceof SingleColumnValueFilter) { toParseableByteArray(byteStream, (SingleColumnValueFilter) filter); + } else if (filter instanceof SingleColumnValueExcludeFilter) { + toParseableByteArray(byteStream, (SingleColumnValueExcludeFilter) filter); } else if (filter instanceof PageFilter) { toParseableByteArray(byteStream, (PageFilter) filter); } else if (filter instanceof ColumnCountGetFilter) { @@ -69,6 +71,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte toParseableByteArray(byteStream, (SkipFilter) filter); } else if (filter instanceof WhileMatchFilter) { toParseableByteArray(byteStream, (WhileMatchFilter) filter); + } else if (filter instanceof DependentColumnFilter) { + toParseableByteArray(byteStream, (DependentColumnFilter) filter); } else { throw new IllegalArgumentException("Invalid filter: " + filter); } @@ -149,6 +153,25 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write(')'); } + // SingleColumnValueExcludeFilter('cf1','col1',=,'binary:123',true,true) + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + SingleColumnValueExcludeFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getFilterIfMissing()).getBytes()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getLatestVersionOnly()).getBytes()); + byteStream.write(')'); + } + // PageFilter(100); private static void toParseableByteArray(ByteArrayOutputStream byteStream, PageFilter filter) throws IOException { @@ -167,6 +190,42 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write(')'); } + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + DependentColumnFilter filter) throws IOException { + boolean dropDependentColumn; + try { + Field field = filter.getClass().getDeclaredField("dropDependentColumn"); + field.setAccessible(true); + dropDependentColumn = (boolean)field.get(filter); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ')' + if (filter.getComparator() == null) { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(dropDependentColumn).getBytes()); + byteStream.write(')'); + } else { // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ',' compare_op ',' comparator ')' + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(dropDependentColumn).getBytes()); + byteStream.write(','); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(')'); + } + } + private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnPaginationFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); From 2aaf7c4d409dbeed6dd6899d1587ce22b7bb63bd Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Wed, 16 Oct 2024 11:54:18 +0800 Subject: [PATCH 02/10] add singleColumnValueExcludeFilter and DependentColumnFilter; add test cases and optimize bufferedMutator test cases --- .../hbase/filter/HBaseFilterUtils.java | 20 +- .../oceanbase/hbase/HTableTestBase.java | 220 ++++++++++++++++++ .../oceanbase/hbase/OHConnectionTest.java | 16 ++ 3 files changed, 242 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index 10c54373..cc512f9e 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -40,13 +40,15 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte throws IOException { if (filter == null) { throw new IllegalArgumentException("Filter is null"); + } else if (filter instanceof DependentColumnFilter) { + toParseableByteArray(byteStream, (DependentColumnFilter) filter); } else if (filter instanceof CompareFilter) { // RowFilter, ValueFilter, QualifierFilter toParseableByteArray(byteStream, (CompareFilter) filter); - } else if (filter instanceof SingleColumnValueFilter) { - toParseableByteArray(byteStream, (SingleColumnValueFilter) filter); } else if (filter instanceof SingleColumnValueExcludeFilter) { toParseableByteArray(byteStream, (SingleColumnValueExcludeFilter) filter); + } else if (filter instanceof SingleColumnValueFilter) { + toParseableByteArray(byteStream, (SingleColumnValueFilter) filter); } else if (filter instanceof PageFilter) { toParseableByteArray(byteStream, (PageFilter) filter); } else if (filter instanceof ColumnCountGetFilter) { @@ -71,8 +73,6 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte toParseableByteArray(byteStream, (SkipFilter) filter); } else if (filter instanceof WhileMatchFilter) { toParseableByteArray(byteStream, (WhileMatchFilter) filter); - } else if (filter instanceof DependentColumnFilter) { - toParseableByteArray(byteStream, (DependentColumnFilter) filter); } else { throw new IllegalArgumentException("Invalid filter: " + filter); } @@ -192,14 +192,6 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, private static void toParseableByteArray(ByteArrayOutputStream byteStream, DependentColumnFilter filter) throws IOException { - boolean dropDependentColumn; - try { - Field field = filter.getClass().getDeclaredField("dropDependentColumn"); - field.setAccessible(true); - dropDependentColumn = (boolean)field.get(filter); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new RuntimeException(e); - } // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ')' if (filter.getComparator() == null) { byteStream.write(filter.getClass().getSimpleName().getBytes()); @@ -208,7 +200,7 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write("','".getBytes()); writeBytesWithEscape(byteStream, filter.getQualifier()); byteStream.write("',".getBytes()); - byteStream.write(Boolean.toString(dropDependentColumn).getBytes()); + byteStream.write(Boolean.toString(filter.getDropDependentColumn()).getBytes()); byteStream.write(')'); } else { // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ',' compare_op ',' comparator ')' byteStream.write(filter.getClass().getSimpleName().getBytes()); @@ -217,7 +209,7 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write("','".getBytes()); writeBytesWithEscape(byteStream, filter.getQualifier()); byteStream.write("',".getBytes()); - byteStream.write(Boolean.toString(dropDependentColumn).getBytes()); + byteStream.write(Boolean.toString(filter.getDropDependentColumn()).getBytes()); byteStream.write(','); byteStream.write(toParseableByteArray(filter.getOperator())); byteStream.write(','); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 4c93efde..a5b891fe 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -509,6 +509,83 @@ public void testFilter() throws Exception { tryPut(hTable, putKey2Column2Value1); tryPut(hTable, putKey2Column2Value2); +// time may be different +// +---------+-----+----------------+--------+ +// | K | Q | T | V | +// +---------+-----+----------------+--------+ +// | getKey1 | abc | -1728834971469 | value1 | +// | getKey1 | abc | -1728834971399 | value2 | +// | getKey1 | abc | -1728834971330 | value1 | +// | getKey1 | def | -1728834971748 | value2 | +// | getKey1 | def | -1728834971679 | value1 | +// | getKey1 | def | -1728834971609 | value2 | +// | getKey1 | def | -1728834971540 | value1 | +// | getKey2 | def | -1728834971887 | value2 | +// | getKey2 | def | -1728834971818 | value1 | +// +---------+-----+----------------+--------+ + + SingleColumnValueFilter singleColumnValueFilter; + singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueFilter); + r = hTable.get(get); + Assert.assertEquals(7, r.raw().length); + + SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; + singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueExcludeFilter); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), true); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + + + filter = new ColumnPrefixFilter(Bytes.toBytes("e")); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -817,6 +894,66 @@ public void testFilter() throws Exception { } Assert.assertEquals(res_count, 12); scanner.close(); + + long timestamp = System.currentTimeMillis(); + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), timestamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timestamp, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + // putKey1Column1Value1 and putKey2Column1Value1 have the same timestamp + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column2Value2); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setStartRow("getKey1".getBytes()); + scan.setStopRow("getKey3".getBytes()); + scan.setMaxVersions(10); + scan.setFilter(dependentColumnFilter); + scanner = hTable.getScanner(scan); + + long prevTimestamp = - 1; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + if (prevTimestamp == -1) { + prevTimestamp = keyValue.getTimestamp(); + } else { + Assert.assertEquals(prevTimestamp, keyValue.getTimestamp()); + } + } + } + scanner.close(); } @Test @@ -1242,6 +1379,15 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(1, r.raw().length); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + // columnCountGetFilter filter 2 get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -1422,6 +1568,67 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(7, r.raw().length); + filterList = new FilterList(); + filterList.addFilter(new SingleColumnValueExcludeFilter(Bytes.toBytes(family), Bytes + .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column1), false)); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), false)); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), true)); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(value2)))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + filterList = new FilterList(); filterList.addFilter(new ColumnCountGetFilter(1)); filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.GREATER, @@ -1547,6 +1754,19 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(7, r.raw().length); + + SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; + singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueExcludeFilter); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index a4a78a61..e8644d7b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -209,6 +209,10 @@ public void testBufferedMutatorWithFlush() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + // only support Put and Delete // for other type of operations, BufferedMutator will not set its type for them Append append = new Append(Bytes.toBytes(key)); @@ -409,6 +413,10 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); @@ -498,6 +506,10 @@ public void testBufferedMutatorWithUserPool() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); @@ -587,6 +599,10 @@ public void testBufferedMutatorConcurrent() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + for (int i = 0; i < 50; ++i) { final int taskId = i; final BufferedMutator thrBufferMutator = ohBufferMutator; From 78df6d8c591558d046f67ad2e039175258133a8b Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 18 Oct 2024 09:52:35 +0800 Subject: [PATCH 03/10] add single cf timerange setting in Get --- src/main/java/com/alipay/oceanbase/hbase/OHTable.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 00d717b3..a4212020 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -598,6 +598,11 @@ public Result call() throws IOException { for (Map.Entry> entry : get.getFamilyMap() .entrySet()) { family = entry.getKey(); + Map colFamTimeRangeMap = get.getColumnFamilyTimeRange(); + if (colFamTimeRangeMap.get(entry.getKey()) != null) { + TimeRange tr = colFamTimeRangeMap.get(entry.getKey()); + get.setTimeRange(tr.getMin(), tr.getMax()); + } obTableQuery = buildObTableQuery(get, entry.getValue()); request = buildObTableQueryAsyncRequest(obTableQuery, getTargetTableName(tableNameString, Bytes.toString(family), From 256fe9ac19c58026d66460f066f08512d8445531 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 18 Oct 2024 15:30:06 +0800 Subject: [PATCH 04/10] single cf setColumnFamilyTimeRange in Get and Scan --- .../com/alipay/oceanbase/hbase/OHTable.java | 17 + .../oceanbase/hbase/HTableTestBase.java | 579 +++++++++++++++++- 2 files changed, 562 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index a4212020..9c321186 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -585,6 +585,9 @@ public Result call() throws IOException { // In a Get operation where the family map is greater than 1 or equal to 0, // we handle this by appending the column family to the qualifier on the client side. // The server can then use this information to filter the appropriate column families and qualifiers. + if (!get.getColumnFamilyTimeRange().isEmpty()) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } NavigableSet columnFilters = new TreeSet<>(Bytes.BYTES_COMPARATOR); processColumnFilters(columnFilters, get.getFamilyMap()); obTableQuery = buildObTableQuery(get, columnFilters); @@ -599,6 +602,9 @@ public Result call() throws IOException { .entrySet()) { family = entry.getKey(); Map colFamTimeRangeMap = get.getColumnFamilyTimeRange(); + if (colFamTimeRangeMap.size() > 1) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } if (colFamTimeRangeMap.get(entry.getKey()) != null) { TimeRange tr = colFamTimeRangeMap.get(entry.getKey()); get.setTimeRange(tr.getMin(), tr.getMax()); @@ -673,6 +679,9 @@ public ResultScanner call() throws IOException { // In a Scan operation where the family map is greater than 1 or equal to 0, // we handle this by appending the column family to the qualifier on the client side. // The server can then use this information to filter the appropriate column families and qualifiers. + if (!scan.getColumnFamilyTimeRange().isEmpty()) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } NavigableSet columnFilters = new TreeSet<>(Bytes.BYTES_COMPARATOR); processColumnFilters(columnFilters, scan.getFamilyMap()); filter = buildObHTableFilter(scan.getFilter(), scan.getTimeRange(), @@ -689,6 +698,14 @@ public ResultScanner call() throws IOException { for (Map.Entry> entry : scan.getFamilyMap() .entrySet()) { family = entry.getKey(); + Map colFamTimeRangeMap = scan.getColumnFamilyTimeRange(); + if (colFamTimeRangeMap.size() > 1) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } + if (colFamTimeRangeMap.get(entry.getKey()) != null) { + TimeRange tr = colFamTimeRangeMap.get(entry.getKey()); + scan.setTimeRange(tr.getMin(), tr.getMax()); + } filter = buildObHTableFilter(scan.getFilter(), scan.getTimeRange(), scan.getMaxVersions(), entry.getValue()); obTableQuery = buildObTableQuery(filter, scan); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 801abd9e..32e76cc7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -2882,23 +2882,25 @@ public void testGet() throws Exception { tryPut(hTable, putzKey2Column1Value1); // show table (time maybe different) - //+-----------+---------+----------------+--------+ - //| K | Q | T | V | - //+-----------+---------+----------------+--------+ - //| scanKey1x | column1 | -1709714409669 | value1 | - //| scanKey1x | column1 | -1709714409637 | value2 | - //| scanKey1x | column1 | -1709714409603 | value1 | - //| scanKey1x | column2 | -1709714409802 | value2 | - //| scanKey1x | column2 | -1709714409768 | value1 | - //| scanKey1x | column2 | -1709714409735 | value2 | - //| scanKey1x | column2 | -1709714409702 | value1 | - //| scanKey2x | column2 | -1709714409869 | value2 | - //| scanKey2x | column2 | -1709714409836 | value1 | - //| scanKey3x | column1 | -1709714409940 | value2 | - //| scanKey3x | column1 | -1709714409904 | value1 | - //| scanKey3x | column2 | -1709714410010 | value2 | - //| scanKey3x | column2 | -1709714409977 | value1 | - //+-----------+---------+----------------+--------+ + // +-----------+---------+----------------+--------+ + // | K | Q | T | V | + // +-----------+---------+----------------+--------+ + // | scanKey1x | column1 | -1729223351579 | value1 | + // | scanKey1x | column1 | -1729223351504 | value2 | + // | scanKey1x | column1 | -1729223351431 | value1 | + // | scanKey1x | column2 | -1729223351867 | value2 | + // | scanKey1x | column2 | -1729223351796 | value1 | + // | scanKey1x | column2 | -1729223351724 | value2 | + // | scanKey1x | column2 | -1729223351651 | value1 | + // | scanKey2x | column2 | -1729223352015 | value2 | + // | scanKey2x | column2 | -1729223351941 | value1 | + // | scanKey3x | column1 | -1729223352159 | value2 | + // | scanKey3x | column1 | -1729223352088 | value1 | + // | scanKey3x | column2 | -1729223352304 | value2 | + // | scanKey3x | column2 | -1729223352232 | value1 | + // | zScanKey1 | column1 | -1729223352378 | value1 | + // | zScanKey2 | column1 | -1729223352450 | value1 | + // +-----------+---------+----------------+--------+ // test closestRowBefore get = new Get("scanKey2x2".getBytes()); @@ -2930,6 +2932,147 @@ public void testGet() throws Exception { assertTrue(booleans[2]); assertFalse(booleans[3]); assertTrue(booleans[4]); + + // test single cf setColumnFamilyTimeRange + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + hTable.delete(deleteZKey1Family); + hTable.delete(deleteZKey2Family); + + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), timeStamp2, toBytes(value1)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timeStamp4, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), timeStamp5, toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), timeStamp7, toBytes(value2)); + + putKey3Column1Value1 = new Put(toBytes(key3)); + putKey3Column1Value1.add(toBytes(family), toBytes(column1), timeStamp8, toBytes(value1)); + + putKey3Column1Value2 = new Put(toBytes(key3)); + putKey3Column1Value2.add(toBytes(family), toBytes(column1), timeStamp9, toBytes(value2)); + + putKey3Column2Value1 = new Put(toBytes(key3)); + putKey3Column2Value1.add(toBytes(family), toBytes(column2), timeStamp10, toBytes(value1)); + + putKey3Column2Value2 = new Put(toBytes(key3)); + putKey3Column2Value2.add(toBytes(family), toBytes(column2), timeStamp11, toBytes(value2)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column1Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + tryPut(hTable, putKey3Column1Value1); + tryPut(hTable, putKey3Column1Value2); + tryPut(hTable, putKey3Column2Value1); + tryPut(hTable, putKey3Column2Value2); + + + get = new Get(toBytes(key1)); + get.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, maxTimeStamp); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + get = new Get(toBytes(key1)); + get.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, timeStamp2 + 1); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + get = new Get(toBytes(key2)); + // set invalid timeRange + get.setTimeRange(minTimeStamp, maxTimeStamp); + get.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, timeStamp5 + 1); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key2)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp5, maxTimeStamp); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp8, timeStamp8); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp8, timeStamp9); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp8, timeStamp9); + get.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp8, timeStamp9); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + final Get errorGet = get; + Assert.assertThrows(IOException.class, () -> { + hTable.get(errorGet); + }); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); } @Test @@ -3029,23 +3172,25 @@ public void testScan() throws Exception { tryPut(hTable, putzKey2Column1Value1); // show table (time maybe different) - //+-----------+---------+----------------+--------+ - //| K | Q | T | V | - //+-----------+---------+----------------+--------+ - //| scanKey1x | column1 | -1709714409669 | value1 | - //| scanKey1x | column1 | -1709714409637 | value2 | - //| scanKey1x | column1 | -1709714409603 | value1 | - //| scanKey1x | column2 | -1709714409802 | value2 | - //| scanKey1x | column2 | -1709714409768 | value1 | - //| scanKey1x | column2 | -1709714409735 | value2 | - //| scanKey1x | column2 | -1709714409702 | value1 | - //| scanKey2x | column2 | -1709714409869 | value2 | - //| scanKey2x | column2 | -1709714409836 | value1 | - //| scanKey3x | column1 | -1709714409940 | value2 | - //| scanKey3x | column1 | -1709714409904 | value1 | - //| scanKey3x | column2 | -1709714410010 | value2 | - //| scanKey3x | column2 | -1709714409977 | value1 | - //+-----------+---------+----------------+--------+ + // +-----------+---------+----------------+--------+ + // | K | Q | T | V | + // +-----------+---------+----------------+--------+ + // | scanKey1x | column1 | -1729236392149 | value1 | + // | scanKey1x | column1 | -1729236392078 | value2 | + // | scanKey1x | column1 | -1729236392008 | value1 | + // | scanKey1x | column2 | -1729236392436 | value2 | + // | scanKey1x | column2 | -1729236392364 | value1 | + // | scanKey1x | column2 | -1729236392291 | value2 | + // | scanKey1x | column2 | -1729236392220 | value1 | + // | scanKey2x | column2 | -1729236392576 | value2 | + // | scanKey2x | column2 | -1729236392506 | value1 | + // | scanKey3x | column1 | -1729236392720 | value2 | + // | scanKey3x | column1 | -1729236392647 | value1 | + // | scanKey3x | column2 | -1729236392861 | value2 | + // | scanKey3x | column2 | -1729236392790 | value1 | + // | zScanKey1 | column1 | -1729236392931 | value1 | + // | zScanKey2 | column1 | -1729236393002 | value1 | + // +-----------+---------+----------------+--------+ scan = new Scan(); scan.addFamily(family.getBytes()); @@ -3232,6 +3377,180 @@ public void testScan() throws Exception { hTable.delete(deleteKey3Family); hTable.delete(deleteZKey1Family); hTable.delete(deleteZKey2Family); + + // test single cf setColumnFamilyTimeRange + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), timeStamp2, toBytes(value1)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timeStamp4, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), timeStamp5, toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), timeStamp7, toBytes(value2)); + + putKey3Column1Value1 = new Put(toBytes(key3)); + putKey3Column1Value1.add(toBytes(family), toBytes(column1), timeStamp8, toBytes(value1)); + + putKey3Column1Value2 = new Put(toBytes(key3)); + putKey3Column1Value2.add(toBytes(family), toBytes(column1), timeStamp9, toBytes(value2)); + + putKey3Column2Value1 = new Put(toBytes(key3)); + putKey3Column2Value1.add(toBytes(family), toBytes(column2), timeStamp10, toBytes(value1)); + + putKey3Column2Value2 = new Put(toBytes(key3)); + putKey3Column2Value2.add(toBytes(family), toBytes(column2), timeStamp11, toBytes(value2)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column1Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + tryPut(hTable, putKey3Column1Value1); + tryPut(hTable, putKey3Column1Value2); + tryPut(hTable, putKey3Column2Value1); + tryPut(hTable, putKey3Column2Value2); + + // scan key1 + key2 + scan = new Scan(toBytes(key1), toBytes(key3)); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, maxTimeStamp); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(8, res_count); + + // scan key1 + scan = new Scan(toBytes(key1), toBytes(key2)); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, maxTimeStamp); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(4, res_count); + + // scan key1 + scan = new Scan(toBytes(key1), toBytes(key2)); + scan.addFamily(toBytes(family)); + // set invalid timeRange + scan.setTimeRange(minTimeStamp, maxTimeStamp); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp1, timeStamp3); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(2, res_count); + + // scan all + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp2, timeStamp7); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(5, res_count); + + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + boolean foundKey3 = false; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (!foundKey3) { + if (key3.equals(Bytes.toString(kv.getRow()))) { + foundKey3 = true; + } + } + ++res_count; + } + } + Assert.assertTrue(foundKey3); + Assert.assertEquals(6, res_count); + + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); + scan.setMaxVersions(); + final Scan errorScan = scan; + Assert.assertThrows(IOException.class, () -> { + hTable.getScanner(errorScan); + }); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); } @Test @@ -3454,6 +3773,198 @@ public void testReversedScan() throws Exception { hTable.delete(deleteKey2Family); hTable.delete(deleteKey3Family); + // test single cf setColumnFamilyTimeRange + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), timeStamp2, toBytes(value1)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timeStamp4, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), timeStamp5, toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), timeStamp7, toBytes(value2)); + + putKey3Column1Value1 = new Put(toBytes(key3)); + putKey3Column1Value1.add(toBytes(family), toBytes(column1), timeStamp8, toBytes(value1)); + + putKey3Column1Value2 = new Put(toBytes(key3)); + putKey3Column1Value2.add(toBytes(family), toBytes(column1), timeStamp9, toBytes(value2)); + + putKey3Column2Value1 = new Put(toBytes(key3)); + putKey3Column2Value1.add(toBytes(family), toBytes(column2), timeStamp10, toBytes(value1)); + + putKey3Column2Value2 = new Put(toBytes(key3)); + putKey3Column2Value2.add(toBytes(family), toBytes(column2), timeStamp11, toBytes(value2)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column1Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + tryPut(hTable, putKey3Column1Value1); + tryPut(hTable, putKey3Column1Value2); + tryPut(hTable, putKey3Column2Value1); + tryPut(hTable, putKey3Column2Value2); + + // scan key1 + scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(family.getBytes()); + scan.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, timeStamp3); + scan.setReversed(true); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(3, res_count); + + // scan key1 + key2 + scan = new Scan(); + scan.setStartRow(toBytes(key2)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp2, timeStamp7); + scan.setReversed(true); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(5, res_count); + + // scan key2 + scan = new Scan(); + scan.setStartRow(toBytes(key2)); + scan.setStopRow(toBytes(key1)); + scan.addFamily(toBytes(family)); + // set invalid timeRange + scan.setTimeRange(minTimeStamp, maxTimeStamp); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp4, timeStamp6); + scan.setReversed(true); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(2, res_count); + + // scan all + scan = new Scan(); + scan.setStartRow(toBytes(key3)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp2, timeStamp7); + scan.setReversed(true); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(5, res_count); + + scan = new Scan(); + scan.setStartRow(toBytes(key3)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setReversed(true); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + boolean foundKey3 = false; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (!foundKey3) { + if (key3.equals(Bytes.toString(kv.getRow()))) { + foundKey3 = true; + } + } + ++res_count; + } + } + Assert.assertTrue(foundKey3); + Assert.assertEquals(6, res_count); + + scan = new Scan(); + scan.setStartRow(toBytes(key3)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); + scan.setReversed(true); + scan.setMaxVersions(); + final Scan errorScan = scan; + Assert.assertThrows(IOException.class, () -> { + hTable.getScanner(errorScan); + }); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + } @Test From a5ec9f6c567bc4b88a5d73a91c54d3fe50426888 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 18 Oct 2024 15:35:02 +0800 Subject: [PATCH 05/10] optimize code --- src/main/java/com/alipay/oceanbase/hbase/OHTable.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 9c321186..e6f04ade 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -605,8 +605,8 @@ public Result call() throws IOException { if (colFamTimeRangeMap.size() > 1) { throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); } - if (colFamTimeRangeMap.get(entry.getKey()) != null) { - TimeRange tr = colFamTimeRangeMap.get(entry.getKey()); + if (colFamTimeRangeMap.get(family) != null) { + TimeRange tr = colFamTimeRangeMap.get(family); get.setTimeRange(tr.getMin(), tr.getMax()); } obTableQuery = buildObTableQuery(get, entry.getValue()); @@ -702,8 +702,8 @@ public ResultScanner call() throws IOException { if (colFamTimeRangeMap.size() > 1) { throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); } - if (colFamTimeRangeMap.get(entry.getKey()) != null) { - TimeRange tr = colFamTimeRangeMap.get(entry.getKey()); + if (colFamTimeRangeMap.get(family) != null) { + TimeRange tr = colFamTimeRangeMap.get(family); scan.setTimeRange(tr.getMin(), tr.getMax()); } filter = buildObHTableFilter(scan.getFilter(), scan.getTimeRange(), From 3123df41b061b0ec77571e0ba98f8557195bedca Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Tue, 22 Oct 2024 09:56:23 +0800 Subject: [PATCH 06/10] add DeleteFamilyVersion and test cases --- .../com/alipay/oceanbase/hbase/OHTable.java | 3 + .../oceanbase/hbase/HTableTestBase.java | 163 ++++++++++++++++++ 2 files changed, 166 insertions(+) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index e6f04ade..758cf3db 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1717,6 +1717,9 @@ private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(KeyValue kv, case DeleteFamily: return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, new Object[] { kv.getRow(), null, -kv.getTimestamp() }, null, null); + case DeleteFamilyVersion: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), null, kv.getTimestamp() }, null, null); default: throw new IllegalArgumentException("illegal mutation type " + kvType); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 32e76cc7..12dcafc8 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -4554,6 +4554,169 @@ public void testCheckAndMutate() throws IOException { Assert.assertEquals(10, r.raw().length); } + @Test + public void testDeleteFamilyVerison() throws Exception { + String key1 = "scanKey1x"; + String key2 = "scanKey2x"; + String key3 = "scanKey3x"; + String column1 = "column1"; + String column2 = "column2"; + String column3 = "column3"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + String family = "family1"; + + // delete previous data + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family)); + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family)); + Delete deleteKey3Family = new Delete(toBytes(key3)); + deleteKey3Family.deleteFamily(toBytes(family)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + + Put putKey1Column1MinTs = new Put(toBytes(key1)); + putKey1Column1MinTs.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + Put putKey3Column1Ts1 = new Put(toBytes(key3)); + putKey3Column1Ts1.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + Put putKey1Column2MinTs = new Put(toBytes(key1)); + putKey1Column2MinTs.add(toBytes(family), toBytes(column2), minTimeStamp, toBytes(value1)); + + Put putKey1Column2Ts3 = new Put(toBytes(key1)); + putKey1Column2Ts3.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Column2Ts3 = new Put(toBytes(key2)); + putKey2Column2Ts3.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Column3Ts1 = new Put(toBytes(key2)); + putKey2Column3Ts1.add(toBytes(family), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Column3Ts1 = new Put(toBytes(key3)); + putKey3Column3Ts1.add(toBytes(family), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Column2Ts6 = new Put(toBytes(key3)); + putKey3Column2Ts6.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + Put putKey2Column3Ts6 = new Put(toBytes(key2)); + putKey2Column3Ts6.add(toBytes(family), toBytes(column3), timeStamp3, toBytes(value1)); + + + tryPut(hTable, putKey1Column1MinTs); + tryPut(hTable, putKey3Column1Ts1); + tryPut(hTable, putKey1Column2MinTs); + tryPut(hTable, putKey1Column2Ts3); + tryPut(hTable, putKey2Column2Ts3); + tryPut(hTable, putKey2Column3Ts1); + tryPut(hTable, putKey3Column3Ts1); + tryPut(hTable, putKey3Column2Ts6); + tryPut(hTable, putKey2Column3Ts6); + + Get get = new Get(toBytes(key1)); + get.addFamily(toBytes(family)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + Result r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + Delete delKey1MinTs = new Delete(toBytes(key1)); + delKey1MinTs.deleteFamilyVersion(toBytes(family), minTimeStamp); + hTable.delete(delKey1MinTs); + + get = new Get(toBytes(key1)); + get.addFamily(toBytes(family)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts1 = new Delete(toBytes(key3)); + delKey3Ts1.deleteFamilyVersion(toBytes(family), timeStamp1); + hTable.delete(delKey3Ts1); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey2Ts3 = new Delete(toBytes(key2)); + delKey2Ts3.deleteFamilyVersion(family.getBytes(), timeStamp3); + hTable.delete(delKey2Ts3); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Scan scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setMaxVersions(10); + ResultScanner scanner = hTable.getScanner(scan); + int cnt = 0; + for (Result result : scanner) { + ++cnt; + for (KeyValue kv : result.raw()) { + System.out.println("Key: " + Bytes.toString(kv.getRow()) + ", Qualifier: " + Bytes.toString(kv.getQualifier()) + + ", Timestamp: " + kv.getTimestamp()); + } + } + Assert.assertEquals(3, cnt); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + } + @Test public void testAppend() throws IOException { String column = "appendColumn"; From e108fa91cc448db38221a597110586c61fa11008 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Tue, 22 Oct 2024 19:21:21 +0800 Subject: [PATCH 07/10] add DeleteFamilyVersion; optimize test cases --- .../com/alipay/oceanbase/hbase/OHTable.java | 3 +- .../oceanbase/hbase/HTableTestBase.java | 236 ++++++++++++++---- .../oceanbase/hbase/OHConnectionTest.java | 131 +++++----- .../hbase/filter/HBaseFilterUtilsTest.java | 33 +++ src/test/java/unit_test_db.sql | 8 + 5 files changed, 304 insertions(+), 107 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 0df31139..2614e712 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -879,7 +879,8 @@ private void innerDelete(Delete delete) throws IOException { // Therefore, this process is handled sequentially. boolean has_delete_family = delete.getFamilyMap().entrySet().stream() .flatMap(entry -> entry.getValue().stream()) - .anyMatch(kv -> KeyValue.Type.codeToType(kv.getType()) == KeyValue.Type.DeleteFamily); + .anyMatch(kv -> (KeyValue.Type.codeToType(kv.getType()) == KeyValue.Type.DeleteFamily + || KeyValue.Type.codeToType(kv.getType()) == KeyValue.Type.DeleteFamilyVersion)); if (!has_delete_family) { BatchOperation batch = buildBatchOperation(tableNameString, delete.getFamilyMap(), false, null); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 69abf542..6c0ed079 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -4871,15 +4871,19 @@ public void testDeleteFamilyVerison() throws Exception { String value1 = "value1"; String value2 = "value2"; String value3 = "value3"; - String family = "family1"; + String family1 = "family_group"; + String family2 = "family_group1"; // delete previous data Delete deleteKey1Family = new Delete(toBytes(key1)); - deleteKey1Family.deleteFamily(toBytes(family)); + deleteKey1Family.deleteFamily(toBytes(family1)); + deleteKey1Family.deleteFamily(toBytes(family2)); Delete deleteKey2Family = new Delete(toBytes(key2)); - deleteKey2Family.deleteFamily(toBytes(family)); + deleteKey2Family.deleteFamily(toBytes(family1)); + deleteKey2Family.deleteFamily(toBytes(family2)); Delete deleteKey3Family = new Delete(toBytes(key3)); - deleteKey3Family.deleteFamily(toBytes(family)); + deleteKey3Family.deleteFamily(toBytes(family1)); + deleteKey3Family.deleteFamily(toBytes(family2)); hTable.delete(deleteKey1Family); hTable.delete(deleteKey2Family); @@ -4912,111 +4916,251 @@ public void testDeleteFamilyVerison() throws Exception { long maxTimeStamp = System.currentTimeMillis(); - Put putKey1Column1MinTs = new Put(toBytes(key1)); - putKey1Column1MinTs.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); + putKey1Fam1Column1MinTs.add(toBytes(family1), toBytes(column1), minTimeStamp, toBytes(value1)); - Put putKey3Column1Ts1 = new Put(toBytes(key3)); - putKey3Column1Ts1.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp1, toBytes(value2)); - Put putKey1Column2MinTs = new Put(toBytes(key1)); - putKey1Column2MinTs.add(toBytes(family), toBytes(column2), minTimeStamp, toBytes(value1)); + Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); + putKey1Fam1Column2MinTs.add(toBytes(family1), toBytes(column2), minTimeStamp, toBytes(value1)); - Put putKey1Column2Ts3 = new Put(toBytes(key1)); - putKey1Column2Ts3.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); - Put putKey2Column2Ts3 = new Put(toBytes(key2)); - putKey2Column2Ts3.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); - Put putKey2Column3Ts1 = new Put(toBytes(key2)); - putKey2Column3Ts1.add(toBytes(family), toBytes(column3), timeStamp1, toBytes(value2)); + Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); - Put putKey3Column3Ts1 = new Put(toBytes(key3)); - putKey3Column3Ts1.add(toBytes(family), toBytes(column3), timeStamp1, toBytes(value2)); + Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); - Put putKey3Column2Ts6 = new Put(toBytes(key3)); - putKey3Column2Ts6.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts6.add(toBytes(family1), toBytes(column2), timeStamp6, toBytes(value1)); - Put putKey2Column3Ts6 = new Put(toBytes(key2)); - putKey2Column3Ts6.add(toBytes(family), toBytes(column3), timeStamp3, toBytes(value1)); + Put putKey2Fam1Column3Ts6 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp3, toBytes(value1)); - tryPut(hTable, putKey1Column1MinTs); - tryPut(hTable, putKey3Column1Ts1); - tryPut(hTable, putKey1Column2MinTs); - tryPut(hTable, putKey1Column2Ts3); - tryPut(hTable, putKey2Column2Ts3); - tryPut(hTable, putKey2Column3Ts1); - tryPut(hTable, putKey3Column3Ts1); - tryPut(hTable, putKey3Column2Ts6); - tryPut(hTable, putKey2Column3Ts6); + tryPut(hTable, putKey1Fam1Column1MinTs); + tryPut(hTable, putKey3Fam1Column1Ts1); + tryPut(hTable, putKey1Fam1Column2MinTs); + tryPut(hTable, putKey1Fam1Column2Ts3); + tryPut(hTable, putKey2Fam1Column2Ts3); + tryPut(hTable, putKey2Fam1Column3Ts1); + tryPut(hTable, putKey3Fam1Column3Ts1); + tryPut(hTable, putKey3Fam1Column2Ts6); + tryPut(hTable, putKey2Fam1Column3Ts6); + // test DeleteFamilyVersion single cf Get get = new Get(toBytes(key1)); - get.addFamily(toBytes(family)); + get.addFamily(toBytes(family1)); get.setTimeStamp(minTimeStamp); get.setMaxVersions(10); Result r = hTable.get(get); Assert.assertEquals(2, r.raw().length); get = new Get(toBytes(key3)); - get.addFamily(toBytes(family)); + get.addFamily(toBytes(family1)); get.setTimeStamp(timeStamp1); get.setMaxVersions(10); r = hTable.get(get); Assert.assertEquals(2, r.raw().length); get = new Get(toBytes(key2)); - get.addFamily(toBytes(family)); + get.addFamily(toBytes(family1)); get.setTimeStamp(timeStamp3); get.setMaxVersions(10); r = hTable.get(get); Assert.assertEquals(2, r.raw().length); Delete delKey1MinTs = new Delete(toBytes(key1)); - delKey1MinTs.deleteFamilyVersion(toBytes(family), minTimeStamp); + delKey1MinTs.deleteFamilyVersion(toBytes(family1), minTimeStamp); hTable.delete(delKey1MinTs); get = new Get(toBytes(key1)); - get.addFamily(toBytes(family)); + get.addFamily(toBytes(family1)); get.setTimeStamp(minTimeStamp); get.setMaxVersions(10); r = hTable.get(get); Assert.assertEquals(0, r.raw().length); Delete delKey3Ts1 = new Delete(toBytes(key3)); - delKey3Ts1.deleteFamilyVersion(toBytes(family), timeStamp1); + delKey3Ts1.deleteFamilyVersion(toBytes(family1), timeStamp1); hTable.delete(delKey3Ts1); get = new Get(toBytes(key3)); - get.addFamily(toBytes(family)); + get.addFamily(toBytes(family1)); get.setTimeStamp(timeStamp1); get.setMaxVersions(10); r = hTable.get(get); Assert.assertEquals(0, r.raw().length); Delete delKey2Ts3 = new Delete(toBytes(key2)); - delKey2Ts3.deleteFamilyVersion(family.getBytes(), timeStamp3); + delKey2Ts3.deleteFamilyVersion(family1.getBytes(), timeStamp3); hTable.delete(delKey2Ts3); get = new Get(toBytes(key2)); - get.addFamily(toBytes(family)); + get.addFamily(toBytes(family1)); get.setTimeStamp(timeStamp3); get.setMaxVersions(10); r = hTable.get(get); Assert.assertEquals(0, r.raw().length); Scan scan = new Scan(); - scan.addFamily(toBytes(family)); + scan.addFamily(toBytes(family1)); scan.setMaxVersions(10); ResultScanner scanner = hTable.getScanner(scan); - int cnt = 0; + int key1Cnt = 0, key2Cnt = 0, key3Cnt = 0; for (Result result : scanner) { - ++cnt; for (KeyValue kv : result.raw()) { - System.out.println("Key: " + Bytes.toString(kv.getRow()) + ", Qualifier: " + Bytes.toString(kv.getQualifier()) - + ", Timestamp: " + kv.getTimestamp()); + if (key1.equals(Bytes.toString(kv.getRow()))) { + ++key1Cnt; + } else if (key2.equals(Bytes.toString(kv.getRow()))) { + ++key2Cnt; + } else { + ++key3Cnt; + } + } + } + Assert.assertEquals(1, key1Cnt); + Assert.assertEquals(1, key2Cnt); + Assert.assertEquals(1, key3Cnt); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + + // test DeleteFamilyVersion multiple cf + Put putKey1Fam1Column3Ts6 = new Put(toBytes(key1)); + putKey1Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp6, toBytes(value3)); + + Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column2Ts2.add(toBytes(family2), toBytes(column2), timeStamp2, toBytes(value1)); + + Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column3Ts2.add(toBytes(family2), toBytes(column3), timeStamp2, toBytes(value1)); + + Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts1.add(toBytes(family1), toBytes(column2), timeStamp1, toBytes(value2)); + + Put putKey2Fam1Column2Ts8 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts8.add(toBytes(family1), toBytes(column2), timeStamp8, toBytes(value2)); + + Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam2Column3Ts1.add(toBytes(family2), toBytes(column3), timeStamp3, toBytes(value3)); + + Put putKey2Fam1Column1Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp8, toBytes(value1)); + + Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); + putKey2Fam2Column1Ts3.add(toBytes(family2), toBytes(column1), timeStamp3, toBytes(value2)); + + Put putKey3Fam1Column2Ts9 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts9.add(toBytes(family1), toBytes(column2), timeStamp9, toBytes(value2)); + + Put putKey3Fam2Column3Ts10 = new Put(toBytes(key3)); + putKey3Fam2Column3Ts10.add(toBytes(family2), toBytes(column3), timeStamp10, toBytes(value1)); + + Put putKey3Fam2Column1Ts10 = new Put(toBytes(key3)); + putKey3Fam2Column1Ts10.add(toBytes(family2), toBytes(column1), timeStamp10, toBytes(value2)); + + Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts2.add(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); + + tryPut(hTable, putKey1Fam1Column3Ts6); + tryPut(hTable, putKey1Fam2Column2Ts2); + tryPut(hTable, putKey1Fam2Column3Ts2); + tryPut(hTable, putKey1Fam1Column2Ts1); + tryPut(hTable, putKey2Fam1Column2Ts8); + tryPut(hTable, putKey2Fam2Column3Ts1); + tryPut(hTable, putKey2Fam1Column1Ts1); + tryPut(hTable, putKey2Fam2Column1Ts3); + tryPut(hTable, putKey3Fam1Column2Ts9); + tryPut(hTable, putKey3Fam2Column3Ts10); + tryPut(hTable, putKey3Fam2Column1Ts10); + tryPut(hTable, putKey3Fam1Column2Ts2); + + Get getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = hTable.get(getKey1); + Assert.assertEquals(4, r.raw().length); + + Get getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = hTable.get(getKey2); + Assert.assertEquals(4, r.raw().length); + + Get getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setMaxVersions(10); + r = hTable.get(getKey3); + Assert.assertEquals(4, r.raw().length); + + Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); + delKey1Ts_6_2.deleteFamilyVersion(toBytes(family1), timeStamp6); + delKey1Ts_6_2.deleteFamilyVersion(toBytes(family2), timeStamp2); + hTable.delete(delKey1Ts_6_2); + + getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = hTable.get(getKey1); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp1, kv.getTimestamp()); + } + + Delete delKey2Ts_8_3 = new Delete(toBytes(key2)); + delKey2Ts_8_3.deleteFamilyVersion(toBytes(family1), timeStamp8); + delKey2Ts_8_3.deleteFamilyVersion(toBytes(family2), timeStamp3); + hTable.delete(delKey2Ts_8_3); + + getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = hTable.get(getKey2); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts_2_10 = new Delete(toBytes(key3)); + delKey3Ts_2_10.deleteFamilyVersion(toBytes(family1), timeStamp2); + delKey3Ts_2_10.deleteFamilyVersion(toBytes(family2), timeStamp10); + hTable.delete(delKey3Ts_2_10); + + getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setTimeStamp(timeStamp9); + getKey3.setMaxVersions(10); + r = hTable.get(getKey3); + Assert.assertEquals(1, r.raw().length); + + scan = new Scan(); + scan.addFamily(toBytes(family1)); + scan.addFamily(toBytes(family2)); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + int ts1Cnt = 0, ts9Cnt = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (kv.getTimestamp() == timeStamp1) { + ++ts1Cnt; + } else if (kv.getTimestamp() == timeStamp9) { + ++ts9Cnt; + } } } - Assert.assertEquals(3, cnt); + Assert.assertEquals(1, ts1Cnt); + Assert.assertEquals(1, ts9Cnt); hTable.delete(deleteKey1Family); hTable.delete(deleteKey2Family); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index 03617c2c..b40b23ea 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -205,22 +205,25 @@ public void testBufferedMutatorWithFlush() throws Exception { conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); BufferedMutator putBufferMutator = null; BufferedMutator delBufferedMutator = null; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // use defualt params putBufferMutator = connection.getBufferedMutator(tableName); delBufferedMutator = connection.getBufferedMutator(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - Delete delete= new Delete(toBytes(key)); - delete.deleteFamily(toBytes("family_group")); - hTable.delete(delete); + long timestamp = System.currentTimeMillis(); // only support Put and Delete // for other type of operations, BufferedMutator will not set its type for them @@ -234,10 +237,10 @@ public void testBufferedMutatorWithFlush() throws Exception { List mutations = new ArrayList<>(); // test Put Put put1 = new Put(Bytes.toBytes(key)); - put1.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); + put1.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); mutations.add(put1); Put put2 = new Put(Bytes.toBytes(key)); - put2.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); + put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); mutations.add(put2); // test add Mutations with List putBufferMutator.mutate(mutations); @@ -253,7 +256,7 @@ public void testBufferedMutatorWithFlush() throws Exception { Assert.assertThrows(FeatureNotSupportedException.class, () -> { noCfMut.mutate(del); }); - del.deleteFamily(Bytes.toBytes("family_group")); + del.deleteFamily(Bytes.toBytes(family)); // test reuse different type bufferedMutator final BufferedMutator difTypeMut = putBufferMutator; Assert.assertThrows(IllegalArgumentException.class, () -> { @@ -281,8 +284,8 @@ public void testBufferedMutatorWithFlush() throws Exception { if (delBufferedMutator != null) { delBufferedMutator.close(); // test add mutations after closed - Delete delete = new Delete(Bytes.toBytes("putKey")); - delete.deleteFamily(Bytes.toBytes("family_group")); + Delete delete = new Delete(Bytes.toBytes(key)); + delete.deleteFamily(Bytes.toBytes(family)); final BufferedMutator closedMutator = delBufferedMutator; Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); @@ -309,24 +312,29 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); BufferedMutator putBufferMutator = null; BufferedMutator delBufferedMutator = null; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + String family = "family_group"; + long timestamp = System.currentTimeMillis(); try { // use n1 database TableName tableName = TableName.valueOf("n1","test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // use defualt params putBufferMutator = connection.getBufferedMutator(tableName); delBufferedMutator = connection.getBufferedMutator(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - // only support Put and Delete // for other type of operations, BufferedMutator will not set its type for them Append append = new Append(Bytes.toBytes(key)); - append.add("family_group".getBytes(), column1.getBytes(), toBytes("_suffix")); + append.add(toBytes(family), toBytes(column1), toBytes("_suffix")); final BufferedMutator apMut = putBufferMutator; Assert.assertThrows(IllegalArgumentException.class, () -> { apMut.mutate(append); @@ -335,10 +343,10 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { List mutations = new ArrayList<>(); // test Put Put put1 = new Put(Bytes.toBytes(key)); - put1.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); + put1.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); mutations.add(put1); Put put2 = new Put(Bytes.toBytes(key)); - put2.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); + put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); mutations.add(put2); // test add Mutations with List putBufferMutator.mutate(mutations); @@ -354,7 +362,7 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { Assert.assertThrows(FeatureNotSupportedException.class, () -> { noCfMut.mutate(del); }); - del.deleteFamily(Bytes.toBytes("family_group")); + del.deleteFamily(Bytes.toBytes(family)); final BufferedMutator difTypeMut = putBufferMutator; // test reuse different type bufferedMutator Assert.assertThrows(IllegalArgumentException.class, () -> { @@ -410,30 +418,31 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { BufferedMutatorParams params = null; long bufferSize = 45000L; int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); putBufferMutator = connection.getBufferedMutator(params); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - - Delete delete= new Delete(toBytes(key)); - delete.deleteFamily(toBytes("family_group")); - hTable.delete(delete); - List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); for (int j = 0; j < 4; ++j) { Put put = new Put(Bytes.toBytes(key)); - put.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "_" + i + "_" + j), + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "_" + i + "_" + j), timestamp, Bytes.toBytes(value + "_" + i + "_" + j)); mutations.add(put); } @@ -449,14 +458,14 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { } finally { if (putBufferMutator != null) { putBufferMutator.close(); - Get get = new Get(toBytes("putKey")); + Get get = new Get(toBytes(key)); Result r = hTable.get(get); for (KeyValue keyValue : r.raw()) { ++count; } Assert.assertEquals(200, count); - Delete delete = new Delete(toBytes("putKey")); - delete.deleteFamily(toBytes("family_group")); + Delete delete = new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); hTable.delete(delete); r = hTable.get(get); Assert.assertEquals(0, r.raw().length); @@ -496,10 +505,20 @@ public void testBufferedMutatorWithUserPool() throws Exception { BufferedMutatorParams params = null; long bufferSize = 45000L; int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); @@ -512,21 +531,12 @@ public void testBufferedMutatorWithUserPool() throws Exception { ohBufferMutator = connection.getBufferedMutator(params); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - - Delete delete= new Delete(toBytes(key)); - delete.deleteFamily(toBytes("family_group")); - hTable.delete(delete); - List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); for (int j = 0; j < 4; ++j) { Put put = new Put(Bytes.toBytes(key)); - put.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "_" + i + "_" + j), + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "_" + i + "_" + j), timestamp, Bytes.toBytes(value + "_" + i + "_" + j)); mutations.add(put); } @@ -542,14 +552,14 @@ public void testBufferedMutatorWithUserPool() throws Exception { } finally { if (ohBufferMutator != null) { ohBufferMutator.close(); - Get get = new Get(toBytes("putKey")); + Get get = new Get(toBytes(key)); Result r = hTable.get(get); for (KeyValue keyValue : r.raw()) { ++count; } Assert.assertEquals(200, count); - Delete delete = new Delete(toBytes("putKey")); - delete.deleteFamily(toBytes("family_group")); + Delete delete = new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); hTable.delete(delete); r = hTable.get(get); @@ -589,10 +599,20 @@ public void testBufferedMutatorConcurrent() throws Exception { ExecutorService executorService = Executors.newFixedThreadPool(10); long bufferSize = 45000L; int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); @@ -605,15 +625,6 @@ public void testBufferedMutatorConcurrent() throws Exception { ohBufferMutator = connection.getBufferedMutator(params); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - - Delete delete= new Delete(toBytes(key)); - delete.deleteFamily(toBytes("family_group")); - hTable.delete(delete); - for (int i = 0; i < 50; ++i) { final int taskId = i; final BufferedMutator thrBufferMutator = ohBufferMutator; @@ -626,7 +637,7 @@ public void testBufferedMutatorConcurrent() throws Exception { long thrTimestamp = timestamp; Put put = new Put(Bytes.toBytes(thrKey)); - put.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(thrColumn), + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(thrColumn), thrTimestamp, Bytes.toBytes(thrValue)); mutations.add(put); } @@ -661,14 +672,14 @@ public void testBufferedMutatorConcurrent() throws Exception { } if (ohBufferMutator != null) { ohBufferMutator.close(); - Get get = new Get(toBytes("putKey")); + Get get = new Get(toBytes(key)); Result r = hTable.get(get); for (KeyValue keyValue : r.raw()) { ++count; } Assert.assertEquals(200, count); - Delete delete = new Delete(toBytes("putKey")); - delete.deleteFamily(toBytes("family_group")); + Delete delete = new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); hTable.delete(delete); r = hTable.get(get); diff --git a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java index f0e66604..9071adbe 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java @@ -138,6 +138,39 @@ public void testSingleColumnValueFilter() throws IOException { } } + @Test + public void testSingleColumnValueExcludeFilter() throws IOException { + for (int i = 0; i < ops.length; i++) { + String expect = String.format( + "SingleColumnValueExcludeFilter('family','qualifier',%s,'binary:value',false,true)", + opFlags[i]); + SingleColumnValueExcludeFilter filter = new SingleColumnValueExcludeFilter("family".getBytes(), + "qualifier".getBytes(), ops[i], "value".getBytes()); + Assert.assertArrayEquals(expect.getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + } + + @Test + public void testDependentColumnFilter() throws IOException { + DependentColumnFilter filter = new DependentColumnFilter("family".getBytes(), "qualifier".getBytes()); + String expect = "DependentColumnFilter('family','qualifier',false)"; + Assert.assertArrayEquals(expect.getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + filter = new DependentColumnFilter("family".getBytes(), "qualifier".getBytes(), true); + expect = "DependentColumnFilter('family','qualifier',true)"; + Assert.assertArrayEquals(expect.getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + for (int i = 0; i < ops.length; ++i) { + filter = new DependentColumnFilter("family".getBytes(), "qualifier".getBytes(), false, + ops[i], new BinaryComparator("value".getBytes())); + expect = String.format("DependentColumnFilter('family','qualifier',false,%s,'binary:value')", + opFlags[i]); + Assert.assertArrayEquals(expect.getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + } + @Test public void testPageFilter() throws IOException { PageFilter filter = new PageFilter(128); diff --git a/src/test/java/unit_test_db.sql b/src/test/java/unit_test_db.sql index c5681bd7..036ccca7 100644 --- a/src/test/java/unit_test_db.sql +++ b/src/test/java/unit_test_db.sql @@ -63,6 +63,14 @@ CREATE TABLE `test$family_group` ( PRIMARY KEY (`K`, `Q`, `T`) ) TABLEGROUP = test; +CREATE TABLE `test$family_group1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test; + CREATE TABLEGROUP test_t SHARDING = 'ADAPTIVE'; CREATE TABLE `test_t$family_group` ( `K` varbinary(1024) NOT NULL, From 56e83f5064d872db438808684e353dd4e3da7c8f Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Wed, 23 Oct 2024 10:19:44 +0800 Subject: [PATCH 08/10] add DeleteFamilyVersion test case and pass --- .../oceanbase/hbase/HTableTestBase.java | 309 ---------------- .../hbase/OHTableDeleteFamilyVersionTest.java | 348 ++++++++++++++++++ 2 files changed, 348 insertions(+), 309 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index ae3776ca..9aa2e1f7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -4841,315 +4841,6 @@ public void testCheckAndMutate() throws IOException { Assert.assertEquals(10, r.raw().length); } - @Test - public void testDeleteFamilyVerison() throws Exception { - String key1 = "scanKey1x"; - String key2 = "scanKey2x"; - String key3 = "scanKey3x"; - String column1 = "column1"; - String column2 = "column2"; - String column3 = "column3"; - String value1 = "value1"; - String value2 = "value2"; - String value3 = "value3"; - String family1 = "family_group"; - String family2 = "family_group1"; - - // delete previous data - Delete deleteKey1Family = new Delete(toBytes(key1)); - deleteKey1Family.deleteFamily(toBytes(family1)); - deleteKey1Family.deleteFamily(toBytes(family2)); - Delete deleteKey2Family = new Delete(toBytes(key2)); - deleteKey2Family.deleteFamily(toBytes(family1)); - deleteKey2Family.deleteFamily(toBytes(family2)); - Delete deleteKey3Family = new Delete(toBytes(key3)); - deleteKey3Family.deleteFamily(toBytes(family1)); - deleteKey3Family.deleteFamily(toBytes(family2)); - - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - - long minTimeStamp = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp1 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp2 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp3 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp4 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp5 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp6 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp7 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp8 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp9 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp10 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp11 = System.currentTimeMillis(); - Thread.sleep(5); - long maxTimeStamp = System.currentTimeMillis(); - - Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); - putKey1Fam1Column1MinTs.add(toBytes(family1), toBytes(column1), minTimeStamp, - toBytes(value1)); - - Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); - putKey3Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp1, toBytes(value2)); - - Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); - putKey1Fam1Column2MinTs.add(toBytes(family1), toBytes(column2), minTimeStamp, - toBytes(value1)); - - Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); - putKey1Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); - - Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); - putKey2Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); - - Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); - putKey2Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); - - Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); - putKey3Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); - - Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts6.add(toBytes(family1), toBytes(column2), timeStamp6, toBytes(value1)); - - Put putKey2Fam1Column3Ts6 = new Put(toBytes(key2)); - putKey2Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp3, toBytes(value1)); - - tryPut(hTable, putKey1Fam1Column1MinTs); - tryPut(hTable, putKey3Fam1Column1Ts1); - tryPut(hTable, putKey1Fam1Column2MinTs); - tryPut(hTable, putKey1Fam1Column2Ts3); - tryPut(hTable, putKey2Fam1Column2Ts3); - tryPut(hTable, putKey2Fam1Column3Ts1); - tryPut(hTable, putKey3Fam1Column3Ts1); - tryPut(hTable, putKey3Fam1Column2Ts6); - tryPut(hTable, putKey2Fam1Column3Ts6); - - // test DeleteFamilyVersion single cf - Get get = new Get(toBytes(key1)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(minTimeStamp); - get.setMaxVersions(10); - Result r = hTable.get(get); - Assert.assertEquals(2, r.raw().length); - - get = new Get(toBytes(key3)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp1); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(2, r.raw().length); - - get = new Get(toBytes(key2)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp3); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(2, r.raw().length); - - Delete delKey1MinTs = new Delete(toBytes(key1)); - delKey1MinTs.deleteFamilyVersion(toBytes(family1), minTimeStamp); - hTable.delete(delKey1MinTs); - - get = new Get(toBytes(key1)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(minTimeStamp); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Delete delKey3Ts1 = new Delete(toBytes(key3)); - delKey3Ts1.deleteFamilyVersion(toBytes(family1), timeStamp1); - hTable.delete(delKey3Ts1); - - get = new Get(toBytes(key3)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp1); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Delete delKey2Ts3 = new Delete(toBytes(key2)); - delKey2Ts3.deleteFamilyVersion(family1.getBytes(), timeStamp3); - hTable.delete(delKey2Ts3); - - get = new Get(toBytes(key2)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp3); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Scan scan = new Scan(); - scan.addFamily(toBytes(family1)); - scan.setMaxVersions(10); - ResultScanner scanner = hTable.getScanner(scan); - int key1Cnt = 0, key2Cnt = 0, key3Cnt = 0; - for (Result result : scanner) { - for (KeyValue kv : result.raw()) { - if (key1.equals(Bytes.toString(kv.getRow()))) { - ++key1Cnt; - } else if (key2.equals(Bytes.toString(kv.getRow()))) { - ++key2Cnt; - } else { - ++key3Cnt; - } - } - } - Assert.assertEquals(1, key1Cnt); - Assert.assertEquals(1, key2Cnt); - Assert.assertEquals(1, key3Cnt); - - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - - // test DeleteFamilyVersion multiple cf - Put putKey1Fam1Column3Ts6 = new Put(toBytes(key1)); - putKey1Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp6, toBytes(value3)); - - Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); - putKey1Fam2Column2Ts2.add(toBytes(family2), toBytes(column2), timeStamp2, toBytes(value1)); - - Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); - putKey1Fam2Column3Ts2.add(toBytes(family2), toBytes(column3), timeStamp2, toBytes(value1)); - - Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); - putKey1Fam1Column2Ts1.add(toBytes(family1), toBytes(column2), timeStamp1, toBytes(value2)); - - Put putKey2Fam1Column2Ts8 = new Put(toBytes(key2)); - putKey2Fam1Column2Ts8.add(toBytes(family1), toBytes(column2), timeStamp8, toBytes(value2)); - - Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); - putKey2Fam2Column3Ts1.add(toBytes(family2), toBytes(column3), timeStamp3, toBytes(value3)); - - Put putKey2Fam1Column1Ts1 = new Put(toBytes(key2)); - putKey2Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp8, toBytes(value1)); - - Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); - putKey2Fam2Column1Ts3.add(toBytes(family2), toBytes(column1), timeStamp3, toBytes(value2)); - - Put putKey3Fam1Column2Ts9 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts9.add(toBytes(family1), toBytes(column2), timeStamp9, toBytes(value2)); - - Put putKey3Fam2Column3Ts10 = new Put(toBytes(key3)); - putKey3Fam2Column3Ts10 - .add(toBytes(family2), toBytes(column3), timeStamp10, toBytes(value1)); - - Put putKey3Fam2Column1Ts10 = new Put(toBytes(key3)); - putKey3Fam2Column1Ts10 - .add(toBytes(family2), toBytes(column1), timeStamp10, toBytes(value2)); - - Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts2.add(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); - - tryPut(hTable, putKey1Fam1Column3Ts6); - tryPut(hTable, putKey1Fam2Column2Ts2); - tryPut(hTable, putKey1Fam2Column3Ts2); - tryPut(hTable, putKey1Fam1Column2Ts1); - tryPut(hTable, putKey2Fam1Column2Ts8); - tryPut(hTable, putKey2Fam2Column3Ts1); - tryPut(hTable, putKey2Fam1Column1Ts1); - tryPut(hTable, putKey2Fam2Column1Ts3); - tryPut(hTable, putKey3Fam1Column2Ts9); - tryPut(hTable, putKey3Fam2Column3Ts10); - tryPut(hTable, putKey3Fam2Column1Ts10); - tryPut(hTable, putKey3Fam1Column2Ts2); - - Get getKey1 = new Get(toBytes(key1)); - getKey1.addFamily(toBytes(family1)); - getKey1.addFamily(toBytes(family2)); - getKey1.setMaxVersions(10); - r = hTable.get(getKey1); - Assert.assertEquals(4, r.raw().length); - - Get getKey2 = new Get(toBytes(key2)); - getKey2.addFamily(toBytes(family1)); - getKey2.addFamily(toBytes(family2)); - getKey2.setMaxVersions(10); - r = hTable.get(getKey2); - Assert.assertEquals(4, r.raw().length); - - Get getKey3 = new Get(toBytes(key3)); - getKey3.addFamily(toBytes(family1)); - getKey3.addFamily(toBytes(family2)); - getKey3.setMaxVersions(10); - r = hTable.get(getKey3); - Assert.assertEquals(4, r.raw().length); - - Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); - delKey1Ts_6_2.deleteFamilyVersion(toBytes(family1), timeStamp6); - delKey1Ts_6_2.deleteFamilyVersion(toBytes(family2), timeStamp2); - hTable.delete(delKey1Ts_6_2); - - getKey1 = new Get(toBytes(key1)); - getKey1.addFamily(toBytes(family1)); - getKey1.addFamily(toBytes(family2)); - getKey1.setMaxVersions(10); - r = hTable.get(getKey1); - Assert.assertEquals(1, r.raw().length); - for (KeyValue kv : r.raw()) { - Assert.assertEquals(timeStamp1, kv.getTimestamp()); - } - - Delete delKey2Ts_8_3 = new Delete(toBytes(key2)); - delKey2Ts_8_3.deleteFamilyVersion(toBytes(family1), timeStamp8); - delKey2Ts_8_3.deleteFamilyVersion(toBytes(family2), timeStamp3); - hTable.delete(delKey2Ts_8_3); - - getKey2 = new Get(toBytes(key2)); - getKey2.addFamily(toBytes(family1)); - getKey2.addFamily(toBytes(family2)); - getKey2.setMaxVersions(10); - r = hTable.get(getKey2); - Assert.assertEquals(0, r.raw().length); - - Delete delKey3Ts_2_10 = new Delete(toBytes(key3)); - delKey3Ts_2_10.deleteFamilyVersion(toBytes(family1), timeStamp2); - delKey3Ts_2_10.deleteFamilyVersion(toBytes(family2), timeStamp10); - hTable.delete(delKey3Ts_2_10); - - getKey3 = new Get(toBytes(key3)); - getKey3.addFamily(toBytes(family1)); - getKey3.addFamily(toBytes(family2)); - getKey3.setTimeStamp(timeStamp9); - getKey3.setMaxVersions(10); - r = hTable.get(getKey3); - Assert.assertEquals(1, r.raw().length); - - scan = new Scan(); - scan.addFamily(toBytes(family1)); - scan.addFamily(toBytes(family2)); - scan.setMaxVersions(10); - scanner = hTable.getScanner(scan); - int ts1Cnt = 0, ts9Cnt = 0; - for (Result result : scanner) { - for (KeyValue kv : result.raw()) { - if (kv.getTimestamp() == timeStamp1) { - ++ts1Cnt; - } else if (kv.getTimestamp() == timeStamp9) { - ++ts9Cnt; - } - } - } - Assert.assertEquals(1, ts1Cnt); - Assert.assertEquals(1, ts9Cnt); - - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - } - @Test public void testAppend() throws IOException { String column = "appendColumn"; diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java new file mode 100644 index 00000000..fefea038 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java @@ -0,0 +1,348 @@ +package com.alipay.oceanbase.hbase; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; +import org.junit.rules.ExpectedException; + +import java.io.IOException; + +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + +public class OHTableDeleteFamilyVersionTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + protected HTableInterface hTable; + private static final String key1 = "scanKey1x"; + private static final String key2 = "scanKey2x"; + private static final String key3 = "scanKey3x"; + private static final String column1 = "column1"; + private static final String column2 = "column2"; + private static final String column3 = "column3"; + private static final String value1 = "value1"; + private static final String value2 = "value2"; + private static final String value3 = "value3"; + private static final String family1 = "family_with_group1"; + private static final String family2 = "family_with_group2"; + + @Before + public void before() throws Exception { + hTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); + ((OHTableClient) hTable).init(); + } + + @After + public void finish() throws IOException { + hTable.close(); + } + + public void tryPut(Table hTable, Put put) throws Exception { + hTable.put(put); + Thread.sleep(1); + } + + @Test + public void testDeleteFamilyVerison() throws Exception { + // delete previous data + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family1)); + deleteKey1Family.deleteFamily(toBytes(family2)); + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family1)); + deleteKey2Family.deleteFamily(toBytes(family2)); + Delete deleteKey3Family = new Delete(toBytes(key3)); + deleteKey3Family.deleteFamily(toBytes(family1)); + deleteKey3Family.deleteFamily(toBytes(family2)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); + putKey1Fam1Column1MinTs.add(toBytes(family1), toBytes(column1), minTimeStamp, + toBytes(value1)); + + Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp1, toBytes(value2)); + + Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); + putKey1Fam1Column2MinTs.add(toBytes(family1), toBytes(column2), minTimeStamp, + toBytes(value1)); + + Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts6.add(toBytes(family1), toBytes(column2), timeStamp6, toBytes(value1)); + + Put putKey2Fam1Column3Ts6 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp3, toBytes(value1)); + + tryPut(hTable, putKey1Fam1Column1MinTs); + tryPut(hTable, putKey3Fam1Column1Ts1); + tryPut(hTable, putKey1Fam1Column2MinTs); + tryPut(hTable, putKey1Fam1Column2Ts3); + tryPut(hTable, putKey2Fam1Column2Ts3); + tryPut(hTable, putKey2Fam1Column3Ts1); + tryPut(hTable, putKey3Fam1Column3Ts1); + tryPut(hTable, putKey3Fam1Column2Ts6); + tryPut(hTable, putKey2Fam1Column3Ts6); + + // test DeleteFamilyVersion single cf + Get get = new Get(toBytes(key1)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + Result r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + Delete delKey1MinTs = new Delete(toBytes(key1)); + delKey1MinTs.deleteFamilyVersion(toBytes(family1), minTimeStamp); + hTable.delete(delKey1MinTs); + + get = new Get(toBytes(key1)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts1 = new Delete(toBytes(key3)); + delKey3Ts1.deleteFamilyVersion(toBytes(family1), timeStamp1); + hTable.delete(delKey3Ts1); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey2Ts3 = new Delete(toBytes(key2)); + delKey2Ts3.deleteFamilyVersion(family1.getBytes(), timeStamp3); + hTable.delete(delKey2Ts3); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Scan scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(toBytes(family1)); + scan.setMaxVersions(10); + ResultScanner scanner = hTable.getScanner(scan); + int key1Cnt = 0, key2Cnt = 0, key3Cnt = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (key1.equals(Bytes.toString(kv.getRow()))) { + ++key1Cnt; + } else if (key2.equals(Bytes.toString(kv.getRow()))) { + ++key2Cnt; + } else { + ++key3Cnt; + } + } + } + Assert.assertEquals(1, key1Cnt); + Assert.assertEquals(1, key2Cnt); + Assert.assertEquals(1, key3Cnt); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + + // test DeleteFamilyVersion multiple cf + Put putKey1Fam1Column3Ts6 = new Put(toBytes(key1)); + putKey1Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp6, toBytes(value3)); + + Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column2Ts2.add(toBytes(family2), toBytes(column2), timeStamp2, toBytes(value1)); + + Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column3Ts2.add(toBytes(family2), toBytes(column3), timeStamp2, toBytes(value1)); + + Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts1.add(toBytes(family1), toBytes(column2), timeStamp1, toBytes(value2)); + + Put putKey2Fam1Column2Ts8 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts8.add(toBytes(family1), toBytes(column2), timeStamp8, toBytes(value2)); + + Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam2Column3Ts1.add(toBytes(family2), toBytes(column3), timeStamp3, toBytes(value3)); + + Put putKey2Fam1Column1Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp8, toBytes(value1)); + + Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); + putKey2Fam2Column1Ts3.add(toBytes(family2), toBytes(column1), timeStamp3, toBytes(value2)); + + Put putKey3Fam1Column2Ts9 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts9.add(toBytes(family1), toBytes(column2), timeStamp9, toBytes(value2)); + + Put putKey3Fam2Column3Ts10 = new Put(toBytes(key3)); + putKey3Fam2Column3Ts10 + .add(toBytes(family2), toBytes(column3), timeStamp10, toBytes(value1)); + + Put putKey3Fam2Column1Ts10 = new Put(toBytes(key3)); + putKey3Fam2Column1Ts10 + .add(toBytes(family2), toBytes(column1), timeStamp10, toBytes(value2)); + + Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts2.add(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); + + tryPut(hTable, putKey1Fam1Column3Ts6); + tryPut(hTable, putKey1Fam2Column2Ts2); + tryPut(hTable, putKey1Fam2Column3Ts2); + tryPut(hTable, putKey1Fam1Column2Ts1); + tryPut(hTable, putKey2Fam1Column2Ts8); + tryPut(hTable, putKey2Fam2Column3Ts1); + tryPut(hTable, putKey2Fam1Column1Ts1); + tryPut(hTable, putKey2Fam2Column1Ts3); + tryPut(hTable, putKey3Fam1Column2Ts9); + tryPut(hTable, putKey3Fam2Column3Ts10); + tryPut(hTable, putKey3Fam2Column1Ts10); + tryPut(hTable, putKey3Fam1Column2Ts2); + + Get getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = hTable.get(getKey1); + Assert.assertEquals(4, r.raw().length); + + Get getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = hTable.get(getKey2); + Assert.assertEquals(4, r.raw().length); + + Get getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setMaxVersions(10); + r = hTable.get(getKey3); + Assert.assertEquals(4, r.raw().length); + + Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); + delKey1Ts_6_2.deleteFamilyVersion(toBytes(family1), timeStamp6); + delKey1Ts_6_2.deleteFamilyVersion(toBytes(family2), timeStamp2); + hTable.delete(delKey1Ts_6_2); + + getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = hTable.get(getKey1); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp1, kv.getTimestamp()); + } + + Delete delKey2Ts_8_3 = new Delete(toBytes(key2)); + delKey2Ts_8_3.deleteFamilyVersion(toBytes(family1), timeStamp8); + delKey2Ts_8_3.deleteFamilyVersion(toBytes(family2), timeStamp3); + hTable.delete(delKey2Ts_8_3); + + getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = hTable.get(getKey2); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts_2_10 = new Delete(toBytes(key3)); + delKey3Ts_2_10.deleteFamilyVersion(toBytes(family1), timeStamp2); + delKey3Ts_2_10.deleteFamilyVersion(toBytes(family2), timeStamp10); + hTable.delete(delKey3Ts_2_10); + + getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setMaxVersions(10); + r = hTable.get(getKey3); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp9, kv.getTimestamp()); + } + + scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(toBytes(family1)); + scan.addFamily(toBytes(family2)); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + int ts1Cnt = 0, ts9Cnt = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (kv.getTimestamp() == timeStamp1) { + ++ts1Cnt; + } else if (kv.getTimestamp() == timeStamp9) { + ++ts9Cnt; + } + } + } + Assert.assertEquals(1, ts1Cnt); + Assert.assertEquals(1, ts9Cnt); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + } +} From b51c991dbbf0b2b6d02e091b03bcbe2cc144b1fe Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Wed, 23 Oct 2024 10:22:43 +0800 Subject: [PATCH 09/10] format code --- .../hbase/OHTableDeleteFamilyVersionTest.java | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java index fefea038..2ed4c190 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java @@ -12,20 +12,20 @@ public class OHTableDeleteFamilyVersionTest { @Rule - public ExpectedException expectedException = ExpectedException.none(); - - protected HTableInterface hTable; - private static final String key1 = "scanKey1x"; - private static final String key2 = "scanKey2x"; - private static final String key3 = "scanKey3x"; - private static final String column1 = "column1"; - private static final String column2 = "column2"; - private static final String column3 = "column3"; - private static final String value1 = "value1"; - private static final String value2 = "value2"; - private static final String value3 = "value3"; - private static final String family1 = "family_with_group1"; - private static final String family2 = "family_with_group2"; + public ExpectedException expectedException = ExpectedException.none(); + + protected HTableInterface hTable; + private static final String key1 = "scanKey1x"; + private static final String key2 = "scanKey2x"; + private static final String key3 = "scanKey3x"; + private static final String column1 = "column1"; + private static final String column2 = "column2"; + private static final String column3 = "column3"; + private static final String value1 = "value1"; + private static final String value2 = "value2"; + private static final String value3 = "value3"; + private static final String family1 = "family_with_group1"; + private static final String family2 = "family_with_group2"; @Before public void before() throws Exception { @@ -88,14 +88,14 @@ public void testDeleteFamilyVerison() throws Exception { Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); putKey1Fam1Column1MinTs.add(toBytes(family1), toBytes(column1), minTimeStamp, - toBytes(value1)); + toBytes(value1)); Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); putKey3Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp1, toBytes(value2)); Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); putKey1Fam1Column2MinTs.add(toBytes(family1), toBytes(column2), minTimeStamp, - toBytes(value1)); + toBytes(value1)); Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); putKey1Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); @@ -236,11 +236,11 @@ public void testDeleteFamilyVerison() throws Exception { Put putKey3Fam2Column3Ts10 = new Put(toBytes(key3)); putKey3Fam2Column3Ts10 - .add(toBytes(family2), toBytes(column3), timeStamp10, toBytes(value1)); + .add(toBytes(family2), toBytes(column3), timeStamp10, toBytes(value1)); Put putKey3Fam2Column1Ts10 = new Put(toBytes(key3)); putKey3Fam2Column1Ts10 - .add(toBytes(family2), toBytes(column1), timeStamp10, toBytes(value2)); + .add(toBytes(family2), toBytes(column1), timeStamp10, toBytes(value2)); Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); putKey3Fam1Column2Ts2.add(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); From da0eface1a3b15adbc210c36f61dedfd33a28652 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Wed, 23 Oct 2024 10:29:25 +0800 Subject: [PATCH 10/10] delete useless self-defined table --- src/test/java/unit_test_db.sql | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/test/java/unit_test_db.sql b/src/test/java/unit_test_db.sql index 036ccca7..c5681bd7 100644 --- a/src/test/java/unit_test_db.sql +++ b/src/test/java/unit_test_db.sql @@ -63,14 +63,6 @@ CREATE TABLE `test$family_group` ( PRIMARY KEY (`K`, `Q`, `T`) ) TABLEGROUP = test; -CREATE TABLE `test$family_group1` ( - `K` varbinary(1024) NOT NULL, - `Q` varbinary(256) NOT NULL, - `T` bigint(20) NOT NULL, - `V` varbinary(1024) DEFAULT NULL, - PRIMARY KEY (`K`, `Q`, `T`) -) TABLEGROUP = test; - CREATE TABLEGROUP test_t SHARDING = 'ADAPTIVE'; CREATE TABLE `test_t$family_group` ( `K` varbinary(1024) NOT NULL,