From 03c8e7149c9e13dcce591a31f8dc64ed04e01838 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 11 Oct 2024 10:55:55 +0800 Subject: [PATCH 1/7] add DepentdentFilter and SingleColumnValueExcludeFilter --- .../hbase/filter/HBaseFilterUtils.java | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index e58d2590..10c54373 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -45,6 +45,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte toParseableByteArray(byteStream, (CompareFilter) filter); } else if (filter instanceof SingleColumnValueFilter) { toParseableByteArray(byteStream, (SingleColumnValueFilter) filter); + } else if (filter instanceof SingleColumnValueExcludeFilter) { + toParseableByteArray(byteStream, (SingleColumnValueExcludeFilter) filter); } else if (filter instanceof PageFilter) { toParseableByteArray(byteStream, (PageFilter) filter); } else if (filter instanceof ColumnCountGetFilter) { @@ -69,6 +71,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte toParseableByteArray(byteStream, (SkipFilter) filter); } else if (filter instanceof WhileMatchFilter) { toParseableByteArray(byteStream, (WhileMatchFilter) filter); + } else if (filter instanceof DependentColumnFilter) { + toParseableByteArray(byteStream, (DependentColumnFilter) filter); } else { throw new IllegalArgumentException("Invalid filter: " + filter); } @@ -149,6 +153,25 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write(')'); } + // SingleColumnValueExcludeFilter('cf1','col1',=,'binary:123',true,true) + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + SingleColumnValueExcludeFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getFilterIfMissing()).getBytes()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getLatestVersionOnly()).getBytes()); + byteStream.write(')'); + } + // PageFilter(100); private static void toParseableByteArray(ByteArrayOutputStream byteStream, PageFilter filter) throws IOException { @@ -167,6 +190,42 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write(')'); } + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + DependentColumnFilter filter) throws IOException { + boolean dropDependentColumn; + try { + Field field = filter.getClass().getDeclaredField("dropDependentColumn"); + field.setAccessible(true); + dropDependentColumn = (boolean)field.get(filter); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ')' + if (filter.getComparator() == null) { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(dropDependentColumn).getBytes()); + byteStream.write(')'); + } else { // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ',' compare_op ',' comparator ')' + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(dropDependentColumn).getBytes()); + byteStream.write(','); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(')'); + } + } + private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnPaginationFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); From 2aaf7c4d409dbeed6dd6899d1587ce22b7bb63bd Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Wed, 16 Oct 2024 11:54:18 +0800 Subject: [PATCH 2/7] add singleColumnValueExcludeFilter and DependentColumnFilter; add test cases and optimize bufferedMutator test cases --- .../hbase/filter/HBaseFilterUtils.java | 20 +- .../oceanbase/hbase/HTableTestBase.java | 220 ++++++++++++++++++ .../oceanbase/hbase/OHConnectionTest.java | 16 ++ 3 files changed, 242 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index 10c54373..cc512f9e 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -40,13 +40,15 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte throws IOException { if (filter == null) { throw new IllegalArgumentException("Filter is null"); + } else if (filter instanceof DependentColumnFilter) { + toParseableByteArray(byteStream, (DependentColumnFilter) filter); } else if (filter instanceof CompareFilter) { // RowFilter, ValueFilter, QualifierFilter toParseableByteArray(byteStream, (CompareFilter) filter); - } else if (filter instanceof SingleColumnValueFilter) { - toParseableByteArray(byteStream, (SingleColumnValueFilter) filter); } else if (filter instanceof SingleColumnValueExcludeFilter) { toParseableByteArray(byteStream, (SingleColumnValueExcludeFilter) filter); + } else if (filter instanceof SingleColumnValueFilter) { + toParseableByteArray(byteStream, (SingleColumnValueFilter) filter); } else if (filter instanceof PageFilter) { toParseableByteArray(byteStream, (PageFilter) filter); } else if (filter instanceof ColumnCountGetFilter) { @@ -71,8 +73,6 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte toParseableByteArray(byteStream, (SkipFilter) filter); } else if (filter instanceof WhileMatchFilter) { toParseableByteArray(byteStream, (WhileMatchFilter) filter); - } else if (filter instanceof DependentColumnFilter) { - toParseableByteArray(byteStream, (DependentColumnFilter) filter); } else { throw new IllegalArgumentException("Invalid filter: " + filter); } @@ -192,14 +192,6 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, private static void toParseableByteArray(ByteArrayOutputStream byteStream, DependentColumnFilter filter) throws IOException { - boolean dropDependentColumn; - try { - Field field = filter.getClass().getDeclaredField("dropDependentColumn"); - field.setAccessible(true); - dropDependentColumn = (boolean)field.get(filter); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new RuntimeException(e); - } // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ')' if (filter.getComparator() == null) { byteStream.write(filter.getClass().getSimpleName().getBytes()); @@ -208,7 +200,7 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write("','".getBytes()); writeBytesWithEscape(byteStream, filter.getQualifier()); byteStream.write("',".getBytes()); - byteStream.write(Boolean.toString(dropDependentColumn).getBytes()); + byteStream.write(Boolean.toString(filter.getDropDependentColumn()).getBytes()); byteStream.write(')'); } else { // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ',' compare_op ',' comparator ')' byteStream.write(filter.getClass().getSimpleName().getBytes()); @@ -217,7 +209,7 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write("','".getBytes()); writeBytesWithEscape(byteStream, filter.getQualifier()); byteStream.write("',".getBytes()); - byteStream.write(Boolean.toString(dropDependentColumn).getBytes()); + byteStream.write(Boolean.toString(filter.getDropDependentColumn()).getBytes()); byteStream.write(','); byteStream.write(toParseableByteArray(filter.getOperator())); byteStream.write(','); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 4c93efde..a5b891fe 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -509,6 +509,83 @@ public void testFilter() throws Exception { tryPut(hTable, putKey2Column2Value1); tryPut(hTable, putKey2Column2Value2); +// time may be different +// +---------+-----+----------------+--------+ +// | K | Q | T | V | +// +---------+-----+----------------+--------+ +// | getKey1 | abc | -1728834971469 | value1 | +// | getKey1 | abc | -1728834971399 | value2 | +// | getKey1 | abc | -1728834971330 | value1 | +// | getKey1 | def | -1728834971748 | value2 | +// | getKey1 | def | -1728834971679 | value1 | +// | getKey1 | def | -1728834971609 | value2 | +// | getKey1 | def | -1728834971540 | value1 | +// | getKey2 | def | -1728834971887 | value2 | +// | getKey2 | def | -1728834971818 | value1 | +// +---------+-----+----------------+--------+ + + SingleColumnValueFilter singleColumnValueFilter; + singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueFilter); + r = hTable.get(get); + Assert.assertEquals(7, r.raw().length); + + SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; + singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueExcludeFilter); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), true); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + + + filter = new ColumnPrefixFilter(Bytes.toBytes("e")); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -817,6 +894,66 @@ public void testFilter() throws Exception { } Assert.assertEquals(res_count, 12); scanner.close(); + + long timestamp = System.currentTimeMillis(); + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), timestamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timestamp, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + // putKey1Column1Value1 and putKey2Column1Value1 have the same timestamp + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column2Value2); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setStartRow("getKey1".getBytes()); + scan.setStopRow("getKey3".getBytes()); + scan.setMaxVersions(10); + scan.setFilter(dependentColumnFilter); + scanner = hTable.getScanner(scan); + + long prevTimestamp = - 1; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + if (prevTimestamp == -1) { + prevTimestamp = keyValue.getTimestamp(); + } else { + Assert.assertEquals(prevTimestamp, keyValue.getTimestamp()); + } + } + } + scanner.close(); } @Test @@ -1242,6 +1379,15 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(1, r.raw().length); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + // columnCountGetFilter filter 2 get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -1422,6 +1568,67 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(7, r.raw().length); + filterList = new FilterList(); + filterList.addFilter(new SingleColumnValueExcludeFilter(Bytes.toBytes(family), Bytes + .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column1), false)); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), false)); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), true)); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(value2)))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + filterList = new FilterList(); filterList.addFilter(new ColumnCountGetFilter(1)); filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.GREATER, @@ -1547,6 +1754,19 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(7, r.raw().length); + + SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; + singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueExcludeFilter); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index a4a78a61..e8644d7b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -209,6 +209,10 @@ public void testBufferedMutatorWithFlush() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + // only support Put and Delete // for other type of operations, BufferedMutator will not set its type for them Append append = new Append(Bytes.toBytes(key)); @@ -409,6 +413,10 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); @@ -498,6 +506,10 @@ public void testBufferedMutatorWithUserPool() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); @@ -587,6 +599,10 @@ public void testBufferedMutatorConcurrent() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + for (int i = 0; i < 50; ++i) { final int taskId = i; final BufferedMutator thrBufferMutator = ohBufferMutator; From 78df6d8c591558d046f67ad2e039175258133a8b Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 18 Oct 2024 09:52:35 +0800 Subject: [PATCH 3/7] add single cf timerange setting in Get --- src/main/java/com/alipay/oceanbase/hbase/OHTable.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 00d717b3..a4212020 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -598,6 +598,11 @@ public Result call() throws IOException { for (Map.Entry> entry : get.getFamilyMap() .entrySet()) { family = entry.getKey(); + Map colFamTimeRangeMap = get.getColumnFamilyTimeRange(); + if (colFamTimeRangeMap.get(entry.getKey()) != null) { + TimeRange tr = colFamTimeRangeMap.get(entry.getKey()); + get.setTimeRange(tr.getMin(), tr.getMax()); + } obTableQuery = buildObTableQuery(get, entry.getValue()); request = buildObTableQueryAsyncRequest(obTableQuery, getTargetTableName(tableNameString, Bytes.toString(family), From 256fe9ac19c58026d66460f066f08512d8445531 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 18 Oct 2024 15:30:06 +0800 Subject: [PATCH 4/7] single cf setColumnFamilyTimeRange in Get and Scan --- .../com/alipay/oceanbase/hbase/OHTable.java | 17 + .../oceanbase/hbase/HTableTestBase.java | 579 +++++++++++++++++- 2 files changed, 562 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index a4212020..9c321186 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -585,6 +585,9 @@ public Result call() throws IOException { // In a Get operation where the family map is greater than 1 or equal to 0, // we handle this by appending the column family to the qualifier on the client side. // The server can then use this information to filter the appropriate column families and qualifiers. + if (!get.getColumnFamilyTimeRange().isEmpty()) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } NavigableSet columnFilters = new TreeSet<>(Bytes.BYTES_COMPARATOR); processColumnFilters(columnFilters, get.getFamilyMap()); obTableQuery = buildObTableQuery(get, columnFilters); @@ -599,6 +602,9 @@ public Result call() throws IOException { .entrySet()) { family = entry.getKey(); Map colFamTimeRangeMap = get.getColumnFamilyTimeRange(); + if (colFamTimeRangeMap.size() > 1) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } if (colFamTimeRangeMap.get(entry.getKey()) != null) { TimeRange tr = colFamTimeRangeMap.get(entry.getKey()); get.setTimeRange(tr.getMin(), tr.getMax()); @@ -673,6 +679,9 @@ public ResultScanner call() throws IOException { // In a Scan operation where the family map is greater than 1 or equal to 0, // we handle this by appending the column family to the qualifier on the client side. // The server can then use this information to filter the appropriate column families and qualifiers. + if (!scan.getColumnFamilyTimeRange().isEmpty()) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } NavigableSet columnFilters = new TreeSet<>(Bytes.BYTES_COMPARATOR); processColumnFilters(columnFilters, scan.getFamilyMap()); filter = buildObHTableFilter(scan.getFilter(), scan.getTimeRange(), @@ -689,6 +698,14 @@ public ResultScanner call() throws IOException { for (Map.Entry> entry : scan.getFamilyMap() .entrySet()) { family = entry.getKey(); + Map colFamTimeRangeMap = scan.getColumnFamilyTimeRange(); + if (colFamTimeRangeMap.size() > 1) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } + if (colFamTimeRangeMap.get(entry.getKey()) != null) { + TimeRange tr = colFamTimeRangeMap.get(entry.getKey()); + scan.setTimeRange(tr.getMin(), tr.getMax()); + } filter = buildObHTableFilter(scan.getFilter(), scan.getTimeRange(), scan.getMaxVersions(), entry.getValue()); obTableQuery = buildObTableQuery(filter, scan); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 801abd9e..32e76cc7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -2882,23 +2882,25 @@ public void testGet() throws Exception { tryPut(hTable, putzKey2Column1Value1); // show table (time maybe different) - //+-----------+---------+----------------+--------+ - //| K | Q | T | V | - //+-----------+---------+----------------+--------+ - //| scanKey1x | column1 | -1709714409669 | value1 | - //| scanKey1x | column1 | -1709714409637 | value2 | - //| scanKey1x | column1 | -1709714409603 | value1 | - //| scanKey1x | column2 | -1709714409802 | value2 | - //| scanKey1x | column2 | -1709714409768 | value1 | - //| scanKey1x | column2 | -1709714409735 | value2 | - //| scanKey1x | column2 | -1709714409702 | value1 | - //| scanKey2x | column2 | -1709714409869 | value2 | - //| scanKey2x | column2 | -1709714409836 | value1 | - //| scanKey3x | column1 | -1709714409940 | value2 | - //| scanKey3x | column1 | -1709714409904 | value1 | - //| scanKey3x | column2 | -1709714410010 | value2 | - //| scanKey3x | column2 | -1709714409977 | value1 | - //+-----------+---------+----------------+--------+ + // +-----------+---------+----------------+--------+ + // | K | Q | T | V | + // +-----------+---------+----------------+--------+ + // | scanKey1x | column1 | -1729223351579 | value1 | + // | scanKey1x | column1 | -1729223351504 | value2 | + // | scanKey1x | column1 | -1729223351431 | value1 | + // | scanKey1x | column2 | -1729223351867 | value2 | + // | scanKey1x | column2 | -1729223351796 | value1 | + // | scanKey1x | column2 | -1729223351724 | value2 | + // | scanKey1x | column2 | -1729223351651 | value1 | + // | scanKey2x | column2 | -1729223352015 | value2 | + // | scanKey2x | column2 | -1729223351941 | value1 | + // | scanKey3x | column1 | -1729223352159 | value2 | + // | scanKey3x | column1 | -1729223352088 | value1 | + // | scanKey3x | column2 | -1729223352304 | value2 | + // | scanKey3x | column2 | -1729223352232 | value1 | + // | zScanKey1 | column1 | -1729223352378 | value1 | + // | zScanKey2 | column1 | -1729223352450 | value1 | + // +-----------+---------+----------------+--------+ // test closestRowBefore get = new Get("scanKey2x2".getBytes()); @@ -2930,6 +2932,147 @@ public void testGet() throws Exception { assertTrue(booleans[2]); assertFalse(booleans[3]); assertTrue(booleans[4]); + + // test single cf setColumnFamilyTimeRange + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + hTable.delete(deleteZKey1Family); + hTable.delete(deleteZKey2Family); + + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), timeStamp2, toBytes(value1)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timeStamp4, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), timeStamp5, toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), timeStamp7, toBytes(value2)); + + putKey3Column1Value1 = new Put(toBytes(key3)); + putKey3Column1Value1.add(toBytes(family), toBytes(column1), timeStamp8, toBytes(value1)); + + putKey3Column1Value2 = new Put(toBytes(key3)); + putKey3Column1Value2.add(toBytes(family), toBytes(column1), timeStamp9, toBytes(value2)); + + putKey3Column2Value1 = new Put(toBytes(key3)); + putKey3Column2Value1.add(toBytes(family), toBytes(column2), timeStamp10, toBytes(value1)); + + putKey3Column2Value2 = new Put(toBytes(key3)); + putKey3Column2Value2.add(toBytes(family), toBytes(column2), timeStamp11, toBytes(value2)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column1Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + tryPut(hTable, putKey3Column1Value1); + tryPut(hTable, putKey3Column1Value2); + tryPut(hTable, putKey3Column2Value1); + tryPut(hTable, putKey3Column2Value2); + + + get = new Get(toBytes(key1)); + get.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, maxTimeStamp); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + get = new Get(toBytes(key1)); + get.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, timeStamp2 + 1); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + get = new Get(toBytes(key2)); + // set invalid timeRange + get.setTimeRange(minTimeStamp, maxTimeStamp); + get.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, timeStamp5 + 1); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key2)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp5, maxTimeStamp); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp8, timeStamp8); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp8, timeStamp9); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp8, timeStamp9); + get.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp8, timeStamp9); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + final Get errorGet = get; + Assert.assertThrows(IOException.class, () -> { + hTable.get(errorGet); + }); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); } @Test @@ -3029,23 +3172,25 @@ public void testScan() throws Exception { tryPut(hTable, putzKey2Column1Value1); // show table (time maybe different) - //+-----------+---------+----------------+--------+ - //| K | Q | T | V | - //+-----------+---------+----------------+--------+ - //| scanKey1x | column1 | -1709714409669 | value1 | - //| scanKey1x | column1 | -1709714409637 | value2 | - //| scanKey1x | column1 | -1709714409603 | value1 | - //| scanKey1x | column2 | -1709714409802 | value2 | - //| scanKey1x | column2 | -1709714409768 | value1 | - //| scanKey1x | column2 | -1709714409735 | value2 | - //| scanKey1x | column2 | -1709714409702 | value1 | - //| scanKey2x | column2 | -1709714409869 | value2 | - //| scanKey2x | column2 | -1709714409836 | value1 | - //| scanKey3x | column1 | -1709714409940 | value2 | - //| scanKey3x | column1 | -1709714409904 | value1 | - //| scanKey3x | column2 | -1709714410010 | value2 | - //| scanKey3x | column2 | -1709714409977 | value1 | - //+-----------+---------+----------------+--------+ + // +-----------+---------+----------------+--------+ + // | K | Q | T | V | + // +-----------+---------+----------------+--------+ + // | scanKey1x | column1 | -1729236392149 | value1 | + // | scanKey1x | column1 | -1729236392078 | value2 | + // | scanKey1x | column1 | -1729236392008 | value1 | + // | scanKey1x | column2 | -1729236392436 | value2 | + // | scanKey1x | column2 | -1729236392364 | value1 | + // | scanKey1x | column2 | -1729236392291 | value2 | + // | scanKey1x | column2 | -1729236392220 | value1 | + // | scanKey2x | column2 | -1729236392576 | value2 | + // | scanKey2x | column2 | -1729236392506 | value1 | + // | scanKey3x | column1 | -1729236392720 | value2 | + // | scanKey3x | column1 | -1729236392647 | value1 | + // | scanKey3x | column2 | -1729236392861 | value2 | + // | scanKey3x | column2 | -1729236392790 | value1 | + // | zScanKey1 | column1 | -1729236392931 | value1 | + // | zScanKey2 | column1 | -1729236393002 | value1 | + // +-----------+---------+----------------+--------+ scan = new Scan(); scan.addFamily(family.getBytes()); @@ -3232,6 +3377,180 @@ public void testScan() throws Exception { hTable.delete(deleteKey3Family); hTable.delete(deleteZKey1Family); hTable.delete(deleteZKey2Family); + + // test single cf setColumnFamilyTimeRange + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), timeStamp2, toBytes(value1)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timeStamp4, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), timeStamp5, toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), timeStamp7, toBytes(value2)); + + putKey3Column1Value1 = new Put(toBytes(key3)); + putKey3Column1Value1.add(toBytes(family), toBytes(column1), timeStamp8, toBytes(value1)); + + putKey3Column1Value2 = new Put(toBytes(key3)); + putKey3Column1Value2.add(toBytes(family), toBytes(column1), timeStamp9, toBytes(value2)); + + putKey3Column2Value1 = new Put(toBytes(key3)); + putKey3Column2Value1.add(toBytes(family), toBytes(column2), timeStamp10, toBytes(value1)); + + putKey3Column2Value2 = new Put(toBytes(key3)); + putKey3Column2Value2.add(toBytes(family), toBytes(column2), timeStamp11, toBytes(value2)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column1Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + tryPut(hTable, putKey3Column1Value1); + tryPut(hTable, putKey3Column1Value2); + tryPut(hTable, putKey3Column2Value1); + tryPut(hTable, putKey3Column2Value2); + + // scan key1 + key2 + scan = new Scan(toBytes(key1), toBytes(key3)); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, maxTimeStamp); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(8, res_count); + + // scan key1 + scan = new Scan(toBytes(key1), toBytes(key2)); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, maxTimeStamp); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(4, res_count); + + // scan key1 + scan = new Scan(toBytes(key1), toBytes(key2)); + scan.addFamily(toBytes(family)); + // set invalid timeRange + scan.setTimeRange(minTimeStamp, maxTimeStamp); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp1, timeStamp3); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(2, res_count); + + // scan all + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp2, timeStamp7); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(5, res_count); + + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + boolean foundKey3 = false; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (!foundKey3) { + if (key3.equals(Bytes.toString(kv.getRow()))) { + foundKey3 = true; + } + } + ++res_count; + } + } + Assert.assertTrue(foundKey3); + Assert.assertEquals(6, res_count); + + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); + scan.setMaxVersions(); + final Scan errorScan = scan; + Assert.assertThrows(IOException.class, () -> { + hTable.getScanner(errorScan); + }); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); } @Test @@ -3454,6 +3773,198 @@ public void testReversedScan() throws Exception { hTable.delete(deleteKey2Family); hTable.delete(deleteKey3Family); + // test single cf setColumnFamilyTimeRange + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), timeStamp2, toBytes(value1)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timeStamp4, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), timeStamp5, toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), timeStamp7, toBytes(value2)); + + putKey3Column1Value1 = new Put(toBytes(key3)); + putKey3Column1Value1.add(toBytes(family), toBytes(column1), timeStamp8, toBytes(value1)); + + putKey3Column1Value2 = new Put(toBytes(key3)); + putKey3Column1Value2.add(toBytes(family), toBytes(column1), timeStamp9, toBytes(value2)); + + putKey3Column2Value1 = new Put(toBytes(key3)); + putKey3Column2Value1.add(toBytes(family), toBytes(column2), timeStamp10, toBytes(value1)); + + putKey3Column2Value2 = new Put(toBytes(key3)); + putKey3Column2Value2.add(toBytes(family), toBytes(column2), timeStamp11, toBytes(value2)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column1Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + tryPut(hTable, putKey3Column1Value1); + tryPut(hTable, putKey3Column1Value2); + tryPut(hTable, putKey3Column2Value1); + tryPut(hTable, putKey3Column2Value2); + + // scan key1 + scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(family.getBytes()); + scan.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, timeStamp3); + scan.setReversed(true); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(3, res_count); + + // scan key1 + key2 + scan = new Scan(); + scan.setStartRow(toBytes(key2)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp2, timeStamp7); + scan.setReversed(true); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(5, res_count); + + // scan key2 + scan = new Scan(); + scan.setStartRow(toBytes(key2)); + scan.setStopRow(toBytes(key1)); + scan.addFamily(toBytes(family)); + // set invalid timeRange + scan.setTimeRange(minTimeStamp, maxTimeStamp); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp4, timeStamp6); + scan.setReversed(true); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(2, res_count); + + // scan all + scan = new Scan(); + scan.setStartRow(toBytes(key3)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp2, timeStamp7); + scan.setReversed(true); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(5, res_count); + + scan = new Scan(); + scan.setStartRow(toBytes(key3)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setReversed(true); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + boolean foundKey3 = false; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (!foundKey3) { + if (key3.equals(Bytes.toString(kv.getRow()))) { + foundKey3 = true; + } + } + ++res_count; + } + } + Assert.assertTrue(foundKey3); + Assert.assertEquals(6, res_count); + + scan = new Scan(); + scan.setStartRow(toBytes(key3)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); + scan.setReversed(true); + scan.setMaxVersions(); + final Scan errorScan = scan; + Assert.assertThrows(IOException.class, () -> { + hTable.getScanner(errorScan); + }); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + } @Test From a5ec9f6c567bc4b88a5d73a91c54d3fe50426888 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 18 Oct 2024 15:35:02 +0800 Subject: [PATCH 5/7] optimize code --- src/main/java/com/alipay/oceanbase/hbase/OHTable.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 9c321186..e6f04ade 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -605,8 +605,8 @@ public Result call() throws IOException { if (colFamTimeRangeMap.size() > 1) { throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); } - if (colFamTimeRangeMap.get(entry.getKey()) != null) { - TimeRange tr = colFamTimeRangeMap.get(entry.getKey()); + if (colFamTimeRangeMap.get(family) != null) { + TimeRange tr = colFamTimeRangeMap.get(family); get.setTimeRange(tr.getMin(), tr.getMax()); } obTableQuery = buildObTableQuery(get, entry.getValue()); @@ -702,8 +702,8 @@ public ResultScanner call() throws IOException { if (colFamTimeRangeMap.size() > 1) { throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); } - if (colFamTimeRangeMap.get(entry.getKey()) != null) { - TimeRange tr = colFamTimeRangeMap.get(entry.getKey()); + if (colFamTimeRangeMap.get(family) != null) { + TimeRange tr = colFamTimeRangeMap.get(family); scan.setTimeRange(tr.getMin(), tr.getMax()); } filter = buildObHTableFilter(scan.getFilter(), scan.getTimeRange(), From c53c3e7decb8f25566ab505567892c8104c04392 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Tue, 22 Oct 2024 10:26:16 +0800 Subject: [PATCH 6/7] report error when setColumnFamilyTimeRange for a column family that is not queryed --- .../com/alipay/oceanbase/hbase/OHTable.java | 34 +++++++++++-------- .../oceanbase/hbase/HTableTestBase.java | 26 +++++++++++--- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index e6f04ade..4ecf0aa5 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -601,13 +601,16 @@ public Result call() throws IOException { for (Map.Entry> entry : get.getFamilyMap() .entrySet()) { family = entry.getKey(); - Map colFamTimeRangeMap = get.getColumnFamilyTimeRange(); - if (colFamTimeRangeMap.size() > 1) { - throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); - } - if (colFamTimeRangeMap.get(family) != null) { - TimeRange tr = colFamTimeRangeMap.get(family); - get.setTimeRange(tr.getMin(), tr.getMax()); + if (!get.getColumnFamilyTimeRange().isEmpty()) { + Map colFamTimeRangeMap = get.getColumnFamilyTimeRange(); + if (colFamTimeRangeMap.size() > 1) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } else if (colFamTimeRangeMap.get(family) == null) { + throw new IllegalArgumentException("Get family is not matched in ColumnFamilyTimeRange"); + } else { + TimeRange tr = colFamTimeRangeMap.get(family); + get.setTimeRange(tr.getMin(), tr.getMax()); + } } obTableQuery = buildObTableQuery(get, entry.getValue()); request = buildObTableQueryAsyncRequest(obTableQuery, @@ -698,13 +701,16 @@ public ResultScanner call() throws IOException { for (Map.Entry> entry : scan.getFamilyMap() .entrySet()) { family = entry.getKey(); - Map colFamTimeRangeMap = scan.getColumnFamilyTimeRange(); - if (colFamTimeRangeMap.size() > 1) { - throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); - } - if (colFamTimeRangeMap.get(family) != null) { - TimeRange tr = colFamTimeRangeMap.get(family); - scan.setTimeRange(tr.getMin(), tr.getMax()); + if (!scan.getColumnFamilyTimeRange().isEmpty()) { + Map colFamTimeRangeMap = scan.getColumnFamilyTimeRange(); + if (colFamTimeRangeMap.size() > 1) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } else if (colFamTimeRangeMap.get(family) == null) { + throw new IllegalArgumentException("Scan family is not matched in ColumnFamilyTimeRange"); + } else { + TimeRange tr = colFamTimeRangeMap.get(family); + scan.setTimeRange(tr.getMin(), tr.getMax()); + } } filter = buildObHTableFilter(scan.getFilter(), scan.getTimeRange(), scan.getMaxVersions(), entry.getValue()); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 32e76cc7..b10fc68d 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -3065,9 +3065,18 @@ public void testGet() throws Exception { get.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp8, timeStamp9); get.addFamily(toBytes(family)); get.setMaxVersions(); - final Get errorGet = get; + final Get multiFamGet = get; Assert.assertThrows(IOException.class, () -> { - hTable.get(errorGet); + hTable.get(multiFamGet); + }); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp8, timeStamp9); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + final Get missFamGet = get; + Assert.assertThrows(IOException.class, () -> { + hTable.get(missFamGet); }); hTable.delete(deleteKey1Family); @@ -3543,9 +3552,18 @@ public void testScan() throws Exception { scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); scan.setMaxVersions(); - final Scan errorScan = scan; + final Scan multiFamScan = scan; Assert.assertThrows(IOException.class, () -> { - hTable.getScanner(errorScan); + hTable.getScanner(multiFamScan); + }); + + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); + scan.setMaxVersions(); + final Scan missScan = scan; + Assert.assertThrows(IOException.class, () -> { + hTable.getScanner(missScan); }); hTable.delete(deleteKey1Family); From 4eb3745ed8682ec1fe830c17fa003fc816ca64d7 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Tue, 22 Oct 2024 10:27:56 +0800 Subject: [PATCH 7/7] correct name --- src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index b10fc68d..072682e6 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -3561,9 +3561,9 @@ public void testScan() throws Exception { scan.addFamily(toBytes(family)); scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); scan.setMaxVersions(); - final Scan missScan = scan; + final Scan missFamScan = scan; Assert.assertThrows(IOException.class, () -> { - hTable.getScanner(missScan); + hTable.getScanner(missFamScan); }); hTable.delete(deleteKey1Family);