diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 00d717b3..4ecf0aa5 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -585,6 +585,9 @@ public Result call() throws IOException { // In a Get operation where the family map is greater than 1 or equal to 0, // we handle this by appending the column family to the qualifier on the client side. // The server can then use this information to filter the appropriate column families and qualifiers. + if (!get.getColumnFamilyTimeRange().isEmpty()) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } NavigableSet columnFilters = new TreeSet<>(Bytes.BYTES_COMPARATOR); processColumnFilters(columnFilters, get.getFamilyMap()); obTableQuery = buildObTableQuery(get, columnFilters); @@ -598,6 +601,17 @@ public Result call() throws IOException { for (Map.Entry> entry : get.getFamilyMap() .entrySet()) { family = entry.getKey(); + if (!get.getColumnFamilyTimeRange().isEmpty()) { + Map colFamTimeRangeMap = get.getColumnFamilyTimeRange(); + if (colFamTimeRangeMap.size() > 1) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } else if (colFamTimeRangeMap.get(family) == null) { + throw new IllegalArgumentException("Get family is not matched in ColumnFamilyTimeRange"); + } else { + TimeRange tr = colFamTimeRangeMap.get(family); + get.setTimeRange(tr.getMin(), tr.getMax()); + } + } obTableQuery = buildObTableQuery(get, entry.getValue()); request = buildObTableQueryAsyncRequest(obTableQuery, getTargetTableName(tableNameString, Bytes.toString(family), @@ -668,6 +682,9 @@ public ResultScanner call() throws IOException { // In a Scan operation where the family map is greater than 1 or equal to 0, // we handle this by appending the column family to the qualifier on the client side. // The server can then use this information to filter the appropriate column families and qualifiers. + if (!scan.getColumnFamilyTimeRange().isEmpty()) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } NavigableSet columnFilters = new TreeSet<>(Bytes.BYTES_COMPARATOR); processColumnFilters(columnFilters, scan.getFamilyMap()); filter = buildObHTableFilter(scan.getFilter(), scan.getTimeRange(), @@ -684,6 +701,17 @@ public ResultScanner call() throws IOException { for (Map.Entry> entry : scan.getFamilyMap() .entrySet()) { family = entry.getKey(); + if (!scan.getColumnFamilyTimeRange().isEmpty()) { + Map colFamTimeRangeMap = scan.getColumnFamilyTimeRange(); + if (colFamTimeRangeMap.size() > 1) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } else if (colFamTimeRangeMap.get(family) == null) { + throw new IllegalArgumentException("Scan family is not matched in ColumnFamilyTimeRange"); + } else { + TimeRange tr = colFamTimeRangeMap.get(family); + scan.setTimeRange(tr.getMin(), tr.getMax()); + } + } filter = buildObHTableFilter(scan.getFilter(), scan.getTimeRange(), scan.getMaxVersions(), entry.getValue()); obTableQuery = buildObTableQuery(filter, scan); diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index 522f2ab2..94812d4c 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -41,9 +41,13 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte throws IOException { if (filter == null) { throw new IllegalArgumentException("Filter is null"); + } else if (filter instanceof DependentColumnFilter) { + toParseableByteArray(byteStream, (DependentColumnFilter) filter); } else if (filter instanceof CompareFilter) { // RowFilter, ValueFilter, QualifierFilter toParseableByteArray(byteStream, (CompareFilter) filter); + } else if (filter instanceof SingleColumnValueExcludeFilter) { + toParseableByteArray(byteStream, (SingleColumnValueExcludeFilter) filter); } else if (filter instanceof SingleColumnValueFilter) { toParseableByteArray(byteStream, (SingleColumnValueFilter) filter); } else if (filter instanceof PageFilter) { @@ -160,6 +164,25 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write(')'); } + // SingleColumnValueExcludeFilter('cf1','col1',=,'binary:123',true,true) + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + SingleColumnValueExcludeFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getFilterIfMissing()).getBytes()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getLatestVersionOnly()).getBytes()); + byteStream.write(')'); + } + // PageFilter(100); private static void toParseableByteArray(ByteArrayOutputStream byteStream, PageFilter filter) throws IOException { @@ -178,6 +201,34 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write(')'); } + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + DependentColumnFilter filter) throws IOException { + // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ')' + if (filter.getComparator() == null) { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(filter.getDropDependentColumn()).getBytes()); + byteStream.write(')'); + } else { // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ',' compare_op ',' comparator ')' + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(filter.getDropDependentColumn()).getBytes()); + byteStream.write(','); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(')'); + } + } + private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnPaginationFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index b8fe6c8c..072682e6 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -524,6 +524,68 @@ public void testFilter() throws Exception { // | getKey2 | def | -1728834971818 | value1 | // +---------+-----+----------------+--------+ + SingleColumnValueFilter singleColumnValueFilter; + singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueFilter); + r = hTable.get(get); + Assert.assertEquals(7, r.raw().length); + + SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; + singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueExcludeFilter); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), true); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + + + filter = new ColumnPrefixFilter(Bytes.toBytes("e")); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -832,6 +894,66 @@ public void testFilter() throws Exception { } Assert.assertEquals(res_count, 12); scanner.close(); + + long timestamp = System.currentTimeMillis(); + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), timestamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timestamp, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + // putKey1Column1Value1 and putKey2Column1Value1 have the same timestamp + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column2Value2); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setStartRow("getKey1".getBytes()); + scan.setStopRow("getKey3".getBytes()); + scan.setMaxVersions(10); + scan.setFilter(dependentColumnFilter); + scanner = hTable.getScanner(scan); + + long prevTimestamp = - 1; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + if (prevTimestamp == -1) { + prevTimestamp = keyValue.getTimestamp(); + } else { + Assert.assertEquals(prevTimestamp, keyValue.getTimestamp()); + } + } + } + scanner.close(); } @Test @@ -1941,6 +2063,15 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(1, r.raw().length); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + // columnCountGetFilter filter 2 get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -2121,6 +2252,67 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(7, r.raw().length); + filterList = new FilterList(); + filterList.addFilter(new SingleColumnValueExcludeFilter(Bytes.toBytes(family), Bytes + .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column1), false)); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), false)); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), true)); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(value2)))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + filterList = new FilterList(); filterList.addFilter(new ColumnCountGetFilter(1)); filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.GREATER, @@ -2246,6 +2438,19 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(7, r.raw().length); + + SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; + singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueExcludeFilter); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); @@ -2677,23 +2882,25 @@ public void testGet() throws Exception { tryPut(hTable, putzKey2Column1Value1); // show table (time maybe different) - //+-----------+---------+----------------+--------+ - //| K | Q | T | V | - //+-----------+---------+----------------+--------+ - //| scanKey1x | column1 | -1709714409669 | value1 | - //| scanKey1x | column1 | -1709714409637 | value2 | - //| scanKey1x | column1 | -1709714409603 | value1 | - //| scanKey1x | column2 | -1709714409802 | value2 | - //| scanKey1x | column2 | -1709714409768 | value1 | - //| scanKey1x | column2 | -1709714409735 | value2 | - //| scanKey1x | column2 | -1709714409702 | value1 | - //| scanKey2x | column2 | -1709714409869 | value2 | - //| scanKey2x | column2 | -1709714409836 | value1 | - //| scanKey3x | column1 | -1709714409940 | value2 | - //| scanKey3x | column1 | -1709714409904 | value1 | - //| scanKey3x | column2 | -1709714410010 | value2 | - //| scanKey3x | column2 | -1709714409977 | value1 | - //+-----------+---------+----------------+--------+ + // +-----------+---------+----------------+--------+ + // | K | Q | T | V | + // +-----------+---------+----------------+--------+ + // | scanKey1x | column1 | -1729223351579 | value1 | + // | scanKey1x | column1 | -1729223351504 | value2 | + // | scanKey1x | column1 | -1729223351431 | value1 | + // | scanKey1x | column2 | -1729223351867 | value2 | + // | scanKey1x | column2 | -1729223351796 | value1 | + // | scanKey1x | column2 | -1729223351724 | value2 | + // | scanKey1x | column2 | -1729223351651 | value1 | + // | scanKey2x | column2 | -1729223352015 | value2 | + // | scanKey2x | column2 | -1729223351941 | value1 | + // | scanKey3x | column1 | -1729223352159 | value2 | + // | scanKey3x | column1 | -1729223352088 | value1 | + // | scanKey3x | column2 | -1729223352304 | value2 | + // | scanKey3x | column2 | -1729223352232 | value1 | + // | zScanKey1 | column1 | -1729223352378 | value1 | + // | zScanKey2 | column1 | -1729223352450 | value1 | + // +-----------+---------+----------------+--------+ // test closestRowBefore get = new Get("scanKey2x2".getBytes()); @@ -2725,6 +2932,156 @@ public void testGet() throws Exception { assertTrue(booleans[2]); assertFalse(booleans[3]); assertTrue(booleans[4]); + + // test single cf setColumnFamilyTimeRange + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + hTable.delete(deleteZKey1Family); + hTable.delete(deleteZKey2Family); + + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), timeStamp2, toBytes(value1)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timeStamp4, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), timeStamp5, toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), timeStamp7, toBytes(value2)); + + putKey3Column1Value1 = new Put(toBytes(key3)); + putKey3Column1Value1.add(toBytes(family), toBytes(column1), timeStamp8, toBytes(value1)); + + putKey3Column1Value2 = new Put(toBytes(key3)); + putKey3Column1Value2.add(toBytes(family), toBytes(column1), timeStamp9, toBytes(value2)); + + putKey3Column2Value1 = new Put(toBytes(key3)); + putKey3Column2Value1.add(toBytes(family), toBytes(column2), timeStamp10, toBytes(value1)); + + putKey3Column2Value2 = new Put(toBytes(key3)); + putKey3Column2Value2.add(toBytes(family), toBytes(column2), timeStamp11, toBytes(value2)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column1Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + tryPut(hTable, putKey3Column1Value1); + tryPut(hTable, putKey3Column1Value2); + tryPut(hTable, putKey3Column2Value1); + tryPut(hTable, putKey3Column2Value2); + + + get = new Get(toBytes(key1)); + get.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, maxTimeStamp); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + get = new Get(toBytes(key1)); + get.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, timeStamp2 + 1); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + get = new Get(toBytes(key2)); + // set invalid timeRange + get.setTimeRange(minTimeStamp, maxTimeStamp); + get.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, timeStamp5 + 1); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key2)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp5, maxTimeStamp); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp8, timeStamp8); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp8, timeStamp9); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp8, timeStamp9); + get.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp8, timeStamp9); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + final Get multiFamGet = get; + Assert.assertThrows(IOException.class, () -> { + hTable.get(multiFamGet); + }); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp8, timeStamp9); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + final Get missFamGet = get; + Assert.assertThrows(IOException.class, () -> { + hTable.get(missFamGet); + }); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); } @Test @@ -2824,23 +3181,25 @@ public void testScan() throws Exception { tryPut(hTable, putzKey2Column1Value1); // show table (time maybe different) - //+-----------+---------+----------------+--------+ - //| K | Q | T | V | - //+-----------+---------+----------------+--------+ - //| scanKey1x | column1 | -1709714409669 | value1 | - //| scanKey1x | column1 | -1709714409637 | value2 | - //| scanKey1x | column1 | -1709714409603 | value1 | - //| scanKey1x | column2 | -1709714409802 | value2 | - //| scanKey1x | column2 | -1709714409768 | value1 | - //| scanKey1x | column2 | -1709714409735 | value2 | - //| scanKey1x | column2 | -1709714409702 | value1 | - //| scanKey2x | column2 | -1709714409869 | value2 | - //| scanKey2x | column2 | -1709714409836 | value1 | - //| scanKey3x | column1 | -1709714409940 | value2 | - //| scanKey3x | column1 | -1709714409904 | value1 | - //| scanKey3x | column2 | -1709714410010 | value2 | - //| scanKey3x | column2 | -1709714409977 | value1 | - //+-----------+---------+----------------+--------+ + // +-----------+---------+----------------+--------+ + // | K | Q | T | V | + // +-----------+---------+----------------+--------+ + // | scanKey1x | column1 | -1729236392149 | value1 | + // | scanKey1x | column1 | -1729236392078 | value2 | + // | scanKey1x | column1 | -1729236392008 | value1 | + // | scanKey1x | column2 | -1729236392436 | value2 | + // | scanKey1x | column2 | -1729236392364 | value1 | + // | scanKey1x | column2 | -1729236392291 | value2 | + // | scanKey1x | column2 | -1729236392220 | value1 | + // | scanKey2x | column2 | -1729236392576 | value2 | + // | scanKey2x | column2 | -1729236392506 | value1 | + // | scanKey3x | column1 | -1729236392720 | value2 | + // | scanKey3x | column1 | -1729236392647 | value1 | + // | scanKey3x | column2 | -1729236392861 | value2 | + // | scanKey3x | column2 | -1729236392790 | value1 | + // | zScanKey1 | column1 | -1729236392931 | value1 | + // | zScanKey2 | column1 | -1729236393002 | value1 | + // +-----------+---------+----------------+--------+ scan = new Scan(); scan.addFamily(family.getBytes()); @@ -3027,6 +3386,189 @@ public void testScan() throws Exception { hTable.delete(deleteKey3Family); hTable.delete(deleteZKey1Family); hTable.delete(deleteZKey2Family); + + // test single cf setColumnFamilyTimeRange + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), timeStamp2, toBytes(value1)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timeStamp4, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), timeStamp5, toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), timeStamp7, toBytes(value2)); + + putKey3Column1Value1 = new Put(toBytes(key3)); + putKey3Column1Value1.add(toBytes(family), toBytes(column1), timeStamp8, toBytes(value1)); + + putKey3Column1Value2 = new Put(toBytes(key3)); + putKey3Column1Value2.add(toBytes(family), toBytes(column1), timeStamp9, toBytes(value2)); + + putKey3Column2Value1 = new Put(toBytes(key3)); + putKey3Column2Value1.add(toBytes(family), toBytes(column2), timeStamp10, toBytes(value1)); + + putKey3Column2Value2 = new Put(toBytes(key3)); + putKey3Column2Value2.add(toBytes(family), toBytes(column2), timeStamp11, toBytes(value2)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column1Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + tryPut(hTable, putKey3Column1Value1); + tryPut(hTable, putKey3Column1Value2); + tryPut(hTable, putKey3Column2Value1); + tryPut(hTable, putKey3Column2Value2); + + // scan key1 + key2 + scan = new Scan(toBytes(key1), toBytes(key3)); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, maxTimeStamp); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(8, res_count); + + // scan key1 + scan = new Scan(toBytes(key1), toBytes(key2)); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, maxTimeStamp); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(4, res_count); + + // scan key1 + scan = new Scan(toBytes(key1), toBytes(key2)); + scan.addFamily(toBytes(family)); + // set invalid timeRange + scan.setTimeRange(minTimeStamp, maxTimeStamp); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp1, timeStamp3); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(2, res_count); + + // scan all + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp2, timeStamp7); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(5, res_count); + + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + boolean foundKey3 = false; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (!foundKey3) { + if (key3.equals(Bytes.toString(kv.getRow()))) { + foundKey3 = true; + } + } + ++res_count; + } + } + Assert.assertTrue(foundKey3); + Assert.assertEquals(6, res_count); + + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); + scan.setMaxVersions(); + final Scan multiFamScan = scan; + Assert.assertThrows(IOException.class, () -> { + hTable.getScanner(multiFamScan); + }); + + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); + scan.setMaxVersions(); + final Scan missFamScan = scan; + Assert.assertThrows(IOException.class, () -> { + hTable.getScanner(missFamScan); + }); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); } @Test @@ -3249,6 +3791,198 @@ public void testReversedScan() throws Exception { hTable.delete(deleteKey2Family); hTable.delete(deleteKey3Family); + // test single cf setColumnFamilyTimeRange + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), timeStamp2, toBytes(value1)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timeStamp4, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), timeStamp5, toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), timeStamp7, toBytes(value2)); + + putKey3Column1Value1 = new Put(toBytes(key3)); + putKey3Column1Value1.add(toBytes(family), toBytes(column1), timeStamp8, toBytes(value1)); + + putKey3Column1Value2 = new Put(toBytes(key3)); + putKey3Column1Value2.add(toBytes(family), toBytes(column1), timeStamp9, toBytes(value2)); + + putKey3Column2Value1 = new Put(toBytes(key3)); + putKey3Column2Value1.add(toBytes(family), toBytes(column2), timeStamp10, toBytes(value1)); + + putKey3Column2Value2 = new Put(toBytes(key3)); + putKey3Column2Value2.add(toBytes(family), toBytes(column2), timeStamp11, toBytes(value2)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column1Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + tryPut(hTable, putKey3Column1Value1); + tryPut(hTable, putKey3Column1Value2); + tryPut(hTable, putKey3Column2Value1); + tryPut(hTable, putKey3Column2Value2); + + // scan key1 + scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(family.getBytes()); + scan.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, timeStamp3); + scan.setReversed(true); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(3, res_count); + + // scan key1 + key2 + scan = new Scan(); + scan.setStartRow(toBytes(key2)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp2, timeStamp7); + scan.setReversed(true); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(5, res_count); + + // scan key2 + scan = new Scan(); + scan.setStartRow(toBytes(key2)); + scan.setStopRow(toBytes(key1)); + scan.addFamily(toBytes(family)); + // set invalid timeRange + scan.setTimeRange(minTimeStamp, maxTimeStamp); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp4, timeStamp6); + scan.setReversed(true); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(2, res_count); + + // scan all + scan = new Scan(); + scan.setStartRow(toBytes(key3)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp2, timeStamp7); + scan.setReversed(true); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(5, res_count); + + scan = new Scan(); + scan.setStartRow(toBytes(key3)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setReversed(true); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + boolean foundKey3 = false; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (!foundKey3) { + if (key3.equals(Bytes.toString(kv.getRow()))) { + foundKey3 = true; + } + } + ++res_count; + } + } + Assert.assertTrue(foundKey3); + Assert.assertEquals(6, res_count); + + scan = new Scan(); + scan.setStartRow(toBytes(key3)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); + scan.setReversed(true); + scan.setMaxVersions(); + final Scan errorScan = scan; + Assert.assertThrows(IOException.class, () -> { + hTable.getScanner(errorScan); + }); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + } @Test diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index f41e6bca..03617c2c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -218,6 +218,10 @@ public void testBufferedMutatorWithFlush() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + // only support Put and Delete // for other type of operations, BufferedMutator will not set its type for them Append append = new Append(Bytes.toBytes(key)); @@ -420,6 +424,10 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); @@ -509,6 +517,10 @@ public void testBufferedMutatorWithUserPool() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); @@ -598,6 +610,10 @@ public void testBufferedMutatorConcurrent() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + for (int i = 0; i < 50; ++i) { final int taskId = i; final BufferedMutator thrBufferMutator = ohBufferMutator;