From 0293b454543665cd4b0ca64eb25c0458d0b0f76f Mon Sep 17 00:00:00 2001 From: miyuan-ljr Date: Wed, 16 Oct 2024 15:24:08 +0800 Subject: [PATCH 01/21] range relevant filter (#80) --- .../hbase/filter/HBaseFilterUtils.java | 29 ++++ .../oceanbase/hbase/HTableTestBase.java | 152 ++++++++++++++++++ .../hbase/filter/HBaseFilterUtilsTest.java | 11 ++ 3 files changed, 192 insertions(+) diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index e58d2590..34d25699 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -26,6 +26,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.List; +import java.util.Set; @InterfaceAudience.Private public class HBaseFilterUtils { @@ -49,6 +50,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte toParseableByteArray(byteStream, (PageFilter) filter); } else if (filter instanceof ColumnCountGetFilter) { toParseableByteArray(byteStream, (ColumnCountGetFilter) filter); + } else if (filter instanceof FirstKeyValueMatchingQualifiersFilter) { + toParseableByteArray(byteStream, (FirstKeyValueMatchingQualifiersFilter) filter); } else if (filter instanceof PrefixFilter) { toParseableByteArray(byteStream, (PrefixFilter) filter); } else if (filter instanceof FilterList) { @@ -243,6 +246,32 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write(')'); } + // FirstKeyValueMatchingQualifiersFilter('q1','q2') + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + FirstKeyValueMatchingQualifiersFilter filter) throws IOException { + Set qualifiers; + try { + Field field = filter.getClass().getDeclaredField("qualifiers"); + field.setAccessible(true); + qualifiers = (Set)field.get(filter); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + int i = 0; + for (byte[] qualifier: qualifiers) { + byteStream.write("'".getBytes()); + byteStream.write(qualifier); + byteStream.write("'".getBytes()); + if (i < qualifiers.size() - 1) { + byteStream.write(','); + } + i++; + } + byteStream.write(')'); + } + // PrefixFilter('prefix'); private static void toParseableByteArray(ByteArrayOutputStream byteStream, PrefixFilter filter) throws IOException { diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 4c93efde..182dd5cb 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -1124,6 +1124,158 @@ public void testFilter2() throws Exception { scanner.close(); } + @Test + public void testFirstKeyValueMatchingQualifiersFilter() throws Exception { + String key1 = "getKey1"; + String key2 = "getKey2"; + String column1 = "c1"; + String column2 = "c2"; + String column3 = "c3"; + String column4 = "c4"; + String column5 = "c5"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + String family = "family1"; + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family)); + + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + + Put putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + Put putKey1Column3Value1 = new Put(toBytes(key1)); + putKey1Column3Value1.add(toBytes(family), toBytes(column3), toBytes(value1)); + + Put putKey1Column4Value1 = new Put(toBytes(key1)); + putKey1Column4Value1.add(toBytes(family), toBytes(column4), toBytes(value1)); + + Put putKey1Column5Value1 = new Put(toBytes(key1)); + putKey1Column5Value1.add(toBytes(family), toBytes(column5), toBytes(value1)); + + Put putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column3Value1); + tryPut(hTable, putKey1Column4Value1); + tryPut(hTable, putKey1Column5Value1); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + + Scan scan; + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + TreeSet qualifiers = new TreeSet<>(Bytes.BYTES_COMPARATOR); + qualifiers.add(Bytes.toBytes("c11")); + qualifiers.add(Bytes.toBytes("c2")); + FirstKeyValueMatchingQualifiersFilter filter = new FirstKeyValueMatchingQualifiersFilter(qualifiers); + scan.setFilter(filter); + ResultScanner scanner = hTable.getScanner(scan); + + int res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 5); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + qualifiers = new TreeSet<>(Bytes.BYTES_COMPARATOR); + qualifiers.add(Bytes.toBytes("c22")); + qualifiers.add(Bytes.toBytes("c4")); + filter = new FirstKeyValueMatchingQualifiersFilter(qualifiers); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 11); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + qualifiers = new TreeSet<>(Bytes.BYTES_COMPARATOR); + qualifiers.add(Bytes.toBytes("c22")); + qualifiers.add(Bytes.toBytes("a")); + filter = new FirstKeyValueMatchingQualifiersFilter(qualifiers); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 12); + scanner.close(); + } + @Test public void testGetFilter() throws Exception { String key1 = "getKey1"; diff --git a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java index b228f0a4..dd57b1df 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java @@ -24,6 +24,7 @@ import org.junit.Test; import java.io.IOException; +import java.util.TreeSet; public class HBaseFilterUtilsTest { private static final CompareFilter.CompareOp[] ops = { CompareFilter.CompareOp.LESS, @@ -166,6 +167,16 @@ public void testColumnCountGetFilter() throws IOException { HBaseFilterUtils.toParseableByteArray(filter)); } + @Test + public void testFirstKeyValueMatchingQualifiersFilter() throws IOException { + TreeSet qualifiers = new TreeSet<>(Bytes.BYTES_COMPARATOR); + qualifiers.add(Bytes.toBytes("q1")); + qualifiers.add(Bytes.toBytes("q2")); + FirstKeyValueMatchingQualifiersFilter filter = new FirstKeyValueMatchingQualifiersFilter(qualifiers); + Assert.assertArrayEquals("FirstKeyValueMatchingQualifiersFilter('q1','q2')".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + @Test public void testPrefixFilter() throws IOException { PrefixFilter filter = new PrefixFilter("prefix".getBytes()); From 3c8c53106ff33f01bb366aa9b1c7900df1937c2b Mon Sep 17 00:00:00 2001 From: miyuan-ljr Date: Wed, 16 Oct 2024 15:36:48 +0800 Subject: [PATCH 02/21] Range relevant filter (#79) * range relevant filter * review --- .../hbase/filter/HBaseFilterUtils.java | 79 +++ .../oceanbase/hbase/HTableTestBase.java | 532 ++++++++++++++++++ .../hbase/filter/HBaseFilterUtilsTest.java | 36 ++ 3 files changed, 647 insertions(+) diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index 34d25699..522f2ab2 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -68,6 +68,14 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte toParseableByteArray(byteStream, (KeyOnlyFilter) filter); } else if (filter instanceof TimestampsFilter) { toParseableByteArray(byteStream, (TimestampsFilter) filter); + } else if (filter instanceof MultiRowRangeFilter) { + toParseableByteArray(byteStream, (MultiRowRangeFilter) filter); + } else if (filter instanceof InclusiveStopFilter) { + toParseableByteArray(byteStream, (InclusiveStopFilter) filter); + } else if (filter instanceof ColumnRangeFilter) { + toParseableByteArray(byteStream, (ColumnRangeFilter) filter); + } else if (filter instanceof MultipleColumnPrefixFilter) { + toParseableByteArray(byteStream, (MultipleColumnPrefixFilter) filter); } else if (filter instanceof SkipFilter) { toParseableByteArray(byteStream, (SkipFilter) filter); } else if (filter instanceof WhileMatchFilter) { @@ -237,6 +245,77 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Times byteStream.write(')'); } + // MultiRowRangeFilter('a',true,'b',false,'c',true,'d',false); + private static void toParseableByteArray(ByteArrayOutputStream byteStream, MultiRowRangeFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + + List ranges = filter.getRowRanges(); + for (int i = 0; i < ranges.size(); i ++) { + MultiRowRangeFilter.RowRange range = ranges.get(i); + byteStream.write("'".getBytes()); + byteStream.write(range.getStartRow()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(range.isStartRowInclusive()).getBytes()); + byteStream.write(','); + + byteStream.write("'".getBytes()); + byteStream.write(range.getStopRow()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(range.isStopRowInclusive()).getBytes()); + if (i < ranges.size() - 1) { + byteStream.write(','); + } + } + byteStream.write(')'); + } + + // InclusiveStopFilter('aaa'); + private static void toParseableByteArray(ByteArrayOutputStream byteStream, InclusiveStopFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + byteStream.write('\''); + byteStream.write(filter.getStopRowKey()); + byteStream.write('\''); + byteStream.write(')'); + } + + // ColumnRangeFilter('a',true,'b',false); + private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnRangeFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + + byteStream.write("'".getBytes()); + byteStream.write(filter.getMinColumn()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(filter.getMinColumnInclusive()).getBytes()); + byteStream.write(','); + + byteStream.write("'".getBytes()); + byteStream.write(filter.getMaxColumn()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(filter.getMaxColumnInclusive()).getBytes()); + byteStream.write(')'); + } + + // MultipleColumnPrefixFilter('a','b','d'); + private static void toParseableByteArray(ByteArrayOutputStream byteStream, MultipleColumnPrefixFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + + byte[][] ranges = filter.getPrefix(); + for (int i = 0; i < ranges.length; i ++) { + byte[] range = ranges[i]; + byteStream.write("'".getBytes()); + byteStream.write(range); + byteStream.write("'".getBytes()); + if (i < ranges.length - 1) { + byteStream.write(','); + } + } + byteStream.write(')'); + } + // ColumnCountGetFilter(100) private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnCountGetFilter filter) throws IOException { diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 182dd5cb..e55e3b3c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -819,6 +819,538 @@ public void testFilter() throws Exception { scanner.close(); } + @Test + public void testRowRangeFilter() throws Exception { + String key1 = "ka1"; + String key2 = "kb2"; + String column1 = "abc"; + String column2 = "def"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + String family = "family1"; + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family)); + + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + + Put putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + Put putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + + Scan scan; + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + List ranges = new ArrayList<>(); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("ka3"), true, Bytes.toBytes("kd4"), false)); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("c"), true, Bytes.toBytes("d$%%"), false)); + MultiRowRangeFilter filter = new MultiRowRangeFilter(ranges); + scan.setFilter(filter); + ResultScanner scanner = hTable.getScanner(scan); + + int res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 2); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + ranges = new ArrayList<>(); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("ka3"), true, Bytes.toBytes("kd4"), false)); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("c"), true, Bytes.toBytes("d$%%"), false)); + filter = new MultiRowRangeFilter(ranges); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 2); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + ranges = new ArrayList<>(); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("kb2"), false, Bytes.toBytes("kd4"), false)); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("kb11"), true, Bytes.toBytes("kb2"), true)); + filter = new MultiRowRangeFilter(ranges); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 2); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + ranges = new ArrayList<>(); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("z4"), true, Bytes.toBytes("zzzsad"), true)); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("kb2"), true, Bytes.toBytes("kd4"), false)); + filter = new MultiRowRangeFilter(ranges); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 2); + scanner.close(); + + // Inclusive stop filter + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + InclusiveStopFilter iFilter = new InclusiveStopFilter(Bytes.toBytes("ka1")); + scan.setFilter(iFilter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + Assert.assertArrayEquals(key1.getBytes(), keyValue.getRow()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 7); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setReversed(true); + scan.setMaxVersions(10); + iFilter = new InclusiveStopFilter(Bytes.toBytes("ka1")); + scan.setFilter(iFilter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + Assert.assertArrayEquals(key1.getBytes(), keyValue.getRow()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 7); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + iFilter = new InclusiveStopFilter(Bytes.toBytes("kb1")); + scan.setFilter(iFilter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 7); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + iFilter = new InclusiveStopFilter(Bytes.toBytes("kb2")); + scan.setFilter(iFilter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 9); + scanner.close(); + } + + @Test + public void testColumnRangeFilter() throws Exception { + String key1 = "ka1"; + String key2 = "kb2"; + String column1 = "abc"; + String column2 = "def"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + String family = "family1"; + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family)); + + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + + Put putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + Put putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + + Scan scan; + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, Bytes.toBytes("b"), false); + scan.setFilter(filter); + ResultScanner scanner = hTable.getScanner(scan); + + int res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 3); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + filter = new ColumnRangeFilter(Bytes.toBytes("abc"), true, Bytes.toBytes("def"), false); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 3); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + filter = new ColumnRangeFilter(Bytes.toBytes("abc"), false, Bytes.toBytes("def"), true); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 6); + scanner.close(); + + // MultipleColumnPrefixFilter + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + byte[][] range = { + Bytes.toBytes("g"), + Bytes.toBytes("3"), + Bytes.toBytes("d"), + }; + MultipleColumnPrefixFilter iFilter = new MultipleColumnPrefixFilter(range); + scan.setFilter(iFilter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 6); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + range = new byte[][]{ + Bytes.toBytes("de"), + Bytes.toBytes("bg"), + Bytes.toBytes("nc"), + Bytes.toBytes("aa"), + Bytes.toBytes("abcd"), + Bytes.toBytes("dea"), + }; + iFilter = new MultipleColumnPrefixFilter(range); + scan.setFilter(iFilter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 6); + scanner.close(); + } + + @Test + public void testFilterNullRange() throws Exception { + String key1 = "ka1"; + String key2 = "kb2"; + String column1 = "abc"; + String column2 = "def"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + String family = "family1"; + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family)); + + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + + Put putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + Put putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + + Scan scan; + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + byte[][] r = {}; + MultipleColumnPrefixFilter iFilter = new MultipleColumnPrefixFilter(r); + scan.setFilter(iFilter); + ResultScanner scanner = hTable.getScanner(scan); + + int res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 0); + scanner.close(); + } + @Test public void testFilter2() throws Exception { String key1 = "getKey1"; diff --git a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java index dd57b1df..fe7a7082 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java @@ -24,6 +24,8 @@ import org.junit.Test; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.TreeSet; public class HBaseFilterUtilsTest { @@ -160,6 +162,40 @@ public void testColumnPrefixFilter() throws IOException { HBaseFilterUtils.toParseableByteArray(filter)); } + @Test + public void testMultiRowRangeFilter() throws IOException { + List ranges = new ArrayList<>(); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("a"), true, Bytes.toBytes("b"), false)); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("c"), true, Bytes.toBytes("d$%%"), false)); + + MultiRowRangeFilter filter = new MultiRowRangeFilter(ranges); + System.out.println(Bytes.toString(HBaseFilterUtils.toParseableByteArray(filter))); + Assert.assertArrayEquals("MultiRowRangeFilter('a',true,'b',false,'c',true,'d$%%',false)".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + } + + @Test + public void testInclusiveStopFilter() throws IOException { + InclusiveStopFilter filter = new InclusiveStopFilter(Bytes.toBytes("aaa")); + Assert.assertArrayEquals("InclusiveStopFilter('aaa')".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + } + + @Test + public void testColumnRangeFilter() throws IOException { + ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, Bytes.toBytes("b"), false); + Assert.assertArrayEquals("ColumnRangeFilter('a',true,'b',false)".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + } + + @Test + public void testMultipleColumnPrefixFilter() throws IOException { + byte[][] prefix = { + Bytes.toBytes("a"), + Bytes.toBytes("b"), + Bytes.toBytes("d"), + }; + MultipleColumnPrefixFilter filter = new MultipleColumnPrefixFilter(prefix); + Assert.assertArrayEquals("MultipleColumnPrefixFilter('a','b','d')".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + } + @Test public void testColumnCountGetFilter() throws IOException { ColumnCountGetFilter filter = new ColumnCountGetFilter(513); From fe9da781a7244c914660741c0d6e769ebb87fcce Mon Sep 17 00:00:00 2001 From: Ziyu Shi <57038180+JackShi148@users.noreply.github.com> Date: Thu, 17 Oct 2024 20:52:47 +0800 Subject: [PATCH 03/21] Namespace mapping to database in observer in one HBase client; set RPC connect timeout to Table Client (#78) * finish connection function * hbase 1.x compatibility: filter relevant (#51) * filter * add scan tests * add scan lease test (#53) * support allowPatialResult (#56) * [Chore] use obkv-table-client 1.2.13-SNAPSHOT for testing (#57) * init bufferedMutator * finish validateFamily and asyncExecute * correct log in OHBufferedMutatorImpl * pass self-test * format code * get bugfix * add retry when batch fails * remove test print * format code * make interface more generalized * test (#59) * random row filter (#62) * format BufferedMutator test case * remove redundancy, add some comments * fix (#66) * fix type of a bufferedMutator. Optimize by review * OHBufferedMutator in OBKV Hbase 1_x_comp (#64) * init bufferedMutator * finish validateFamily and asyncExecute * correct log in OHBufferedMutatorImpl * pass self-test * format code * add retry when batch fails * remove test print * format code * make interface more generalized * format BufferedMutator test case * remove redundancy, add some comments * fix type of a bufferedMutator. Optimize by review * hbase_multi_column_family_dev (#67) * hbase_multi_column_family_dev * fix * fix revierw * OHBufferedMutator set and use runtimeBatchExecutor in ObTableClient * OHBufferedMutator set and add runtimeBatchExecutor in ObTableClient (#68) * init bufferedMutator * finish validateFamily and asyncExecute * correct log in OHBufferedMutatorImpl * pass self-test * format code * add retry when batch fails * remove test print * format code * make interface more generalized * format BufferedMutator test case * remove redundancy, add some comments * fix type of a bufferedMutator. Optimize by review * OHBufferedMutator set and use runtimeBatchExecutor in ObTableClient * Bugfix/sql and conflict fixes (#69) * add sql for multi-cf test; fix conflict error * fix buffered mutation family_violation check * Remove multi-family checks for buffered mutations * fix check family empty * [Chore] refresh code from master branch and upgrade table client version (#71) * [Fix] checkAndMutate and get/Scan with filter return -5006 when include special character (#58) * [Fix] checkAndMutate and get/Scan with filter return -5006 when include special character * [Chore] fix review * [Fix] fix review * [Fix] fix test * [Fix] fix test testScanSessionClean * multi-namespace in one HBase client * format code * set rpcConnectTimeout to tableClient * multi-namespace in ocp mode and odp mode * revert self-defined pom xml * use the test as initial testing case * format code * use 'default' database when param_url misses the database parameter * add new test case testing multi-namespace, remove some comments --------- Co-authored-by: stuBirdFly <1065492934@qq.com> Co-authored-by: stuBirdFly <84010733+stuBirdFly@users.noreply.github.com> Co-authored-by: miyuan-ljr Co-authored-by: Shen Yunlong <44497386+shenyunlong@users.noreply.github.com> Co-authored-by: DingZhen --- .../com/alipay/oceanbase/hbase/OHTable.java | 45 ++++++++-- .../alipay/oceanbase/hbase/OHTablePool.java | 35 ++++++++ .../hbase/constants/OHConstants.java | 4 + .../hbase/util/OHConnectionConfiguration.java | 38 ++++++++- .../oceanbase/hbase/util/OHTableFactory.java | 2 + .../hbase/util/ObTableClientManager.java | 15 +++- .../oceanbase/hbase/HTableTestBase.java | 15 ++++ .../oceanbase/hbase/OHConnectionTest.java | 13 ++- .../oceanbase/hbase/OHMultiNamespaceTest.java | 82 +++++++++++++++++++ .../oceanbase/hbase/OHTableClientTest.java | 4 + src/test/java/unit_test_db.sql | 52 ++++++++++++ 11 files changed, 292 insertions(+), 13 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHMultiNamespaceTest.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 3defa3c7..00d717b3 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -35,6 +35,7 @@ import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.*; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncRequest; import com.alipay.oceanbase.rpc.stream.ObTableClientQueryAsyncStreamResult; +import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult; import com.alipay.oceanbase.rpc.table.ObHBaseParams; import com.alipay.oceanbase.rpc.table.ObKVParams; import com.alipay.sofa.common.thread.SofaThreadPoolExecutor; @@ -195,8 +196,9 @@ public OHTable(Configuration configuration, String tableName) throws IOException long keepAliveTime = configuration.getLong(HBASE_HTABLE_THREAD_KEEP_ALIVE_TIME, DEFAULT_HBASE_HTABLE_THREAD_KEEP_ALIVE_TIME); this.executePool = createDefaultThreadPoolExecutor(1, maxThreads, keepAliveTime); - this.obTableClient = ObTableClientManager - .getOrCreateObTableClient(new OHConnectionConfiguration(configuration)); + OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); + this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( + this.tableNameString, ohConnectionConf)); finishSetUp(); } @@ -242,8 +244,9 @@ public OHTable(Configuration configuration, final byte[] tableName, this.tableNameString = Bytes.toString(tableName); this.executePool = executePool; this.cleanupPoolOnClose = false; - this.obTableClient = ObTableClientManager - .getOrCreateObTableClient(new OHConnectionConfiguration(configuration)); + OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); + this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( + this.tableNameString, ohConnectionConf)); finishSetUp(); } @@ -260,6 +263,7 @@ public OHTable(Configuration configuration, final byte[] tableName, * @param executePool ExecutorService to be used. * @throws IllegalArgumentException if the param error */ + @InterfaceAudience.Private public OHTable(final byte[] tableName, final ObTableClient obTableClient, final ExecutorService executePool) { checkArgument(tableName != null, "tableNameString is blank."); @@ -306,7 +310,8 @@ public OHTable(TableName tableName, Connection connection, DEFAULT_HBASE_HTABLE_PUT_WRITE_BUFFER_CHECK); this.writeBufferSize = connectionConfig.getWriteBufferSize(); this.tableName = tableName.getName(); - this.obTableClient = ObTableClientManager.getOrCreateObTableClient(connectionConfig); + this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( + this.tableNameString, connectionConfig)); } /** @@ -364,6 +369,36 @@ private void finishSetUp() { WRITE_BUFFER_SIZE_DEFAULT); } + private OHConnectionConfiguration setUserDefinedNamespace(String tableNameString, + OHConnectionConfiguration ohConnectionConf) + throws IOException { + if (tableNameString.indexOf(':') != -1) { + String[] params = tableNameString.split(":"); + if (params.length != 2) { + throw new IllegalArgumentException("Please check the format of self-defined " + + "namespace and qualifier: { " + + tableNameString + " }"); + } + String database = params[0]; + checkArgument(isNotBlank(database), "self-defined namespace cannot be blank or null { " + + tableNameString + " }"); + if (ohConnectionConf.isOdpMode()) { + ohConnectionConf.setDatabase(database); + } else { + String databaseSuffix = "database=" + database; + String paramUrl = ohConnectionConf.getParamUrl(); + int databasePos = paramUrl.indexOf("database"); + if (databasePos == -1) { + paramUrl += "&" + databaseSuffix; + } else { + paramUrl = paramUrl.substring(0, databasePos) + databaseSuffix; + } + ohConnectionConf.setParamUrl(paramUrl); + } + } + return ohConnectionConf; + } + @Override public byte[] getTableName() { return tableName; diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java b/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java index d4b92d08..507a09b1 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java @@ -49,6 +49,7 @@ public class OHTablePool implements Closeable { + private String originTabelName = null; private final PoolMap tables; private final int maxSize; private final PoolMap.PoolType poolType; @@ -314,6 +315,9 @@ int getCurrentPoolSize(String tableName) { * @param paramUrl the table root server http url */ public void setParamUrl(final String tableName, String paramUrl) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_PARAM_URL, Bytes.toBytes(paramUrl)); } @@ -334,6 +338,9 @@ public String getParamUrl(final String tableName) { * @param fullUserName the table login username */ public void setFullUserName(final String tableName, final String fullUserName) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_FULL_USER_NAME, Bytes.toBytes(fullUserName)); } @@ -354,6 +361,9 @@ public String getFullUserName(final String tableName) { * @param password the table login password */ public void setPassword(final String tableName, final String password) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_PASSWORD, Bytes.toBytes(password)); } @@ -374,6 +384,9 @@ public String getPassword(final String tableName) { * @param sysUserName the sys username */ public void setSysUserName(final String tableName, final String sysUserName) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_SYS_USER_NAME, Bytes.toBytes(sysUserName)); } @@ -394,6 +407,9 @@ public String getSysUserName(final String tableName) { * @param sysPassword the sys user password */ public void setSysPassword(final String tableName, final String sysPassword) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_SYS_PASSWORD, Bytes.toBytes(sysPassword)); } @@ -481,6 +497,10 @@ public long getWriteBufferSize(String tableName) { .toLong(attr); } + public String getOriginTableName() { + return this.originTabelName; + } + /** * Sets the operation timeout for the specified tables in this pool. * @@ -520,6 +540,9 @@ public int getOperationTimeout(String tableName) { * @param odpAddr ODP address */ public void setOdpAddr(final String tableName, String odpAddr) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_ODP_ADDR, Bytes.toBytes(odpAddr)); } @@ -545,6 +568,9 @@ public String getOdpAddr(String tableName) { * @param odpPort ODP port */ public void setOdpPort(final String tableName, int odpPort) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_ODP_PORT, Bytes.toBytes(odpPort)); } @@ -564,6 +590,9 @@ public int getOdpPort(String tableName) { * @param odpMode ODP mode */ public void setOdpMode(final String tableName, boolean odpMode) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_ODP_MODE, Bytes.toBytes(odpMode)); } @@ -583,6 +612,9 @@ public boolean getOdpMode(String tableName) { * @param database ODP database name */ public void setDatabase(final String tableName, String database) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_DATABASE, Bytes.toBytes(database)); } @@ -653,6 +685,9 @@ public Object getTableExtendAttribute(String tableName, String attributeName) { } public void setRuntimeBatchExecutor(String tableName, ExecutorService runtimeBatchExecutor) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableExtendAttribute(tableName, HBASE_OCEANBASE_BATCH_EXECUTOR, runtimeBatchExecutor); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java index bda64927..e10ef0b1 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java +++ b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java @@ -159,4 +159,8 @@ public final class OHConstants { public static final int MAX_KEYVALUE_SIZE_DEFAULT = -1; + public static final String SOCKET_TIMEOUT = "ipc.socket.timeout"; + + public static final int DEFAULT_SOCKET_TIMEOUT = 20000; // 20 seconds + } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java index e4789df3..f602fdda 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java @@ -25,11 +25,15 @@ import java.util.Properties; import static com.alipay.oceanbase.hbase.constants.OHConstants.*; +import static org.apache.commons.lang.StringUtils.isBlank; +import static org.apache.hadoop.hbase.ipc.RpcClient.DEFAULT_SOCKET_TIMEOUT_CONNECT; +import static org.apache.hadoop.hbase.ipc.RpcClient.SOCKET_TIMEOUT_CONNECT; @InterfaceAudience.Private public class OHConnectionConfiguration { + private String paramUrl; + private String database; private final Properties properties; - private final String paramUrl; private final String fullUsername; private final String password; private final String sysUsername; @@ -37,13 +41,13 @@ public class OHConnectionConfiguration { private final String odpAddr; private final int odpPort; private final boolean odpMode; - private final String database; private final long writeBufferSize; private final int operationTimeout; private final int scannerCaching; private final long scannerMaxResultSize; private final int maxKeyValueSize; private final int rpcTimeout; + private final int rpcConnectTimeout; public OHConnectionConfiguration(Configuration conf) { this.paramUrl = conf.get(HBASE_OCEANBASE_PARAM_URL); @@ -54,11 +58,27 @@ public OHConnectionConfiguration(Configuration conf) { this.odpAddr = conf.get(HBASE_OCEANBASE_ODP_ADDR); this.odpPort = conf.getInt(HBASE_OCEANBASE_ODP_PORT, -1); this.odpMode = conf.getBoolean(HBASE_OCEANBASE_ODP_MODE, false); - this.database = conf.get(HBASE_OCEANBASE_DATABASE); + String database = conf.get(HBASE_OCEANBASE_DATABASE); + if (isBlank(database)) { + database = "default"; + } + this.database = database; this.writeBufferSize = conf.getLong(WRITE_BUFFER_SIZE_KEY, WRITE_BUFFER_SIZE_DEFAULT); this.operationTimeout = conf.getInt("hbase.client.operation.timeout", 1200000); this.rpcTimeout = conf.getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT); + int rpcConnectTimeout = -1; + if (conf.get(SOCKET_TIMEOUT_CONNECT) != null) { + rpcConnectTimeout = conf.getInt(SOCKET_TIMEOUT_CONNECT, DEFAULT_SOCKET_TIMEOUT_CONNECT); + } else { + if (conf.get(SOCKET_TIMEOUT) != null) { + rpcConnectTimeout = conf.getInt(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT); + } else { + rpcConnectTimeout = conf.getInt(SOCKET_TIMEOUT_CONNECT, + DEFAULT_SOCKET_TIMEOUT_CONNECT); + } + } + this.rpcConnectTimeout = rpcConnectTimeout; this.scannerCaching = conf.getInt("hbase.client.scanner.caching", Integer.MAX_VALUE); this.scannerMaxResultSize = conf.getLong("hbase.client.scanner.max.result.size", WRITE_BUFFER_SIZE_DEFAULT); @@ -72,6 +92,14 @@ public OHConnectionConfiguration(Configuration conf) { } } + public void setParamUrl(String paramUrl) { + this.paramUrl = paramUrl; + } + + public void setDatabase(String database) { + this.database = database; + } + public long getWriteBufferSize() { return this.writeBufferSize; } @@ -92,6 +120,10 @@ public int getRpcTimeout() { return this.rpcTimeout; } + public int getRpcConnectTimeout() { + return this.rpcConnectTimeout; + } + public long getScannerMaxResultSize() { return this.scannerMaxResultSize; } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableFactory.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableFactory.java index 06c88892..8d42daa4 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableFactory.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableFactory.java @@ -65,6 +65,8 @@ public OHTableFactory(Configuration conf, OHTablePool tablePool, public HTableInterface createHTableInterface(Configuration config, byte[] tableName) { try { String tableNameStr = Bytes.toString(tableName); + tableNameStr = tableNameStr.equals(this.tablePool.getOriginTableName()) ? tableNameStr + : this.tablePool.getOriginTableName(); OHTable ht = new OHTable(adjustConfiguration(copyConfiguration(config), tableNameStr), tableName, this.threadPool); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java index 9b0c8182..036ce265 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java @@ -21,6 +21,8 @@ import com.alipay.oceanbase.rpc.constant.Constants; import com.google.common.base.Objects; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.ConnectionConfiguration; import java.io.IOException; import java.util.Map; @@ -58,7 +60,11 @@ public static ObTableClient getOrCreateObTableClient(OHConnectionConfiguration c checkArgument(isNotBlank(connectionConfig.getParamUrl()), HBASE_OCEANBASE_PARAM_URL + " is blank"); obTableClientKey = new ObTableClientKey(); - obTableClientKey.setParamUrl(connectionConfig.getParamUrl()); + String paramUrl = connectionConfig.getParamUrl(); + if (!paramUrl.contains("database")) { + paramUrl += "&database=default"; + } + obTableClientKey.setParamUrl(paramUrl); obTableClientKey.setSysUserName(connectionConfig.getSysUsername()); if (connectionConfig.getSysPassword() == null) { obTableClientKey.setSysPassword(Constants.EMPTY_STRING); @@ -80,11 +86,11 @@ public static ObTableClient getOrCreateObTableClient(OHConnectionConfiguration c obTableClientKey.getProperties().put(property.getKey(), property.getValue()); } - return getOrCreateObTableClient(obTableClientKey); + return getOrCreateObTableClient(obTableClientKey, connectionConfig.getRpcConnectTimeout()); } - public static ObTableClient getOrCreateObTableClient(ObTableClientKey obTableClientKey) - throws IOException { + public static ObTableClient getOrCreateObTableClient(ObTableClientKey obTableClientKey, + int rpcConnectTimeout) throws IOException { if (OB_TABLE_CLIENT_INSTANCE.get(obTableClientKey) == null) { ReentrantLock tmp = new ReentrantLock(); ReentrantLock lock = OB_TABLE_CLIENT_LOCK.putIfAbsent(obTableClientKey, tmp); @@ -109,6 +115,7 @@ public static ObTableClient getOrCreateObTableClient(ObTableClientKey obTableCli } obTableClient.setFullUserName(obTableClientKey.getFullUserName()); obTableClient.setPassword(obTableClientKey.getPassword()); + obTableClient.setRpcConnectTimeout(rpcConnectTimeout); obTableClient.init(); OB_TABLE_CLIENT_INSTANCE.put(obTableClientKey, obTableClient); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index e55e3b3c..b8fe6c8c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -509,6 +509,21 @@ public void testFilter() throws Exception { tryPut(hTable, putKey2Column2Value1); tryPut(hTable, putKey2Column2Value2); +// time may be different +// +---------+-----+----------------+--------+ +// | K | Q | T | V | +// +---------+-----+----------------+--------+ +// | getKey1 | abc | -1728834971469 | value1 | +// | getKey1 | abc | -1728834971399 | value2 | +// | getKey1 | abc | -1728834971330 | value1 | +// | getKey1 | def | -1728834971748 | value2 | +// | getKey1 | def | -1728834971679 | value1 | +// | getKey1 | def | -1728834971609 | value2 | +// | getKey1 | def | -1728834971540 | value1 | +// | getKey2 | def | -1728834971887 | value2 | +// | getKey2 | def | -1728834971818 | value1 | +// +---------+-----+----------------+--------+ + filter = new ColumnPrefixFilter(Bytes.toBytes("e")); get = new Get(toBytes(key1)); get.setMaxVersions(10); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index a4a78a61..f41e6bca 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -18,6 +18,7 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; +import com.alipay.oceanbase.hbase.util.OHBufferedMutatorImpl; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; @@ -31,6 +32,8 @@ import java.util.List; import java.util.concurrent.*; +import static com.alipay.oceanbase.hbase.constants.OHConstants.SOCKET_TIMEOUT; +import static org.apache.hadoop.hbase.ipc.RpcClient.SOCKET_TIMEOUT_CONNECT; import static org.apache.hadoop.hbase.util.Bytes.toBytes; public class OHConnectionTest { @@ -43,6 +46,10 @@ public void testConnectionBySet() throws Exception { c.set(ClusterConnection.HBASE_CLIENT_CONNECTION_IMPL, "com.alipay.oceanbase.hbase.util.OHConnectionImpl"); c.set("rs.list.acquire.read.timeout", "10000"); + // test set rpc connection timeout, the first one is the latest version + c.set(SOCKET_TIMEOUT_CONNECT, "15000"); + // the second one is the deprecated version + c.set(SOCKET_TIMEOUT, "12000"); connection = ConnectionFactory.createConnection(c); TableName tableName = TableName.valueOf("test"); hTable = connection.getTable(tableName); @@ -54,6 +61,7 @@ public void testConnectionByXml() throws Exception { Configuration c = ObHTableTestUtil.newConfiguration(); c.set("rs.list.acquire.read.timeout", "10000"); + // can set rpc connection timeout in xml connection = ConnectionFactory.createConnection(c); TableName tableName = TableName.valueOf("test"); hTable = connection.getTable(tableName); @@ -194,6 +202,7 @@ PRIMARY KEY (`K`, `Q`, `T`) public void testBufferedMutatorWithFlush() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); conf.set("rs.list.acquire.read.timeout", "10000"); + conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); BufferedMutator putBufferMutator = null; BufferedMutator delBufferedMutator = null; try { @@ -279,6 +288,7 @@ public void testBufferedMutatorWithFlush() throws Exception { } /* + USE n1; CREATE TABLEGROUP `n1:test` SHARDING = 'ADAPTIVE'; CREATE TABLE `n1:test$family_group` ( `K` varbinary(1024) NOT NULL, @@ -292,11 +302,12 @@ PRIMARY KEY (`K`, `Q`, `T`) public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); conf.set("rs.list.acquire.read.timeout", "10000"); + conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); BufferedMutator putBufferMutator = null; BufferedMutator delBufferedMutator = null; try { // use n1 database - TableName tableName = TableName.valueOf("n1:test"); + TableName tableName = TableName.valueOf("n1","test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); // use defualt params diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHMultiNamespaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHMultiNamespaceTest.java new file mode 100644 index 00000000..9c585c12 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/OHMultiNamespaceTest.java @@ -0,0 +1,82 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase; + +import org.apache.hadoop.hbase.client.Get; +import org.junit.*; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class OHMultiNamespaceTest extends HTableTestBase { + @Before + public void before() throws Exception { + // use self-defined namespace "n1" + hTable = ObHTableTestUtil.newOHTableClient("n1:test"); + ((OHTableClient) hTable).init(); + } + + @After + public void finish() throws IOException { + hTable.close(); + } + + @Test + public void testRefreshTableEntry() throws Exception { + ((OHTableClient) hTable).refreshTableEntry("family1", false); + ((OHTableClient) hTable).refreshTableEntry("family1", true); + } + + @Test + public void testGetColumnFamilyNotExists() throws Exception { + /** family 不存在时提示不友好，*/ + Get get = new Get(("key_c_f").getBytes()); + get.addFamily("family_not_exists".getBytes()); + expectedException.expect(IOException.class); + expectedException.expectMessage("query table:n1:test family family_not_exists error."); + hTable.get(get); + } + + @Test + public void testNew() throws Exception { + OHTableClient hTable2 = ObHTableTestUtil.newOHTableClient("n1:test"); + hTable2.init(); + hTable2.getConfiguration().set("rs.list.acquire.read.timeout", "10000"); + + assertTrue(hTable2.isAutoFlush()); + hTable2.setAutoFlush(false); + assertFalse(hTable2.isAutoFlush()); + hTable2.setAutoFlush(true, true); + assertTrue(hTable2.isAutoFlush()); + hTable2.setWriteBufferSize(10000000L); + assertEquals(10000000L, hTable2.getWriteBufferSize()); + assertEquals("n1:test", hTable2.getTableNameString()); + assertEquals("n1:test", new String(hTable2.getTableName())); + hTable2.flushCommits(); + hTable2.close(); + assertTrue(true); + } + + @After + public void after() throws IOException { + hTable.close(); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java index 044f45d4..59d4aac5 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java @@ -29,6 +29,7 @@ public class OHTableClientTest extends HTableTestBase { @Before public void before() throws Exception { hTable = ObHTableTestUtil.newOHTableClient("test"); + // hTable = ObHTableTestUtil.newOHTableClient("n1:test"); ((OHTableClient) hTable).init(); } @@ -46,6 +47,7 @@ public void testRefreshTableEntry() throws Exception { @Test public void testNew() throws Exception { OHTableClient hTable2 = ObHTableTestUtil.newOHTableClient("test"); + // OHTableClient hTable2 = ObHTableTestUtil.newOHTableClient("n1:test"); hTable2.init(); hTable2.getConfiguration().set("rs.list.acquire.read.timeout", "10000"); @@ -57,7 +59,9 @@ public void testNew() throws Exception { hTable2.setWriteBufferSize(10000000L); assertEquals(10000000L, hTable2.getWriteBufferSize()); assertEquals("test", hTable2.getTableNameString()); + // assertEquals("n1:test", hTable2.getTableNameString()); assertEquals("test", new String(hTable2.getTableName())); + // assertEquals("n1:test", new String(hTable2.getTableName())); hTable2.flushCommits(); hTable2.close(); assertTrue(true); diff --git a/src/test/java/unit_test_db.sql b/src/test/java/unit_test_db.sql index 44b37a1c..c5681bd7 100644 --- a/src/test/java/unit_test_db.sql +++ b/src/test/java/unit_test_db.sql @@ -203,3 +203,55 @@ CREATE TABLE `test_multi_cf$family_with_group3` ( `V` varbinary(1024) DEFAULT NULL, PRIMARY KEY (`K`, `Q`, `T`) ) TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3; + +USE `n1`; +CREATE TABLE `n1:test$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +); + +CREATE TABLE `n1:test_t$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +); + +CREATE TABLEGROUP `n1:test` SHARDING = 'ADAPTIVE'; +CREATE TABLE `n1:test$family'1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = `n1:test`; + +CREATE TABLE `n1:test$family_with_local_index` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + key `idx1`(T) local, + PRIMARY KEY (`K`, `Q`, `T`) +); + +CREATE TABLE `n1:test$family_group` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = `n1:test`; + +CREATE TABLE `n1:test$partitionFamily1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) partition by key(`K`) partitions 17; + From e8a6ce5c04a8db8a674a6770d1c8935855f676f5 Mon Sep 17 00:00:00 2001 From: miyuan-ljr Date: Mon, 21 Oct 2024 19:51:30 +0800 Subject: [PATCH 04/21] fuzzy (#81) --- .../hbase/filter/HBaseFilterUtils.java | 32 ++ .../oceanbase/hbase/HTableTestBase.java | 288 ++++++++++++++++++ .../hbase/filter/HBaseFilterUtilsTest.java | 12 + 3 files changed, 332 insertions(+) diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index 522f2ab2..2eebd870 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -20,6 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.filter.*; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import java.lang.reflect.Field; @@ -66,6 +67,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte toParseableByteArray(byteStream, (FirstKeyOnlyFilter) filter); } else if (filter instanceof KeyOnlyFilter) { toParseableByteArray(byteStream, (KeyOnlyFilter) filter); + } else if (filter instanceof FuzzyRowFilter) { + toParseableByteArray(byteStream, (FuzzyRowFilter) filter); } else if (filter instanceof TimestampsFilter) { toParseableByteArray(byteStream, (TimestampsFilter) filter); } else if (filter instanceof MultiRowRangeFilter) { @@ -224,6 +227,35 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, KeyOn byteStream.write(')'); } + // FuzzyRowFilter('abc','101','ddd','010'); + private static void toParseableByteArray(ByteArrayOutputStream byteStream, FuzzyRowFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + + List> fuzzyKeysData; + try { + Field field = filter.getClass().getDeclaredField("fuzzyKeysData"); + field.setAccessible(true); + fuzzyKeysData = (List>)field.get(filter); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + for (int i = 0; i < fuzzyKeysData.size(); i ++) { + Pair data = fuzzyKeysData.get(i); + byteStream.write("'".getBytes()); + byteStream.write(data.getFirst()); + byteStream.write("'".getBytes()); + byteStream.write(','); + byteStream.write("'".getBytes()); + byteStream.write(data.getSecond()); + byteStream.write("'".getBytes()); + if (i < fuzzyKeysData.size() - 1) { + byteStream.write(','); + } + } + byteStream.write(')'); + } + private static void toParseableByteArray(ByteArrayOutputStream byteStream, TimestampsFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index b8fe6c8c..ed663321 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.*; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.junit.Assert; import org.junit.Ignore; import org.junit.Rule; @@ -1671,6 +1672,293 @@ public void testFilter2() throws Exception { scanner.close(); } + @Test + public void testFuzzyRowFilter() throws Exception { + String key1 = "abab"; + String key2 = "abcc"; + String column1 = "c1"; + String column2 = "c2"; + String column3 = "c3"; + String column4 = "c4"; + String column5 = "c5"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + String family = "family1"; + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family)); + + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + + Put putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + Put putKey1Column3Value1 = new Put(toBytes(key1)); + putKey1Column3Value1.add(toBytes(family), toBytes(column3), toBytes(value1)); + + Put putKey1Column4Value1 = new Put(toBytes(key1)); + putKey1Column4Value1.add(toBytes(family), toBytes(column4), toBytes(value1)); + + Put putKey1Column5Value1 = new Put(toBytes(key1)); + putKey1Column5Value1.add(toBytes(family), toBytes(column5), toBytes(value1)); + + Put putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column3Value1); + tryPut(hTable, putKey1Column4Value1); + tryPut(hTable, putKey1Column5Value1); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + + Scan scan; + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + List> fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("abab"), Bytes.toBytes("0000"))); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + FuzzyRowFilter filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + ResultScanner scanner = hTable.getScanner(scan); + + int res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 10); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + fuzzyKey.add(new Pair(Bytes.toBytes("abcc"), Bytes.toBytes("0000"))); + filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 2); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("ccab"), Bytes.toBytes("1100"))); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 10); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("cccc"), Bytes.toBytes("1100"))); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 2); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("ab##"), Bytes.toBytes("0011"))); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 12); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("azc"), Bytes.toBytes("010"))); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 2); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("azccd"), Bytes.toBytes("01001"))); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 2); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes(""), Bytes.toBytes(""))); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 12); + scanner.close(); + } + @Test public void testFirstKeyValueMatchingQualifiersFilter() throws Exception { String key1 = "getKey1"; diff --git a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java index fe7a7082..f0e66604 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java @@ -19,6 +19,7 @@ import org.apache.hadoop.hbase.filter.*; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -162,6 +163,17 @@ public void testColumnPrefixFilter() throws IOException { HBaseFilterUtils.toParseableByteArray(filter)); } + @Test + public void testFuzzyRowFilter() throws IOException { + List> fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("abc"), Bytes.toBytes("101"))); + fuzzyKey.add(new Pair(Bytes.toBytes("ddd"), Bytes.toBytes("010"))); + + FuzzyRowFilter filter = new FuzzyRowFilter(fuzzyKey); + System.out.println(Bytes.toString(HBaseFilterUtils.toParseableByteArray(filter))); + Assert.assertArrayEquals("FuzzyRowFilter('abc','101','ddd','010')".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + } + @Test public void testMultiRowRangeFilter() throws IOException { List ranges = new ArrayList<>(); From 868f52ba38d7f42bc4e4a988063884723afa2005 Mon Sep 17 00:00:00 2001 From: Ziyu Shi <57038180+JackShi148@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:36:52 +0800 Subject: [PATCH 05/21] DependentColumnFilter, singleColumnValueExcludeFilter and single cf setColumnFamilyTimeRange development (#82) * add DepentdentFilter and SingleColumnValueExcludeFilter * add singleColumnValueExcludeFilter and DependentColumnFilter; add test cases and optimize bufferedMutator test cases * add single cf timerange setting in Get * single cf setColumnFamilyTimeRange in Get and Scan * optimize code * report error when setColumnFamilyTimeRange for a column family that is not queryed * correct name --- .../com/alipay/oceanbase/hbase/OHTable.java | 28 + .../hbase/filter/HBaseFilterUtils.java | 51 ++ .../oceanbase/hbase/HTableTestBase.java | 802 +++++++++++++++++- .../oceanbase/hbase/OHConnectionTest.java | 16 + 4 files changed, 863 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 00d717b3..4ecf0aa5 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -585,6 +585,9 @@ public Result call() throws IOException { // In a Get operation where the family map is greater than 1 or equal to 0, // we handle this by appending the column family to the qualifier on the client side. // The server can then use this information to filter the appropriate column families and qualifiers. + if (!get.getColumnFamilyTimeRange().isEmpty()) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } NavigableSet columnFilters = new TreeSet<>(Bytes.BYTES_COMPARATOR); processColumnFilters(columnFilters, get.getFamilyMap()); obTableQuery = buildObTableQuery(get, columnFilters); @@ -598,6 +601,17 @@ public Result call() throws IOException { for (Map.Entry> entry : get.getFamilyMap() .entrySet()) { family = entry.getKey(); + if (!get.getColumnFamilyTimeRange().isEmpty()) { + Map colFamTimeRangeMap = get.getColumnFamilyTimeRange(); + if (colFamTimeRangeMap.size() > 1) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } else if (colFamTimeRangeMap.get(family) == null) { + throw new IllegalArgumentException("Get family is not matched in ColumnFamilyTimeRange"); + } else { + TimeRange tr = colFamTimeRangeMap.get(family); + get.setTimeRange(tr.getMin(), tr.getMax()); + } + } obTableQuery = buildObTableQuery(get, entry.getValue()); request = buildObTableQueryAsyncRequest(obTableQuery, getTargetTableName(tableNameString, Bytes.toString(family), @@ -668,6 +682,9 @@ public ResultScanner call() throws IOException { // In a Scan operation where the family map is greater than 1 or equal to 0, // we handle this by appending the column family to the qualifier on the client side. // The server can then use this information to filter the appropriate column families and qualifiers. + if (!scan.getColumnFamilyTimeRange().isEmpty()) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } NavigableSet columnFilters = new TreeSet<>(Bytes.BYTES_COMPARATOR); processColumnFilters(columnFilters, scan.getFamilyMap()); filter = buildObHTableFilter(scan.getFilter(), scan.getTimeRange(), @@ -684,6 +701,17 @@ public ResultScanner call() throws IOException { for (Map.Entry> entry : scan.getFamilyMap() .entrySet()) { family = entry.getKey(); + if (!scan.getColumnFamilyTimeRange().isEmpty()) { + Map colFamTimeRangeMap = scan.getColumnFamilyTimeRange(); + if (colFamTimeRangeMap.size() > 1) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } else if (colFamTimeRangeMap.get(family) == null) { + throw new IllegalArgumentException("Scan family is not matched in ColumnFamilyTimeRange"); + } else { + TimeRange tr = colFamTimeRangeMap.get(family); + scan.setTimeRange(tr.getMin(), tr.getMax()); + } + } filter = buildObHTableFilter(scan.getFilter(), scan.getTimeRange(), scan.getMaxVersions(), entry.getValue()); obTableQuery = buildObTableQuery(filter, scan); diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index 2eebd870..85c4f593 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -42,9 +42,13 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte throws IOException { if (filter == null) { throw new IllegalArgumentException("Filter is null"); + } else if (filter instanceof DependentColumnFilter) { + toParseableByteArray(byteStream, (DependentColumnFilter) filter); } else if (filter instanceof CompareFilter) { // RowFilter, ValueFilter, QualifierFilter toParseableByteArray(byteStream, (CompareFilter) filter); + } else if (filter instanceof SingleColumnValueExcludeFilter) { + toParseableByteArray(byteStream, (SingleColumnValueExcludeFilter) filter); } else if (filter instanceof SingleColumnValueFilter) { toParseableByteArray(byteStream, (SingleColumnValueFilter) filter); } else if (filter instanceof PageFilter) { @@ -163,6 +167,25 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write(')'); } + // SingleColumnValueExcludeFilter('cf1','col1',=,'binary:123',true,true) + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + SingleColumnValueExcludeFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getFilterIfMissing()).getBytes()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getLatestVersionOnly()).getBytes()); + byteStream.write(')'); + } + // PageFilter(100); private static void toParseableByteArray(ByteArrayOutputStream byteStream, PageFilter filter) throws IOException { @@ -181,6 +204,34 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write(')'); } + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + DependentColumnFilter filter) throws IOException { + // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ')' + if (filter.getComparator() == null) { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(filter.getDropDependentColumn()).getBytes()); + byteStream.write(')'); + } else { // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ',' compare_op ',' comparator ')' + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(filter.getDropDependentColumn()).getBytes()); + byteStream.write(','); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(')'); + } + } + private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnPaginationFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index ed663321..4a1aea84 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -525,6 +525,68 @@ public void testFilter() throws Exception { // | getKey2 | def | -1728834971818 | value1 | // +---------+-----+----------------+--------+ + SingleColumnValueFilter singleColumnValueFilter; + singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueFilter); + r = hTable.get(get); + Assert.assertEquals(7, r.raw().length); + + SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; + singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueExcludeFilter); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), true); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + + + filter = new ColumnPrefixFilter(Bytes.toBytes("e")); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -833,6 +895,66 @@ public void testFilter() throws Exception { } Assert.assertEquals(res_count, 12); scanner.close(); + + long timestamp = System.currentTimeMillis(); + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), timestamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timestamp, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + // putKey1Column1Value1 and putKey2Column1Value1 have the same timestamp + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column2Value2); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setStartRow("getKey1".getBytes()); + scan.setStopRow("getKey3".getBytes()); + scan.setMaxVersions(10); + scan.setFilter(dependentColumnFilter); + scanner = hTable.getScanner(scan); + + long prevTimestamp = - 1; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + if (prevTimestamp == -1) { + prevTimestamp = keyValue.getTimestamp(); + } else { + Assert.assertEquals(prevTimestamp, keyValue.getTimestamp()); + } + } + } + scanner.close(); } @Test @@ -2229,6 +2351,15 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(1, r.raw().length); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + // columnCountGetFilter filter 2 get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -2409,6 +2540,67 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(7, r.raw().length); + filterList = new FilterList(); + filterList.addFilter(new SingleColumnValueExcludeFilter(Bytes.toBytes(family), Bytes + .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column1), false)); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), false)); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), true)); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(value2)))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + filterList = new FilterList(); filterList.addFilter(new ColumnCountGetFilter(1)); filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.GREATER, @@ -2534,6 +2726,19 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(7, r.raw().length); + + SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; + singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueExcludeFilter); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); @@ -2965,23 +3170,25 @@ public void testGet() throws Exception { tryPut(hTable, putzKey2Column1Value1); // show table (time maybe different) - //+-----------+---------+----------------+--------+ - //| K | Q | T | V | - //+-----------+---------+----------------+--------+ - //| scanKey1x | column1 | -1709714409669 | value1 | - //| scanKey1x | column1 | -1709714409637 | value2 | - //| scanKey1x | column1 | -1709714409603 | value1 | - //| scanKey1x | column2 | -1709714409802 | value2 | - //| scanKey1x | column2 | -1709714409768 | value1 | - //| scanKey1x | column2 | -1709714409735 | value2 | - //| scanKey1x | column2 | -1709714409702 | value1 | - //| scanKey2x | column2 | -1709714409869 | value2 | - //| scanKey2x | column2 | -1709714409836 | value1 | - //| scanKey3x | column1 | -1709714409940 | value2 | - //| scanKey3x | column1 | -1709714409904 | value1 | - //| scanKey3x | column2 | -1709714410010 | value2 | - //| scanKey3x | column2 | -1709714409977 | value1 | - //+-----------+---------+----------------+--------+ + // +-----------+---------+----------------+--------+ + // | K | Q | T | V | + // +-----------+---------+----------------+--------+ + // | scanKey1x | column1 | -1729223351579 | value1 | + // | scanKey1x | column1 | -1729223351504 | value2 | + // | scanKey1x | column1 | -1729223351431 | value1 | + // | scanKey1x | column2 | -1729223351867 | value2 | + // | scanKey1x | column2 | -1729223351796 | value1 | + // | scanKey1x | column2 | -1729223351724 | value2 | + // | scanKey1x | column2 | -1729223351651 | value1 | + // | scanKey2x | column2 | -1729223352015 | value2 | + // | scanKey2x | column2 | -1729223351941 | value1 | + // | scanKey3x | column1 | -1729223352159 | value2 | + // | scanKey3x | column1 | -1729223352088 | value1 | + // | scanKey3x | column2 | -1729223352304 | value2 | + // | scanKey3x | column2 | -1729223352232 | value1 | + // | zScanKey1 | column1 | -1729223352378 | value1 | + // | zScanKey2 | column1 | -1729223352450 | value1 | + // +-----------+---------+----------------+--------+ // test closestRowBefore get = new Get("scanKey2x2".getBytes()); @@ -3013,6 +3220,156 @@ public void testGet() throws Exception { assertTrue(booleans[2]); assertFalse(booleans[3]); assertTrue(booleans[4]); + + // test single cf setColumnFamilyTimeRange + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + hTable.delete(deleteZKey1Family); + hTable.delete(deleteZKey2Family); + + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), timeStamp2, toBytes(value1)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timeStamp4, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), timeStamp5, toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), timeStamp7, toBytes(value2)); + + putKey3Column1Value1 = new Put(toBytes(key3)); + putKey3Column1Value1.add(toBytes(family), toBytes(column1), timeStamp8, toBytes(value1)); + + putKey3Column1Value2 = new Put(toBytes(key3)); + putKey3Column1Value2.add(toBytes(family), toBytes(column1), timeStamp9, toBytes(value2)); + + putKey3Column2Value1 = new Put(toBytes(key3)); + putKey3Column2Value1.add(toBytes(family), toBytes(column2), timeStamp10, toBytes(value1)); + + putKey3Column2Value2 = new Put(toBytes(key3)); + putKey3Column2Value2.add(toBytes(family), toBytes(column2), timeStamp11, toBytes(value2)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column1Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + tryPut(hTable, putKey3Column1Value1); + tryPut(hTable, putKey3Column1Value2); + tryPut(hTable, putKey3Column2Value1); + tryPut(hTable, putKey3Column2Value2); + + + get = new Get(toBytes(key1)); + get.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, maxTimeStamp); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + get = new Get(toBytes(key1)); + get.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, timeStamp2 + 1); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + get = new Get(toBytes(key2)); + // set invalid timeRange + get.setTimeRange(minTimeStamp, maxTimeStamp); + get.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, timeStamp5 + 1); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key2)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp5, maxTimeStamp); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp8, timeStamp8); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp8, timeStamp9); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp8, timeStamp9); + get.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp8, timeStamp9); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + final Get multiFamGet = get; + Assert.assertThrows(IOException.class, () -> { + hTable.get(multiFamGet); + }); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp8, timeStamp9); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + final Get missFamGet = get; + Assert.assertThrows(IOException.class, () -> { + hTable.get(missFamGet); + }); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); } @Test @@ -3112,23 +3469,25 @@ public void testScan() throws Exception { tryPut(hTable, putzKey2Column1Value1); // show table (time maybe different) - //+-----------+---------+----------------+--------+ - //| K | Q | T | V | - //+-----------+---------+----------------+--------+ - //| scanKey1x | column1 | -1709714409669 | value1 | - //| scanKey1x | column1 | -1709714409637 | value2 | - //| scanKey1x | column1 | -1709714409603 | value1 | - //| scanKey1x | column2 | -1709714409802 | value2 | - //| scanKey1x | column2 | -1709714409768 | value1 | - //| scanKey1x | column2 | -1709714409735 | value2 | - //| scanKey1x | column2 | -1709714409702 | value1 | - //| scanKey2x | column2 | -1709714409869 | value2 | - //| scanKey2x | column2 | -1709714409836 | value1 | - //| scanKey3x | column1 | -1709714409940 | value2 | - //| scanKey3x | column1 | -1709714409904 | value1 | - //| scanKey3x | column2 | -1709714410010 | value2 | - //| scanKey3x | column2 | -1709714409977 | value1 | - //+-----------+---------+----------------+--------+ + // +-----------+---------+----------------+--------+ + // | K | Q | T | V | + // +-----------+---------+----------------+--------+ + // | scanKey1x | column1 | -1729236392149 | value1 | + // | scanKey1x | column1 | -1729236392078 | value2 | + // | scanKey1x | column1 | -1729236392008 | value1 | + // | scanKey1x | column2 | -1729236392436 | value2 | + // | scanKey1x | column2 | -1729236392364 | value1 | + // | scanKey1x | column2 | -1729236392291 | value2 | + // | scanKey1x | column2 | -1729236392220 | value1 | + // | scanKey2x | column2 | -1729236392576 | value2 | + // | scanKey2x | column2 | -1729236392506 | value1 | + // | scanKey3x | column1 | -1729236392720 | value2 | + // | scanKey3x | column1 | -1729236392647 | value1 | + // | scanKey3x | column2 | -1729236392861 | value2 | + // | scanKey3x | column2 | -1729236392790 | value1 | + // | zScanKey1 | column1 | -1729236392931 | value1 | + // | zScanKey2 | column1 | -1729236393002 | value1 | + // +-----------+---------+----------------+--------+ scan = new Scan(); scan.addFamily(family.getBytes()); @@ -3315,6 +3674,189 @@ public void testScan() throws Exception { hTable.delete(deleteKey3Family); hTable.delete(deleteZKey1Family); hTable.delete(deleteZKey2Family); + + // test single cf setColumnFamilyTimeRange + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), timeStamp2, toBytes(value1)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timeStamp4, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), timeStamp5, toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), timeStamp7, toBytes(value2)); + + putKey3Column1Value1 = new Put(toBytes(key3)); + putKey3Column1Value1.add(toBytes(family), toBytes(column1), timeStamp8, toBytes(value1)); + + putKey3Column1Value2 = new Put(toBytes(key3)); + putKey3Column1Value2.add(toBytes(family), toBytes(column1), timeStamp9, toBytes(value2)); + + putKey3Column2Value1 = new Put(toBytes(key3)); + putKey3Column2Value1.add(toBytes(family), toBytes(column2), timeStamp10, toBytes(value1)); + + putKey3Column2Value2 = new Put(toBytes(key3)); + putKey3Column2Value2.add(toBytes(family), toBytes(column2), timeStamp11, toBytes(value2)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column1Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + tryPut(hTable, putKey3Column1Value1); + tryPut(hTable, putKey3Column1Value2); + tryPut(hTable, putKey3Column2Value1); + tryPut(hTable, putKey3Column2Value2); + + // scan key1 + key2 + scan = new Scan(toBytes(key1), toBytes(key3)); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, maxTimeStamp); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(8, res_count); + + // scan key1 + scan = new Scan(toBytes(key1), toBytes(key2)); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, maxTimeStamp); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(4, res_count); + + // scan key1 + scan = new Scan(toBytes(key1), toBytes(key2)); + scan.addFamily(toBytes(family)); + // set invalid timeRange + scan.setTimeRange(minTimeStamp, maxTimeStamp); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp1, timeStamp3); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(2, res_count); + + // scan all + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp2, timeStamp7); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(5, res_count); + + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + boolean foundKey3 = false; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (!foundKey3) { + if (key3.equals(Bytes.toString(kv.getRow()))) { + foundKey3 = true; + } + } + ++res_count; + } + } + Assert.assertTrue(foundKey3); + Assert.assertEquals(6, res_count); + + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); + scan.setMaxVersions(); + final Scan multiFamScan = scan; + Assert.assertThrows(IOException.class, () -> { + hTable.getScanner(multiFamScan); + }); + + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); + scan.setMaxVersions(); + final Scan missFamScan = scan; + Assert.assertThrows(IOException.class, () -> { + hTable.getScanner(missFamScan); + }); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); } @Test @@ -3537,6 +4079,198 @@ public void testReversedScan() throws Exception { hTable.delete(deleteKey2Family); hTable.delete(deleteKey3Family); + // test single cf setColumnFamilyTimeRange + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), timeStamp2, toBytes(value1)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timeStamp4, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), timeStamp5, toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), timeStamp7, toBytes(value2)); + + putKey3Column1Value1 = new Put(toBytes(key3)); + putKey3Column1Value1.add(toBytes(family), toBytes(column1), timeStamp8, toBytes(value1)); + + putKey3Column1Value2 = new Put(toBytes(key3)); + putKey3Column1Value2.add(toBytes(family), toBytes(column1), timeStamp9, toBytes(value2)); + + putKey3Column2Value1 = new Put(toBytes(key3)); + putKey3Column2Value1.add(toBytes(family), toBytes(column2), timeStamp10, toBytes(value1)); + + putKey3Column2Value2 = new Put(toBytes(key3)); + putKey3Column2Value2.add(toBytes(family), toBytes(column2), timeStamp11, toBytes(value2)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column1Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + tryPut(hTable, putKey3Column1Value1); + tryPut(hTable, putKey3Column1Value2); + tryPut(hTable, putKey3Column2Value1); + tryPut(hTable, putKey3Column2Value2); + + // scan key1 + scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(family.getBytes()); + scan.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, timeStamp3); + scan.setReversed(true); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(3, res_count); + + // scan key1 + key2 + scan = new Scan(); + scan.setStartRow(toBytes(key2)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp2, timeStamp7); + scan.setReversed(true); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(5, res_count); + + // scan key2 + scan = new Scan(); + scan.setStartRow(toBytes(key2)); + scan.setStopRow(toBytes(key1)); + scan.addFamily(toBytes(family)); + // set invalid timeRange + scan.setTimeRange(minTimeStamp, maxTimeStamp); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp4, timeStamp6); + scan.setReversed(true); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(2, res_count); + + // scan all + scan = new Scan(); + scan.setStartRow(toBytes(key3)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp2, timeStamp7); + scan.setReversed(true); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(5, res_count); + + scan = new Scan(); + scan.setStartRow(toBytes(key3)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setReversed(true); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + boolean foundKey3 = false; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (!foundKey3) { + if (key3.equals(Bytes.toString(kv.getRow()))) { + foundKey3 = true; + } + } + ++res_count; + } + } + Assert.assertTrue(foundKey3); + Assert.assertEquals(6, res_count); + + scan = new Scan(); + scan.setStartRow(toBytes(key3)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); + scan.setReversed(true); + scan.setMaxVersions(); + final Scan errorScan = scan; + Assert.assertThrows(IOException.class, () -> { + hTable.getScanner(errorScan); + }); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + } @Test diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index f41e6bca..03617c2c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -218,6 +218,10 @@ public void testBufferedMutatorWithFlush() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + // only support Put and Delete // for other type of operations, BufferedMutator will not set its type for them Append append = new Append(Bytes.toBytes(key)); @@ -420,6 +424,10 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); @@ -509,6 +517,10 @@ public void testBufferedMutatorWithUserPool() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); @@ -598,6 +610,10 @@ public void testBufferedMutatorConcurrent() throws Exception { String value = "value333444"; long timestamp = System.currentTimeMillis(); + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes("family_group")); + hTable.delete(delete); + for (int i = 0; i < 50; ++i) { final int taskId = i; final BufferedMutator thrBufferMutator = ohBufferMutator; From be654df67e8812a255cbbec1babf9dc7abe608b9 Mon Sep 17 00:00:00 2001 From: stuBirdFly <84010733+stuBirdFly@users.noreply.github.com> Date: Tue, 22 Oct 2024 20:35:40 +0800 Subject: [PATCH 06/21] hbase support batch (#84) --- .../com/alipay/oceanbase/hbase/OHTable.java | 380 ++++++++---------- .../oceanbase/hbase/util/BatchError.java | 40 ++ .../oceanbase/hbase/HTableTestBase.java | 18 +- .../hbase/OHTableMultiColumnFamilyTest.java | 109 ++++- 4 files changed, 331 insertions(+), 216 deletions(-) create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 4ecf0aa5..d38eaa46 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -24,6 +24,7 @@ import com.alipay.oceanbase.hbase.result.ClientStreamScanner; import com.alipay.oceanbase.hbase.util.*; import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.mutation.BatchOperation; import com.alipay.oceanbase.rpc.mutation.result.BatchOperationResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; @@ -131,7 +132,7 @@ public class OHTable implements HTableInterface { /** * the buffer of put request */ - private final ArrayList writeBuffer = new ArrayList(); + private final ArrayList writeBuffer = new ArrayList<>(); /** * when the put request reach the write buffer size the do put will * flush commits automatically @@ -336,7 +337,7 @@ public static ThreadPoolExecutor createDefaultThreadPoolExecutor(int coreSize, i System.setProperty(SOFA_THREAD_POOL_LOGGING_CAPABILITY, "false"); } SofaThreadPoolExecutor executor = new SofaThreadPoolExecutor(coreSize, maxThreads, - keepAliveTime, SECONDS, new SynchronousQueue(), "OHTableDefaultExecutePool", + keepAliveTime, SECONDS, new SynchronousQueue<>(), "OHTableDefaultExecutePool", TABLE_HBASE_LOGGER_SPACE); executor.allowCoreThreadTimeOut(true); return executor; @@ -463,13 +464,63 @@ public Boolean[] exists(List gets) throws IOException { } @Override - public void batch(List actions, Object[] results) { - throw new FeatureNotSupportedException("not supported yet."); + public void batch(final List actions, final Object[] results) throws IOException { + BatchError batchError = new BatchError(); + try { + List resultMapSingleOp = new LinkedList<>(); + String realTableName = getTargetTableName(actions); + obTableClient.setRuntimeBatchExecutor(executePool); + BatchOperation batch = buildBatchOperation(realTableName, actions, tableNameString.equals(realTableName), resultMapSingleOp); + BatchOperationResult tmpResults = batch.execute(); + if (results != null) { + if (results.length != actions.size()) { + throw new AssertionError("results.length"); + } + int index = 0; + for (int i = 0; i != results.length; ++i) { + results[i] = tmpResults.getResults().get(index); + index += resultMapSingleOp.get(i); + if (results[i] instanceof ObTableException) { + batchError.add((ObTableException) results[i], actions.get(i), null); + } + } + if (batchError.hasErrors()) { + throw batchError.makeException(); + } + } + } catch (Exception e) { + logger.error(LCD.convert("01-000010"), tableNameString, actions, e); + throw new IOException("batch table " + tableNameString + " error", e); + } + } + + private String getTargetTableName(List actions) { + byte[] family = null; + for (Row action : actions) { + if (action instanceof RowMutations || action instanceof RegionCoprocessorServiceExec) { + throw new FeatureNotSupportedException("not supported yet'"); + } else { + Mutation mutation = (Mutation) action; + if (mutation.getFamilyCellMap().size() != 1) { + return getTargetTableName(tableNameString); + } else { + byte[] nextFamily = mutation.getFamilyCellMap().keySet().iterator().next(); + if (family != null && !Arrays.equals(family, nextFamily)) { + return getTargetTableName(tableNameString); + } else if (family == null) { + family = nextFamily; + } + } + } + } + return getTargetTableName(tableNameString, Bytes.toString(family), configuration); } @Override - public Object[] batch(List actions) { - throw new FeatureNotSupportedException("not supported yet."); + public Object[] batch(List actions) throws IOException { + Object[] results = new Object[actions.size()]; + batch(actions, results); + return results; } @Override @@ -573,7 +624,7 @@ public Result get(final Get get) throws IOException { ServerCallable serverCallable = new ServerCallable(configuration, obTableClient, tableNameString, get.getRow(), get.getRow(), operationTimeout) { public Result call() throws IOException { - List keyValueList = new ArrayList(); + List keyValueList = new ArrayList<>(); byte[] family = new byte[] {}; ObTableClientQueryAsyncStreamResult clientQueryStreamResult; ObTableQueryAsyncRequest request; @@ -860,57 +911,11 @@ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, private void innerDelete(Delete delete) throws IOException { checkArgument(delete.getRow() != null, "row is null"); - List errorCodeList = new ArrayList(); - BatchOperationResult results = null; - try { - checkFamilyViolation(delete.getFamilyMap().keySet(), false); - if (delete.getFamilyMap().isEmpty()) { - // For a Delete operation without any qualifiers, we construct a DeleteFamily request. - // The server then performs the operation on all column families. - KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), - KeyValue.Type.DeleteFamily); - - BatchOperation batch = buildBatchOperation(tableNameString, Arrays.asList(kv), false, null); - results = batch.execute(); - } else if (delete.getFamilyMap().size() > 1) { - // Currently, the Delete Family operation type cannot transmit qualifiers to the server. - // As a result, the server cannot identify which families need to be deleted. - // Therefore, this process is handled sequentially. - boolean has_delete_family = delete.getFamilyMap().entrySet().stream() - .flatMap(entry -> entry.getValue().stream()) - .anyMatch(kv -> KeyValue.Type.codeToType(kv.getType()) == KeyValue.Type.DeleteFamily); - if (!has_delete_family) { - BatchOperation batch = buildBatchOperation(tableNameString, - delete.getFamilyMap(), false, null); - results = batch.execute(); - } else { - for (Map.Entry> entry : delete.getFamilyMap().entrySet()) { - BatchOperation batch = buildBatchOperation( - getTargetTableName(tableNameString, Bytes.toString(entry.getKey()), configuration), - entry.getValue(), false, null); - results = batch.execute(); - } - } - } else { - Map.Entry> entry = delete.getFamilyMap().entrySet().iterator() - .next(); - - BatchOperation batch = buildBatchOperation( - getTargetTableName(tableNameString, Bytes.toString(entry.getKey()), configuration), - entry.getValue(), false, null); - results = batch.execute(); - } - - errorCodeList = results.getErrorCodeList(); - boolean hasError = results.hasError(); - if (hasError) { - throw results.getFirstException(); - } + batch(Collections.singletonList(delete)); } catch (Exception e) { - logger.error(LCD.convert("01-00004"), tableNameString, errorCodeList, e); - throw new IOException("delete table " + tableNameString + " error codes " - + errorCodeList, e); + logger.error(LCD.convert("01-00004"), tableNameString, e); + throw new IOException("delete table " + tableNameString + "error" , e); } } @@ -1187,77 +1192,34 @@ public boolean isAutoFlush() { public void flushCommits() throws IOException { try { + if (writeBuffer.isEmpty()){ + return; + } + Map exceptionRowMap = new LinkedHashMap(); boolean[] resultSuccess = new boolean[writeBuffer.size()]; try { - Map, List>> familyMap = new HashMap, List>>(); - for (int i = 0; i < writeBuffer.size(); i++) { - Put aPut = writeBuffer.get(i); - Map> innerFamilyMap = aPut.getFamilyMap(); - if (innerFamilyMap.size() > 1) { - // Bypass logic: directly construct BatchOperation for puts with family map size > 1 - try { - BatchOperation batch = buildBatchOperation(this.tableNameString, - innerFamilyMap, false, null); - BatchOperationResult results = batch.execute(); - - boolean hasError = results.hasError(); - resultSuccess[i] = !hasError; - if (hasError) { - throw results.getFirstException(); - } - } catch (Exception e) { - logger.error(LCD.convert("01-00008"), tableNameString, null, autoFlush, - writeBuffer.size(), e); - throw new IOException("put table " + tableNameString + " error codes " - + null + "auto flush " + autoFlush - + " current buffer size " + writeBuffer.size(), e); - } - } else { - // Existing logic for puts with family map size = 1 - for (Map.Entry> entry : innerFamilyMap.entrySet()) { - String family = Bytes.toString(entry.getKey()); - Pair, List> keyValueWithIndex = familyMap - .get(family); - if (keyValueWithIndex == null) { - keyValueWithIndex = new Pair, List>( - new ArrayList(), new ArrayList()); - familyMap.put(family, keyValueWithIndex); - } - keyValueWithIndex.getFirst().add(i); - keyValueWithIndex.getSecond().addAll(entry.getValue()); - } - } - } - for (Map.Entry, List>> entry : familyMap - .entrySet()) { - List errorCodeList = new ArrayList(entry.getValue() - .getSecond().size()); - try { - String targetTableName = getTargetTableName(this.tableNameString, - entry.getKey(), configuration); - - BatchOperation batch = buildBatchOperation(targetTableName, entry - .getValue().getSecond(), false, null); - BatchOperationResult results = batch.execute(); - - errorCodeList = results.getErrorCodeList(); - boolean hasError = results.hasError(); - for (Integer index : entry.getValue().getFirst()) { - resultSuccess[index] = !hasError; - } - if (hasError) { - throw results.getFirstException(); + String realTableName = getTargetTableName(writeBuffer); + List resultMapSingleOp = new LinkedList<>(); + BatchOperation batch = buildBatchOperation(realTableName, writeBuffer, tableNameString.equals(realTableName), resultMapSingleOp); + BatchOperationResult results = batch.execute(); + if (results != null) { + int index = 0; + for (int i = 0; i != resultSuccess.length; ++i) { + if (results.getResults().get(index) instanceof ObTableException) { + resultSuccess[i] = false; + exceptionRowMap.put((ObTableException)results.getResults().get(index), writeBuffer.get(i)); + } else { + resultSuccess[i] = true; } - } catch (Exception e) { - logger.error(LCD.convert("01-00008"), tableNameString, errorCodeList, - autoFlush, writeBuffer.size(), e); - throw new IOException("put table " + tableNameString + " error codes " - + errorCodeList + "auto flush " + autoFlush - + " current buffer size " + writeBuffer.size(), e); + index += resultMapSingleOp.get(i); } - } - + } catch (Exception e) { + logger.error(LCD.convert("01-00008"), tableNameString, null, autoFlush, + writeBuffer.size(), e); + throw new IOException("put table " + tableNameString + " error codes " + null + + "auto flush " + autoFlush + " current buffer size " + + writeBuffer.size(), e); } finally { // mutate list so that it is empty for complete success, or contains // only failed records results are returned in the same order as the @@ -1269,6 +1231,12 @@ public void flushCommits() throws IOException { writeBuffer.remove(i); } } + if (!exceptionRowMap.isEmpty()) { + exceptionRowMap.forEach((e, row)->{ + logger.error(LCD.convert("01-00008"), row, tableNameString, autoFlush, + writeBuffer.size(), e); + }); + } } } finally { if (clearBufferOnFail) { @@ -1674,62 +1642,48 @@ public static ObTableBatchOperation buildObTableBatchOperation(List ke return batch; } - private ObTableBatchOperation buildObTableBatchOperation(Map> familyMap, - boolean putToAppend, - List qualifiers) { - ObTableBatchOperation batch = new ObTableBatchOperation(); - for (Map.Entry> entry : familyMap.entrySet()) { - byte[] family = entry.getKey(); - List keyValueList = entry.getValue(); - for (KeyValue kv : keyValueList) { - if (qualifiers != null) { - qualifiers - .add((Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())) - .getBytes()); - } - KeyValue new_kv = modifyQualifier(kv, - (Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())).getBytes()); - batch.addTableOperation(buildObTableOperation(new_kv, putToAppend)); - } - } - batch.setSameType(true); - batch.setSamePropertiesNames(true); - return batch; - } - private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(KeyValue kv, - boolean putToAppend) { + ObTableOperationType operationType, boolean isTableGroup) { KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getType()); - switch (kvType) { - case Put: - ObTableOperationType operationType; - if (putToAppend) { - operationType = APPEND; - } else { - operationType = INSERT_OR_UPDATE; - } + switch (operationType) { + case INSERT_OR_UPDATE: + case APPEND: return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(operationType, ROW_KEY_COLUMNS, new Object[] { kv.getRow(), kv.getQualifier(), kv.getTimestamp() }, V_COLUMNS, new Object[] { kv.getValue() }); - case Delete: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), kv.getQualifier(), kv.getTimestamp() }, null, null); - case DeleteColumn: - return com.alipay.oceanbase.rpc.mutation.Mutation - .getInstance(DEL, ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), kv.getQualifier(), -kv.getTimestamp() }, null, - null); - case DeleteFamily: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), null, -kv.getTimestamp() }, null, null); + case DEL: + switch (kvType) { + case Delete: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), kv.getQualifier(), kv.getTimestamp() }, + null, null); + case Maximum: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), null, -kv.getTimestamp() }, + null, null); + case DeleteColumn: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), kv.getQualifier(), -kv.getTimestamp() }, + null, null); + case DeleteFamily: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), isTableGroup?kv.getQualifier():null, -kv.getTimestamp() }, + null, null); + default: + throw new IllegalArgumentException("illegal mutation type " + kvType); + } default: - throw new IllegalArgumentException("illegal mutation type " + kvType); + throw new IllegalArgumentException("illegal mutation type " + operationType); } } private KeyValue modifyQualifier(KeyValue original, byte[] newQualifier) { - // Extract existing components + // Extract existing components byte[] row = original.getRow(); byte[] family = original.getFamily(); byte[] value = original.getValue(); @@ -1740,36 +1694,65 @@ private KeyValue modifyQualifier(KeyValue original, byte[] newQualifier) { value); } - private BatchOperation buildBatchOperation(String tableName, - Map> familyMap, - boolean putToAppend, List qualifiers) { + private BatchOperation buildBatchOperation(String tableName, List actions, boolean isTableGroup, List resultMapSingleOp) { BatchOperation batch = obTableClient.batchOperation(tableName); - - for (Map.Entry> entry : familyMap.entrySet()) { - byte[] family = entry.getKey(); - List keyValueList = entry.getValue(); - for (KeyValue kv : keyValueList) { - if (qualifiers != null) { - qualifiers.add(kv.getQualifier()); + if (actions != null) { + int posInList = -1; + int singleOpResultNum; + for (Row row : actions) { + singleOpResultNum = 0; + posInList++; + if (!(row instanceof Put) && !(row instanceof Delete)) { + throw new FeatureNotSupportedException( + "not supported other type in batch yet,only support put and delete"); + } else if (row instanceof Put) { + Put put = (Put) row; + if (put.isEmpty()) { + throw new IllegalArgumentException("No columns to insert for #" + + (posInList + 1) + " item"); + } + for (Map.Entry> entry : put.getFamilyMap().entrySet()) { + byte[] family = entry.getKey(); + List keyValueList = entry.getValue(); + for (KeyValue kv : keyValueList) { + singleOpResultNum++; + if(isTableGroup){ + KeyValue new_kv = modifyQualifier(kv,(Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())) + .getBytes()); + batch.addOperation(buildMutation(new_kv, INSERT_OR_UPDATE, isTableGroup)); + } else { + batch.addOperation(buildMutation(kv, INSERT_OR_UPDATE, isTableGroup)); + } + } + } + } else { + Delete delete = (Delete) row; + if (delete.isEmpty()) { + singleOpResultNum++; + KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), + KeyValue.Type.Maximum); + batch.addOperation(buildMutation(kv, DEL, isTableGroup)); + } else { + for (Map.Entry> entry : delete.getFamilyMap() + .entrySet()) { + byte[] family = entry.getKey(); + List keyValueList = entry.getValue(); + List mutations = new LinkedList<>(); + for (KeyValue kv : keyValueList) { + singleOpResultNum++; + if(isTableGroup){ + KeyValue new_kv = modifyQualifier(kv,(Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())) + .getBytes()); + batch.addOperation(buildMutation(new_kv, DEL, true)); + } else { + batch.addOperation(buildMutation(kv, DEL, false)); + } + } + } + } } - KeyValue new_kv = modifyQualifier(kv, - (Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())).getBytes()); - batch.addOperation(buildMutation(new_kv, putToAppend)); - } - } - - batch.setEntityType(ObTableEntityType.HKV); - return batch; - } - - private BatchOperation buildBatchOperation(String tableName, List keyValueList, - boolean putToAppend, List qualifiers) { - BatchOperation batch = obTableClient.batchOperation(tableName); - for (KeyValue kv : keyValueList) { - if (qualifiers != null) { - qualifiers.add(kv.getQualifier()); + resultMapSingleOp.add(singleOpResultNum); } - batch.addOperation(buildMutation(kv, putToAppend)); } batch.setEntityType(ObTableEntityType.HKV); return batch; @@ -1802,15 +1785,6 @@ public static ObTableOperation buildObTableOperation(KeyValue kv, boolean putToA } } - private ObTableQueryRequest buildObTableQueryRequest(ObTableQuery obTableQuery, - String targetTableName) { - ObTableQueryRequest request = new ObTableQueryRequest(); - request.setEntityType(ObTableEntityType.HKV); - request.setTableQuery(obTableQuery); - request.setTableName(targetTableName); - return request; - } - private ObTableQueryAsyncRequest buildObTableQueryAsyncRequest(ObTableQuery obTableQuery, String targetTableName) { ObTableQueryRequest request = new ObTableQueryRequest(); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java b/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java new file mode 100644 index 00000000..640e432d --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java @@ -0,0 +1,40 @@ +package com.alipay.oceanbase.hbase.util; + +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; +import org.apache.hadoop.hbase.client.Row; + +import java.util.ArrayList; +import java.util.List; + +public class BatchError { + private final List throwables = new ArrayList(); + private final List actions = new ArrayList(); + private final List addresses = new ArrayList(); + + public synchronized void add(Throwable ex, Row row, ServerName serverName) { + if (row == null){ + throw new IllegalArgumentException("row cannot be null. location=" + serverName); + } + + throwables.add(ex); + actions.add(row); + addresses.add(serverName != null ? serverName.toString() : "null"); + } + + public boolean hasErrors() { + return !throwables.isEmpty(); + } + + public synchronized RetriesExhaustedWithDetailsException makeException() { + return new RetriesExhaustedWithDetailsException( + new ArrayList(throwables), + new ArrayList(actions), new ArrayList(addresses)); + } + + public synchronized void clear() { + throwables.clear(); + actions.clear(); + addresses.clear(); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 4a1aea84..676dc2a5 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -104,7 +104,7 @@ PRIMARY KEY (`K`, `Q`, `T`) } // test scan with empty family - Scan scan = new Scan(); + Scan scan = new Scan(toBytes(key)); ResultScanner scanner = hTable.getScanner(scan); for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { @@ -3980,7 +3980,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); // reverse scan with MaxVersion @@ -3998,7 +3998,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 3); + Assert.assertEquals(3, res_count); scanner.close(); // reverse scan with pageFilter @@ -4018,7 +4018,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); // reverse scan with not_exist_start_row @@ -4036,7 +4036,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); // reverse scan with abnormal range @@ -4054,7 +4054,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 0); + Assert.assertEquals(0, res_count); scanner.close(); // reverse scan with abnormal range @@ -4072,7 +4072,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 13); + Assert.assertEquals(13, res_count); scanner.close(); hTable.delete(deleteKey1Family); @@ -4421,7 +4421,7 @@ public void testPartitionScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 7); + Assert.assertEquals(7, res_count); scanner.close(); // scan with prefixFilter @@ -4441,7 +4441,7 @@ public void testPartitionScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 2); + Assert.assertEquals(2, res_count); scanner.close(); // scan with singleColumnValueFilter diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java index 1793bb7f..f8d4eed3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java @@ -24,9 +24,7 @@ import org.junit.rules.ExpectedException; import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.*; @@ -48,6 +46,108 @@ public void finish() throws IOException { hTable.close(); } + @Test + public void testMultiColumnFamilyBatch() throws Exception { + byte[] family1 = "family_with_group1".getBytes(); + byte[] family2 = "family_with_group2".getBytes(); + byte[] family3 = "family_with_group3".getBytes(); + + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family1_value = "VVV1".getBytes(); + byte[] family2_value = "VVV2".getBytes(); + byte[] family3_value = "VVV3".getBytes(); + + int rows = 10; + List batchLsit = new LinkedList<>(); + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + Delete delete = new Delete(toBytes("Key" + i)); + batchLsit.add(delete); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + batchLsit.add(put); + } + + // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 + Delete delete = new Delete(toBytes("Key1")); + delete.deleteColumns(family1, family1_column1); + delete.deleteColumns(family2, family2_column1); + batchLsit.add(delete); + hTable.batch(batchLsit); + // f1c2 f1c3 f2c2 f3c1 + Get get = new Get(toBytes("Key1")); + Result result = hTable.get(get); + KeyValue[] keyValues = result.raw(); + assertEquals(4, keyValues.length); + assertFalse(result.containsColumn(family1, family1_column1)); + assertFalse(result.containsColumn(family2, family2_column1)); + + assertTrue(result.containsColumn(family1, family1_column2)); + assertArrayEquals(result.getValue(family1, family1_column2), family1_value); + assertTrue(result.containsColumn(family1, family1_column3)); + assertArrayEquals(result.getValue(family1, family1_column3), family1_value); + assertTrue(result.containsColumn(family2, family2_column2)); + assertArrayEquals(result.getValue(family2, family2_column2), family2_value); + assertTrue(result.containsColumn(family3, family3_column1)); + assertArrayEquals(result.getValue(family3, family3_column1), family3_value); + + // f1c1 f2c1 f2c2 + delete = new Delete(toBytes("Key2")); + delete.deleteColumns(family1, family1_column2); + delete.deleteColumns(family1, family1_column3); + delete.deleteColumns(family3, family3_column1); + batchLsit.add(delete); + // null + hTable.batch(batchLsit); + get = new Get(toBytes("Key2")); + result = hTable.get(get); + keyValues = result.raw(); + assertEquals(3, keyValues.length); + batchLsit.clear(); + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + batchLsit.add(put); + } + + delete = new Delete(toBytes("Key3")); + delete.deleteColumn(family1, family1_column2); + delete.deleteColumn(family2, family2_column1); + batchLsit.add(delete); + hTable.batch(batchLsit); + get = new Get(toBytes("Key3")); + result = hTable.get(get); + keyValues = result.raw(); + assertEquals(6, keyValues.length); + + batchLsit.clear(); + delete = new Delete(toBytes("Key4")); + delete.deleteColumns(family1, family1_column2); + delete.deleteColumns(family2, family2_column1); + delete.deleteFamily(family3); + batchLsit.add(delete); + hTable.batch(batchLsit); + get = new Get(toBytes("Key4")); + get.setMaxVersions(10); + result = hTable.get(get); + keyValues = result.raw(); + assertEquals(6, keyValues.length); + } + @Test public void testMultiColumnFamilyPut() throws Exception { byte[] family1 = "family_with_group1".getBytes(); @@ -487,7 +587,6 @@ public void testMultiColumnFamilyGet() throws Exception { assertEquals(expectedValue, keyValues[i].getValue()); } } - System.out.println(Arrays.toString(result2.raw())); assertEquals(3, keyValues.length); //f2c1 f2c2 @@ -529,6 +628,8 @@ public void testMultiColumnFamilyDelete() throws Exception { for (int i = 0; i < rows; ++i) { Put put = new Put(toBytes("Key" + i)); + Delete delete = new Delete(toBytes("Key" + i)); + hTable.delete(delete); put.add(family1, family1_column1, family1_value); put.add(family1, family1_column2, family1_value); put.add(family1, family1_column3, family1_value); From 697c5d07671b3026cbada259efd297094102e1fe Mon Sep 17 00:00:00 2001 From: Ziyu Shi <57038180+JackShi148@users.noreply.github.com> Date: Wed, 23 Oct 2024 10:49:17 +0800 Subject: [PATCH 07/21] Add DeleteFamilyVersion function and corresponding test cases (#85) * add DepentdentFilter and SingleColumnValueExcludeFilter * add singleColumnValueExcludeFilter and DependentColumnFilter; add test cases and optimize bufferedMutator test cases * add single cf timerange setting in Get * single cf setColumnFamilyTimeRange in Get and Scan * optimize code * add DeleteFamilyVersion and test cases * add DeleteFamilyVersion; optimize test cases * add DeleteFamilyVersion test case and pass * format code * delete useless self-defined table --- .../com/alipay/oceanbase/hbase/OHTable.java | 17 +- .../hbase/filter/HBaseFilterUtils.java | 19 +- .../oceanbase/hbase/util/BatchError.java | 11 +- .../oceanbase/hbase/HTableTestBase.java | 167 ++++----- .../oceanbase/hbase/OHConnectionTest.java | 131 ++++--- .../hbase/OHTableDeleteFamilyVersionTest.java | 348 ++++++++++++++++++ .../hbase/filter/HBaseFilterUtilsTest.java | 51 ++- 7 files changed, 563 insertions(+), 181 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index d38eaa46..6d1cc873 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1656,9 +1656,9 @@ private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(KeyValue kv, switch (kvType) { case Delete: return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), kv.getQualifier(), kv.getTimestamp() }, - null, null); + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), kv.getQualifier(), kv.getTimestamp() }, + null, null); case Maximum: return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, @@ -1671,9 +1671,14 @@ private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(KeyValue kv, null, null); case DeleteFamily: return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), isTableGroup?kv.getQualifier():null, -kv.getTimestamp() }, - null, null); + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), isTableGroup ? kv.getQualifier() : null, -kv.getTimestamp() }, + null, null); + case DeleteFamilyVersion: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), isTableGroup ? kv.getQualifier() : null, kv.getTimestamp() }, + null, null); default: throw new IllegalArgumentException("illegal mutation type " + kvType); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index 85c4f593..ed98d1b3 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -169,7 +169,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, // SingleColumnValueExcludeFilter('cf1','col1',=,'binary:123',true,true) private static void toParseableByteArray(ByteArrayOutputStream byteStream, - SingleColumnValueExcludeFilter filter) throws IOException { + SingleColumnValueExcludeFilter filter) + throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write("('".getBytes()); writeBytesWithEscape(byteStream, filter.getFamily()); @@ -329,12 +330,13 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Times } // MultiRowRangeFilter('a',true,'b',false,'c',true,'d',false); - private static void toParseableByteArray(ByteArrayOutputStream byteStream, MultiRowRangeFilter filter) throws IOException { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + MultiRowRangeFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); List ranges = filter.getRowRanges(); - for (int i = 0; i < ranges.size(); i ++) { + for (int i = 0; i < ranges.size(); i++) { MultiRowRangeFilter.RowRange range = ranges.get(i); byteStream.write("'".getBytes()); byteStream.write(range.getStartRow()); @@ -354,7 +356,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Multi } // InclusiveStopFilter('aaa'); - private static void toParseableByteArray(ByteArrayOutputStream byteStream, InclusiveStopFilter filter) throws IOException { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + InclusiveStopFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); byteStream.write('\''); @@ -364,7 +367,8 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Inclu } // ColumnRangeFilter('a',true,'b',false); - private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnRangeFilter filter) throws IOException { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + ColumnRangeFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); @@ -382,12 +386,13 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Colum } // MultipleColumnPrefixFilter('a','b','d'); - private static void toParseableByteArray(ByteArrayOutputStream byteStream, MultipleColumnPrefixFilter filter) throws IOException { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + MultipleColumnPrefixFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); byte[][] ranges = filter.getPrefix(); - for (int i = 0; i < ranges.length; i ++) { + for (int i = 0; i < ranges.length; i++) { byte[] range = ranges[i]; byteStream.write("'".getBytes()); byteStream.write(range); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java b/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java index 640e432d..49fe17e9 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java @@ -9,11 +9,11 @@ public class BatchError { private final List throwables = new ArrayList(); - private final List actions = new ArrayList(); - private final List addresses = new ArrayList(); + private final List actions = new ArrayList(); + private final List addresses = new ArrayList(); public synchronized void add(Throwable ex, Row row, ServerName serverName) { - if (row == null){ + if (row == null) { throw new IllegalArgumentException("row cannot be null. location=" + serverName); } @@ -27,9 +27,8 @@ public boolean hasErrors() { } public synchronized RetriesExhaustedWithDetailsException makeException() { - return new RetriesExhaustedWithDetailsException( - new ArrayList(throwables), - new ArrayList(actions), new ArrayList(addresses)); + return new RetriesExhaustedWithDetailsException(new ArrayList(throwables), + new ArrayList(actions), new ArrayList(addresses)); } public synchronized void clear() { diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 676dc2a5..9aa2e1f7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -510,24 +510,24 @@ public void testFilter() throws Exception { tryPut(hTable, putKey2Column2Value1); tryPut(hTable, putKey2Column2Value2); -// time may be different -// +---------+-----+----------------+--------+ -// | K | Q | T | V | -// +---------+-----+----------------+--------+ -// | getKey1 | abc | -1728834971469 | value1 | -// | getKey1 | abc | -1728834971399 | value2 | -// | getKey1 | abc | -1728834971330 | value1 | -// | getKey1 | def | -1728834971748 | value2 | -// | getKey1 | def | -1728834971679 | value1 | -// | getKey1 | def | -1728834971609 | value2 | -// | getKey1 | def | -1728834971540 | value1 | -// | getKey2 | def | -1728834971887 | value2 | -// | getKey2 | def | -1728834971818 | value1 | -// +---------+-----+----------------+--------+ + // time may be different + // +---------+-----+----------------+--------+ + // | K | Q | T | V | + // +---------+-----+----------------+--------+ + // | getKey1 | abc | -1728834971469 | value1 | + // | getKey1 | abc | -1728834971399 | value2 | + // | getKey1 | abc | -1728834971330 | value1 | + // | getKey1 | def | -1728834971748 | value2 | + // | getKey1 | def | -1728834971679 | value1 | + // | getKey1 | def | -1728834971609 | value2 | + // | getKey1 | def | -1728834971540 | value1 | + // | getKey2 | def | -1728834971887 | value2 | + // | getKey2 | def | -1728834971818 | value1 | + // +---------+-----+----------------+--------+ SingleColumnValueFilter singleColumnValueFilter; singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -538,7 +538,7 @@ public void testFilter() throws Exception { SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -547,8 +547,8 @@ public void testFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(4, r.raw().length); - DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), false); + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter( + Bytes.toBytes(family), Bytes.toBytes(column1), false); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -557,7 +557,7 @@ public void testFilter() throws Exception { Assert.assertEquals(3, r.raw().length); dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), true); + Bytes.toBytes(column1), true); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -566,7 +566,7 @@ public void testFilter() throws Exception { Assert.assertEquals(0, r.raw().length); dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); get = new Get(toBytes(key2)); get.setMaxVersions(10); @@ -576,7 +576,7 @@ public void testFilter() throws Exception { Assert.assertEquals(0, r.raw().length); dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); get = new Get(toBytes(key2)); get.setMaxVersions(10); @@ -585,8 +585,6 @@ public void testFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(1, r.raw().length); - - filter = new ColumnPrefixFilter(Bytes.toBytes("e")); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -934,7 +932,7 @@ public void testFilter() throws Exception { tryPut(hTable, putKey2Column2Value2); dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); scan = new Scan(); scan.addFamily(family.getBytes()); @@ -944,7 +942,7 @@ public void testFilter() throws Exception { scan.setFilter(dependentColumnFilter); scanner = hTable.getScanner(scan); - long prevTimestamp = - 1; + long prevTimestamp = -1; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { if (prevTimestamp == -1) { @@ -1280,20 +1278,20 @@ public void testColumnRangeFilter() throws Exception { scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); - ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, Bytes.toBytes("b"), false); + ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, + Bytes.toBytes("b"), false); scan.setFilter(filter); ResultScanner scanner = hTable.getScanner(scan); int res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); res_count += 1; } } @@ -1310,13 +1308,12 @@ public void testColumnRangeFilter() throws Exception { res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); res_count += 1; } } @@ -1333,13 +1330,12 @@ public void testColumnRangeFilter() throws Exception { res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); res_count += 1; } } @@ -1350,11 +1346,7 @@ public void testColumnRangeFilter() throws Exception { scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); - byte[][] range = { - Bytes.toBytes("g"), - Bytes.toBytes("3"), - Bytes.toBytes("d"), - }; + byte[][] range = { Bytes.toBytes("g"), Bytes.toBytes("3"), Bytes.toBytes("d"), }; MultipleColumnPrefixFilter iFilter = new MultipleColumnPrefixFilter(range); scan.setFilter(iFilter); scanner = hTable.getScanner(scan); @@ -1362,13 +1354,12 @@ public void testColumnRangeFilter() throws Exception { res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); res_count += 1; } } @@ -1378,14 +1369,8 @@ public void testColumnRangeFilter() throws Exception { scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); - range = new byte[][]{ - Bytes.toBytes("de"), - Bytes.toBytes("bg"), - Bytes.toBytes("nc"), - Bytes.toBytes("aa"), - Bytes.toBytes("abcd"), - Bytes.toBytes("dea"), - }; + range = new byte[][] { Bytes.toBytes("de"), Bytes.toBytes("bg"), Bytes.toBytes("nc"), + Bytes.toBytes("aa"), Bytes.toBytes("abcd"), Bytes.toBytes("dea"), }; iFilter = new MultipleColumnPrefixFilter(range); scan.setFilter(iFilter); scanner = hTable.getScanner(scan); @@ -1393,13 +1378,12 @@ public void testColumnRangeFilter() throws Exception { res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); res_count += 1; } } @@ -1474,13 +1458,12 @@ public void testFilterNullRange() throws Exception { int res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); res_count += 1; } @@ -2354,8 +2337,8 @@ public void testGetFilter() throws Exception { get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); - DependentColumnFilter dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), - Bytes.toBytes(column1)); + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter( + Bytes.toBytes(family), Bytes.toBytes(column1)); get.setFilter(dependentColumnFilter); r = hTable.get(get); Assert.assertEquals(3, r.raw().length); @@ -2542,7 +2525,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new SingleColumnValueExcludeFilter(Bytes.toBytes(family), Bytes - .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); + .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2552,7 +2535,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column1), false)); + .toBytes(column1), false)); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2562,7 +2545,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2), false)); + .toBytes(column2), false)); get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2572,7 +2555,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2))); + .toBytes(column2))); get = new Get(toBytes(key2)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2582,7 +2565,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2), true)); + .toBytes(column2), true)); get = new Get(toBytes(key2)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2592,8 +2575,8 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes - .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(value2)))); + .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2)))); get = new Get(toBytes(key2)); get.setMaxVersions(10); get.addFamily(toBytes(family)); @@ -2726,10 +2709,9 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(7, r.raw().length); - SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), - Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value1))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -2738,7 +2720,6 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(4, r.raw().length); - singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index 03617c2c..b40b23ea 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -205,22 +205,25 @@ public void testBufferedMutatorWithFlush() throws Exception { conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); BufferedMutator putBufferMutator = null; BufferedMutator delBufferedMutator = null; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // use defualt params putBufferMutator = connection.getBufferedMutator(tableName); delBufferedMutator = connection.getBufferedMutator(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - Delete delete= new Delete(toBytes(key)); - delete.deleteFamily(toBytes("family_group")); - hTable.delete(delete); + long timestamp = System.currentTimeMillis(); // only support Put and Delete // for other type of operations, BufferedMutator will not set its type for them @@ -234,10 +237,10 @@ public void testBufferedMutatorWithFlush() throws Exception { List mutations = new ArrayList<>(); // test Put Put put1 = new Put(Bytes.toBytes(key)); - put1.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); + put1.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); mutations.add(put1); Put put2 = new Put(Bytes.toBytes(key)); - put2.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); + put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); mutations.add(put2); // test add Mutations with List putBufferMutator.mutate(mutations); @@ -253,7 +256,7 @@ public void testBufferedMutatorWithFlush() throws Exception { Assert.assertThrows(FeatureNotSupportedException.class, () -> { noCfMut.mutate(del); }); - del.deleteFamily(Bytes.toBytes("family_group")); + del.deleteFamily(Bytes.toBytes(family)); // test reuse different type bufferedMutator final BufferedMutator difTypeMut = putBufferMutator; Assert.assertThrows(IllegalArgumentException.class, () -> { @@ -281,8 +284,8 @@ public void testBufferedMutatorWithFlush() throws Exception { if (delBufferedMutator != null) { delBufferedMutator.close(); // test add mutations after closed - Delete delete = new Delete(Bytes.toBytes("putKey")); - delete.deleteFamily(Bytes.toBytes("family_group")); + Delete delete = new Delete(Bytes.toBytes(key)); + delete.deleteFamily(Bytes.toBytes(family)); final BufferedMutator closedMutator = delBufferedMutator; Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); @@ -309,24 +312,29 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); BufferedMutator putBufferMutator = null; BufferedMutator delBufferedMutator = null; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + String family = "family_group"; + long timestamp = System.currentTimeMillis(); try { // use n1 database TableName tableName = TableName.valueOf("n1","test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // use defualt params putBufferMutator = connection.getBufferedMutator(tableName); delBufferedMutator = connection.getBufferedMutator(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - // only support Put and Delete // for other type of operations, BufferedMutator will not set its type for them Append append = new Append(Bytes.toBytes(key)); - append.add("family_group".getBytes(), column1.getBytes(), toBytes("_suffix")); + append.add(toBytes(family), toBytes(column1), toBytes("_suffix")); final BufferedMutator apMut = putBufferMutator; Assert.assertThrows(IllegalArgumentException.class, () -> { apMut.mutate(append); @@ -335,10 +343,10 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { List mutations = new ArrayList<>(); // test Put Put put1 = new Put(Bytes.toBytes(key)); - put1.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); + put1.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); mutations.add(put1); Put put2 = new Put(Bytes.toBytes(key)); - put2.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); + put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); mutations.add(put2); // test add Mutations with List putBufferMutator.mutate(mutations); @@ -354,7 +362,7 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { Assert.assertThrows(FeatureNotSupportedException.class, () -> { noCfMut.mutate(del); }); - del.deleteFamily(Bytes.toBytes("family_group")); + del.deleteFamily(Bytes.toBytes(family)); final BufferedMutator difTypeMut = putBufferMutator; // test reuse different type bufferedMutator Assert.assertThrows(IllegalArgumentException.class, () -> { @@ -410,30 +418,31 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { BufferedMutatorParams params = null; long bufferSize = 45000L; int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); putBufferMutator = connection.getBufferedMutator(params); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - - Delete delete= new Delete(toBytes(key)); - delete.deleteFamily(toBytes("family_group")); - hTable.delete(delete); - List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); for (int j = 0; j < 4; ++j) { Put put = new Put(Bytes.toBytes(key)); - put.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "_" + i + "_" + j), + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "_" + i + "_" + j), timestamp, Bytes.toBytes(value + "_" + i + "_" + j)); mutations.add(put); } @@ -449,14 +458,14 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { } finally { if (putBufferMutator != null) { putBufferMutator.close(); - Get get = new Get(toBytes("putKey")); + Get get = new Get(toBytes(key)); Result r = hTable.get(get); for (KeyValue keyValue : r.raw()) { ++count; } Assert.assertEquals(200, count); - Delete delete = new Delete(toBytes("putKey")); - delete.deleteFamily(toBytes("family_group")); + Delete delete = new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); hTable.delete(delete); r = hTable.get(get); Assert.assertEquals(0, r.raw().length); @@ -496,10 +505,20 @@ public void testBufferedMutatorWithUserPool() throws Exception { BufferedMutatorParams params = null; long bufferSize = 45000L; int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); @@ -512,21 +531,12 @@ public void testBufferedMutatorWithUserPool() throws Exception { ohBufferMutator = connection.getBufferedMutator(params); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - - Delete delete= new Delete(toBytes(key)); - delete.deleteFamily(toBytes("family_group")); - hTable.delete(delete); - List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); for (int j = 0; j < 4; ++j) { Put put = new Put(Bytes.toBytes(key)); - put.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "_" + i + "_" + j), + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "_" + i + "_" + j), timestamp, Bytes.toBytes(value + "_" + i + "_" + j)); mutations.add(put); } @@ -542,14 +552,14 @@ public void testBufferedMutatorWithUserPool() throws Exception { } finally { if (ohBufferMutator != null) { ohBufferMutator.close(); - Get get = new Get(toBytes("putKey")); + Get get = new Get(toBytes(key)); Result r = hTable.get(get); for (KeyValue keyValue : r.raw()) { ++count; } Assert.assertEquals(200, count); - Delete delete = new Delete(toBytes("putKey")); - delete.deleteFamily(toBytes("family_group")); + Delete delete = new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); hTable.delete(delete); r = hTable.get(get); @@ -589,10 +599,20 @@ public void testBufferedMutatorConcurrent() throws Exception { ExecutorService executorService = Executors.newFixedThreadPool(10); long bufferSize = 45000L; int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); @@ -605,15 +625,6 @@ public void testBufferedMutatorConcurrent() throws Exception { ohBufferMutator = connection.getBufferedMutator(params); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - - Delete delete= new Delete(toBytes(key)); - delete.deleteFamily(toBytes("family_group")); - hTable.delete(delete); - for (int i = 0; i < 50; ++i) { final int taskId = i; final BufferedMutator thrBufferMutator = ohBufferMutator; @@ -626,7 +637,7 @@ public void testBufferedMutatorConcurrent() throws Exception { long thrTimestamp = timestamp; Put put = new Put(Bytes.toBytes(thrKey)); - put.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(thrColumn), + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(thrColumn), thrTimestamp, Bytes.toBytes(thrValue)); mutations.add(put); } @@ -661,14 +672,14 @@ public void testBufferedMutatorConcurrent() throws Exception { } if (ohBufferMutator != null) { ohBufferMutator.close(); - Get get = new Get(toBytes("putKey")); + Get get = new Get(toBytes(key)); Result r = hTable.get(get); for (KeyValue keyValue : r.raw()) { ++count; } Assert.assertEquals(200, count); - Delete delete = new Delete(toBytes("putKey")); - delete.deleteFamily(toBytes("family_group")); + Delete delete = new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); hTable.delete(delete); r = hTable.get(get); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java new file mode 100644 index 00000000..2ed4c190 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java @@ -0,0 +1,348 @@ +package com.alipay.oceanbase.hbase; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; +import org.junit.rules.ExpectedException; + +import java.io.IOException; + +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + +public class OHTableDeleteFamilyVersionTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + protected HTableInterface hTable; + private static final String key1 = "scanKey1x"; + private static final String key2 = "scanKey2x"; + private static final String key3 = "scanKey3x"; + private static final String column1 = "column1"; + private static final String column2 = "column2"; + private static final String column3 = "column3"; + private static final String value1 = "value1"; + private static final String value2 = "value2"; + private static final String value3 = "value3"; + private static final String family1 = "family_with_group1"; + private static final String family2 = "family_with_group2"; + + @Before + public void before() throws Exception { + hTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); + ((OHTableClient) hTable).init(); + } + + @After + public void finish() throws IOException { + hTable.close(); + } + + public void tryPut(Table hTable, Put put) throws Exception { + hTable.put(put); + Thread.sleep(1); + } + + @Test + public void testDeleteFamilyVerison() throws Exception { + // delete previous data + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family1)); + deleteKey1Family.deleteFamily(toBytes(family2)); + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family1)); + deleteKey2Family.deleteFamily(toBytes(family2)); + Delete deleteKey3Family = new Delete(toBytes(key3)); + deleteKey3Family.deleteFamily(toBytes(family1)); + deleteKey3Family.deleteFamily(toBytes(family2)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); + putKey1Fam1Column1MinTs.add(toBytes(family1), toBytes(column1), minTimeStamp, + toBytes(value1)); + + Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp1, toBytes(value2)); + + Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); + putKey1Fam1Column2MinTs.add(toBytes(family1), toBytes(column2), minTimeStamp, + toBytes(value1)); + + Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts6.add(toBytes(family1), toBytes(column2), timeStamp6, toBytes(value1)); + + Put putKey2Fam1Column3Ts6 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp3, toBytes(value1)); + + tryPut(hTable, putKey1Fam1Column1MinTs); + tryPut(hTable, putKey3Fam1Column1Ts1); + tryPut(hTable, putKey1Fam1Column2MinTs); + tryPut(hTable, putKey1Fam1Column2Ts3); + tryPut(hTable, putKey2Fam1Column2Ts3); + tryPut(hTable, putKey2Fam1Column3Ts1); + tryPut(hTable, putKey3Fam1Column3Ts1); + tryPut(hTable, putKey3Fam1Column2Ts6); + tryPut(hTable, putKey2Fam1Column3Ts6); + + // test DeleteFamilyVersion single cf + Get get = new Get(toBytes(key1)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + Result r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + Delete delKey1MinTs = new Delete(toBytes(key1)); + delKey1MinTs.deleteFamilyVersion(toBytes(family1), minTimeStamp); + hTable.delete(delKey1MinTs); + + get = new Get(toBytes(key1)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts1 = new Delete(toBytes(key3)); + delKey3Ts1.deleteFamilyVersion(toBytes(family1), timeStamp1); + hTable.delete(delKey3Ts1); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey2Ts3 = new Delete(toBytes(key2)); + delKey2Ts3.deleteFamilyVersion(family1.getBytes(), timeStamp3); + hTable.delete(delKey2Ts3); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Scan scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(toBytes(family1)); + scan.setMaxVersions(10); + ResultScanner scanner = hTable.getScanner(scan); + int key1Cnt = 0, key2Cnt = 0, key3Cnt = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (key1.equals(Bytes.toString(kv.getRow()))) { + ++key1Cnt; + } else if (key2.equals(Bytes.toString(kv.getRow()))) { + ++key2Cnt; + } else { + ++key3Cnt; + } + } + } + Assert.assertEquals(1, key1Cnt); + Assert.assertEquals(1, key2Cnt); + Assert.assertEquals(1, key3Cnt); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + + // test DeleteFamilyVersion multiple cf + Put putKey1Fam1Column3Ts6 = new Put(toBytes(key1)); + putKey1Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp6, toBytes(value3)); + + Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column2Ts2.add(toBytes(family2), toBytes(column2), timeStamp2, toBytes(value1)); + + Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column3Ts2.add(toBytes(family2), toBytes(column3), timeStamp2, toBytes(value1)); + + Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts1.add(toBytes(family1), toBytes(column2), timeStamp1, toBytes(value2)); + + Put putKey2Fam1Column2Ts8 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts8.add(toBytes(family1), toBytes(column2), timeStamp8, toBytes(value2)); + + Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam2Column3Ts1.add(toBytes(family2), toBytes(column3), timeStamp3, toBytes(value3)); + + Put putKey2Fam1Column1Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp8, toBytes(value1)); + + Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); + putKey2Fam2Column1Ts3.add(toBytes(family2), toBytes(column1), timeStamp3, toBytes(value2)); + + Put putKey3Fam1Column2Ts9 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts9.add(toBytes(family1), toBytes(column2), timeStamp9, toBytes(value2)); + + Put putKey3Fam2Column3Ts10 = new Put(toBytes(key3)); + putKey3Fam2Column3Ts10 + .add(toBytes(family2), toBytes(column3), timeStamp10, toBytes(value1)); + + Put putKey3Fam2Column1Ts10 = new Put(toBytes(key3)); + putKey3Fam2Column1Ts10 + .add(toBytes(family2), toBytes(column1), timeStamp10, toBytes(value2)); + + Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts2.add(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); + + tryPut(hTable, putKey1Fam1Column3Ts6); + tryPut(hTable, putKey1Fam2Column2Ts2); + tryPut(hTable, putKey1Fam2Column3Ts2); + tryPut(hTable, putKey1Fam1Column2Ts1); + tryPut(hTable, putKey2Fam1Column2Ts8); + tryPut(hTable, putKey2Fam2Column3Ts1); + tryPut(hTable, putKey2Fam1Column1Ts1); + tryPut(hTable, putKey2Fam2Column1Ts3); + tryPut(hTable, putKey3Fam1Column2Ts9); + tryPut(hTable, putKey3Fam2Column3Ts10); + tryPut(hTable, putKey3Fam2Column1Ts10); + tryPut(hTable, putKey3Fam1Column2Ts2); + + Get getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = hTable.get(getKey1); + Assert.assertEquals(4, r.raw().length); + + Get getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = hTable.get(getKey2); + Assert.assertEquals(4, r.raw().length); + + Get getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setMaxVersions(10); + r = hTable.get(getKey3); + Assert.assertEquals(4, r.raw().length); + + Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); + delKey1Ts_6_2.deleteFamilyVersion(toBytes(family1), timeStamp6); + delKey1Ts_6_2.deleteFamilyVersion(toBytes(family2), timeStamp2); + hTable.delete(delKey1Ts_6_2); + + getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = hTable.get(getKey1); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp1, kv.getTimestamp()); + } + + Delete delKey2Ts_8_3 = new Delete(toBytes(key2)); + delKey2Ts_8_3.deleteFamilyVersion(toBytes(family1), timeStamp8); + delKey2Ts_8_3.deleteFamilyVersion(toBytes(family2), timeStamp3); + hTable.delete(delKey2Ts_8_3); + + getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = hTable.get(getKey2); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts_2_10 = new Delete(toBytes(key3)); + delKey3Ts_2_10.deleteFamilyVersion(toBytes(family1), timeStamp2); + delKey3Ts_2_10.deleteFamilyVersion(toBytes(family2), timeStamp10); + hTable.delete(delKey3Ts_2_10); + + getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setMaxVersions(10); + r = hTable.get(getKey3); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp9, kv.getTimestamp()); + } + + scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(toBytes(family1)); + scan.addFamily(toBytes(family2)); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + int ts1Cnt = 0, ts9Cnt = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (kv.getTimestamp() == timeStamp1) { + ++ts1Cnt; + } else if (kv.getTimestamp() == timeStamp9) { + ++ts9Cnt; + } + } + } + Assert.assertEquals(1, ts1Cnt); + Assert.assertEquals(1, ts9Cnt); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java index f0e66604..4fac8402 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java @@ -138,6 +138,39 @@ public void testSingleColumnValueFilter() throws IOException { } } + @Test + public void testSingleColumnValueExcludeFilter() throws IOException { + for (int i = 0; i < ops.length; i++) { + String expect = String + .format( + "SingleColumnValueExcludeFilter('family','qualifier',%s,'binary:value',false,true)", + opFlags[i]); + SingleColumnValueExcludeFilter filter = new SingleColumnValueExcludeFilter( + "family".getBytes(), "qualifier".getBytes(), ops[i], "value".getBytes()); + Assert.assertArrayEquals(expect.getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + } + + @Test + public void testDependentColumnFilter() throws IOException { + DependentColumnFilter filter = new DependentColumnFilter("family".getBytes(), + "qualifier".getBytes()); + String expect = "DependentColumnFilter('family','qualifier',false)"; + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + filter = new DependentColumnFilter("family".getBytes(), "qualifier".getBytes(), true); + expect = "DependentColumnFilter('family','qualifier',true)"; + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + for (int i = 0; i < ops.length; ++i) { + filter = new DependentColumnFilter("family".getBytes(), "qualifier".getBytes(), false, + ops[i], new BinaryComparator("value".getBytes())); + expect = String.format( + "DependentColumnFilter('family','qualifier',false,%s,'binary:value')", opFlags[i]); + Assert.assertArrayEquals(expect.getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + } + @Test public void testPageFilter() throws IOException { PageFilter filter = new PageFilter(128); @@ -188,24 +221,24 @@ public void testMultiRowRangeFilter() throws IOException { @Test public void testInclusiveStopFilter() throws IOException { InclusiveStopFilter filter = new InclusiveStopFilter(Bytes.toBytes("aaa")); - Assert.assertArrayEquals("InclusiveStopFilter('aaa')".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + Assert.assertArrayEquals("InclusiveStopFilter('aaa')".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test public void testColumnRangeFilter() throws IOException { - ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, Bytes.toBytes("b"), false); - Assert.assertArrayEquals("ColumnRangeFilter('a',true,'b',false)".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, + Bytes.toBytes("b"), false); + Assert.assertArrayEquals("ColumnRangeFilter('a',true,'b',false)".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test public void testMultipleColumnPrefixFilter() throws IOException { - byte[][] prefix = { - Bytes.toBytes("a"), - Bytes.toBytes("b"), - Bytes.toBytes("d"), - }; + byte[][] prefix = { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("d"), }; MultipleColumnPrefixFilter filter = new MultipleColumnPrefixFilter(prefix); - Assert.assertArrayEquals("MultipleColumnPrefixFilter('a','b','d')".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + Assert.assertArrayEquals("MultipleColumnPrefixFilter('a','b','d')".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test From 34114912efc1876b7a8b9cf81de061f08792e061 Mon Sep 17 00:00:00 2001 From: stuBirdFly <84010733+stuBirdFly@users.noreply.github.com> Date: Wed, 23 Oct 2024 11:42:52 +0800 Subject: [PATCH 08/21] hbase support batchCallBack (#86) --- .../com/alipay/oceanbase/hbase/OHTable.java | 23 +++++++++++---- .../hbase/OHTableMultiColumnFamilyTest.java | 28 +++++++++++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 6d1cc873..4712527b 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -526,15 +526,26 @@ public Object[] batch(List actions) throws IOException { @Override public void batchCallback(List actions, Object[] results, Batch.Callback callback) throws IOException, - InterruptedException { - throw new FeatureNotSupportedException("not supported yet'"); + InterruptedException { + try { + batch(actions, results); + } finally { + if (results != null) { + for (int i = 0; i < results.length; i++) { + if (!(results[i] instanceof ObTableException)) { + callback.update(null, actions.get(i).getRow(), (R) results[i]); + } + } + } + } } @Override - public Object[] batchCallback(List actions, Batch.Callback callback) - throws IOException, - InterruptedException { - throw new FeatureNotSupportedException("not supported yet'"); + public Object[] batchCallback( + final List actions, final Batch.Callback callback) throws IOException, InterruptedException { + Object[] results = new Object[actions.size()]; + batchCallback(actions, results, callback); + return results; } public static int compareByteArray(byte[] bt1, byte[] bt2) { diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java index f8d4eed3..a649e58e 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java @@ -17,8 +17,10 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.filter.PrefixFilter; import org.junit.*; import org.junit.rules.ExpectedException; @@ -146,6 +148,32 @@ public void testMultiColumnFamilyBatch() throws Exception { result = hTable.get(get); keyValues = result.raw(); assertEquals(6, keyValues.length); + + batchLsit.clear(); + final long[] updateCounter = new long[] { 0L }; + delete = new Delete(toBytes("Key5")); + delete.deleteColumns(family1, family1_column2); + delete.deleteColumns(family2, family2_column1); + delete.deleteFamily(family3); + batchLsit.add(delete); + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + batchLsit.add(put); + } + hTable.batchCallback(batchLsit, new Batch.Callback() { + @Override + public void update(byte[] region, byte[] row, MutationResult result) { + updateCounter[0]++; + } + }); + assertEquals(11, updateCounter[0]); + } @Test From 71ff69ddf32fe38d5e79c52d462a540d94a297c7 Mon Sep 17 00:00:00 2001 From: stuBirdFly <84010733+stuBirdFly@users.noreply.github.com> Date: Mon, 28 Oct 2024 18:03:17 +0800 Subject: [PATCH 09/21] add native hbase test (#90) --- .../com/alipay/oceanbase/hbase/OHTable.java | 7 +- ...lyTest.java => HTableMultiCFTestBase.java} | 406 +++++++++++++++--- .../oceanbase/hbase/HTableTestBase.java | 160 +------ .../alipay/oceanbase/hbase/LoggerTest.java | 5 +- .../oceanbase/hbase/NativeHBaseTest.java | 46 ++ .../oceanbase/hbase/OHConnectionTest.java | 56 ++- .../oceanbase/hbase/OHMultiNamespaceTest.java | 82 ---- .../hbase/OHTableAdminInterfaceTest.java | 1 + .../oceanbase/hbase/OHTableClientTest.java | 26 +- .../hbase/OHTableClientTestLoadTest.java | 30 +- .../hbase/OHTableDeleteFamilyVersionTest.java | 348 --------------- .../oceanbase/hbase/OHTablePoolLoadTest.java | 31 +- .../oceanbase/hbase/OHTablePoolTest.java | 33 +- .../alipay/oceanbase/hbase/OHTableTest.java | 30 +- .../oceanbase/hbase/ObHTableTestUtil.java | 59 --- .../oceanbase/hbase/util/NativeHBaseUtil.java | 105 +++++ .../hbase/util/OHTableHotkeyThrottleUtil.java | 1 - .../hbase/util/ObHTableTestUtil.java | 148 +++++++ 18 files changed, 843 insertions(+), 731 deletions(-) rename src/test/java/com/alipay/oceanbase/hbase/{OHTableMultiColumnFamilyTest.java => HTableMultiCFTestBase.java} (67%) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/NativeHBaseTest.java delete mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHMultiNamespaceTest.java delete mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java delete mode 100644 src/test/java/com/alipay/oceanbase/hbase/ObHTableTestUtil.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/util/NativeHBaseUtil.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 4712527b..630b008f 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1558,14 +1558,16 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, byte[] start, boolean includeStart, byte[] stop, boolean includeStop, boolean isReversed) { ObNewRange obNewRange = new ObNewRange(); - + ObBorderFlag obBorderFlag = new ObBorderFlag(); if (Arrays.equals(start, HConstants.EMPTY_BYTE_ARRAY)) { obNewRange.setStartKey(ObRowKey.getInstance(ObObj.getMin(), ObObj.getMin(), ObObj.getMin())); } else if (includeStart) { obNewRange.setStartKey(ObRowKey.getInstance(start, ObObj.getMin(), ObObj.getMin())); + obBorderFlag.setInclusiveStart(); } else { obNewRange.setStartKey(ObRowKey.getInstance(start, ObObj.getMax(), ObObj.getMax())); + obBorderFlag.unsetInclusiveStart(); } if (Arrays.equals(stop, HConstants.EMPTY_BYTE_ARRAY)) { @@ -1573,10 +1575,13 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, byte[] start, ObObj.getMax())); } else if (includeStop) { obNewRange.setEndKey(ObRowKey.getInstance(stop, ObObj.getMax(), ObObj.getMax())); + obBorderFlag.setInclusiveEnd(); } else { obNewRange.setEndKey(ObRowKey.getInstance(stop, ObObj.getMin(), ObObj.getMin())); + obBorderFlag.unsetInclusiveEnd(); } ObTableQuery obTableQuery = new ObTableQuery(); + obNewRange.setBorderFlag(obBorderFlag); if (isReversed) { obTableQuery.setScanOrder(ObScanOrder.Reverse); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java similarity index 67% rename from src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java rename to src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java index a649e58e..fa28a810 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java @@ -22,30 +22,329 @@ import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.util.Bytes; import org.junit.*; import org.junit.rules.ExpectedException; -import java.io.IOException; import java.util.*; import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.*; -public class OHTableMultiColumnFamilyTest { +public abstract class HTableMultiCFTestBase { @Rule - public ExpectedException expectedException = ExpectedException.none(); + public ExpectedException expectedException = ExpectedException.none(); - protected HTableInterface hTable; + protected static Table multiCfHTable; - @Before - public void before() throws Exception { - hTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); - ((OHTableClient) hTable).init(); + public void tryPut(Table multiCfHTable, Put put) throws Exception { + multiCfHTable.put(put); + Thread.sleep(1); } - @After - public void finish() throws IOException { - hTable.close(); + @Test + public void testDeleteFamilyVerison() throws Exception { + String key1 = "scanKey1x"; + String key2 = "scanKey2x"; + String key3 = "scanKey3x"; + String column1 = "column1"; + String column2 = "column2"; + String column3 = "column3"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + String family1 = "family_with_group1"; + String family2 = "family_with_group2"; + // delete previous data + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family1)); + deleteKey1Family.deleteFamily(toBytes(family2)); + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family1)); + deleteKey2Family.deleteFamily(toBytes(family2)); + Delete deleteKey3Family = new Delete(toBytes(key3)); + deleteKey3Family.deleteFamily(toBytes(family1)); + deleteKey3Family.deleteFamily(toBytes(family2)); + + multiCfHTable.delete(deleteKey1Family); + multiCfHTable.delete(deleteKey2Family); + multiCfHTable.delete(deleteKey3Family); + + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + + Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); + putKey1Fam1Column1MinTs.add(toBytes(family1), toBytes(column1), minTimeStamp, + toBytes(value1)); + + Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp1, toBytes(value2)); + + Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); + putKey1Fam1Column2MinTs.add(toBytes(family1), toBytes(column2), minTimeStamp, + toBytes(value1)); + + Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column2Ts4 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts4.add(toBytes(family1), toBytes(column2), timeStamp4, toBytes(value1)); + + Put putKey2Fam1Column3Ts3 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts3.add(toBytes(family1), toBytes(column3), timeStamp3, toBytes(value1)); + + tryPut(multiCfHTable, putKey1Fam1Column1MinTs); + tryPut(multiCfHTable, putKey3Fam1Column1Ts1); + tryPut(multiCfHTable, putKey1Fam1Column2MinTs); + tryPut(multiCfHTable, putKey1Fam1Column2Ts3); + tryPut(multiCfHTable, putKey2Fam1Column2Ts3); + tryPut(multiCfHTable, putKey2Fam1Column3Ts1); + tryPut(multiCfHTable, putKey3Fam1Column3Ts1); + tryPut(multiCfHTable, putKey3Fam1Column2Ts4); + tryPut(multiCfHTable, putKey2Fam1Column3Ts3); + + // test DeleteFamilyVersion single cf + Get get = new Get(toBytes(key1)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + Result r = multiCfHTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(2, r.raw().length); + + Delete delKey1MinTs = new Delete(toBytes(key1)); + delKey1MinTs.deleteFamilyVersion(toBytes(family1), minTimeStamp); + multiCfHTable.delete(delKey1MinTs); + + get = new Get(toBytes(key1)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts1 = new Delete(toBytes(key3)); + delKey3Ts1.deleteFamilyVersion(toBytes(family1), timeStamp1); + multiCfHTable.delete(delKey3Ts1); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey2Ts3 = new Delete(toBytes(key2)); + delKey2Ts3.deleteFamilyVersion(family1.getBytes(), timeStamp3); + multiCfHTable.delete(delKey2Ts3); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Scan scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(toBytes(family1)); + scan.setMaxVersions(10); + ResultScanner scanner = multiCfHTable.getScanner(scan); + int key1Cnt = 0, key2Cnt = 0, key3Cnt = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (key1.equals(Bytes.toString(kv.getRow()))) { + ++key1Cnt; + } else if (key2.equals(Bytes.toString(kv.getRow()))) { + ++key2Cnt; + } else { + ++key3Cnt; + } + } + } + Assert.assertEquals(1, key1Cnt); + Assert.assertEquals(1, key2Cnt); + Assert.assertEquals(1, key3Cnt); + + multiCfHTable.delete(deleteKey1Family); + multiCfHTable.delete(deleteKey2Family); + multiCfHTable.delete(deleteKey3Family); + + // test DeleteFamilyVersion multiple cf + Put putKey1Fam1Column3Ts4 = new Put(toBytes(key1)); + putKey1Fam1Column3Ts4.add(toBytes(family1), toBytes(column3), timeStamp4, toBytes(value3)); + + Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column2Ts2.add(toBytes(family2), toBytes(column2), timeStamp2, toBytes(value1)); + + Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column3Ts2.add(toBytes(family2), toBytes(column3), timeStamp2, toBytes(value1)); + + Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts1.add(toBytes(family1), toBytes(column2), timeStamp1, toBytes(value2)); + + Put putKey2Fam1Column2Ts5 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts5.add(toBytes(family1), toBytes(column2), timeStamp5, toBytes(value2)); + + Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam2Column3Ts1.add(toBytes(family2), toBytes(column3), timeStamp3, toBytes(value3)); + + Put putKey2Fam1Column1Ts5 = new Put(toBytes(key2)); + putKey2Fam1Column1Ts5.add(toBytes(family1), toBytes(column1), timeStamp5, toBytes(value1)); + + Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); + putKey2Fam2Column1Ts3.add(toBytes(family2), toBytes(column1), timeStamp3, toBytes(value2)); + + Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts6.add(toBytes(family1), toBytes(column2), timeStamp6, toBytes(value2)); + + Put putKey3Fam2Column3Ts7 = new Put(toBytes(key3)); + putKey3Fam2Column3Ts7 + .add(toBytes(family2), toBytes(column3), timeStamp7, toBytes(value1)); + + Put putKey3Fam2Column1Ts7 = new Put(toBytes(key3)); + putKey3Fam2Column1Ts7 + .add(toBytes(family2), toBytes(column1), timeStamp7, toBytes(value2)); + + Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts2.add(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); + + tryPut(multiCfHTable, putKey1Fam1Column3Ts4); + tryPut(multiCfHTable, putKey1Fam2Column2Ts2); + tryPut(multiCfHTable, putKey1Fam2Column3Ts2); + tryPut(multiCfHTable, putKey1Fam1Column2Ts1); + tryPut(multiCfHTable, putKey2Fam1Column2Ts5); + tryPut(multiCfHTable, putKey2Fam2Column3Ts1); + tryPut(multiCfHTable, putKey2Fam1Column1Ts5); + tryPut(multiCfHTable, putKey2Fam2Column1Ts3); + tryPut(multiCfHTable, putKey3Fam1Column2Ts6); + tryPut(multiCfHTable, putKey3Fam2Column3Ts7); + tryPut(multiCfHTable, putKey3Fam2Column1Ts7); + tryPut(multiCfHTable, putKey3Fam1Column2Ts2); + + Get getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = multiCfHTable.get(getKey1); + Assert.assertEquals(4, r.raw().length); + + Get getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = multiCfHTable.get(getKey2); + Assert.assertEquals(4, r.raw().length); + + Get getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setMaxVersions(10); + r = multiCfHTable.get(getKey3); + Assert.assertEquals(4, r.raw().length); + + Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); + delKey1Ts_6_2.deleteFamilyVersion(toBytes(family1), timeStamp4); + delKey1Ts_6_2.deleteFamilyVersion(toBytes(family2), timeStamp2); + multiCfHTable.delete(delKey1Ts_6_2); + + getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = multiCfHTable.get(getKey1); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp1, kv.getTimestamp()); + } + + Delete delKey2Ts_5_3 = new Delete(toBytes(key2)); + delKey2Ts_5_3.deleteFamilyVersion(toBytes(family1), timeStamp5); + delKey2Ts_5_3.deleteFamilyVersion(toBytes(family2), timeStamp3); + multiCfHTable.delete(delKey2Ts_5_3); + + getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = multiCfHTable.get(getKey2); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts_2_7 = new Delete(toBytes(key3)); + delKey3Ts_2_7.deleteFamilyVersion(toBytes(family1), timeStamp2); + delKey3Ts_2_7.deleteFamilyVersion(toBytes(family2), timeStamp7); + multiCfHTable.delete(delKey3Ts_2_7); + + getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setMaxVersions(10); + r = multiCfHTable.get(getKey3); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp6, kv.getTimestamp()); + } + + scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(toBytes(family1)); + scan.addFamily(toBytes(family2)); + scan.setMaxVersions(10); + scanner = multiCfHTable.getScanner(scan); + int ts1Cnt = 0, ts9Cnt = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (kv.getTimestamp() == timeStamp1) { + ++ts1Cnt; + } else if (kv.getTimestamp() == timeStamp6) { + ++ts9Cnt; + } + } + } + Assert.assertEquals(1, ts1Cnt); + Assert.assertEquals(1, ts9Cnt); + + multiCfHTable.delete(deleteKey1Family); + multiCfHTable.delete(deleteKey2Family); + multiCfHTable.delete(deleteKey3Family); } @Test @@ -84,10 +383,10 @@ public void testMultiColumnFamilyBatch() throws Exception { delete.deleteColumns(family1, family1_column1); delete.deleteColumns(family2, family2_column1); batchLsit.add(delete); - hTable.batch(batchLsit); + multiCfHTable.batch(batchLsit); // f1c2 f1c3 f2c2 f3c1 Get get = new Get(toBytes("Key1")); - Result result = hTable.get(get); + Result result = multiCfHTable.get(get); KeyValue[] keyValues = result.raw(); assertEquals(4, keyValues.length); assertFalse(result.containsColumn(family1, family1_column1)); @@ -109,9 +408,9 @@ public void testMultiColumnFamilyBatch() throws Exception { delete.deleteColumns(family3, family3_column1); batchLsit.add(delete); // null - hTable.batch(batchLsit); + multiCfHTable.batch(batchLsit); get = new Get(toBytes("Key2")); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(3, keyValues.length); batchLsit.clear(); @@ -130,9 +429,9 @@ public void testMultiColumnFamilyBatch() throws Exception { delete.deleteColumn(family1, family1_column2); delete.deleteColumn(family2, family2_column1); batchLsit.add(delete); - hTable.batch(batchLsit); + multiCfHTable.batch(batchLsit); get = new Get(toBytes("Key3")); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(6, keyValues.length); @@ -142,10 +441,10 @@ public void testMultiColumnFamilyBatch() throws Exception { delete.deleteColumns(family2, family2_column1); delete.deleteFamily(family3); batchLsit.add(delete); - hTable.batch(batchLsit); + multiCfHTable.batch(batchLsit); get = new Get(toBytes("Key4")); get.setMaxVersions(10); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(6, keyValues.length); @@ -166,7 +465,7 @@ public void testMultiColumnFamilyBatch() throws Exception { put.add(family3, family3_column1, family3_value); batchLsit.add(put); } - hTable.batchCallback(batchLsit, new Batch.Callback() { + multiCfHTable.batchCallback(batchLsit, new Batch.Callback() { @Override public void update(byte[] region, byte[] row, MutationResult result) { updateCounter[0]++; @@ -210,14 +509,13 @@ public void testMultiColumnFamilyPut() throws Exception { put.add(family2, family2_column1, family2_value); put.add(family2, family2_column2, family2_value); put.add(family3, family3_column1, family3_value); - hTable.put(put); + multiCfHTable.put(put); } - hTable.flushCommits(); Scan scan = new Scan(); scan.setStartRow(toBytes("Key")); scan.setStopRow(toBytes("Kf")); - ResultScanner scanner = hTable.getScanner(scan); + ResultScanner scanner = multiCfHTable.getScanner(scan); int count = 0; for (Result result : scanner) { @@ -270,14 +568,13 @@ public void testMultiColumnFamilyAppend() throws Exception { append.add(family2, family2_column1, family2_value); append.add(family2, family2_column2, family2_value); append.add(family3, family3_column1, family3_value); - hTable.append(append); + multiCfHTable.append(append); } - hTable.flushCommits(); Scan scan = new Scan(); scan.setStartRow(toBytes("Key")); scan.setStopRow(toBytes("Kf")); - ResultScanner scanner = hTable.getScanner(scan); + ResultScanner scanner = multiCfHTable.getScanner(scan); int count = 0; for (Result result : scanner) { @@ -330,14 +627,14 @@ public void testMultiColumnFamilyReverseScan() throws Exception { put.add(family2, family2_column1, family2_value); put.add(family2, family2_column2, family2_value); put.add(family3, family3_column1, family3_value); - hTable.put(put); + multiCfHTable.put(put); } Scan scan = new Scan(); scan.addFamily(family1); scan.addFamily(family2); scan.setReversed(true); - ResultScanner scanner2 = hTable.getScanner(scan); + ResultScanner scanner2 = multiCfHTable.getScanner(scan); for (Result result : scanner2) { KeyValue[] keyValues = result.raw(); @@ -387,7 +684,7 @@ public void testMultiColumnFamilyScanWithColumns() throws Exception { put.add(family2, family2_column1, family2_value); put.add(family2, family2_column2, family2_value); put.add(family3, family3_column1, family3_value); - hTable.put(put); + multiCfHTable.put(put); } Scan scan = new Scan(); @@ -395,7 +692,7 @@ public void testMultiColumnFamilyScanWithColumns() throws Exception { scan.setStopRow(toBytes("Kf")); scan.addColumn(family1, family1_column1); scan.addColumn(family2, family2_column1); - ResultScanner scanner = hTable.getScanner(scan); + ResultScanner scanner = multiCfHTable.getScanner(scan); for (Result result : scanner) { KeyValue[] keyValues = result.raw(); @@ -419,7 +716,7 @@ public void testMultiColumnFamilyScanWithColumns() throws Exception { scan.addColumn(family1, family1_column3); scan.addColumn(family2, family2_column1); scan.addColumn(family2, family2_column2); - scanner = hTable.getScanner(scan); + scanner = multiCfHTable.getScanner(scan); for (Result result : scanner) { KeyValue[] keyValues = result.raw(); @@ -441,7 +738,7 @@ public void testMultiColumnFamilyScanWithColumns() throws Exception { scan.addFamily(family1); scan.addFamily(family2); - scanner = hTable.getScanner(scan); + scanner = multiCfHTable.getScanner(scan); for (Result result : scanner) { KeyValue[] keyValues = result.raw(); @@ -463,7 +760,7 @@ public void testMultiColumnFamilyScanWithColumns() throws Exception { scan.addFamily(family1); scan.addFamily(family3); - scanner = hTable.getScanner(scan); + scanner = multiCfHTable.getScanner(scan); for (Result result : scanner) { KeyValue[] keyValues = result.raw(); @@ -516,7 +813,7 @@ public void testMultiColumnFamilyScanWithFilter() throws Exception { put.add(family2, family2_column1, family2_value); put.add(family2, family2_column2, family2_value); put.add(family3, family3_column1, family3_value); - hTable.put(put); + multiCfHTable.put(put); } PrefixFilter filter = new PrefixFilter(toBytes("Key1")); @@ -524,7 +821,7 @@ public void testMultiColumnFamilyScanWithFilter() throws Exception { scan.setStartRow(toBytes("Key")); scan.setStopRow(toBytes("Kf")); scan.setFilter(filter); - ResultScanner scanner = hTable.getScanner(scan); + ResultScanner scanner = multiCfHTable.getScanner(scan); // Key1, Key10, Key11, Key12, Key13, Key14, Key15, Key16, Key17, Key18, Key19 int count = 0; @@ -579,14 +876,13 @@ public void testMultiColumnFamilyGet() throws Exception { put.add(family2, family2_column1, family2_value); put.add(family2, family2_column2, family2_value); put.add(family3, family3_column1, family3_value); - hTable.put(put); + multiCfHTable.put(put); } - hTable.flushCommits(); // get with empty family // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 Get get = new Get(toBytes("Key1")); - Result result = hTable.get(get); + Result result = multiCfHTable.get(get); KeyValue[] keyValues = result.raw(); long timestamp = keyValues[0].getTimestamp(); for (int i = 1; i < keyValues.length; ++i) { @@ -604,7 +900,7 @@ public void testMultiColumnFamilyGet() throws Exception { get2.addColumn(family1, family1_column1); get2.addColumn(family2, family2_column1); get2.addColumn(family2, family2_column2); - Result result2 = hTable.get(get2); + Result result2 = multiCfHTable.get(get2); keyValues = result2.raw(); timestamp = keyValues[0].getTimestamp(); for (int i = 1; i < keyValues.length; ++i) { @@ -622,7 +918,7 @@ public void testMultiColumnFamilyGet() throws Exception { get3.addFamily(family1); get3.addColumn(family2, family2_column1); get3.addColumn(family2, family2_column2); - Result result3 = hTable.get(get3); + Result result3 = multiCfHTable.get(get3); keyValues = result3.raw(); timestamp = keyValues[0].getTimestamp(); for (int i = 1; i < keyValues.length; ++i) { @@ -657,24 +953,24 @@ public void testMultiColumnFamilyDelete() throws Exception { for (int i = 0; i < rows; ++i) { Put put = new Put(toBytes("Key" + i)); Delete delete = new Delete(toBytes("Key" + i)); - hTable.delete(delete); + multiCfHTable.delete(delete); put.add(family1, family1_column1, family1_value); put.add(family1, family1_column2, family1_value); put.add(family1, family1_column3, family1_value); put.add(family2, family2_column1, family2_value); put.add(family2, family2_column2, family2_value); put.add(family3, family3_column1, family3_value); - hTable.put(put); + multiCfHTable.put(put); } // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 Delete delete = new Delete(toBytes("Key1")); delete.deleteColumns(family1, family1_column1); delete.deleteColumns(family2, family2_column1); - hTable.delete(delete); + multiCfHTable.delete(delete); // f1c2 f1c3 f2c2 f3c1 Get get = new Get(toBytes("Key1")); - Result result = hTable.get(get); + Result result = multiCfHTable.get(get); KeyValue[] keyValues = result.raw(); assertEquals(4, keyValues.length); assertFalse(result.containsColumn(family1, family1_column1)); @@ -694,9 +990,9 @@ public void testMultiColumnFamilyDelete() throws Exception { delete.deleteFamily(family1); delete.deleteFamily(family2); // f3c1 - hTable.delete(delete); + multiCfHTable.delete(delete); get = new Get(toBytes("Key2")); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(1, keyValues.length); @@ -704,19 +1000,19 @@ public void testMultiColumnFamilyDelete() throws Exception { delete = new Delete(toBytes("Key3")); delete.deleteFamily(family1); delete.deleteColumns(family2, family2_column1); - hTable.delete(delete); + multiCfHTable.delete(delete); // f2c2 f3c1 get = new Get(toBytes("Key3")); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(2, keyValues.length); // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 delete = new Delete(toBytes("Key4")); - hTable.delete(delete); + multiCfHTable.delete(delete); // null get = new Get(toBytes("Key4")); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(0, keyValues.length); @@ -725,10 +1021,10 @@ public void testMultiColumnFamilyDelete() throws Exception { delete.deleteColumns(family1, family1_column2); delete.deleteColumns(family1, family1_column3); delete.deleteColumns(family3, family3_column1); - hTable.delete(delete); + multiCfHTable.delete(delete); // null get = new Get(toBytes("Key5")); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(3, keyValues.length); @@ -740,15 +1036,15 @@ public void testMultiColumnFamilyDelete() throws Exception { put.add(family2, family2_column1, family2_value); put.add(family2, family2_column2, family2_value); put.add(family3, family3_column1, family3_value); - hTable.put(put); + multiCfHTable.put(put); } delete = new Delete(toBytes("Key6")); delete.deleteColumn(family1, family1_column2); delete.deleteColumn(family2, family2_column1); - hTable.delete(delete); + multiCfHTable.delete(delete); get = new Get(toBytes("Key6")); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(6, keyValues.length); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 9aa2e1f7..4131a572 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.*; +import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.junit.Assert; @@ -43,12 +44,12 @@ import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.*; -public abstract class HTableTestBase { +public abstract class HTableTestBase extends HTableMultiCFTestBase { @Rule public ExpectedException expectedException = ExpectedException.none(); - protected Table hTable; + protected static Table hTable; @Test public void testTableGroup() throws IOError, IOException { @@ -112,11 +113,11 @@ PRIMARY KEY (`K`, `Q`, `T`) Assert.assertEquals(column1, Bytes.toString(keyValue.getQualifier())); Assert.assertEquals(timestamp, keyValue.getTimestamp()); Assert.assertEquals(value + "1", Bytes.toString(keyValue.getValue())); - System.out.println("rowKey: " + new String(keyValue.getRow()) + " family :" - + new String(keyValue.getFamily()) + " columnQualifier:" - + new String(keyValue.getQualifier()) + " timestamp:" - + keyValue.getTimestamp() + " value:" - + new String(keyValue.getValue())); + System.out.println( + "rowKey: " + new String(keyValue.getRow()) + " family :" + new String( + keyValue.getFamily()) + " columnQualifier:" + new String( + keyValue.getQualifier()) + " timestamp:" + keyValue.getTimestamp() + " value:" + new String( + keyValue.getValue())); } } } @@ -132,9 +133,6 @@ private void testBasic(String family) throws Exception { String column2 = "putColumn2"; String value = "value"; long timestamp = System.currentTimeMillis(); - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); Put put = new Put(toBytes(key)); put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(value)); @@ -168,7 +166,7 @@ private void testBasic(String family) throws Exception { r = hTable.get(get); Assert.assertEquals(1, r.raw().length); - delete = new Delete(key.getBytes()); + Delete delete = new Delete(key.getBytes()); delete.deleteFamily(family.getBytes()); hTable.delete(delete); @@ -406,27 +404,7 @@ public void testMultiPartitionDel() throws IOException { String column1 = "column1"; String column2 = "column2"; String column3 = "column3"; - String value = "value"; String family = "familyPartition"; - // delete - { - List deletes = new ArrayList(); - for (String key : keys) { - Delete del = new Delete(Bytes.toBytes(key)); - del.deleteColumns(toBytes(family), toBytes(column1)); - del.deleteColumns(toBytes(family), toBytes(column2), System.currentTimeMillis()); - deletes.add(del); - } - - for (String key : keys) { - // del same k, q, t - Delete del = new Delete(Bytes.toBytes(key)); - del.deleteColumn(toBytes(family), toBytes(column3), 100L); - del.deleteColumn(toBytes(family), toBytes(column3), 100L); - deletes.add(del); - } - hTable.delete(deletes); - } // get { List gets = new ArrayList(); @@ -459,7 +437,6 @@ public void testFilter() throws Exception { String column2 = "def"; String value1 = "value1"; String value2 = "value2"; - String value3 = "value3"; String family = "family1"; Delete deleteKey1Family = new Delete(toBytes(key1)); deleteKey1Family.deleteFamily(toBytes(family)); @@ -467,9 +444,6 @@ public void testFilter() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -498,8 +472,6 @@ public void testFilter() throws Exception { Result r; ColumnPrefixFilter filter; - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); tryPut(hTable, putKey1Column1Value1); tryPut(hTable, putKey1Column1Value2); tryPut(hTable, putKey1Column1Value1); @@ -971,9 +943,6 @@ public void testRowRangeFilter() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -1227,7 +1196,6 @@ public void testColumnRangeFilter() throws Exception { String column2 = "def"; String value1 = "value1"; String value2 = "value2"; - String value3 = "value3"; String family = "family1"; Delete deleteKey1Family = new Delete(toBytes(key1)); deleteKey1Family.deleteFamily(toBytes(family)); @@ -1235,9 +1203,6 @@ public void testColumnRangeFilter() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -1262,8 +1227,6 @@ public void testColumnRangeFilter() throws Exception { Put putKey2Column2Value1 = new Put(toBytes(key2)); putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); tryPut(hTable, putKey1Column1Value1); tryPut(hTable, putKey1Column1Value2); tryPut(hTable, putKey1Column1Value1); @@ -1295,7 +1258,7 @@ public void testColumnRangeFilter() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 3); + Assert.assertEquals(3, res_count); scanner.close(); scan = new Scan(); @@ -1317,7 +1280,7 @@ public void testColumnRangeFilter() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 3); + Assert.assertEquals(3, res_count); scanner.close(); scan = new Scan(); @@ -1339,7 +1302,7 @@ public void testColumnRangeFilter() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); // MultipleColumnPrefixFilter @@ -1363,12 +1326,13 @@ public void testColumnRangeFilter() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); + // 和原生hbase不一致，已知 range = new byte[][] { Bytes.toBytes("de"), Bytes.toBytes("bg"), Bytes.toBytes("nc"), Bytes.toBytes("aa"), Bytes.toBytes("abcd"), Bytes.toBytes("dea"), }; iFilter = new MultipleColumnPrefixFilter(range); @@ -1387,7 +1351,7 @@ public void testColumnRangeFilter() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); } @@ -1407,9 +1371,6 @@ public void testFilterNullRange() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -1489,9 +1450,6 @@ public void testFilter2() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -1796,9 +1754,6 @@ public void testFuzzyRowFilter() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -2083,9 +2038,6 @@ public void testFirstKeyValueMatchingQualifiersFilter() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -2232,9 +2184,6 @@ public void testGetFilter() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -2894,11 +2843,6 @@ public void testScanWithObParams() throws Exception { Delete deleteKey4Family = new Delete(toBytes(key4)); deleteKey4Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteKey4Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -2998,12 +2942,6 @@ public void testScanSessionClean() throws Exception { Delete deleteKey5Family = new Delete(toBytes(key5)); deleteKey5Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteKey4Family); - hTable.delete(deleteKey5Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -3067,7 +3005,6 @@ public void testGet() throws Exception { String column2 = "column2"; String value1 = "value1"; String value2 = "value2"; - String value3 = "value3"; String family = "family1"; // delete previous data @@ -3082,12 +3019,6 @@ public void testGet() throws Exception { Delete deleteZKey2Family = new Delete(toBytes(zKey2)); deleteZKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteZKey1Family); - hTable.delete(deleteZKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -3364,7 +3295,6 @@ public void testScan() throws Exception { String column2 = "column2"; String value1 = "value1"; String value2 = "value2"; - String value3 = "value3"; String family = "family1"; // delete previous data @@ -3379,12 +3309,6 @@ public void testScan() throws Exception { Delete deleteZKey2Family = new Delete(toBytes(zKey2)); deleteZKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteZKey1Family); - hTable.delete(deleteZKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -3866,12 +3790,6 @@ public void testReversedScan() throws Exception { Delete deleteZKey2Family = new Delete(toBytes(zKey2)); deleteZKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteZKey1Family); - hTable.delete(deleteZKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -4265,7 +4183,6 @@ public void testPartitionScan() throws Exception { String column2 = "column2"; String value1 = "value1"; String value2 = "value2"; - String value3 = "value3"; String family = "partitionFamily1"; // delete previous data @@ -4280,12 +4197,6 @@ public void testPartitionScan() throws Exception { Delete deleteZKey2Family = new Delete(toBytes(zKey2)); deleteZKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteZKey1Family); - hTable.delete(deleteZKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -4553,9 +4464,6 @@ public void testCheckAndPut() throws IOException, InterruptedException { String column = "checkAndPut"; String value = "value"; String family = "family1"; - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); Get get = new Get(key.getBytes()); get.setMaxVersions(Integer.MAX_VALUE); get.addColumn(family.getBytes(), column.getBytes()); @@ -4605,15 +4513,12 @@ public void testCheckAndDelete() throws IOException { String column2 = "checkAndDeleteColumn2"; String value = "value"; String family = "family1"; - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); Put put = new Put(key.getBytes()); put.add(family.getBytes(), column.getBytes(), value.getBytes()); hTable.put(put); // check delete column - delete = new Delete(key.getBytes()); + Delete delete = new Delete(key.getBytes()); delete.deleteColumn(family.getBytes(), column.getBytes()); boolean ret = hTable.checkAndDelete(key.getBytes(), family.getBytes(), column.getBytes(), value.getBytes(), delete); @@ -4703,9 +4608,6 @@ public void testCheckAndMutate() throws IOException { String value1 = "value1"; String value2 = "value2"; String family = "family1"; - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); long t = System.currentTimeMillis(); // put @@ -4845,9 +4747,6 @@ public void testCheckAndMutate() throws IOException { public void testAppend() throws IOException { String column = "appendColumn"; String key = "appendKey"; - Delete delete = new Delete(key.getBytes()); - delete.deleteColumns("family1".getBytes(), column.getBytes()); - hTable.delete(delete); // append an absent column is not supported yet // Append append = new Append(key.getBytes()); @@ -4874,9 +4773,6 @@ public void testAppend() throws IOException { public void testIncrement() throws IOException { String column = "incrementColumn"; String key = "incrementKey"; - Delete delete = new Delete(key.getBytes()); - delete.deleteColumns("family1".getBytes(), column.getBytes()); - hTable.delete(delete); // increment an absent column is not supported yet // Increment increment = new Increment(key.getBytes()); @@ -4922,9 +4818,6 @@ public void testIncrement() throws IOException { public void testExist() throws IOException { String column = "existColumn"; String key = "existKey"; - Delete delete = new Delete(key.getBytes()); - delete.deleteColumns("family1".getBytes(), column.getBytes()); - hTable.delete(delete); Get get = new Get(key.getBytes()); get.addFamily("family1".getBytes()); @@ -4948,8 +4841,6 @@ public void testExist() throws IOException { get.setTimeStamp(timestamp + 1); Assert.assertFalse(hTable.exists(get)); - - hTable.delete(delete); } @Ignore @@ -4959,7 +4850,6 @@ public void testMutateRow() throws IOException { String column2 = "mutationRowColumn2"; String key = "mutationRowKey"; String family1 = "family1"; - String family2 = "family2"; String value = "value"; Delete deleteFamily = new Delete(key.getBytes()); @@ -5039,9 +4929,6 @@ public void testQualifyNull() throws Exception { String value = "value"; String value1 = "value1"; String family = "family1"; - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); Put put = new Put(key.getBytes()); put.add(family.getBytes(), null, value.getBytes()); hTable.put(put); @@ -5129,6 +5016,8 @@ public void testFamilyBlank() throws Exception { fail(); } catch (IllegalArgumentException e) { Assert.assertTrue(e.getMessage().contains("family is blank")); + } catch (NoSuchColumnFamilyException e) { + Assert.assertTrue(e.getMessage().contains("does not exist")); } Put put = new Put(key.getBytes()); put.add(null, null, value.getBytes()); @@ -5166,9 +5055,6 @@ public void testScannerMultiVersion() throws Exception { String column = "column"; String value1 = "value1"; String family = "family1"; - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); Put put = new Put(key.getBytes()); put.add(family.getBytes(), Bytes.toBytes(column), value.getBytes()); hTable.put(put); @@ -5439,9 +5325,6 @@ public static byte[] toByteArray(long value) { public void testIncrementConcurrency() throws Exception { String column = "incrementColumn"; String key = "incrementKey"; - Delete delete = new Delete(key.getBytes()); - delete.deleteColumns("family1".getBytes(), column.getBytes()); - hTable.delete(delete); for (int i = 0; i < 100; i++) { Increment increment = new Increment(key.getBytes()); @@ -5487,16 +5370,12 @@ public void testFilterSpecialValue() throws IOException { byte[] columnBytes = specialBytes; byte[] valueBytes = specialBytes; - Delete delete = new Delete(keyBytes); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); - Put put = new Put(keyBytes); put.add(family.getBytes(), columnBytes, valueBytes); hTable.put(put); // check delete column - delete = new Delete(keyBytes); + Delete delete = new Delete(keyBytes); delete.deleteColumn(family.getBytes(), columnBytes); boolean ret = hTable.checkAndDelete(keyBytes, family.getBytes(), columnBytes, valueBytes, delete); @@ -5532,6 +5411,5 @@ public void testFilterSpecialValue() throws IOException { result = hTable.get(get); Assert.assertEquals(1, result.raw().length); Assert.assertArrayEquals(keyBytes, result.getRow()); - } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/LoggerTest.java b/src/test/java/com/alipay/oceanbase/hbase/LoggerTest.java index d1909ea0..0ab0617b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/LoggerTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/LoggerTest.java @@ -17,12 +17,13 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Increment; import org.apache.hadoop.hbase.client.Scan; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.api.support.membermodification.MemberModifier; @@ -42,7 +43,7 @@ public class LoggerTest { HTableInterface hTableMock; - @Before + @BeforeClass public void setup() throws IOException { Configuration c = ObHTableTestUtil.newConfiguration(); c.set("rs.list.acquire.read.timeout", "10000"); diff --git a/src/test/java/com/alipay/oceanbase/hbase/NativeHBaseTest.java b/src/test/java/com/alipay/oceanbase/hbase/NativeHBaseTest.java new file mode 100644 index 00000000..545d95b9 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/NativeHBaseTest.java @@ -0,0 +1,46 @@ +package com.alipay.oceanbase.hbase; + +import com.alipay.oceanbase.hbase.util.NativeHBaseUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.*; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class NativeHBaseTest extends HTableTestBase { + + static Admin admin; + static TableName tableName1 = TableName.valueOf("test"); + static TableName tableName2 = TableName.valueOf("test_multi_cf"); + + static { + try { + admin = NativeHBaseUtil.getAdmin(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @BeforeClass + public static void setup() throws IOException { + hTable = NativeHBaseUtil.getTable(tableName1); + multiCfHTable = NativeHBaseUtil.getTable(tableName2); + } + + @Before + public void cleanData() throws IOException { + admin.disableTable(tableName1); + admin.disableTable(tableName2); + admin.truncateTable(tableName1, true); + admin.truncateTable(tableName2, true); + } + + @AfterClass + public static void finish() throws IOException { + hTable.close(); + multiCfHTable.close(); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index b40b23ea..79848a1e 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -18,16 +18,16 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; -import com.alipay.oceanbase.hbase.util.OHBufferedMutatorImpl; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; -import org.junit.Assert; -import org.junit.Test; +import org.junit.*; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.*; @@ -35,10 +35,11 @@ import static com.alipay.oceanbase.hbase.constants.OHConstants.SOCKET_TIMEOUT; import static org.apache.hadoop.hbase.ipc.RpcClient.SOCKET_TIMEOUT_CONNECT; import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.*; public class OHConnectionTest { - protected Table hTable; - protected Connection connection; + protected static Table hTable; + protected Connection connection; @Test public void testConnectionBySet() throws Exception { @@ -54,6 +55,7 @@ public void testConnectionBySet() throws Exception { TableName tableName = TableName.valueOf("test"); hTable = connection.getTable(tableName); testBasic(); + hTable.close(); } @Test @@ -66,6 +68,50 @@ public void testConnectionByXml() throws Exception { TableName tableName = TableName.valueOf("test"); hTable = connection.getTable(tableName); testBasic(); + hTable.close(); + } + + @BeforeClass + public static void before() throws Exception { + // use self-defined namespace "n1" + hTable = ObHTableTestUtil.newOHTableClient("n1:test"); + ((OHTableClient) hTable).init(); + } + + @AfterClass + public static void finish() throws IOException { + hTable.close(); + } + + @Test + public void testRefreshTableEntry() throws Exception { + ((OHTableClient) hTable).refreshTableEntry("family1", false); + ((OHTableClient) hTable).refreshTableEntry("family1", true); + } + + @Test + public void testNew() throws Exception { + OHTableClient hTable2 = ObHTableTestUtil.newOHTableClient("n1:test"); + hTable2.init(); + hTable2.getConfiguration().set("rs.list.acquire.read.timeout", "10000"); + + assertTrue(hTable2.isAutoFlush()); + hTable2.setAutoFlush(false); + assertFalse(hTable2.isAutoFlush()); + hTable2.setAutoFlush(true, true); + assertTrue(hTable2.isAutoFlush()); + hTable2.setWriteBufferSize(10000000L); + assertEquals(10000000L, hTable2.getWriteBufferSize()); + assertEquals("n1:test", hTable2.getTableNameString()); + assertEquals("n1:test", new String(hTable2.getTableName())); + hTable2.flushCommits(); + hTable2.close(); + assertTrue(true); + } + + @After + public void after() throws IOException { + hTable.close(); } private void testBasic() throws Exception { diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHMultiNamespaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHMultiNamespaceTest.java deleted file mode 100644 index 9c585c12..00000000 --- a/src/test/java/com/alipay/oceanbase/hbase/OHMultiNamespaceTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/*- - * #%L - * OBKV HBase Client Framework - * %% - * Copyright (C) 2022 OceanBase Group - * %% - * OBKV HBase Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.hbase; - -import org.apache.hadoop.hbase.client.Get; -import org.junit.*; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class OHMultiNamespaceTest extends HTableTestBase { - @Before - public void before() throws Exception { - // use self-defined namespace "n1" - hTable = ObHTableTestUtil.newOHTableClient("n1:test"); - ((OHTableClient) hTable).init(); - } - - @After - public void finish() throws IOException { - hTable.close(); - } - - @Test - public void testRefreshTableEntry() throws Exception { - ((OHTableClient) hTable).refreshTableEntry("family1", false); - ((OHTableClient) hTable).refreshTableEntry("family1", true); - } - - @Test - public void testGetColumnFamilyNotExists() throws Exception { - /** family 不存在时提示不友好，*/ - Get get = new Get(("key_c_f").getBytes()); - get.addFamily("family_not_exists".getBytes()); - expectedException.expect(IOException.class); - expectedException.expectMessage("query table:n1:test family family_not_exists error."); - hTable.get(get); - } - - @Test - public void testNew() throws Exception { - OHTableClient hTable2 = ObHTableTestUtil.newOHTableClient("n1:test"); - hTable2.init(); - hTable2.getConfiguration().set("rs.list.acquire.read.timeout", "10000"); - - assertTrue(hTable2.isAutoFlush()); - hTable2.setAutoFlush(false); - assertFalse(hTable2.isAutoFlush()); - hTable2.setAutoFlush(true, true); - assertTrue(hTable2.isAutoFlush()); - hTable2.setWriteBufferSize(10000000L); - assertEquals(10000000L, hTable2.getWriteBufferSize()); - assertEquals("n1:test", hTable2.getTableNameString()); - assertEquals("n1:test", new String(hTable2.getTableName())); - hTable2.flushCommits(); - hTable2.close(); - assertTrue(true); - } - - @After - public void after() throws IOException { - hTable.close(); - } -} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 380125f1..6d53616b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -17,6 +17,7 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.util.Pair; import org.junit.Assert; diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java index 59d4aac5..01c4057f 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java @@ -17,25 +17,33 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.junit.*; -import java.io.IOException; +import java.util.LinkedList; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class OHTableClientTest extends HTableTestBase { - @Before - public void before() throws Exception { + @BeforeClass + public static void before() throws Exception { hTable = ObHTableTestUtil.newOHTableClient("test"); // hTable = ObHTableTestUtil.newOHTableClient("n1:test"); ((OHTableClient) hTable).init(); + multiCfHTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); + ((OHTableClient) multiCfHTable).init(); + List tableGroups = new LinkedList<>(); + tableGroups.add("test"); + tableGroups.add("test_multi_cf"); + ObHTableTestUtil.prepareClean(tableGroups); } - @After - public void finish() throws IOException { - hTable.close(); + @Before + public void prepareCase() { + ObHTableTestUtil.cleanData(); } @Test @@ -67,8 +75,10 @@ public void testNew() throws Exception { assertTrue(true); } - @After - public void after() throws IOException { + @AfterClass + public static void finish() throws Exception { hTable.close(); + multiCfHTable.close(); + ObHTableTestUtil.closeConn(); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTestLoadTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTestLoadTest.java index 2c636436..ed9df2fa 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTestLoadTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTestLoadTest.java @@ -17,24 +17,36 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.rpc.exception.ObTableNotExistException; import org.apache.hadoop.hbase.client.Delete; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import java.io.IOException; +import java.util.LinkedList; +import java.util.List; import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_ENABLE; import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_SUFFIX; public class OHTableClientTestLoadTest extends HTableTestBase { - @Before - public void before() throws Exception { + @BeforeClass + public static void before() throws Exception { hTable = ObHTableTestUtil.newOHTableClient("test"); ((OHTableClient) hTable).init(); hTable.getConfiguration().set(HBASE_HTABLE_TEST_LOAD_ENABLE, "true"); + multiCfHTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); + ((OHTableClient) multiCfHTable).init(); + multiCfHTable.getConfiguration().set(HBASE_HTABLE_TEST_LOAD_ENABLE, "true"); + List tableGroups = new LinkedList<>(); + tableGroups.add("test"); + tableGroups.add("test_multi_cf"); + ObHTableTestUtil.prepareClean(tableGroups); + } + + @Before + public void prepareCase() { + ObHTableTestUtil.cleanData(); } @Test @@ -53,9 +65,11 @@ public void test_refresh_table_entry() throws Exception { } - @After - public void after() throws IOException { + @AfterClass + public static void after() throws Exception { hTable.close(); + multiCfHTable.close(); + ObHTableTestUtil.closeConn(); } @Test diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java deleted file mode 100644 index 2ed4c190..00000000 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java +++ /dev/null @@ -1,348 +0,0 @@ -package com.alipay.oceanbase.hbase; - -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.*; -import org.junit.rules.ExpectedException; - -import java.io.IOException; - -import static org.apache.hadoop.hbase.util.Bytes.toBytes; - -public class OHTableDeleteFamilyVersionTest { - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - protected HTableInterface hTable; - private static final String key1 = "scanKey1x"; - private static final String key2 = "scanKey2x"; - private static final String key3 = "scanKey3x"; - private static final String column1 = "column1"; - private static final String column2 = "column2"; - private static final String column3 = "column3"; - private static final String value1 = "value1"; - private static final String value2 = "value2"; - private static final String value3 = "value3"; - private static final String family1 = "family_with_group1"; - private static final String family2 = "family_with_group2"; - - @Before - public void before() throws Exception { - hTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); - ((OHTableClient) hTable).init(); - } - - @After - public void finish() throws IOException { - hTable.close(); - } - - public void tryPut(Table hTable, Put put) throws Exception { - hTable.put(put); - Thread.sleep(1); - } - - @Test - public void testDeleteFamilyVerison() throws Exception { - // delete previous data - Delete deleteKey1Family = new Delete(toBytes(key1)); - deleteKey1Family.deleteFamily(toBytes(family1)); - deleteKey1Family.deleteFamily(toBytes(family2)); - Delete deleteKey2Family = new Delete(toBytes(key2)); - deleteKey2Family.deleteFamily(toBytes(family1)); - deleteKey2Family.deleteFamily(toBytes(family2)); - Delete deleteKey3Family = new Delete(toBytes(key3)); - deleteKey3Family.deleteFamily(toBytes(family1)); - deleteKey3Family.deleteFamily(toBytes(family2)); - - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - - long minTimeStamp = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp1 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp2 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp3 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp4 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp5 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp6 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp7 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp8 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp9 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp10 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp11 = System.currentTimeMillis(); - Thread.sleep(5); - long maxTimeStamp = System.currentTimeMillis(); - - Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); - putKey1Fam1Column1MinTs.add(toBytes(family1), toBytes(column1), minTimeStamp, - toBytes(value1)); - - Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); - putKey3Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp1, toBytes(value2)); - - Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); - putKey1Fam1Column2MinTs.add(toBytes(family1), toBytes(column2), minTimeStamp, - toBytes(value1)); - - Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); - putKey1Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); - - Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); - putKey2Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); - - Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); - putKey2Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); - - Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); - putKey3Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); - - Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts6.add(toBytes(family1), toBytes(column2), timeStamp6, toBytes(value1)); - - Put putKey2Fam1Column3Ts6 = new Put(toBytes(key2)); - putKey2Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp3, toBytes(value1)); - - tryPut(hTable, putKey1Fam1Column1MinTs); - tryPut(hTable, putKey3Fam1Column1Ts1); - tryPut(hTable, putKey1Fam1Column2MinTs); - tryPut(hTable, putKey1Fam1Column2Ts3); - tryPut(hTable, putKey2Fam1Column2Ts3); - tryPut(hTable, putKey2Fam1Column3Ts1); - tryPut(hTable, putKey3Fam1Column3Ts1); - tryPut(hTable, putKey3Fam1Column2Ts6); - tryPut(hTable, putKey2Fam1Column3Ts6); - - // test DeleteFamilyVersion single cf - Get get = new Get(toBytes(key1)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(minTimeStamp); - get.setMaxVersions(10); - Result r = hTable.get(get); - Assert.assertEquals(2, r.raw().length); - - get = new Get(toBytes(key3)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp1); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(2, r.raw().length); - - get = new Get(toBytes(key2)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp3); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(2, r.raw().length); - - Delete delKey1MinTs = new Delete(toBytes(key1)); - delKey1MinTs.deleteFamilyVersion(toBytes(family1), minTimeStamp); - hTable.delete(delKey1MinTs); - - get = new Get(toBytes(key1)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(minTimeStamp); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Delete delKey3Ts1 = new Delete(toBytes(key3)); - delKey3Ts1.deleteFamilyVersion(toBytes(family1), timeStamp1); - hTable.delete(delKey3Ts1); - - get = new Get(toBytes(key3)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp1); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Delete delKey2Ts3 = new Delete(toBytes(key2)); - delKey2Ts3.deleteFamilyVersion(family1.getBytes(), timeStamp3); - hTable.delete(delKey2Ts3); - - get = new Get(toBytes(key2)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp3); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Scan scan = new Scan(); - scan.setStartRow(toBytes(key1)); - scan.setStopRow("scanKey4x".getBytes()); - scan.addFamily(toBytes(family1)); - scan.setMaxVersions(10); - ResultScanner scanner = hTable.getScanner(scan); - int key1Cnt = 0, key2Cnt = 0, key3Cnt = 0; - for (Result result : scanner) { - for (KeyValue kv : result.raw()) { - if (key1.equals(Bytes.toString(kv.getRow()))) { - ++key1Cnt; - } else if (key2.equals(Bytes.toString(kv.getRow()))) { - ++key2Cnt; - } else { - ++key3Cnt; - } - } - } - Assert.assertEquals(1, key1Cnt); - Assert.assertEquals(1, key2Cnt); - Assert.assertEquals(1, key3Cnt); - - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - - // test DeleteFamilyVersion multiple cf - Put putKey1Fam1Column3Ts6 = new Put(toBytes(key1)); - putKey1Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp6, toBytes(value3)); - - Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); - putKey1Fam2Column2Ts2.add(toBytes(family2), toBytes(column2), timeStamp2, toBytes(value1)); - - Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); - putKey1Fam2Column3Ts2.add(toBytes(family2), toBytes(column3), timeStamp2, toBytes(value1)); - - Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); - putKey1Fam1Column2Ts1.add(toBytes(family1), toBytes(column2), timeStamp1, toBytes(value2)); - - Put putKey2Fam1Column2Ts8 = new Put(toBytes(key2)); - putKey2Fam1Column2Ts8.add(toBytes(family1), toBytes(column2), timeStamp8, toBytes(value2)); - - Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); - putKey2Fam2Column3Ts1.add(toBytes(family2), toBytes(column3), timeStamp3, toBytes(value3)); - - Put putKey2Fam1Column1Ts1 = new Put(toBytes(key2)); - putKey2Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp8, toBytes(value1)); - - Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); - putKey2Fam2Column1Ts3.add(toBytes(family2), toBytes(column1), timeStamp3, toBytes(value2)); - - Put putKey3Fam1Column2Ts9 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts9.add(toBytes(family1), toBytes(column2), timeStamp9, toBytes(value2)); - - Put putKey3Fam2Column3Ts10 = new Put(toBytes(key3)); - putKey3Fam2Column3Ts10 - .add(toBytes(family2), toBytes(column3), timeStamp10, toBytes(value1)); - - Put putKey3Fam2Column1Ts10 = new Put(toBytes(key3)); - putKey3Fam2Column1Ts10 - .add(toBytes(family2), toBytes(column1), timeStamp10, toBytes(value2)); - - Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts2.add(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); - - tryPut(hTable, putKey1Fam1Column3Ts6); - tryPut(hTable, putKey1Fam2Column2Ts2); - tryPut(hTable, putKey1Fam2Column3Ts2); - tryPut(hTable, putKey1Fam1Column2Ts1); - tryPut(hTable, putKey2Fam1Column2Ts8); - tryPut(hTable, putKey2Fam2Column3Ts1); - tryPut(hTable, putKey2Fam1Column1Ts1); - tryPut(hTable, putKey2Fam2Column1Ts3); - tryPut(hTable, putKey3Fam1Column2Ts9); - tryPut(hTable, putKey3Fam2Column3Ts10); - tryPut(hTable, putKey3Fam2Column1Ts10); - tryPut(hTable, putKey3Fam1Column2Ts2); - - Get getKey1 = new Get(toBytes(key1)); - getKey1.addFamily(toBytes(family1)); - getKey1.addFamily(toBytes(family2)); - getKey1.setMaxVersions(10); - r = hTable.get(getKey1); - Assert.assertEquals(4, r.raw().length); - - Get getKey2 = new Get(toBytes(key2)); - getKey2.addFamily(toBytes(family1)); - getKey2.addFamily(toBytes(family2)); - getKey2.setMaxVersions(10); - r = hTable.get(getKey2); - Assert.assertEquals(4, r.raw().length); - - Get getKey3 = new Get(toBytes(key3)); - getKey3.addFamily(toBytes(family1)); - getKey3.addFamily(toBytes(family2)); - getKey3.setMaxVersions(10); - r = hTable.get(getKey3); - Assert.assertEquals(4, r.raw().length); - - Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); - delKey1Ts_6_2.deleteFamilyVersion(toBytes(family1), timeStamp6); - delKey1Ts_6_2.deleteFamilyVersion(toBytes(family2), timeStamp2); - hTable.delete(delKey1Ts_6_2); - - getKey1 = new Get(toBytes(key1)); - getKey1.addFamily(toBytes(family1)); - getKey1.addFamily(toBytes(family2)); - getKey1.setMaxVersions(10); - r = hTable.get(getKey1); - Assert.assertEquals(1, r.raw().length); - for (KeyValue kv : r.raw()) { - Assert.assertEquals(timeStamp1, kv.getTimestamp()); - } - - Delete delKey2Ts_8_3 = new Delete(toBytes(key2)); - delKey2Ts_8_3.deleteFamilyVersion(toBytes(family1), timeStamp8); - delKey2Ts_8_3.deleteFamilyVersion(toBytes(family2), timeStamp3); - hTable.delete(delKey2Ts_8_3); - - getKey2 = new Get(toBytes(key2)); - getKey2.addFamily(toBytes(family1)); - getKey2.addFamily(toBytes(family2)); - getKey2.setMaxVersions(10); - r = hTable.get(getKey2); - Assert.assertEquals(0, r.raw().length); - - Delete delKey3Ts_2_10 = new Delete(toBytes(key3)); - delKey3Ts_2_10.deleteFamilyVersion(toBytes(family1), timeStamp2); - delKey3Ts_2_10.deleteFamilyVersion(toBytes(family2), timeStamp10); - hTable.delete(delKey3Ts_2_10); - - getKey3 = new Get(toBytes(key3)); - getKey3.addFamily(toBytes(family1)); - getKey3.addFamily(toBytes(family2)); - getKey3.setMaxVersions(10); - r = hTable.get(getKey3); - Assert.assertEquals(1, r.raw().length); - for (KeyValue kv : r.raw()) { - Assert.assertEquals(timeStamp9, kv.getTimestamp()); - } - - scan = new Scan(); - scan.setStartRow(toBytes(key1)); - scan.setStopRow("scanKey4x".getBytes()); - scan.addFamily(toBytes(family1)); - scan.addFamily(toBytes(family2)); - scan.setMaxVersions(10); - scanner = hTable.getScanner(scan); - int ts1Cnt = 0, ts9Cnt = 0; - for (Result result : scanner) { - for (KeyValue kv : result.raw()) { - if (kv.getTimestamp() == timeStamp1) { - ++ts1Cnt; - } else if (kv.getTimestamp() == timeStamp9) { - ++ts9Cnt; - } - } - } - Assert.assertEquals(1, ts1Cnt); - Assert.assertEquals(1, ts9Cnt); - - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - } -} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolLoadTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolLoadTest.java index 4e66fc0b..d639d4a4 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolLoadTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolLoadTest.java @@ -17,15 +17,17 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.rpc.exception.ObTableNotExistException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.HTableInterface; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import java.io.IOException; +import java.sql.SQLException; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.Executors; import static com.alipay.oceanbase.hbase.constants.OHConstants.*; @@ -34,10 +36,10 @@ import static org.junit.Assert.assertTrue; public class OHTablePoolLoadTest extends HTableTestBase { - private OHTablePool ohTablePool; + private static OHTablePool ohTablePool; - @Before - public void setup() throws IOException { + @BeforeClass + public static void setup() throws Exception { Configuration c = new Configuration(); c.set(HBASE_HTABLE_TEST_LOAD_ENABLE, "true"); ohTablePool = new OHTablePool(c, 10); @@ -56,6 +58,16 @@ public void setup() throws IOException { } ohTablePool.setRuntimeBatchExecutor("test", Executors.newFixedThreadPool(3)); hTable = ohTablePool.getTable("test"); + multiCfHTable = ohTablePool.getTable("test_multi_cf"); + List tableGroups = new LinkedList<>(); + tableGroups.add("test"); + tableGroups.add("test_multi_cf"); + ObHTableTestUtil.prepareClean(tableGroups); + } + + @Before + public void prepareCase() { + ObHTableTestUtil.cleanData(); } @Test @@ -152,4 +164,11 @@ public void testNew() throws IOException { hTable2.close(); assertTrue(true); } + + @AfterClass + public static void finish() throws IOException, SQLException { + hTable.close(); + multiCfHTable.close(); + ObHTableTestUtil.closeConn(); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolTest.java index 7ed492d3..d915afdc 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolTest.java @@ -17,25 +17,26 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.remoting.util.ConcurrentHashSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.util.PoolMap; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import java.io.IOException; +import java.sql.SQLException; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import static com.alipay.oceanbase.hbase.util.ObTableClientManager.OB_TABLE_CLIENT_INSTANCE; public class OHTablePoolTest extends HTableTestBase { - protected OHTablePool ohTablePool; + protected static OHTablePool ohTablePool; - private OHTablePool newOHTablePool(final int maxSize, final PoolMap.PoolType poolType) { + private static OHTablePool newOHTablePool(final int maxSize, final PoolMap.PoolType poolType) { OHTablePool pool = new OHTablePool(new Configuration(), maxSize, poolType); pool.setFullUserName("test", ObHTableTestUtil.FULL_USER_NAME); pool.setPassword("test", ObHTableTestUtil.PASSWORD); @@ -52,17 +53,29 @@ private OHTablePool newOHTablePool(final int maxSize, final PoolMap.PoolType poo return pool; } - @Before - public void setup() throws IOException { + @BeforeClass + public static void setup() throws Exception { Configuration c = new Configuration(); ohTablePool = newOHTablePool(10, null); ohTablePool.setRuntimeBatchExecutor("test", Executors.newFixedThreadPool(3)); hTable = ohTablePool.getTable("test"); + multiCfHTable = ohTablePool.getTable("test_multi_cf"); + List tableGroups = new LinkedList<>(); + tableGroups.add("test"); + tableGroups.add("test_multi_cf"); + ObHTableTestUtil.prepareClean(tableGroups); + } + + @Before + public void prepareCase() { + ObHTableTestUtil.cleanData(); } - @After - public void finish() throws IOException { + @AfterClass + public static void finish() throws IOException, SQLException { hTable.close(); + multiCfHTable.close(); + ObHTableTestUtil.closeConn(); } public void test_current_get_close(final OHTablePool ohTablePool, int concurrency, int maxSize) { diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java index b14ca220..f36969ca 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java @@ -17,16 +17,18 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.exception.ObTableNotExistException; import com.alipay.sofa.common.thread.SofaThreadPoolExecutor; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTableInterface; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import java.io.IOException; +import java.sql.SQLException; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.SynchronousQueue; import static com.alipay.oceanbase.hbase.util.TableHBaseLoggerFactory.TABLE_HBASE_LOGGER_SPACE; @@ -35,12 +37,23 @@ import static org.junit.Assert.fail; public class OHTableTest extends HTableTestBase { - @Before - public void setup() throws IOException { + @BeforeClass + public static void setup() throws Exception { Configuration c = ObHTableTestUtil.newConfiguration(); c.set("rs.list.acquire.read.timeout", "10000"); + hTable = new OHTable(c, "test"); + multiCfHTable = new OHTable(c, "test_multi_cf"); + List tableGroups = new LinkedList<>(); + tableGroups.add("test"); +// tableGroups.add("test_multi_cf"); + ObHTableTestUtil.prepareClean(tableGroups); + } + + @Before + public void prepareCase() { + ObHTableTestUtil.cleanData(); } @Test @@ -93,4 +106,11 @@ public void testNew() throws Exception { } + @AfterClass + public static void finish() throws IOException, SQLException { + hTable.close(); + multiCfHTable.close(); + ObHTableTestUtil.closeConn(); + } + } diff --git a/src/test/java/com/alipay/oceanbase/hbase/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/ObHTableTestUtil.java deleted file mode 100644 index ba9d25f2..00000000 --- a/src/test/java/com/alipay/oceanbase/hbase/ObHTableTestUtil.java +++ /dev/null @@ -1,59 +0,0 @@ -/*- - * #%L - * OBKV HBase Client Framework - * %% - * Copyright (C) 2022 OceanBase Group - * %% - * OBKV HBase Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.hbase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; - -import static com.alipay.oceanbase.hbase.constants.OHConstants.*; - -public class ObHTableTestUtil { - // please consult your dba for the following configuration. - public static String PARAM_URL = ""; - public static String FULL_USER_NAME = ""; - public static String PASSWORD = ""; - public static String SYS_USER_NAME = ""; - public static String SYS_PASSWORD = ""; - public static String ODP_ADDR = ""; - public static int ODP_PORT = 0; - public static boolean ODP_MODE = false; - public static String DATABASE = ""; - - public static Configuration newConfiguration() { - Configuration conf = HBaseConfiguration.create(); - conf.set(HBASE_OCEANBASE_FULL_USER_NAME, FULL_USER_NAME); - conf.set(HBASE_OCEANBASE_PASSWORD, PASSWORD); - if (ODP_MODE) { - // ODP mode - conf.set(HBASE_OCEANBASE_ODP_ADDR, ODP_ADDR); - conf.setInt(HBASE_OCEANBASE_ODP_PORT, ODP_PORT); - conf.setBoolean(HBASE_OCEANBASE_ODP_MODE, ODP_MODE); - conf.set(HBASE_OCEANBASE_DATABASE, DATABASE); - } else { - // OCP mode - conf.set(HBASE_OCEANBASE_PARAM_URL, PARAM_URL); - conf.set(HBASE_OCEANBASE_SYS_USER_NAME, SYS_USER_NAME); - conf.set(HBASE_OCEANBASE_SYS_PASSWORD, SYS_PASSWORD); - } - return conf; - } - - public static OHTableClient newOHTableClient(String tableName) { - return new OHTableClient(tableName, newConfiguration()); - } -} \ No newline at end of file diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/NativeHBaseUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/NativeHBaseUtil.java new file mode 100644 index 00000000..7d6d82b5 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/NativeHBaseUtil.java @@ -0,0 +1,105 @@ +package com.alipay.oceanbase.hbase.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Table; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.LinkedHashMap; +import java.util.Map; + +public class NativeHBaseUtil { + public static String SQL_PATH = "src/test/java/unit_test_db.sql"; + public static String MASTER_IP_PORT = ""; + public static String ZK_QUORUM = ""; + public static String ZK_PORT = ""; + + @Test + public void createTable() throws IOException { + Configuration config = new Configuration(); + if (!MASTER_IP_PORT.isEmpty()) { + config.set("hbase.master", MASTER_IP_PORT); + } + config.set("hbase.zookeeper.quorum", ZK_QUORUM); + config.set("hbase.zookeeper.property.clientPort", ZK_PORT); + // 建立连接 + Connection connection = ConnectionFactory.createConnection(config); + Admin admin = connection.getAdmin(); + + // 读取建表语句 + String sql = new String(Files.readAllBytes(Paths.get(SQL_PATH))); + String[] sqlList = sql.split(";"); + Map tableMap = new LinkedHashMap<>(); + for (String singleSql : sqlList) { + String namespace = null; + String realTableName; + String family; + + if (singleSql.contains("CREATE TABLE ")) { + singleSql.trim(); + String[] splits = singleSql.split(" "); + realTableName = splits[2].substring(1, splits[2].length() - 1); + if (realTableName.contains(":")) { + String[] tmpStr = realTableName.split(":", 2); + namespace = tmpStr[0]; + realTableName = tmpStr[1]; + } + String[] tmpStr = realTableName.split("\\$", 2); + realTableName = tmpStr[0]; + family = tmpStr[1]; + HTableDescriptor hTableDescriptor = tableMap.get(realTableName); + if (hTableDescriptor == null) { + hTableDescriptor = new HTableDescriptor(TableName.valueOf(namespace, realTableName)); + tableMap.put(realTableName, hTableDescriptor); + } + if (family != null) { + try { + HColumnDescriptor hColumnDescriptor = new HColumnDescriptor(family); + hColumnDescriptor.setMaxVersions(1000); + hTableDescriptor.addFamily(hColumnDescriptor); + + } catch (Exception e) { + System.out.println("family has exist"); + } + } + // 不分区，结果不会有变化 + } + } + for (Map.Entry entry : tableMap.entrySet()) { + if (admin.tableExists(entry.getValue().getTableName())) { + admin.disableTable(entry.getValue().getTableName()); + admin.deleteTable(entry.getValue().getTableName()); + } + admin.createTable(entry.getValue()); + } + } + + static public Table getTable(TableName tableName) throws IOException { + Configuration config = new Configuration(); + config.set("hbase.master", MASTER_IP_PORT); + config.set("hbase.zookeeper.quorum", ZK_QUORUM); + config.set("hbase.zookeeper.property.clientPort", ZK_PORT); + // 建立连接 + Connection connection = ConnectionFactory.createConnection(config); + return connection.getTable(tableName); + } + + static public Admin getAdmin() throws IOException { + Configuration config = new Configuration(); + config.set("hbase.master", MASTER_IP_PORT); + config.set("hbase.zookeeper.quorum", ZK_QUORUM); + config.set("hbase.zookeeper.property.clientPort", ZK_PORT); + // 建立连接 + Connection connection = ConnectionFactory.createConnection(config); + return connection.getAdmin(); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/OHTableHotkeyThrottleUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/OHTableHotkeyThrottleUtil.java index 950ce5db..aaa3c5a9 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/OHTableHotkeyThrottleUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/OHTableHotkeyThrottleUtil.java @@ -18,7 +18,6 @@ package com.alipay.oceanbase.hbase.util; import com.alipay.oceanbase.hbase.OHTableClient; -import com.alipay.oceanbase.hbase.ObHTableTestUtil; import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; import org.apache.hadoop.hbase.client.*; import org.junit.Assert; diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java new file mode 100644 index 00000000..2e5e8f10 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -0,0 +1,148 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import com.alipay.oceanbase.hbase.OHTableClient; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HTableDescriptor; + +import java.sql.Connection; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.constants.OHConstants.*; + +public class ObHTableTestUtil { + // please consult your dba for the following configuration. + public static String PARAM_URL = ""; + public static String FULL_USER_NAME = ""; + public static String PASSWORD = ""; + public static String SYS_USER_NAME = ""; + public static String SYS_PASSWORD = ""; + public static String ODP_ADDR = ""; + public static int ODP_PORT = 0; + public static boolean ODP_MODE = false; + public static String DATABASE = ""; + public static String JDBC_IP = ""; + public static String JDBC_PORT = ""; + public static String JDBC_DATABASE = ""; + public static String JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " + JDBC_DATABASE + "?" + "useUnicode=TRUE&" + "characterEncoding=utf-8&" + "socketTimeout=3000000&" + "connectTimeout=60000"; + + public static String SQL_FORMAT = "truncate %s"; + public static List tableNameList = new LinkedList<>(); + public static Connection conn; + public static Statement stmt; + + public static void prepareClean(List tableGroupList) throws Exception { + for (String tableGroup : tableGroupList) { + tableNameList.addAll(getOTableNameList(tableGroup)); + } + conn = getConnection(); + stmt = conn.createStatement(); + } + + public static void cleanData() { + try { + for (String realTableName : tableNameList) { + try { + stmt.execute(String.format(SQL_FORMAT, realTableName)); + } catch (Exception e) { + System.out.println( + "clean table data error ." + realTableName + " exception:" + e); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static void closeConn() throws SQLException { + stmt.close(); + conn.close(); + } + + public static Configuration newConfiguration() { + Configuration conf = HBaseConfiguration.create(); + conf.set(HBASE_OCEANBASE_FULL_USER_NAME, FULL_USER_NAME); + conf.set(HBASE_OCEANBASE_PASSWORD, PASSWORD); + if (ODP_MODE) { + // ODP mode + conf.set(HBASE_OCEANBASE_ODP_ADDR, ODP_ADDR); + conf.setInt(HBASE_OCEANBASE_ODP_PORT, ODP_PORT); + conf.setBoolean(HBASE_OCEANBASE_ODP_MODE, ODP_MODE); + conf.set(HBASE_OCEANBASE_DATABASE, DATABASE); + } else { + // OCP mode + conf.set(HBASE_OCEANBASE_PARAM_URL, PARAM_URL); + conf.set(HBASE_OCEANBASE_SYS_USER_NAME, SYS_USER_NAME); + conf.set(HBASE_OCEANBASE_SYS_PASSWORD, SYS_PASSWORD); + } + return conf; + } + + public static OHTableClient newOHTableClient(String tableName) { + return new OHTableClient(tableName, newConfiguration()); + } + + static public List getOTableNameList(String tableGroup) throws IOException { + // 读取建表语句 + List res = new LinkedList<>(); + String sql = new String(Files.readAllBytes(Paths.get(NativeHBaseUtil.SQL_PATH))); + String[] sqlList = sql.split(";"); + Map tableMap = new LinkedHashMap<>(); + for (String singleSql : sqlList) { + String realTableName; + if (singleSql.contains("CREATE TABLE ")) { + singleSql.trim(); + String[] splits = singleSql.split(" "); + String tableGroupName = splits[2].substring(1, splits[2].length() - 1); + if (tableGroupName.contains(":")) { + String[] tmpStr = tableGroupName.split(":", 2); + tableGroupName = tmpStr[1]; + } + realTableName = tableGroupName.split("\\$", 2)[0]; + if (realTableName.equals(tableGroup)) { + res.add(tableGroupName); + } + } + } + return res; + } + + static public Connection getConnection() { + try { + Class.forName("com.mysql.cj.jdbc.Driver"); + String[] userNames = FULL_USER_NAME.split("#"); + Connection conn = DriverManager.getConnection(JDBC_URL, userNames[0], PASSWORD); + + return conn; + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file From 8848394d9a096c505e441829f72c41d4925f483f Mon Sep 17 00:00:00 2001 From: Ziyu Shi <57038180+JackShi148@users.noreply.github.com> Date: Wed, 30 Oct 2024 10:59:48 +0800 Subject: [PATCH 10/21] BufferedMutator 1.3 batch (#88) * add DepentdentFilter and SingleColumnValueExcludeFilter * add singleColumnValueExcludeFilter and DependentColumnFilter; add test cases and optimize bufferedMutator test cases * remove DeleteFamilyVersion file and move all cases to MultiColumnFamilyTest * adjust bufferdMutatorImpl 1.x to new batch * bufferedMutator do not retry, batch retry in table client * add rpcTimeout and operationTimetout setting in bufferedMutator * clean execBuffer if all commits have succeed * format code * change connectionConfig to local variable; add deprecated interfaces in 1.x * add check null --- .../com/alipay/oceanbase/hbase/OHTable.java | 18 + .../hbase/util/OHBufferedMutatorImpl.java | 228 ++++---- .../hbase/HTableMultiCFTestBase.java | 504 +++++++++++++++++- .../oceanbase/hbase/HTableTestBase.java | 10 +- .../oceanbase/hbase/OHConnectionTest.java | 212 ++++---- 5 files changed, 741 insertions(+), 231 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 630b008f..145e97de 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -198,8 +198,13 @@ public OHTable(Configuration configuration, String tableName) throws IOException DEFAULT_HBASE_HTABLE_THREAD_KEEP_ALIVE_TIME); this.executePool = createDefaultThreadPoolExecutor(1, maxThreads, keepAliveTime); OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); + int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, ohConnectionConf)); + this.obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); + this.obTableClient.setRuntimeRetryTimes(numRetries); + setOperationTimeout(ohConnectionConf.getOperationTimeout()); finishSetUp(); } @@ -246,8 +251,13 @@ public OHTable(Configuration configuration, final byte[] tableName, this.executePool = executePool; this.cleanupPoolOnClose = false; OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); + int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, ohConnectionConf)); + this.obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); + this.obTableClient.setRuntimeRetryTimes(numRetries); + setOperationTimeout(ohConnectionConf.getOperationTimeout()); finishSetUp(); } @@ -301,6 +311,7 @@ public OHTable(TableName tableName, Connection connection, } else { this.cleanupPoolOnClose = false; } + this.rpcTimeout = connectionConfig.getRpcTimeout(); this.operationTimeout = connectionConfig.getOperationTimeout(); this.operationExecuteInPool = this.configuration.getBoolean( @@ -311,8 +322,15 @@ public OHTable(TableName tableName, Connection connection, DEFAULT_HBASE_HTABLE_PUT_WRITE_BUFFER_CHECK); this.writeBufferSize = connectionConfig.getWriteBufferSize(); this.tableName = tableName.getName(); + int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, connectionConfig)); + this.obTableClient.setRpcExecuteTimeout(rpcTimeout); + this.obTableClient.setRuntimeRetryTimes(numRetries); + setOperationTimeout(operationTimeout); + + finishSetUp(); } /** diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index 0cef7e62..42e44737 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -18,14 +18,11 @@ package com.alipay.oceanbase.hbase.util; import com.alipay.oceanbase.hbase.OHTable; -import com.alipay.oceanbase.rpc.ObTableClient; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.*; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger; import java.io.IOException; @@ -34,54 +31,53 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.LCD; @InterfaceAudience.Private public class OHBufferedMutatorImpl implements BufferedMutator { - private static final Logger LOGGER = TableHBaseLoggerFactory - .getLogger(OHBufferedMutatorImpl.class); + private static final Logger LOGGER = TableHBaseLoggerFactory + .getLogger(OHBufferedMutatorImpl.class); - private final ExceptionListener listener; + private final ExceptionListener listener; - protected final ObTableClient obTableClient; - private final TableName tableName; - private volatile Configuration conf; - private final OHConnectionConfiguration connectionConfig; + private final OHTable ohTable; + private final TableName tableName; + private volatile Configuration conf; - final ConcurrentLinkedQueue asyncWriteBuffer = new ConcurrentLinkedQueue(); - AtomicLong currentAsyncBufferSize = new AtomicLong(0); + @VisibleForTesting + final ConcurrentLinkedQueue asyncWriteBuffer = new ConcurrentLinkedQueue(); + @VisibleForTesting + AtomicLong currentAsyncBufferSize = new AtomicLong(0); - private AtomicReference> type = new AtomicReference<>(null); - private final long writeBufferSize; - private final int maxKeyValueSize; - private boolean closed = false; - private final ExecutorService pool; - private final int rpcTimeout; + private long writeBufferSize; + private final int maxKeyValueSize; + private boolean closed = false; + private final ExecutorService pool; + private int rpcTimeout; + private int operationTimeout; public OHBufferedMutatorImpl(OHConnectionImpl ohConnection, BufferedMutatorParams params) throws IOException { if (ohConnection == null || ohConnection.isClosed()) { throw new IllegalArgumentException("Connection is null or closed."); } - // create a ObTableClient to do rpc operations - this.obTableClient = ObTableClientManager.getOrCreateObTableClient(ohConnection - .getOHConnectionConfiguration()); - // init params in OHBufferedMutatorImpl this.tableName = params.getTableName(); this.conf = ohConnection.getConfiguration(); - this.connectionConfig = ohConnection.getOHConnectionConfiguration(); this.listener = params.getListener(); + + OHConnectionConfiguration connectionConfig = ohConnection.getOHConnectionConfiguration(); this.pool = params.getPool(); - this.obTableClient.setRuntimeBatchExecutor(pool); + this.rpcTimeout = connectionConfig.getRpcTimeout(); + this.operationTimeout = connectionConfig.getOperationTimeout(); this.writeBufferSize = params.getWriteBufferSize() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params .getWriteBufferSize() : connectionConfig.getWriteBufferSize(); this.maxKeyValueSize = params.getMaxKeyValueSize() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params .getMaxKeyValueSize() : connectionConfig.getMaxKeyValueSize(); - this.rpcTimeout = connectionConfig.getRpcTimeout(); - this.obTableClient.setRpcExecuteTimeout(rpcTimeout); + + // create an OHTable object to do batch work + this.ohTable = new OHTable(tableName, ohConnection, connectionConfig, pool); } @Override @@ -119,38 +115,37 @@ public void mutate(List mutations) throws IOException { } long toAddSize = 0; - // check if every mutation's family is the same - // check if mutations are the same type for (Mutation m : mutations) { - OHTable.checkFamilyViolation(m.getFamilyMap().keySet(), true); - validateInsUpAndDelete(m); - Class curType = m.getClass(); - // set the type of this BufferedMutator - if (type.get() == null) { - type.compareAndSet(null, mutations.get(0).getClass()); - } - if (!type.get().equals(curType)) { - throw new IllegalArgumentException("Not support different type in one batch."); - } + validateOperation(m); toAddSize += m.heapSize(); } currentAsyncBufferSize.addAndGet(toAddSize); asyncWriteBuffer.addAll(mutations); - asyncExecute(false); + if (currentAsyncBufferSize.get() > writeBufferSize) { + execute(false); + } + } /** * Check whether the mutation is Put or Delete in 1.x * @param mt - mutation operation */ - private void validateInsUpAndDelete(Mutation mt) throws IllegalArgumentException { + private void validateOperation(Mutation mt) throws IllegalArgumentException { + if (mt == null) { + throw new IllegalArgumentException("Mutation operation cannot be null"); + } if (!(mt instanceof Put) && !(mt instanceof Delete)) { throw new IllegalArgumentException("Only support for Put and Delete for now."); } if (mt instanceof Put) { + // family empty check is in validatePut HTable.validatePut((Put) mt, maxKeyValueSize); + OHTable.checkFamilyViolation(mt.getFamilyMap().keySet(), true); + } else { + OHTable.checkFamilyViolation(mt.getFamilyMap().keySet(), false); } } @@ -161,91 +156,49 @@ private void validateInsUpAndDelete(Mutation mt) throws IllegalArgumentException * @param flushAll - if true, sends all the writes and wait for all of them to finish before * returning. */ - private void asyncExecute(boolean flushAll) throws IOException { + private void execute(boolean flushAll) throws IOException { LinkedList execBuffer = new LinkedList<>(); - ObTableBatchOperationRequest request = null; - // namespace n1, n1:table_name - // namespace default, table_name - String tableNameString = tableName.getNameAsString(); + long dequeuedSize = 0L; try { - while (true) { - try{ - if (!flushAll || asyncWriteBuffer.isEmpty()) { - if (currentAsyncBufferSize.get() <= writeBufferSize) { - break; - } - } - Mutation m; - while ((m = asyncWriteBuffer.poll()) != null) { - execBuffer.add(m); - long size = m.heapSize(); - currentAsyncBufferSize.addAndGet(-size); - } - // in concurrent situation, asyncWriteBuffer may be empty here - // for other threads flush all buffer - if (execBuffer.isEmpty()) { - break; - } - // for now, operations' family is the same - byte[] family = execBuffer.getFirst().getFamilyMap().firstKey(); - ObTableBatchOperation batch = buildObTableBatchOperation(execBuffer); - // table_name$cf_name - String targetTableName = OHTable.getTargetTableName(tableNameString, Bytes.toString(family), conf); - request = OHTable.buildObTableBatchOperationRequest(batch, targetTableName); - } catch (Exception ex) { - LOGGER.error("Errors occur before mutation operation", ex); - throw new IllegalArgumentException("Errors occur before mutation operation", ex); - } - try { - ObTableBatchOperationResult result = (ObTableBatchOperationResult) obTableClient.execute(request); - } catch (Exception ex) { - LOGGER.debug("Errors occur during mutation operation", ex); - Mutation m = null; - try { - // retry every single operation - while (!execBuffer.isEmpty()) { - // poll elements from execBuffer to recollect remaining operations - m = execBuffer.poll(); - byte[] family = m.getFamilyMap().firstKey(); - ObTableBatchOperation batch = buildObTableBatchOperation(Collections.singletonList(m)); - String targetTableName = OHTable.getTargetTableName(tableNameString, Bytes.toString(family), conf); - request = OHTable.buildObTableBatchOperationRequest(batch, targetTableName); - ObTableBatchOperationResult result = (ObTableBatchOperationResult) obTableClient.execute(request); - } - } catch (Exception newEx) { - if (m != null) { - execBuffer.addFirst(m); - } - // if retry fails, only recollect remaining operations - while(!execBuffer.isEmpty()) { - m = execBuffer.poll(); - long size = m.heapSize(); - asyncWriteBuffer.add(m); - currentAsyncBufferSize.addAndGet(size); - } - throw newEx; - } - } + Mutation m; + while ((writeBufferSize <= 0 || dequeuedSize < (writeBufferSize * 2) || flushAll) + && (m = asyncWriteBuffer.poll()) != null) { + execBuffer.add(m); + long size = m.heapSize(); + currentAsyncBufferSize.addAndGet(-size); + dequeuedSize += size; + } + + if (execBuffer.isEmpty()) { + return; } + ohTable.batch(execBuffer); + // if commit all successfully, clean execBuffer + execBuffer.clear(); } catch (Exception ex) { LOGGER.error(LCD.convert("01-00026"), ex); - // if the cause is illegal argument, directly throw to user - if (ex instanceof IllegalArgumentException) { - throw (IllegalArgumentException) ex; - } - // TODO: need to collect error information and actions during batch operations - // TODO: maybe keep in ObTableBatchOperationResult - List throwables = new ArrayList(); - List actions = new ArrayList(); - List addresses = new ArrayList(); - throwables.add(ex); - RetriesExhaustedWithDetailsException error = new RetriesExhaustedWithDetailsException( - new ArrayList(throwables), - new ArrayList(actions), new ArrayList(addresses)); - if (listener == null) { - throw error; + if (ex.getCause() instanceof RetriesExhaustedWithDetailsException) { + LOGGER.error(tableName + ": One or more of the operations have failed after retries."); + RetriesExhaustedWithDetailsException retryException = (RetriesExhaustedWithDetailsException) ex.getCause(); + // recollect mutations + execBuffer.clear(); + for (int i = 0; i < retryException.getNumExceptions(); ++i) { + execBuffer.add((Mutation) retryException.getRow(i)); + } + if (listener != null) { + listener.onException(retryException, this); + } else { + throw retryException; + } } else { - listener.onException(error, this); + LOGGER.error("Errors unrelated to operations occur during mutation operation", ex); + throw ex; + } + } finally { + for (Mutation mutation : execBuffer) { + long size = mutation.heapSize(); + currentAsyncBufferSize.addAndGet(size); + asyncWriteBuffer.add(mutation); } } } @@ -256,7 +209,7 @@ public void close() throws IOException { return; } try { - asyncExecute(true); + execute(true); } finally { // the pool in ObTableClient will be shut down too this.pool.shutdown(); @@ -273,13 +226,21 @@ public void close() throws IOException { } } + @Deprecated + public void setWriteBufferSize(long writeBufferSize) throws IOException { + this.writeBufferSize = writeBufferSize; + if (currentAsyncBufferSize.get() > writeBufferSize) { + flush(); + } + } + /** * Force to commit all operations * do not care whether the pool is shut down or this BufferedMutator is closed */ @Override public void flush() throws IOException { - asyncExecute(true); + execute(true); } @Override @@ -287,13 +248,18 @@ public long getWriteBufferSize() { return this.writeBufferSize; } - private ObTableBatchOperation buildObTableBatchOperation(List execBuffer) { - List keyValueList = new LinkedList<>(); - for (Mutation mutation : execBuffer) { - for (Map.Entry> entry : mutation.getFamilyMap().entrySet()) { - keyValueList.addAll(entry.getValue()); - } - } - return OHTable.buildObTableBatchOperation(keyValueList, false, null); + public void setRpcTimeout(int rpcTimeout) { + this.rpcTimeout = rpcTimeout; + this.ohTable.setRpcTimeout(rpcTimeout); + } + + public void setOperationTimeout(int operationTimeout) { + this.operationTimeout = operationTimeout; + this.ohTable.setOperationTimeout(operationTimeout); + } + + @Deprecated + public List getWriteBuffer() { + return Arrays.asList(asyncWriteBuffer.toArray(new Row[0])); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java index fa28a810..c318b37f 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java @@ -17,8 +17,11 @@ package com.alipay.oceanbase.hbase; +import org.apache.hadoop.conf.Configuration; import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.filter.PrefixFilter; @@ -236,12 +239,10 @@ public void testDeleteFamilyVerison() throws Exception { putKey3Fam1Column2Ts6.add(toBytes(family1), toBytes(column2), timeStamp6, toBytes(value2)); Put putKey3Fam2Column3Ts7 = new Put(toBytes(key3)); - putKey3Fam2Column3Ts7 - .add(toBytes(family2), toBytes(column3), timeStamp7, toBytes(value1)); + putKey3Fam2Column3Ts7.add(toBytes(family2), toBytes(column3), timeStamp7, toBytes(value1)); Put putKey3Fam2Column1Ts7 = new Put(toBytes(key3)); - putKey3Fam2Column1Ts7 - .add(toBytes(family2), toBytes(column1), timeStamp7, toBytes(value2)); + putKey3Fam2Column1Ts7.add(toBytes(family2), toBytes(column1), timeStamp7, toBytes(value2)); Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); putKey3Fam1Column2Ts2.add(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); @@ -347,6 +348,197 @@ public void testDeleteFamilyVerison() throws Exception { multiCfHTable.delete(deleteKey3Family); } + @Test + public void testMulfiColumnFamilyBufferedMutator() throws Exception { + byte[] family1 = "family_with_group1".getBytes(); + byte[] family2 = "family_with_group2".getBytes(); + byte[] family3 = "family_with_group3".getBytes(); + + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family3_column2 = "family3_column2".getBytes(); + byte[] family1_value = "VVV1".getBytes(); + byte[] family2_value = "VVV2".getBytes(); + byte[] family3_value = "VVV3".getBytes(); + + Configuration conf = ObHTableTestUtil.newConfiguration(); + TableName tableName = TableName.valueOf("test_multi_cf"); + Connection connection = ConnectionFactory.createConnection(conf); + BufferedMutator mutator = connection.getBufferedMutator(tableName); + + int rows = 10; + List keys = new ArrayList<>(); + List mutations = new ArrayList<>(); + for (int i = 0; i < rows; ++i) { + String key = "Key" + i; + keys.add(key); + Delete delete = new Delete(toBytes(key)); + mutations.add(delete); + Put put = new Put(toBytes(key)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + mutations.add(put); + } + mutator.mutate(mutations); + + // test force flush + mutator.flush(); + Get get = new Get(toBytes("Key2")); + get.addFamily(family1); + get.addFamily(family2); + Result result = multiCfHTable.get(get); + Assert.assertEquals(5, result.raw().length); + + mutations.clear(); + for (int i = 0; i < rows; ++i) { + if (i % 5 == 0) { // 0, 5 + Delete delete = new Delete(toBytes("Key" + i)); + delete.deleteFamily(family2); + delete.deleteFamily(family3); + mutations.add(delete); + } + } + mutator.mutate(mutations); + mutator.flush(); + + get = new Get(toBytes("Key0")); + result = multiCfHTable.get(get); + Assert.assertEquals(3, result.raw().length); + Assert.assertFalse(result.containsColumn(family2, family2_column1)); + Assert.assertFalse(result.containsColumn(family2, family2_column2)); + Assert.assertFalse(result.containsColumn(family3, family3_column1)); + + get = new Get(toBytes("Key5")); + result = multiCfHTable.get(get); + Assert.assertEquals(3, result.raw().length); + Assert.assertFalse(result.containsColumn(family2, family2_column1)); + Assert.assertFalse(result.containsColumn(family2, family2_column2)); + Assert.assertFalse(result.containsColumn(family3, family3_column1)); + + mutations.clear(); + for (String key : keys) { + Delete delete = new Delete(toBytes(key)); + mutations.add(delete); + } + mutator.mutate(mutations); + mutator.flush(); + + Scan scan = new Scan(); + scan.setStartRow(toBytes("Key0")); + scan.setStopRow(toBytes("Key10")); + scan.addFamily(family1); + scan.addFamily(family2); + scan.addFamily(family3); + ResultScanner scanner = multiCfHTable.getScanner(scan); + int count = 0; + for (Result r : scanner) { + count += r.raw().length; + } + Assert.assertEquals(0, count); + + // test auto flush + long bufferSize = 45000L; + BufferedMutatorParams params = new BufferedMutatorParams(tableName); + params.writeBufferSize(bufferSize); + mutator = connection.getBufferedMutator(params); + + while (true) { + for (int i = 0; i < rows; ++i) { + mutations.clear(); + Put put = new Put(toBytes(keys.get(i))); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family3, family3_column1, family2_value); + put.add(family3, family3_column2, family3_value); + mutations.add(put); + if (i % 3 == 0) { // 0, 3, 6, 9 + Delete delete = new Delete(toBytes(keys.get(i))); + delete.deleteFamily(family1); + delete.deleteFamily(family2); + mutations.add(delete); + } + mutator.mutate(mutations); + } + + get = new Get(toBytes("Key0")); + result = multiCfHTable.get(get); + if (!result.isEmpty()) { + break; + } + } + get = new Get(toBytes("Key2")); + result = multiCfHTable.get(get); + Assert.assertEquals(6 , result.raw().length); + Assert.assertTrue(result.containsColumn(family1, family1_column1)); + Assert.assertTrue(result.containsColumn(family1, family1_column2)); + Assert.assertTrue(result.containsColumn(family1, family1_column3)); + Assert.assertTrue(result.containsColumn(family2, family2_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column2)); + + get = new Get(toBytes("Key3")); + result = multiCfHTable.get(get); + if (result.containsColumn(family1, family1_column1) || result.containsColumn(family2, family2_column1)) { + mutator.flush(); + } + get = new Get(toBytes("Key3")); + result = multiCfHTable.get(get); + Assert.assertEquals(2, result.raw().length); + Assert.assertFalse(result.containsColumn(family1, family1_column1)); + Assert.assertFalse(result.containsColumn(family1, family1_column2)); + Assert.assertFalse(result.containsColumn(family1, family1_column3)); + Assert.assertFalse(result.containsColumn(family2, family2_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column2)); + + get = new Get(toBytes("Key9")); + result = multiCfHTable.get(get); + if (result.containsColumn(family1, family1_column1) || result.containsColumn(family2, family2_column1)) { + mutator.flush(); + } + get = new Get(toBytes("Key9")); + result = multiCfHTable.get(get); + Assert.assertEquals(2, result.raw().length); + Assert.assertFalse(result.containsColumn(family1, family1_column1)); + Assert.assertFalse(result.containsColumn(family1, family1_column2)); + Assert.assertFalse(result.containsColumn(family1, family1_column3)); + Assert.assertFalse(result.containsColumn(family2, family2_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column2)); + + // clean data + mutations.clear(); + for (String key : keys) { + Delete delete = new Delete(toBytes(key)); + mutations.add(delete); + } + mutator.mutate(mutations); + mutator.flush(); + + scan = new Scan(); + scan.setStartRow(toBytes("Key0")); + scan.setStopRow(toBytes("Key10")); + scan.addFamily(family1); + scan.addFamily(family2); + scan.addFamily(family3); + scanner = multiCfHTable.getScanner(scan); + count = 0; + for (Result r : scanner) { + count += r.raw().length; + } + Assert.assertEquals(0, count); + } + @Test public void testMultiColumnFamilyBatch() throws Exception { byte[] family1 = "family_with_group1".getBytes(); @@ -934,6 +1126,13 @@ public void testMultiColumnFamilyGet() throws Exception { @Test public void testMultiColumnFamilyDelete() throws Exception { + String key1 = "scanKey1x"; + String key2 = "scanKey2x"; + String key3 = "scanKey3x"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + byte[] family1 = "family_with_group1".getBytes(); byte[] family2 = "family_with_group2".getBytes(); byte[] family3 = "family_with_group3".getBytes(); @@ -943,6 +1142,7 @@ public void testMultiColumnFamilyDelete() throws Exception { byte[] family1_column3 = "family1_column3".getBytes(); byte[] family2_column1 = "family2_column1".getBytes(); byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family2_column3 = "family2_column3".getBytes(); byte[] family3_column1 = "family3_column1".getBytes(); byte[] family1_value = "VVV1".getBytes(); byte[] family2_value = "VVV2".getBytes(); @@ -1060,5 +1260,301 @@ public void testMultiColumnFamilyDelete() throws Exception { assertEquals(oldTimestamp, result.getColumnCells(family2, family2_column1).get(0) .getTimestamp()); assertTrue(lastTimestamp > oldTimestamp); + + // delete previous data + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(family1); + deleteKey1Family.deleteFamily(family2); + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(family1); + deleteKey2Family.deleteFamily(family2); + Delete deleteKey3Family = new Delete(toBytes(key3)); + deleteKey3Family.deleteFamily(family1); + deleteKey3Family.deleteFamily(family2); + + multiCfHTable.delete(deleteKey1Family); + multiCfHTable.delete(deleteKey2Family); + multiCfHTable.delete(deleteKey3Family); + + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); + putKey1Fam1Column1MinTs.add(family1, family1_column1, minTimeStamp, toBytes(value1)); + + Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column1Ts1.add(family1, family1_column1, timeStamp1, toBytes(value2)); + + Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); + putKey1Fam1Column2MinTs.add(family1, family1_column2, minTimeStamp, toBytes(value1)); + + Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts3.add(family1, family1_column2, timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts3.add(family1, family1_column2, timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts1.add(family1, family1_column3, timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column3Ts1.add(family1, family1_column3, timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts6.add(family1, family1_column2, timeStamp6, toBytes(value1)); + + Put putKey2Fam1Column3Ts6 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts6.add(family1, family1_column3, timeStamp3, toBytes(value1)); + + multiCfHTable.put(putKey1Fam1Column1MinTs); + multiCfHTable.put(putKey3Fam1Column1Ts1); + multiCfHTable.put(putKey1Fam1Column2MinTs); + multiCfHTable.put(putKey1Fam1Column2Ts3); + multiCfHTable.put(putKey2Fam1Column2Ts3); + multiCfHTable.put(putKey2Fam1Column3Ts1); + multiCfHTable.put(putKey3Fam1Column3Ts1); + multiCfHTable.put(putKey3Fam1Column2Ts6); + multiCfHTable.put(putKey2Fam1Column3Ts6); + + // test DeleteFamilyVersion single cf + get = new Get(toBytes(key1)); + get.addFamily(family1); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + Result r = multiCfHTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key3)); + get.addFamily(family1); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key2)); + get.addFamily(family1); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(2, r.raw().length); + + Delete delKey1MinTs = new Delete(toBytes(key1)); + delKey1MinTs.deleteFamilyVersion(family1, minTimeStamp); + multiCfHTable.delete(delKey1MinTs); + + get = new Get(toBytes(key1)); + get.addFamily(family1); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts1 = new Delete(toBytes(key3)); + delKey3Ts1.deleteFamilyVersion(family1, timeStamp1); + multiCfHTable.delete(delKey3Ts1); + + get = new Get(toBytes(key3)); + get.addFamily(family1); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey2Ts3 = new Delete(toBytes(key2)); + delKey2Ts3.deleteFamilyVersion(family1, timeStamp3); + multiCfHTable.delete(delKey2Ts3); + + get = new Get(toBytes(key2)); + get.addFamily(family1); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Scan scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(family1); + scan.setMaxVersions(10); + ResultScanner scanner = multiCfHTable.getScanner(scan); + int key1Cnt = 0, key2Cnt = 0, key3Cnt = 0; + for (Result res : scanner) { + for (KeyValue kv : res.raw()) { + if (key1.equals(Bytes.toString(kv.getRow()))) { + ++key1Cnt; + } else if (key2.equals(Bytes.toString(kv.getRow()))) { + ++key2Cnt; + } else { + ++key3Cnt; + } + } + } + Assert.assertEquals(1, key1Cnt); + Assert.assertEquals(1, key2Cnt); + Assert.assertEquals(1, key3Cnt); + + multiCfHTable.delete(deleteKey1Family); + multiCfHTable.delete(deleteKey2Family); + multiCfHTable.delete(deleteKey3Family); + + // test DeleteFamilyVersion multiple cf + Put putKey1Fam1Column3Ts6 = new Put(toBytes(key1)); + putKey1Fam1Column3Ts6.add(family1, family1_column3, timeStamp6, toBytes(value3)); + + Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column2Ts2.add(family2, family2_column2, timeStamp2, toBytes(value1)); + + Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column3Ts2.add(family2, family2_column3, timeStamp2, toBytes(value1)); + + Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts1.add(family1, family1_column2, timeStamp1, toBytes(value2)); + + Put putKey2Fam1Column2Ts8 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts8.add(family1, family1_column2, timeStamp8, toBytes(value2)); + + Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam2Column3Ts1.add(family2, family2_column3, timeStamp3, toBytes(value3)); + + Put putKey2Fam1Column1Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column1Ts1.add(family1, family1_column1, timeStamp8, toBytes(value1)); + + Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); + putKey2Fam2Column1Ts3.add(family2, family2_column1, timeStamp3, toBytes(value2)); + + Put putKey3Fam1Column2Ts9 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts9.add(family1, family1_column2, timeStamp9, toBytes(value2)); + + Put putKey3Fam2Column3Ts10 = new Put(toBytes(key3)); + putKey3Fam2Column3Ts10.add(family2, family2_column3, timeStamp10, toBytes(value1)); + + Put putKey3Fam2Column1Ts10 = new Put(toBytes(key3)); + putKey3Fam2Column1Ts10.add(family2, family2_column1, timeStamp10, toBytes(value2)); + + Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts2.add(family1, family1_column2, timeStamp2, toBytes(value1)); + + multiCfHTable.put(putKey1Fam1Column3Ts6); + multiCfHTable.put(putKey1Fam2Column2Ts2); + multiCfHTable.put(putKey1Fam2Column3Ts2); + multiCfHTable.put(putKey1Fam1Column2Ts1); + multiCfHTable.put(putKey2Fam1Column2Ts8); + multiCfHTable.put(putKey2Fam2Column3Ts1); + multiCfHTable.put(putKey2Fam1Column1Ts1); + multiCfHTable.put(putKey2Fam2Column1Ts3); + multiCfHTable.put(putKey3Fam1Column2Ts9); + multiCfHTable.put(putKey3Fam2Column3Ts10); + multiCfHTable.put(putKey3Fam2Column1Ts10); + multiCfHTable.put(putKey3Fam1Column2Ts2); + + Get getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(family1); + getKey1.addFamily(family2); + getKey1.setMaxVersions(10); + r = multiCfHTable.get(getKey1); + Assert.assertEquals(4, r.raw().length); + + Get getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(family1); + getKey2.addFamily(family2); + getKey2.setMaxVersions(10); + r = multiCfHTable.get(getKey2); + Assert.assertEquals(4, r.raw().length); + + Get getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(family1); + getKey3.addFamily(family2); + getKey3.setMaxVersions(10); + r = multiCfHTable.get(getKey3); + Assert.assertEquals(4, r.raw().length); + + Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); + delKey1Ts_6_2.deleteFamilyVersion(family1, timeStamp6); + delKey1Ts_6_2.deleteFamilyVersion(family2, timeStamp2); + multiCfHTable.delete(delKey1Ts_6_2); + + getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(family1); + getKey1.addFamily(family2); + getKey1.setMaxVersions(10); + r = multiCfHTable.get(getKey1); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp1, kv.getTimestamp()); + } + + Delete delKey2Ts_8_3 = new Delete(toBytes(key2)); + delKey2Ts_8_3.deleteFamilyVersion(family1, timeStamp8); + delKey2Ts_8_3.deleteFamilyVersion(family2, timeStamp3); + multiCfHTable.delete(delKey2Ts_8_3); + + getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(family1); + getKey2.addFamily(family2); + getKey2.setMaxVersions(10); + r = multiCfHTable.get(getKey2); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts_2_10 = new Delete(toBytes(key3)); + delKey3Ts_2_10.deleteFamilyVersion(family1, timeStamp2); + delKey3Ts_2_10.deleteFamilyVersion(family2, timeStamp10); + multiCfHTable.delete(delKey3Ts_2_10); + + getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(family1); + getKey3.addFamily(family2); + getKey3.setMaxVersions(10); + r = multiCfHTable.get(getKey3); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp9, kv.getTimestamp()); + } + + scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(family1); + scan.addFamily(family2); + scan.setMaxVersions(10); + scanner = multiCfHTable.getScanner(scan); + int ts1Cnt = 0, ts9Cnt = 0; + for (Result res : scanner) { + for (KeyValue kv : res.raw()) { + if (kv.getTimestamp() == timeStamp1) { + ++ts1Cnt; + } else if (kv.getTimestamp() == timeStamp9) { + ++ts9Cnt; + } + } + } + Assert.assertEquals(1, ts1Cnt); + Assert.assertEquals(1, ts9Cnt); + + multiCfHTable.delete(deleteKey1Family); + multiCfHTable.delete(deleteKey2Family); + multiCfHTable.delete(deleteKey3Family); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 4131a572..9d32921d 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -113,11 +113,11 @@ PRIMARY KEY (`K`, `Q`, `T`) Assert.assertEquals(column1, Bytes.toString(keyValue.getQualifier())); Assert.assertEquals(timestamp, keyValue.getTimestamp()); Assert.assertEquals(value + "1", Bytes.toString(keyValue.getValue())); - System.out.println( - "rowKey: " + new String(keyValue.getRow()) + " family :" + new String( - keyValue.getFamily()) + " columnQualifier:" + new String( - keyValue.getQualifier()) + " timestamp:" + keyValue.getTimestamp() + " value:" + new String( - keyValue.getValue())); + System.out.println("rowKey: " + new String(keyValue.getRow()) + " family :" + + new String(keyValue.getFamily()) + " columnQualifier:" + + new String(keyValue.getQualifier()) + " timestamp:" + + keyValue.getTimestamp() + " value:" + + new String(keyValue.getValue())); } } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index 79848a1e..42de6ef7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -17,7 +17,6 @@ package com.alipay.oceanbase.hbase; -import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; @@ -71,20 +70,10 @@ public void testConnectionByXml() throws Exception { hTable.close(); } - @BeforeClass - public static void before() throws Exception { - // use self-defined namespace "n1" - hTable = ObHTableTestUtil.newOHTableClient("n1:test"); - ((OHTableClient) hTable).init(); - } - - @AfterClass - public static void finish() throws IOException { - hTable.close(); - } - @Test public void testRefreshTableEntry() throws Exception { + hTable = ObHTableTestUtil.newOHTableClient("n1:test"); + ((OHTableClient) hTable).init(); ((OHTableClient) hTable).refreshTableEntry("family1", false); ((OHTableClient) hTable).refreshTableEntry("family1", true); } @@ -249,8 +238,7 @@ public void testBufferedMutatorWithFlush() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); conf.set("rs.list.acquire.read.timeout", "10000"); conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); - BufferedMutator putBufferMutator = null; - BufferedMutator delBufferedMutator = null; + BufferedMutator bufferMutator = null; String key = "putKey"; String column1 = "putColumn1"; String value = "value333444"; @@ -265,17 +253,15 @@ public void testBufferedMutatorWithFlush() throws Exception { hTable.delete(delete); // use defualt params - putBufferMutator = connection.getBufferedMutator(tableName); - delBufferedMutator = connection.getBufferedMutator(tableName); + bufferMutator = connection.getBufferedMutator(tableName); long timestamp = System.currentTimeMillis(); // only support Put and Delete - // for other type of operations, BufferedMutator will not set its type for them Append append = new Append(Bytes.toBytes(key)); append.add("family_group".getBytes(), column1.getBytes(), toBytes("_suffix")); - final BufferedMutator apMut = putBufferMutator; + final BufferedMutator apMut = bufferMutator; Assert.assertThrows(IllegalArgumentException.class, () -> { apMut.mutate(append); }); @@ -289,31 +275,46 @@ public void testBufferedMutatorWithFlush() throws Exception { put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); mutations.add(put2); // test add Mutations with List - putBufferMutator.mutate(mutations); - putBufferMutator.flush(); + bufferMutator.mutate(mutations); + bufferMutator.flush(); Get get = new Get(toBytes(key)); Result r = hTable.get(get); Assert.assertEquals(2, r.raw().length); - Delete del = new Delete(Bytes.toBytes(key)); - final BufferedMutator noCfMut = putBufferMutator; - // test mutation without setting family - Assert.assertThrows(FeatureNotSupportedException.class, () -> { - noCfMut.mutate(del); - }); - del.deleteFamily(Bytes.toBytes(family)); - // test reuse different type bufferedMutator - final BufferedMutator difTypeMut = putBufferMutator; + Put put3 = new Put(Bytes.toBytes(key)); + final BufferedMutator noCfMut = bufferMutator; + // test Put without setting family Assert.assertThrows(IllegalArgumentException.class, () -> { - difTypeMut.mutate(del); + noCfMut.mutate(put3); }); + put3.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "2"), timestamp, Bytes.toBytes(value)); // test add Mutation directly - delBufferedMutator.mutate(del); - delBufferedMutator.flush(); + bufferMutator.mutate(put3); + bufferMutator.flush(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + // test Delete + Delete del = new Delete(toBytes(key)); + // test without setting family, delete all + bufferMutator.mutate(del); + bufferMutator.flush(); r = hTable.get(get); Assert.assertEquals(0, r.raw().length); + + // test hybrid mutations + mutations.clear(); + mutations.add(put1); + mutations.add(put2); + mutations.add(del); + mutations.add(put3); + bufferMutator.mutate(mutations); + bufferMutator.flush(); + + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); } catch (Exception ex) { if (ex instanceof RetriesExhaustedWithDetailsException) { ((RetriesExhaustedWithDetailsException) ex).getCauses().get(0).printStackTrace(); @@ -322,17 +323,14 @@ public void testBufferedMutatorWithFlush() throws Exception { } Assert.assertTrue(false); } finally { - if (putBufferMutator != null ) { - putBufferMutator.close(); + if (bufferMutator != null ) { + bufferMutator.close(); // test flush after closed - putBufferMutator.flush(); - } - if (delBufferedMutator != null) { - delBufferedMutator.close(); + bufferMutator.flush(); // test add mutations after closed Delete delete = new Delete(Bytes.toBytes(key)); delete.deleteFamily(Bytes.toBytes(family)); - final BufferedMutator closedMutator = delBufferedMutator; + final BufferedMutator closedMutator = bufferMutator; Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); }); @@ -356,32 +354,31 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); conf.set("rs.list.acquire.read.timeout", "10000"); conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); - BufferedMutator putBufferMutator = null; - BufferedMutator delBufferedMutator = null; + BufferedMutator bufferMutator = null; String key = "putKey"; String column1 = "putColumn1"; String value = "value333444"; String family = "family_group"; - long timestamp = System.currentTimeMillis(); try { // use n1 database TableName tableName = TableName.valueOf("n1","test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + Assert.assertEquals("n1:test", Bytes.toString(((OHTable) hTable).getTableName())); Delete delete= new Delete(toBytes(key)); delete.deleteFamily(toBytes(family)); hTable.delete(delete); // use defualt params - putBufferMutator = connection.getBufferedMutator(tableName); - delBufferedMutator = connection.getBufferedMutator(tableName); + bufferMutator = connection.getBufferedMutator(tableName); + + long timestamp = System.currentTimeMillis(); // only support Put and Delete - // for other type of operations, BufferedMutator will not set its type for them Append append = new Append(Bytes.toBytes(key)); - append.add(toBytes(family), toBytes(column1), toBytes("_suffix")); - final BufferedMutator apMut = putBufferMutator; + append.add("family_group".getBytes(), column1.getBytes(), toBytes("_suffix")); + final BufferedMutator apMut = bufferMutator; Assert.assertThrows(IllegalArgumentException.class, () -> { apMut.mutate(append); }); @@ -395,31 +392,46 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); mutations.add(put2); // test add Mutations with List - putBufferMutator.mutate(mutations); - putBufferMutator.flush(); + bufferMutator.mutate(mutations); + bufferMutator.flush(); Get get = new Get(toBytes(key)); Result r = hTable.get(get); Assert.assertEquals(2, r.raw().length); - Delete del = new Delete(Bytes.toBytes(key)); - final BufferedMutator noCfMut = putBufferMutator; - // test mutation without setting family - Assert.assertThrows(FeatureNotSupportedException.class, () -> { - noCfMut.mutate(del); - }); - del.deleteFamily(Bytes.toBytes(family)); - final BufferedMutator difTypeMut = putBufferMutator; - // test reuse different type bufferedMutator + Put put3 = new Put(Bytes.toBytes(key)); + final BufferedMutator noCfMut = bufferMutator; + // test Put without setting family Assert.assertThrows(IllegalArgumentException.class, () -> { - difTypeMut.mutate(del); + noCfMut.mutate(put3); }); + put3.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "2"), timestamp, Bytes.toBytes(value)); // test add Mutation directly - delBufferedMutator.mutate(del); - delBufferedMutator.flush(); + bufferMutator.mutate(put3); + bufferMutator.flush(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + // test Delete + Delete del = new Delete(toBytes(key)); + // test without setting family, delete all + bufferMutator.mutate(del); + bufferMutator.flush(); r = hTable.get(get); Assert.assertEquals(0, r.raw().length); + + // test hybrid mutations + mutations.clear(); + mutations.add(put1); + mutations.add(put2); + mutations.add(del); + mutations.add(put3); + bufferMutator.mutate(mutations); + bufferMutator.flush(); + + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); } catch (Exception ex) { if (ex instanceof RetriesExhaustedWithDetailsException) { ((RetriesExhaustedWithDetailsException) ex).getCauses().get(0).printStackTrace(); @@ -428,17 +440,14 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { } Assert.assertTrue(false); } finally { - if (putBufferMutator != null ) { - putBufferMutator.close(); + if (bufferMutator != null ) { + bufferMutator.close(); // test flush after closed - putBufferMutator.flush(); - } - if (delBufferedMutator != null) { - delBufferedMutator.close(); + bufferMutator.flush(); // test add mutations after closed - Delete delete = new Delete(Bytes.toBytes("putKey")); - delete.deleteFamily(Bytes.toBytes("family_group")); - final BufferedMutator closedMutator = delBufferedMutator; + Delete delete = new Delete(Bytes.toBytes(key)); + delete.deleteFamily(Bytes.toBytes(family)); + final BufferedMutator closedMutator = bufferMutator; Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); }); @@ -460,7 +469,7 @@ PRIMARY KEY (`K`, `Q`, `T`) public void testBufferedMutatorWithAutoFlush() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); conf.set("rs.list.acquire.read.timeout", "10000"); - BufferedMutator putBufferMutator = null; + BufferedMutator bufferMutator = null; BufferedMutatorParams params = null; long bufferSize = 45000L; int count = 0; @@ -481,7 +490,7 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); - putBufferMutator = connection.getBufferedMutator(params); + bufferMutator = connection.getBufferedMutator(params); List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { @@ -492,8 +501,20 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { timestamp, Bytes.toBytes(value + "_" + i + "_" + j)); mutations.add(put); } - putBufferMutator.mutate(mutations); + if (i % 10 == 0) { // 0, 10, 20, 30, 40 + for(int j = 0; j < 4; ++j) { + Delete del = new Delete(Bytes.toBytes(key)); + del.addColumns(toBytes(family), toBytes(column1 + "_" + i + "_" + j)); + mutations.add(del); + } + } + bufferMutator.mutate(mutations); } + Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); + Result r = hTable.get(get); + count = r.raw().length; + Assert.assertTrue(count > 0); } catch (Exception ex) { if (ex instanceof RetriesExhaustedWithDetailsException) { ((RetriesExhaustedWithDetailsException) ex).getCauses().get(0).printStackTrace(); @@ -502,14 +523,13 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { } Assert.assertTrue(false); } finally { - if (putBufferMutator != null) { - putBufferMutator.close(); + if (bufferMutator != null) { + bufferMutator.close(); Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); Result r = hTable.get(get); - for (KeyValue keyValue : r.raw()) { - ++count; - } - Assert.assertEquals(200, count); + count = r.raw().length; + Assert.assertEquals(180, count); Delete delete = new Delete(toBytes(key)); delete.deleteFamily(toBytes(family)); hTable.delete(delete); @@ -517,12 +537,12 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { Assert.assertEquals(0, r.raw().length); // test add mutations after closed - final BufferedMutator closedMutator = putBufferMutator; + final BufferedMutator closedMutator = bufferMutator; Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); }); // test flush after closed - putBufferMutator.flush(); + bufferMutator.flush(); } if (params != null) { @@ -586,8 +606,20 @@ public void testBufferedMutatorWithUserPool() throws Exception { timestamp, Bytes.toBytes(value + "_" + i + "_" + j)); mutations.add(put); } + if (i % 10 == 0) { // 0, 10, 20, 30, 40 + for(int j = 0; j < 4; ++j) { + Delete del = new Delete(Bytes.toBytes(key)); + del.addColumns(toBytes(family), toBytes(column1 + "_" + i + "_" + j)); + mutations.add(del); + } + } ohBufferMutator.mutate(mutations); } + Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); + Result r = hTable.get(get); + count = r.raw().length; + Assert.assertTrue(count > 0); } catch (Exception ex) { if (ex instanceof RetriesExhaustedWithDetailsException) { ((RetriesExhaustedWithDetailsException) ex).getCauses().get(0).printStackTrace(); @@ -599,11 +631,10 @@ public void testBufferedMutatorWithUserPool() throws Exception { if (ohBufferMutator != null) { ohBufferMutator.close(); Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); Result r = hTable.get(get); - for (KeyValue keyValue : r.raw()) { - ++count; - } - Assert.assertEquals(200, count); + count = r.raw().length; + Assert.assertEquals(180, count); Delete delete = new Delete(toBytes(key)); delete.deleteFamily(toBytes(family)); hTable.delete(delete); @@ -671,18 +702,18 @@ public void testBufferedMutatorConcurrent() throws Exception { ohBufferMutator = connection.getBufferedMutator(params); + // BufferedMutator is not concurrently safe for (int i = 0; i < 50; ++i) { final int taskId = i; final BufferedMutator thrBufferMutator = ohBufferMutator; executorService.submit(() -> { List mutations = new ArrayList<>(); for (int j = 0; j < 4; ++j) { - String thrKey = key; String thrColumn = column1 + "_" + taskId + "_" + j; String thrValue = value + "_" + taskId + "_" + j; long thrTimestamp = timestamp; - Put put = new Put(Bytes.toBytes(thrKey)); + Put put = new Put(Bytes.toBytes(key)); put.addColumn(Bytes.toBytes(family), Bytes.toBytes(thrColumn), thrTimestamp, Bytes.toBytes(thrValue)); mutations.add(put); @@ -719,10 +750,9 @@ public void testBufferedMutatorConcurrent() throws Exception { if (ohBufferMutator != null) { ohBufferMutator.close(); Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); Result r = hTable.get(get); - for (KeyValue keyValue : r.raw()) { - ++count; - } + count = r.raw().length; Assert.assertEquals(200, count); Delete delete = new Delete(toBytes(key)); delete.deleteFamily(toBytes(family)); From 3520de0467289640bc8d7c571a0c60c05d12ba0c Mon Sep 17 00:00:00 2001 From: Ziyu Shi <57038180+JackShi148@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:41:48 +0800 Subject: [PATCH 11/21] Correct test case for the multi-cf bug (#100) * add DepentdentFilter and SingleColumnValueExcludeFilter * add singleColumnValueExcludeFilter and DependentColumnFilter; add test cases and optimize bufferedMutator test cases * merge obkv branch-1.3 and correct test case for the mulfi-cf bug --- src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index 42de6ef7..7490141c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -297,7 +297,7 @@ public void testBufferedMutatorWithFlush() throws Exception { // test Delete Delete del = new Delete(toBytes(key)); - // test without setting family, delete all + del.deleteFamily(toBytes(family)); bufferMutator.mutate(del); bufferMutator.flush(); From 9776399630c476a16833e7157f3ac113156991be Mon Sep 17 00:00:00 2001 From: Ziyu Shi <57038180+JackShi148@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:37:02 +0800 Subject: [PATCH 12/21] Optimize test case and add sql script for multi-namespace test (#101) * add DepentdentFilter and SingleColumnValueExcludeFilter * add singleColumnValueExcludeFilter and DependentColumnFilter; add test cases and optimize bufferedMutator test cases * merge obkv branch-1.3 and correct test case for the mulfi-cf bug * optimize test case and add sql to create self-defined database --- src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java | 2 +- src/test/java/unit_test_db.sql | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index 7490141c..1fa79204 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -414,7 +414,7 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { // test Delete Delete del = new Delete(toBytes(key)); - // test without setting family, delete all + del.deleteFamily(toBytes(family)); bufferMutator.mutate(del); bufferMutator.flush(); diff --git a/src/test/java/unit_test_db.sql b/src/test/java/unit_test_db.sql index c5681bd7..022177dc 100644 --- a/src/test/java/unit_test_db.sql +++ b/src/test/java/unit_test_db.sql @@ -204,6 +204,7 @@ CREATE TABLE `test_multi_cf$family_with_group3` ( PRIMARY KEY (`K`, `Q`, `T`) ) TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3; +CREATE DATABASE IF NOT EXISTS `n1`; USE `n1`; CREATE TABLE `n1:test$family1` ( `K` varbinary(1024) NOT NULL, From f1588f8b90866faaa27e610c4fb6566984d587b2 Mon Sep 17 00:00:00 2001 From: stuBirdFly <84010733+stuBirdFly@users.noreply.github.com> Date: Thu, 7 Nov 2024 11:32:46 +0800 Subject: [PATCH 13/21] batch compat old server (#106) --- .../com/alipay/oceanbase/hbase/OHTable.java | 286 +++++++++++------ .../hbase/util/CompatibilityUtil.java | 9 + .../hbase/HTableMultiCFTestBase.java | 296 ------------------ .../hbase/util/ObHTableTestUtil.java | 99 +++--- src/{main => test}/resources/hbase-site.xml | 0 5 files changed, 259 insertions(+), 431 deletions(-) create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/CompatibilityUtil.java rename src/{main => test}/resources/hbase-site.xml (100%) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 145e97de..35a9e087 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -36,7 +36,6 @@ import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.*; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncRequest; import com.alipay.oceanbase.rpc.stream.ObTableClientQueryAsyncStreamResult; -import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult; import com.alipay.oceanbase.rpc.table.ObHBaseParams; import com.alipay.oceanbase.rpc.table.ObKVParams; import com.alipay.sofa.common.thread.SofaThreadPoolExecutor; @@ -199,7 +198,7 @@ public OHTable(Configuration configuration, String tableName) throws IOException this.executePool = createDefaultThreadPoolExecutor(1, maxThreads, keepAliveTime); OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, ohConnectionConf)); this.obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); @@ -252,7 +251,7 @@ public OHTable(Configuration configuration, final byte[] tableName, this.cleanupPoolOnClose = false; OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, ohConnectionConf)); this.obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); @@ -323,7 +322,7 @@ public OHTable(TableName tableName, Connection connection, this.writeBufferSize = connectionConfig.getWriteBufferSize(); this.tableName = tableName.getName(); int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, connectionConfig)); this.obTableClient.setRpcExecuteTimeout(rpcTimeout); @@ -481,29 +480,130 @@ public Boolean[] exists(List gets) throws IOException { return objectResults; } - @Override - public void batch(final List actions, final Object[] results) throws IOException { + private void compatOldServerBatch(final List actions, final Object[] results) + throws Exception { + for (Row row : actions) { + if (!(row instanceof Put) && !(row instanceof Delete)) { + throw new FeatureNotSupportedException( + "not supported other type in batch yet,only support put and delete"); + } + } BatchError batchError = new BatchError(); - try { - List resultMapSingleOp = new LinkedList<>(); - String realTableName = getTargetTableName(actions); - obTableClient.setRuntimeBatchExecutor(executePool); - BatchOperation batch = buildBatchOperation(realTableName, actions, tableNameString.equals(realTableName), resultMapSingleOp); - BatchOperationResult tmpResults = batch.execute(); - if (results != null) { - if (results.length != actions.size()) { - throw new AssertionError("results.length"); + List resultMapSingleOp = new LinkedList<>(); + for (int i = 0; i < actions.size(); i++) { + BatchOperationResult tmpResults; + BatchOperation batch = null; + if (actions.get(i) instanceof Put) { + Put put = (Put) actions.get(i); + if (put.getFamilyCellMap().size() > 1) { + String realTableName = getTargetTableName(tableNameString); + batch = buildBatchOperation(realTableName, + Collections.singletonList(put), + true, resultMapSingleOp); + } else { + byte[] family = put.getFamilyCellMap().firstKey(); + String realTableName = getTargetTableName(tableNameString, Bytes.toString(family), configuration); + batch = buildBatchOperation(realTableName, + Collections.singletonList(put), + false, resultMapSingleOp); } - int index = 0; - for (int i = 0; i != results.length; ++i) { - results[i] = tmpResults.getResults().get(index); - index += resultMapSingleOp.get(i); - if (results[i] instanceof ObTableException) { - batchError.add((ObTableException) results[i], actions.get(i), null); + } else if (actions.get(i) instanceof Delete) { + Delete delete = (Delete)actions.get(i); + if (delete.isEmpty()) { + batch = buildBatchOperation(tableNameString, + Collections.singletonList(delete), true, + resultMapSingleOp); + } else if (delete.getFamilyCellMap().size() > 1) { + boolean has_delete_family = delete.getFamilyMap().entrySet().stream() + .flatMap(entry -> entry.getValue().stream()).anyMatch( + kv -> KeyValue.Type.codeToType( + kv.getType()) == KeyValue.Type.DeleteFamily || KeyValue.Type.codeToType( + kv.getType()) == KeyValue.Type.DeleteFamilyVersion); + if (!has_delete_family) { + batch = buildBatchOperation(tableNameString, + Collections.singletonList(delete), true, + resultMapSingleOp); + } else { + for (Map.Entry> entry : delete.getFamilyCellMap() + .entrySet()) { + byte[] family = entry.getKey(); + String realTableName = getTargetTableName(tableNameString, + Bytes.toString(family), configuration); + // split delete + List cells = entry.getValue(); + Delete del = new Delete(delete.getRow()); + del.getFamilyCellMap().put(family, cells); + batch = buildBatchOperation(realTableName, + Collections.singletonList(del), false, + resultMapSingleOp); + tmpResults = batch.execute(); + if (tmpResults.getResults().get(0) instanceof ObTableException) { + results[i] = tmpResults.getResults().get(0); + } + } + if (results[i] == null) { + results[i] = new Result(); + } else { + batchError.add((ObTableException) results[i], actions.get(i), null); + } + continue; } + } else { + byte[] family = delete.getFamilyCellMap().firstKey(); + String realTableName = getTargetTableName(tableNameString, Bytes.toString(family), configuration); + batch = buildBatchOperation(realTableName, + Collections.singletonList(delete), + false, resultMapSingleOp); } - if (batchError.hasErrors()) { - throw batchError.makeException(); + } + if (batch != null) { + tmpResults = batch.execute(); + if (tmpResults.getResults().get(0) instanceof ObTableException) { + results[i] = tmpResults.getResults().get(0); + batchError.add((ObTableException) results[i], actions.get(i), null); + } else { + results[i] = new Result(); + } + } + } + } + + @Override + public void batch(final List actions, final Object[] results) + throws IOException, FeatureNotSupportedException, IllegalArgumentException { + if (actions == null) { + return; + } + if (results != null) { + if (results.length != actions.size()) { + throw new AssertionError("results.length"); + } + } + BatchError batchError = new BatchError(); + obTableClient.setRuntimeBatchExecutor(executePool); + List resultMapSingleOp = new LinkedList<>(); + try { + if (!CompatibilityUtil.isBatchSupport()) { + compatOldServerBatch(actions, results); + } else { + String realTableName = getTargetTableName(actions); + BatchOperation batch = buildBatchOperation(realTableName, actions, + tableNameString.equals(realTableName), resultMapSingleOp); + BatchOperationResult tmpResults = batch.execute(); + if (results != null) { + int index = 0; + for (int i = 0; i != results.length; ++i) { + if (tmpResults.getResults().get(index) instanceof ObTableException) { + results[i] = tmpResults.getResults().get(index); + batchError.add((ObTableException) results[i], actions.get(i), null); + } else { + results[i] = new Result(); + } + index += resultMapSingleOp.get(i); + } + if (batchError.hasErrors()) { + throw batchError.makeException(); + } } } } catch (Exception e) { @@ -544,7 +644,7 @@ public Object[] batch(List actions) throws IOException { @Override public void batchCallback(List actions, Object[] results, Batch.Callback callback) throws IOException, - InterruptedException { + InterruptedException { try { batch(actions, results); } finally { @@ -559,8 +659,9 @@ public void batchCallback(List actions, Object[] results, } @Override - public Object[] batchCallback( - final List actions, final Batch.Callback callback) throws IOException, InterruptedException { + public Object[] batchCallback(final List actions, + final Batch.Callback callback) throws IOException, + InterruptedException { Object[] results = new Object[actions.size()]; batchCallback(actions, results, callback); return results; @@ -944,7 +1045,7 @@ private void innerDelete(Delete delete) throws IOException { batch(Collections.singletonList(delete)); } catch (Exception e) { logger.error(LCD.convert("01-00004"), tableNameString, e); - throw new IOException("delete table " + tableNameString + "error" , e); + throw new IOException("delete table " + tableNameString + " error.", e); } } @@ -1677,7 +1778,8 @@ public static ObTableBatchOperation buildObTableBatchOperation(List ke } private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(KeyValue kv, - ObTableOperationType operationType, boolean isTableGroup) { + ObTableOperationType operationType, + boolean isTableGroup) { KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getType()); switch (operationType) { case INSERT_OR_UPDATE: @@ -1690,29 +1792,29 @@ private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(KeyValue kv, switch (kvType) { case Delete: return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), kv.getQualifier(), kv.getTimestamp() }, - null, null); + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), kv.getQualifier(), kv.getTimestamp() }, + null, null); case Maximum: return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), null, -kv.getTimestamp() }, - null, null); + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), null, -kv.getTimestamp() }, null, null); case DeleteColumn: return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), kv.getQualifier(), -kv.getTimestamp() }, - null, null); + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), kv.getQualifier(), -kv.getTimestamp() }, + null, null); case DeleteFamily: return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), isTableGroup ? kv.getQualifier() : null, -kv.getTimestamp() }, - null, null); + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), isTableGroup ? kv.getQualifier() : null, + -kv.getTimestamp() }, null, null); case DeleteFamilyVersion: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, - ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), isTableGroup ? kv.getQualifier() : null, kv.getTimestamp() }, - null, null); + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance( + DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), isTableGroup ? kv.getQualifier() : null, + kv.getTimestamp() }, null, null); default: throw new IllegalArgumentException("illegal mutation type " + kvType); } @@ -1733,65 +1835,67 @@ private KeyValue modifyQualifier(KeyValue original, byte[] newQualifier) { value); } - private BatchOperation buildBatchOperation(String tableName, List actions, boolean isTableGroup, List resultMapSingleOp) { + private BatchOperation buildBatchOperation(String tableName, List actions, + boolean isTableGroup, List resultMapSingleOp) + throws FeatureNotSupportedException, + IllegalArgumentException { BatchOperation batch = obTableClient.batchOperation(tableName); - if (actions != null) { - int posInList = -1; - int singleOpResultNum; - for (Row row : actions) { - singleOpResultNum = 0; - posInList++; - if (!(row instanceof Put) && !(row instanceof Delete)) { - throw new FeatureNotSupportedException( - "not supported other type in batch yet,only support put and delete"); - } else if (row instanceof Put) { - Put put = (Put) row; - if (put.isEmpty()) { - throw new IllegalArgumentException("No columns to insert for #" - + (posInList + 1) + " item"); + int posInList = -1; + int singleOpResultNum; + for (Row row : actions) { + singleOpResultNum = 0; + posInList++; + if (row instanceof Put) { + Put put = (Put) row; + if (put.isEmpty()) { + throw new IllegalArgumentException("No columns to insert for #" + + (posInList + 1) + " item"); + } + for (Map.Entry> entry : put.getFamilyMap().entrySet()) { + byte[] family = entry.getKey(); + List keyValueList = entry.getValue(); + for (KeyValue kv : keyValueList) { + singleOpResultNum++; + if (isTableGroup) { + KeyValue new_kv = modifyQualifier(kv, + (Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())) + .getBytes()); + batch + .addOperation(buildMutation(new_kv, INSERT_OR_UPDATE, isTableGroup)); + } else { + batch.addOperation(buildMutation(kv, INSERT_OR_UPDATE, isTableGroup)); + } } - for (Map.Entry> entry : put.getFamilyMap().entrySet()) { + } + } else if (row instanceof Delete) { + Delete delete = (Delete) row; + if (delete.isEmpty()) { + singleOpResultNum++; + KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), + KeyValue.Type.Maximum); + batch.addOperation(buildMutation(kv, DEL, isTableGroup)); + } else { + for (Map.Entry> entry : delete.getFamilyMap().entrySet()) { byte[] family = entry.getKey(); List keyValueList = entry.getValue(); for (KeyValue kv : keyValueList) { singleOpResultNum++; - if(isTableGroup){ - KeyValue new_kv = modifyQualifier(kv,(Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())) - .getBytes()); - batch.addOperation(buildMutation(new_kv, INSERT_OR_UPDATE, isTableGroup)); + if (isTableGroup) { + KeyValue new_kv = modifyQualifier(kv, + (Bytes.toString(family) + "." + Bytes.toString(kv + .getQualifier())).getBytes()); + batch.addOperation(buildMutation(new_kv, DEL, true)); } else { - batch.addOperation(buildMutation(kv, INSERT_OR_UPDATE, isTableGroup)); - } - } - } - } else { - Delete delete = (Delete) row; - if (delete.isEmpty()) { - singleOpResultNum++; - KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), - KeyValue.Type.Maximum); - batch.addOperation(buildMutation(kv, DEL, isTableGroup)); - } else { - for (Map.Entry> entry : delete.getFamilyMap() - .entrySet()) { - byte[] family = entry.getKey(); - List keyValueList = entry.getValue(); - List mutations = new LinkedList<>(); - for (KeyValue kv : keyValueList) { - singleOpResultNum++; - if(isTableGroup){ - KeyValue new_kv = modifyQualifier(kv,(Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())) - .getBytes()); - batch.addOperation(buildMutation(new_kv, DEL, true)); - } else { - batch.addOperation(buildMutation(kv, DEL, false)); - } + batch.addOperation(buildMutation(kv, DEL, false)); } } } } - resultMapSingleOp.add(singleOpResultNum); + } else { + throw new FeatureNotSupportedException( + "not supported other type in batch yet,only support put and delete"); } + resultMapSingleOp.add(singleOpResultNum); } batch.setEntityType(ObTableEntityType.HKV); return batch; diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/CompatibilityUtil.java b/src/main/java/com/alipay/oceanbase/hbase/util/CompatibilityUtil.java new file mode 100644 index 00000000..39db637c --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/CompatibilityUtil.java @@ -0,0 +1,9 @@ +package com.alipay.oceanbase.hbase.util; + +import com.alipay.oceanbase.rpc.ObGlobal; + +public class CompatibilityUtil { + public static boolean isBatchSupport() { + return ObGlobal.OB_VERSION > ObGlobal.OB_VERSION_4_3_4_0; + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java index c318b37f..89660ddb 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java @@ -1260,301 +1260,5 @@ public void testMultiColumnFamilyDelete() throws Exception { assertEquals(oldTimestamp, result.getColumnCells(family2, family2_column1).get(0) .getTimestamp()); assertTrue(lastTimestamp > oldTimestamp); - - // delete previous data - Delete deleteKey1Family = new Delete(toBytes(key1)); - deleteKey1Family.deleteFamily(family1); - deleteKey1Family.deleteFamily(family2); - Delete deleteKey2Family = new Delete(toBytes(key2)); - deleteKey2Family.deleteFamily(family1); - deleteKey2Family.deleteFamily(family2); - Delete deleteKey3Family = new Delete(toBytes(key3)); - deleteKey3Family.deleteFamily(family1); - deleteKey3Family.deleteFamily(family2); - - multiCfHTable.delete(deleteKey1Family); - multiCfHTable.delete(deleteKey2Family); - multiCfHTable.delete(deleteKey3Family); - - long minTimeStamp = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp1 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp2 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp3 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp4 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp5 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp6 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp7 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp8 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp9 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp10 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp11 = System.currentTimeMillis(); - Thread.sleep(5); - long maxTimeStamp = System.currentTimeMillis(); - - Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); - putKey1Fam1Column1MinTs.add(family1, family1_column1, minTimeStamp, toBytes(value1)); - - Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); - putKey3Fam1Column1Ts1.add(family1, family1_column1, timeStamp1, toBytes(value2)); - - Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); - putKey1Fam1Column2MinTs.add(family1, family1_column2, minTimeStamp, toBytes(value1)); - - Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); - putKey1Fam1Column2Ts3.add(family1, family1_column2, timeStamp3, toBytes(value2)); - - Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); - putKey2Fam1Column2Ts3.add(family1, family1_column2, timeStamp3, toBytes(value2)); - - Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); - putKey2Fam1Column3Ts1.add(family1, family1_column3, timeStamp1, toBytes(value2)); - - Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); - putKey3Fam1Column3Ts1.add(family1, family1_column3, timeStamp1, toBytes(value2)); - - Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts6.add(family1, family1_column2, timeStamp6, toBytes(value1)); - - Put putKey2Fam1Column3Ts6 = new Put(toBytes(key2)); - putKey2Fam1Column3Ts6.add(family1, family1_column3, timeStamp3, toBytes(value1)); - - multiCfHTable.put(putKey1Fam1Column1MinTs); - multiCfHTable.put(putKey3Fam1Column1Ts1); - multiCfHTable.put(putKey1Fam1Column2MinTs); - multiCfHTable.put(putKey1Fam1Column2Ts3); - multiCfHTable.put(putKey2Fam1Column2Ts3); - multiCfHTable.put(putKey2Fam1Column3Ts1); - multiCfHTable.put(putKey3Fam1Column3Ts1); - multiCfHTable.put(putKey3Fam1Column2Ts6); - multiCfHTable.put(putKey2Fam1Column3Ts6); - - // test DeleteFamilyVersion single cf - get = new Get(toBytes(key1)); - get.addFamily(family1); - get.setTimeStamp(minTimeStamp); - get.setMaxVersions(10); - Result r = multiCfHTable.get(get); - Assert.assertEquals(2, r.raw().length); - - get = new Get(toBytes(key3)); - get.addFamily(family1); - get.setTimeStamp(timeStamp1); - get.setMaxVersions(10); - r = multiCfHTable.get(get); - Assert.assertEquals(2, r.raw().length); - - get = new Get(toBytes(key2)); - get.addFamily(family1); - get.setTimeStamp(timeStamp3); - get.setMaxVersions(10); - r = multiCfHTable.get(get); - Assert.assertEquals(2, r.raw().length); - - Delete delKey1MinTs = new Delete(toBytes(key1)); - delKey1MinTs.deleteFamilyVersion(family1, minTimeStamp); - multiCfHTable.delete(delKey1MinTs); - - get = new Get(toBytes(key1)); - get.addFamily(family1); - get.setTimeStamp(minTimeStamp); - get.setMaxVersions(10); - r = multiCfHTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Delete delKey3Ts1 = new Delete(toBytes(key3)); - delKey3Ts1.deleteFamilyVersion(family1, timeStamp1); - multiCfHTable.delete(delKey3Ts1); - - get = new Get(toBytes(key3)); - get.addFamily(family1); - get.setTimeStamp(timeStamp1); - get.setMaxVersions(10); - r = multiCfHTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Delete delKey2Ts3 = new Delete(toBytes(key2)); - delKey2Ts3.deleteFamilyVersion(family1, timeStamp3); - multiCfHTable.delete(delKey2Ts3); - - get = new Get(toBytes(key2)); - get.addFamily(family1); - get.setTimeStamp(timeStamp3); - get.setMaxVersions(10); - r = multiCfHTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Scan scan = new Scan(); - scan.setStartRow(toBytes(key1)); - scan.setStopRow("scanKey4x".getBytes()); - scan.addFamily(family1); - scan.setMaxVersions(10); - ResultScanner scanner = multiCfHTable.getScanner(scan); - int key1Cnt = 0, key2Cnt = 0, key3Cnt = 0; - for (Result res : scanner) { - for (KeyValue kv : res.raw()) { - if (key1.equals(Bytes.toString(kv.getRow()))) { - ++key1Cnt; - } else if (key2.equals(Bytes.toString(kv.getRow()))) { - ++key2Cnt; - } else { - ++key3Cnt; - } - } - } - Assert.assertEquals(1, key1Cnt); - Assert.assertEquals(1, key2Cnt); - Assert.assertEquals(1, key3Cnt); - - multiCfHTable.delete(deleteKey1Family); - multiCfHTable.delete(deleteKey2Family); - multiCfHTable.delete(deleteKey3Family); - - // test DeleteFamilyVersion multiple cf - Put putKey1Fam1Column3Ts6 = new Put(toBytes(key1)); - putKey1Fam1Column3Ts6.add(family1, family1_column3, timeStamp6, toBytes(value3)); - - Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); - putKey1Fam2Column2Ts2.add(family2, family2_column2, timeStamp2, toBytes(value1)); - - Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); - putKey1Fam2Column3Ts2.add(family2, family2_column3, timeStamp2, toBytes(value1)); - - Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); - putKey1Fam1Column2Ts1.add(family1, family1_column2, timeStamp1, toBytes(value2)); - - Put putKey2Fam1Column2Ts8 = new Put(toBytes(key2)); - putKey2Fam1Column2Ts8.add(family1, family1_column2, timeStamp8, toBytes(value2)); - - Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); - putKey2Fam2Column3Ts1.add(family2, family2_column3, timeStamp3, toBytes(value3)); - - Put putKey2Fam1Column1Ts1 = new Put(toBytes(key2)); - putKey2Fam1Column1Ts1.add(family1, family1_column1, timeStamp8, toBytes(value1)); - - Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); - putKey2Fam2Column1Ts3.add(family2, family2_column1, timeStamp3, toBytes(value2)); - - Put putKey3Fam1Column2Ts9 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts9.add(family1, family1_column2, timeStamp9, toBytes(value2)); - - Put putKey3Fam2Column3Ts10 = new Put(toBytes(key3)); - putKey3Fam2Column3Ts10.add(family2, family2_column3, timeStamp10, toBytes(value1)); - - Put putKey3Fam2Column1Ts10 = new Put(toBytes(key3)); - putKey3Fam2Column1Ts10.add(family2, family2_column1, timeStamp10, toBytes(value2)); - - Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts2.add(family1, family1_column2, timeStamp2, toBytes(value1)); - - multiCfHTable.put(putKey1Fam1Column3Ts6); - multiCfHTable.put(putKey1Fam2Column2Ts2); - multiCfHTable.put(putKey1Fam2Column3Ts2); - multiCfHTable.put(putKey1Fam1Column2Ts1); - multiCfHTable.put(putKey2Fam1Column2Ts8); - multiCfHTable.put(putKey2Fam2Column3Ts1); - multiCfHTable.put(putKey2Fam1Column1Ts1); - multiCfHTable.put(putKey2Fam2Column1Ts3); - multiCfHTable.put(putKey3Fam1Column2Ts9); - multiCfHTable.put(putKey3Fam2Column3Ts10); - multiCfHTable.put(putKey3Fam2Column1Ts10); - multiCfHTable.put(putKey3Fam1Column2Ts2); - - Get getKey1 = new Get(toBytes(key1)); - getKey1.addFamily(family1); - getKey1.addFamily(family2); - getKey1.setMaxVersions(10); - r = multiCfHTable.get(getKey1); - Assert.assertEquals(4, r.raw().length); - - Get getKey2 = new Get(toBytes(key2)); - getKey2.addFamily(family1); - getKey2.addFamily(family2); - getKey2.setMaxVersions(10); - r = multiCfHTable.get(getKey2); - Assert.assertEquals(4, r.raw().length); - - Get getKey3 = new Get(toBytes(key3)); - getKey3.addFamily(family1); - getKey3.addFamily(family2); - getKey3.setMaxVersions(10); - r = multiCfHTable.get(getKey3); - Assert.assertEquals(4, r.raw().length); - - Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); - delKey1Ts_6_2.deleteFamilyVersion(family1, timeStamp6); - delKey1Ts_6_2.deleteFamilyVersion(family2, timeStamp2); - multiCfHTable.delete(delKey1Ts_6_2); - - getKey1 = new Get(toBytes(key1)); - getKey1.addFamily(family1); - getKey1.addFamily(family2); - getKey1.setMaxVersions(10); - r = multiCfHTable.get(getKey1); - Assert.assertEquals(1, r.raw().length); - for (KeyValue kv : r.raw()) { - Assert.assertEquals(timeStamp1, kv.getTimestamp()); - } - - Delete delKey2Ts_8_3 = new Delete(toBytes(key2)); - delKey2Ts_8_3.deleteFamilyVersion(family1, timeStamp8); - delKey2Ts_8_3.deleteFamilyVersion(family2, timeStamp3); - multiCfHTable.delete(delKey2Ts_8_3); - - getKey2 = new Get(toBytes(key2)); - getKey2.addFamily(family1); - getKey2.addFamily(family2); - getKey2.setMaxVersions(10); - r = multiCfHTable.get(getKey2); - Assert.assertEquals(0, r.raw().length); - - Delete delKey3Ts_2_10 = new Delete(toBytes(key3)); - delKey3Ts_2_10.deleteFamilyVersion(family1, timeStamp2); - delKey3Ts_2_10.deleteFamilyVersion(family2, timeStamp10); - multiCfHTable.delete(delKey3Ts_2_10); - - getKey3 = new Get(toBytes(key3)); - getKey3.addFamily(family1); - getKey3.addFamily(family2); - getKey3.setMaxVersions(10); - r = multiCfHTable.get(getKey3); - Assert.assertEquals(1, r.raw().length); - for (KeyValue kv : r.raw()) { - Assert.assertEquals(timeStamp9, kv.getTimestamp()); - } - - scan = new Scan(); - scan.setStartRow(toBytes(key1)); - scan.setStopRow("scanKey4x".getBytes()); - scan.addFamily(family1); - scan.addFamily(family2); - scan.setMaxVersions(10); - scanner = multiCfHTable.getScanner(scan); - int ts1Cnt = 0, ts9Cnt = 0; - for (Result res : scanner) { - for (KeyValue kv : res.raw()) { - if (kv.getTimestamp() == timeStamp1) { - ++ts1Cnt; - } else if (kv.getTimestamp() == timeStamp9) { - ++ts9Cnt; - } - } - } - Assert.assertEquals(1, ts1Cnt); - Assert.assertEquals(1, ts9Cnt); - - multiCfHTable.delete(deleteKey1Family); - multiCfHTable.delete(deleteKey2Family); - multiCfHTable.delete(deleteKey3Family); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index 2e5e8f10..41e68bb2 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -30,50 +30,61 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; import static com.alipay.oceanbase.hbase.constants.OHConstants.*; public class ObHTableTestUtil { // please consult your dba for the following configuration. - public static String PARAM_URL = ""; - public static String FULL_USER_NAME = ""; - public static String PASSWORD = ""; - public static String SYS_USER_NAME = ""; - public static String SYS_PASSWORD = ""; - public static String ODP_ADDR = ""; - public static int ODP_PORT = 0; - public static boolean ODP_MODE = false; - public static String DATABASE = ""; - public static String JDBC_IP = ""; - public static String JDBC_PORT = ""; - public static String JDBC_DATABASE = ""; - public static String JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " + JDBC_DATABASE + "?" + "useUnicode=TRUE&" + "characterEncoding=utf-8&" + "socketTimeout=3000000&" + "connectTimeout=60000"; - - public static String SQL_FORMAT = "truncate %s"; - public static List tableNameList = new LinkedList<>(); + public static String PARAM_URL = ""; + public static String FULL_USER_NAME = ""; + public static String PASSWORD = ""; + public static String SYS_USER_NAME = ""; + public static String SYS_PASSWORD = ""; + public static String ODP_ADDR = ""; + public static int ODP_PORT = 0; + public static boolean ODP_MODE = false; + public static String DATABASE = ""; + public static String JDBC_IP = ""; + public static String JDBC_PORT = ""; + public static String JDBC_DATABASE = ""; + public static String JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " + + JDBC_DATABASE + "?" + "useUnicode=TRUE&" + + "characterEncoding=utf-8&" + + "socketTimeout=3000000&" + "connectTimeout=60000"; + + public static String SQL_FORMAT = "truncate %s"; + public static List tableNameList; public static Connection conn; public static Statement stmt; + static { + tableNameList = new LinkedList<>(); + conn = getConnection(); + try { + stmt = conn.createStatement(); + } catch (SQLException e) { + System.out.println("sql error " + e); + } + } + public static void prepareClean(List tableGroupList) throws Exception { for (String tableGroup : tableGroupList) { tableNameList.addAll(getOTableNameList(tableGroup)); } - conn = getConnection(); - stmt = conn.createStatement(); } public static void cleanData() { try { for (String realTableName : tableNameList) { try { + if (realTableName.contains("'")) { + realTableName = "`" + realTableName + "`"; + } stmt.execute(String.format(SQL_FORMAT, realTableName)); } catch (Exception e) { - System.out.println( - "clean table data error ." + realTableName + " exception:" + e); + System.out.println("clean table data error ." + realTableName + " exception:" + + e); } } } catch (Exception e) { @@ -110,29 +121,29 @@ public static OHTableClient newOHTableClient(String tableName) { } static public List getOTableNameList(String tableGroup) throws IOException { - // 读取建表语句 - List res = new LinkedList<>(); - String sql = new String(Files.readAllBytes(Paths.get(NativeHBaseUtil.SQL_PATH))); - String[] sqlList = sql.split(";"); - Map tableMap = new LinkedHashMap<>(); - for (String singleSql : sqlList) { - String realTableName; - if (singleSql.contains("CREATE TABLE ")) { - singleSql.trim(); - String[] splits = singleSql.split(" "); - String tableGroupName = splits[2].substring(1, splits[2].length() - 1); - if (tableGroupName.contains(":")) { - String[] tmpStr = tableGroupName.split(":", 2); - tableGroupName = tmpStr[1]; - } - realTableName = tableGroupName.split("\\$", 2)[0]; - if (realTableName.equals(tableGroup)) { - res.add(tableGroupName); + // 读取建表语句 + List res = new LinkedList<>(); + String sql = new String(Files.readAllBytes(Paths.get(NativeHBaseUtil.SQL_PATH))); + String[] sqlList = sql.split(";"); + Map tableMap = new LinkedHashMap<>(); + for (String singleSql : sqlList) { + String realTableName; + if (singleSql.contains("CREATE TABLE ")) { + singleSql.trim(); + String[] splits = singleSql.split(" "); + String tableGroupName = splits[2].substring(1, splits[2].length() - 1); + if (tableGroupName.contains(":")) { + String[] tmpStr = tableGroupName.split(":", 2); + tableGroupName = tmpStr[1]; + } + realTableName = tableGroupName.split("\\$", 2)[0]; + if (realTableName.equals(tableGroup)) { + res.add(tableGroupName); + } } } + return res; } - return res; - } static public Connection getConnection() { try { diff --git a/src/main/resources/hbase-site.xml b/src/test/resources/hbase-site.xml similarity index 100% rename from src/main/resources/hbase-site.xml rename to src/test/resources/hbase-site.xml From 1d6548c0915b59eaad4228af5dd2acd88b52cbb3 Mon Sep 17 00:00:00 2001 From: miyuan-ljr Date: Fri, 15 Nov 2024 15:27:56 +0800 Subject: [PATCH 14/21] refine (#115) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e6658921..dace701b 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ ${project.encoding} UTF-8 1.7.21 - 1.2.14.1-SNAPSHOT + 1.2.14.4-SNAPSHOT From 972ac6250d8338e2a73d38e79fc2114787756168 Mon Sep 17 00:00:00 2001 From: miyuan-ljr Date: Fri, 15 Nov 2024 16:50:20 +0800 Subject: [PATCH 15/21] family filter (#83) --- .../hbase/filter/HBaseFilterUtils.java | 2 +- .../hbase/HTableMultiCFTestBase.java | 154 +++++++++++++++++- .../hbase/filter/HBaseFilterUtilsTest.java | 6 + 3 files changed, 160 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index ed98d1b3..833a539d 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -45,7 +45,7 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte } else if (filter instanceof DependentColumnFilter) { toParseableByteArray(byteStream, (DependentColumnFilter) filter); } else if (filter instanceof CompareFilter) { - // RowFilter, ValueFilter, QualifierFilter + // RowFilter, ValueFilter, QualifierFilter, FamilyFilter toParseableByteArray(byteStream, (CompareFilter) filter); } else if (filter instanceof SingleColumnValueExcludeFilter) { toParseableByteArray(byteStream, (SingleColumnValueExcludeFilter) filter); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java index 89660ddb..6297bad9 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.client.coprocessor.Batch; -import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.filter.*; import org.apache.hadoop.hbase.util.Bytes; import org.junit.*; import org.junit.rules.ExpectedException; @@ -1261,4 +1261,156 @@ public void testMultiColumnFamilyDelete() throws Exception { .getTimestamp()); assertTrue(lastTimestamp > oldTimestamp); } + + @Test + public void testFamilyFilter() throws Exception { + String key1 = "getKey1"; + String key2 = "getKey2"; + String column1 = "abc"; + String column2 = "def"; + String value1 = "value1"; + String value2 = "value2"; + String family1 = "family_with_group1"; + String family2 = "family_with_group2"; + String family3 = "family_with_group3"; + + Delete deleteKey1 = new Delete(toBytes(key1)); + deleteKey1.deleteFamily(toBytes(family1)); + deleteKey1.deleteFamily(toBytes(family2)); + deleteKey1.deleteFamily(toBytes(family3)); + Delete deleteKey2 = new Delete(toBytes(key2)); + deleteKey2.deleteFamily(toBytes(family1)); + deleteKey2.deleteFamily(toBytes(family2)); + deleteKey2.deleteFamily(toBytes(family3)); + + Put putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family1), toBytes(column1), toBytes(value1)); + + Put putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family2), toBytes(column1), toBytes(value2)); + + Put putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family2), toBytes(column2), toBytes(value2)); + + Put putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family3), toBytes(column2), toBytes(value1)); + + Put putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family3), toBytes(column1), toBytes(value1)); + + Put putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family3), toBytes(column1), toBytes(value2)); + + multiCfHTable.delete(deleteKey1); + multiCfHTable.delete(deleteKey2); + multiCfHTable.put(putKey1Column1Value1); + multiCfHTable.put(putKey1Column1Value2); + multiCfHTable.put(putKey1Column2Value2); + multiCfHTable.put(putKey2Column2Value1); + multiCfHTable.put(putKey2Column1Value1); + multiCfHTable.put(putKey2Column1Value2); + + Scan scan; + scan = new Scan(); + scan.addFamily(family1.getBytes()); + scan.addFamily(family2.getBytes()); + scan.addFamily(family3.getBytes()); + scan.setMaxVersions(10); + FamilyFilter f = new FamilyFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(family2))); + scan.setFilter(f); + ResultScanner scanner = multiCfHTable.getScanner(scan); + + int res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + Assert.assertArrayEquals(family2.getBytes(), keyValue.getFamily()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 2); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family1.getBytes()); + scan.addFamily(family2.getBytes()); + scan.addFamily(family3.getBytes()); + scan.setMaxVersions(10); + f = new FamilyFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator(Bytes.toBytes(family2))); + scan.setFilter(f); + scanner = multiCfHTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 4); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family1.getBytes()); + scan.addFamily(family2.getBytes()); + scan.addFamily(family3.getBytes()); + scan.setMaxVersions(10); + f = new FamilyFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator(Bytes.toBytes(family2))); + scan.setFilter(f); + scanner = multiCfHTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + Assert.assertArrayEquals(family3.getBytes(), keyValue.getFamily()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 3); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family1.getBytes()); + scan.addFamily(family2.getBytes()); + scan.addFamily(family3.getBytes()); + scan.setMaxVersions(10); + f = new FamilyFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(Bytes.toBytes("family_with_group"))); + scan.setFilter(f); + scanner = multiCfHTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 6); + scanner.close(); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java index 4fac8402..f35f72a2 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java @@ -241,6 +241,12 @@ public void testMultipleColumnPrefixFilter() throws IOException { HBaseFilterUtils.toParseableByteArray(filter)); } + @Test + public void testFamilyFilter() throws IOException { + FamilyFilter filter = new FamilyFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator(Bytes.toBytes("cf"))); + Assert.assertArrayEquals("FamilyFilter(!=,'binary:cf')".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + } + @Test public void testColumnCountGetFilter() throws IOException { ColumnCountGetFilter filter = new ColumnCountGetFilter(513); From 1c85fe3909241426f024d16513e2d824fe22021d Mon Sep 17 00:00:00 2001 From: miyuan-ljr Date: Fri, 15 Nov 2024 17:07:32 +0800 Subject: [PATCH 16/21] fix encode (#111) --- .../hbase/filter/HBaseFilterUtils.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index 833a539d..296f70bc 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -295,11 +295,11 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Fuzzy for (int i = 0; i < fuzzyKeysData.size(); i ++) { Pair data = fuzzyKeysData.get(i); byteStream.write("'".getBytes()); - byteStream.write(data.getFirst()); + writeBytesWithEscape(byteStream, data.getFirst()); byteStream.write("'".getBytes()); byteStream.write(','); byteStream.write("'".getBytes()); - byteStream.write(data.getSecond()); + writeBytesWithEscape(byteStream, data.getSecond()); byteStream.write("'".getBytes()); if (i < fuzzyKeysData.size() - 1) { byteStream.write(','); @@ -339,13 +339,13 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, for (int i = 0; i < ranges.size(); i++) { MultiRowRangeFilter.RowRange range = ranges.get(i); byteStream.write("'".getBytes()); - byteStream.write(range.getStartRow()); + writeBytesWithEscape(byteStream, range.getStartRow()); byteStream.write("',".getBytes()); byteStream.write(Boolean.toString(range.isStartRowInclusive()).getBytes()); byteStream.write(','); byteStream.write("'".getBytes()); - byteStream.write(range.getStopRow()); + writeBytesWithEscape(byteStream, range.getStopRow()); byteStream.write("',".getBytes()); byteStream.write(Boolean.toString(range.isStopRowInclusive()).getBytes()); if (i < ranges.size() - 1) { @@ -361,7 +361,7 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); byteStream.write('\''); - byteStream.write(filter.getStopRowKey()); + writeBytesWithEscape(byteStream, filter.getStopRowKey()); byteStream.write('\''); byteStream.write(')'); } @@ -373,13 +373,13 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write('('); byteStream.write("'".getBytes()); - byteStream.write(filter.getMinColumn()); + writeBytesWithEscape(byteStream, filter.getMinColumn()); byteStream.write("',".getBytes()); byteStream.write(Boolean.toString(filter.getMinColumnInclusive()).getBytes()); byteStream.write(','); byteStream.write("'".getBytes()); - byteStream.write(filter.getMaxColumn()); + writeBytesWithEscape(byteStream, filter.getMaxColumn()); byteStream.write("',".getBytes()); byteStream.write(Boolean.toString(filter.getMaxColumnInclusive()).getBytes()); byteStream.write(')'); @@ -395,7 +395,7 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, for (int i = 0; i < ranges.length; i++) { byte[] range = ranges[i]; byteStream.write("'".getBytes()); - byteStream.write(range); + writeBytesWithEscape(byteStream, range); byteStream.write("'".getBytes()); if (i < ranges.length - 1) { byteStream.write(','); @@ -429,7 +429,7 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, int i = 0; for (byte[] qualifier: qualifiers) { byteStream.write("'".getBytes()); - byteStream.write(qualifier); + writeBytesWithEscape(byteStream, qualifier); byteStream.write("'".getBytes()); if (i < qualifiers.size() - 1) { byteStream.write(','); From 9334ea09909893c7b66239884cca7c94c589438f Mon Sep 17 00:00:00 2001 From: maochongxin Date: Mon, 4 Nov 2024 17:53:33 +0800 Subject: [PATCH 17/21] fix: Change single column family get operation to sync query (#96) * fix: Change single column family get operation to sync query * fix: resolve Maven build failure --- .../java/com/alipay/oceanbase/hbase/OHTable.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 35a9e087..99de07f4 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -36,6 +36,7 @@ import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.*; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncRequest; import com.alipay.oceanbase.rpc.stream.ObTableClientQueryAsyncStreamResult; +import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult; import com.alipay.oceanbase.rpc.table.ObHBaseParams; import com.alipay.oceanbase.rpc.table.ObKVParams; import com.alipay.sofa.common.thread.SofaThreadPoolExecutor; @@ -756,8 +757,6 @@ public Result get(final Get get) throws IOException { public Result call() throws IOException { List keyValueList = new ArrayList<>(); byte[] family = new byte[] {}; - ObTableClientQueryAsyncStreamResult clientQueryStreamResult; - ObTableQueryAsyncRequest request; ObTableQuery obTableQuery; try { if (get.getFamilyMap().keySet() == null @@ -772,10 +771,10 @@ public Result call() throws IOException { NavigableSet columnFilters = new TreeSet<>(Bytes.BYTES_COMPARATOR); processColumnFilters(columnFilters, get.getFamilyMap()); obTableQuery = buildObTableQuery(get, columnFilters); - request = buildObTableQueryAsyncRequest(obTableQuery, + ObTableQueryAsyncRequest request = buildObTableQueryAsyncRequest(obTableQuery, getTargetTableName(tableNameString)); - clientQueryStreamResult = (ObTableClientQueryAsyncStreamResult) obTableClient + ObTableClientQueryAsyncStreamResult clientQueryStreamResult = (ObTableClientQueryAsyncStreamResult) obTableClient .execute(request); getMaxRowFromResult(clientQueryStreamResult, keyValueList, true, family); } else { @@ -794,10 +793,10 @@ public Result call() throws IOException { } } obTableQuery = buildObTableQuery(get, entry.getValue()); - request = buildObTableQueryAsyncRequest(obTableQuery, + ObTableQueryRequest request = buildObTableQueryRequest(obTableQuery, getTargetTableName(tableNameString, Bytes.toString(family), configuration)); - clientQueryStreamResult = (ObTableClientQueryAsyncStreamResult) obTableClient + ObTableClientQueryStreamResult clientQueryStreamResult = (ObTableClientQueryStreamResult) obTableClient .execute(request); getMaxRowFromResult(clientQueryStreamResult, keyValueList, false, family); @@ -1218,12 +1217,12 @@ public Result increment(Increment increment) throws IOException { byte[] f = entry.getKey(); ObTableBatchOperation batch = new ObTableBatchOperation(); - entry.getValue().forEach(cell -> { + for (Cell cell : entry.getValue()) { byte[] qualifier = cell.getQualifier(); qualifiers.add(qualifier); batch.addTableOperation(getInstance(INCREMENT, new Object[] { rowKey, qualifier, Long.MAX_VALUE }, V_COLUMNS, new Object[] { cell.getValue() })); - }); + } ObHTableFilter filter = buildObHTableFilter(null, increment.getTimeRange(), 1, qualifiers); From 548846f758171509d4b33be03e8d79966cb8e0a2 Mon Sep 17 00:00:00 2001 From: Shen Yunlong <44497386+shenyunlong@users.noreply.github.com> Date: Fri, 15 Nov 2024 14:19:54 +0800 Subject: [PATCH 18/21] [Fix] dependency conflict (#116) --- pom.xml | 100 ++++++++++++++++-- .../com/alipay/oceanbase/hbase/OHTable.java | 4 +- 2 files changed, 91 insertions(+), 13 deletions(-) diff --git a/pom.xml b/pom.xml index dace701b..9077b8a5 100644 --- a/pom.xml +++ b/pom.xml @@ -57,6 +57,77 @@ 1.2.14.4-SNAPSHOT + + + + com.google.guava + guava + 11.0.2 + + + com.google.protobuf + protobuf-java + 3.11.4 + + + commons-lang + commons-lang + 2.6 + + + commons-codec + commons-codec + 1.9 + + + org.codehaus.jackson + jackson-core-asl + 1.9.13 + + + commons-logging + commons-logging + 1.2 + + + log4j + log4j + 1.2.17 + + + io.netty + netty-all + 4.1.100.Final + + + org.slf4j + slf4j-api + ${slf4j.version} + + + io.netty + netty + 3.7.0.Final + + + junit + junit + ${junit.version} + test + + + com.alipay.sofa.common + sofa-common-tools + 1.3.12 + + + org.objenesis + objenesis + 3.0.1 + + + + org.apache.hadoop @@ -173,22 +244,11 @@ - - org.slf4j - slf4j-api - ${slf4j.version} - org.slf4j slf4j-nop ${slf4j.version} - - junit - junit - ${junit.version} - test - org.powermock powermock-api-mockito2 @@ -405,6 +465,24 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + 3.0.0-M3 + + + enforce-dependency-convergence + + enforce + + + + + + + + + diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 99de07f4..6b813ee7 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1211,8 +1211,8 @@ public Result increment(Increment increment) throws IOException { List qualifiers = new ArrayList(); byte[] rowKey = increment.getRow(); - Map.Entry> entry = increment.getFamilyCellMap() - .entrySet().iterator().next(); + Map.Entry> entry = increment.getFamilyCellMap().entrySet() + .iterator().next(); byte[] f = entry.getKey(); From 96be1694400801a46e8672f3650296a1cf773295 Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Tue, 19 Nov 2024 19:36:08 +0800 Subject: [PATCH 19/21] [Chore] remove the use of sofa common log --- pom.xml | 11 +++--- .../hbase/util/TableHBaseLoggerFactory.java | 4 +-- .../log/log4j/log-conf.xml | 22 ------------ .../log/log4j2/log-conf.xml | 32 ----------------- .../log/logback/log-conf.xml | 22 ------------ src/test/resources/logback.xml | 34 +++++++++++++++++++ 6 files changed, 42 insertions(+), 83 deletions(-) delete mode 100644 src/main/resources/oceanbase-table-hbase/log/log4j/log-conf.xml delete mode 100644 src/main/resources/oceanbase-table-hbase/log/log4j2/log-conf.xml delete mode 100644 src/main/resources/oceanbase-table-hbase/log/logback/log-conf.xml create mode 100644 src/test/resources/logback.xml diff --git a/pom.xml b/pom.xml index 9077b8a5..ac53a096 100644 --- a/pom.xml +++ b/pom.xml @@ -244,11 +244,6 @@ - - org.slf4j - slf4j-nop - ${slf4j.version} - org.powermock powermock-api-mockito2 @@ -261,6 +256,12 @@ ${powermock.version} test + + ch.qos.logback + logback-classic + 1.2.12 + test + diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/TableHBaseLoggerFactory.java b/src/main/java/com/alipay/oceanbase/hbase/util/TableHBaseLoggerFactory.java index 4c669bf7..f90b5d0a 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/TableHBaseLoggerFactory.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/TableHBaseLoggerFactory.java @@ -18,9 +18,9 @@ package com.alipay.oceanbase.hbase.util; import com.alipay.sofa.common.code.LogCode2Description; -import com.alipay.sofa.common.log.LoggerSpaceManager; import org.apache.hadoop.classification.InterfaceAudience; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @InterfaceAudience.Private public class TableHBaseLoggerFactory { @@ -33,7 +33,7 @@ public static Logger getLogger(String name) { if (name == null || name.isEmpty()) { return null; } - return LoggerSpaceManager.getLoggerBySpace(name, TABLE_HBASE_LOGGER_SPACE); + return LoggerFactory.getLogger(name); } public static Logger getLogger(Class klass) { diff --git a/src/main/resources/oceanbase-table-hbase/log/log4j/log-conf.xml b/src/main/resources/oceanbase-table-hbase/log/log4j/log-conf.xml deleted file mode 100644 index ccf115d2..00000000 --- a/src/main/resources/oceanbase-table-hbase/log/log4j/log-conf.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - - - - - - - - - - diff --git a/src/main/resources/oceanbase-table-hbase/log/log4j2/log-conf.xml b/src/main/resources/oceanbase-table-hbase/log/log4j2/log-conf.xml deleted file mode 100644 index 4775dcff..00000000 --- a/src/main/resources/oceanbase-table-hbase/log/log4j2/log-conf.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - ${sys:logging.level.oceanbase-table-hbase} - ${sys:logging.path.oceanbase-table-hbase} - ${sys:file.encoding} - - - - - - - %d-%m%n - - - - - - - - - - - - - - - diff --git a/src/main/resources/oceanbase-table-hbase/log/logback/log-conf.xml b/src/main/resources/oceanbase-table-hbase/log/logback/log-conf.xml deleted file mode 100644 index f7f7d910..00000000 --- a/src/main/resources/oceanbase-table-hbase/log/logback/log-conf.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - ${logging.path.com.alipay.oceanbase-table-hbase}/oceanbase-table-hbase/oceanbase-table-hbase.log - - - %d{yyyy-MM-dd HH:mm:ss}-%msg%n - - - - - ${logging.path.oceanbase-table-hbase}/oceanbase-table-hbase/oceanbase-table-hbase.%d{yyyy-MM-dd}.log - - - - - - - - - \ No newline at end of file diff --git a/src/test/resources/logback.xml b/src/test/resources/logback.xml new file mode 100644 index 00000000..3d2e01c7 --- /dev/null +++ b/src/test/resources/logback.xml @@ -0,0 +1,34 @@ + + + + %d{yyyy-MM-dd HH:mm:ss} [%t] %-5level %logger{36} - %msg%n + + + + + logs/oceanbase-table-hbase/ceanbase-table-hbase.log + + logs/ceanbase-table-hbase-%d{yyyy-MM-dd-HH}-%i.log + 10MB + 5GB + 48 + + + + %d{yyyy-MM-dd HH:mm:ss} [%t] %-5level %logger{36} - %msg%n + + + + 0 + true + 1024 + 0 + false + + + + + + + + \ No newline at end of file From 855faca47d4c10103fd67ca2e08d08c0b91afb84 Mon Sep 17 00:00:00 2001 From: miyuan-ljr Date: Thu, 21 Nov 2024 15:43:50 +0800 Subject: [PATCH 20/21] add version control (#108) --- src/main/java/com/alipay/oceanbase/hbase/OHTable.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 6b813ee7..f449abd6 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.VersionInfo; import org.slf4j.Logger; import java.io.ByteArrayOutputStream; @@ -993,6 +994,7 @@ private ObKVParams buildOBKVParams(final Scan scan) { obHBaseParams.setCallTimeout(scannerTimeout); obHBaseParams.setCacheBlock(scan.isGetScan()); obHBaseParams.setAllowPartialResults(scan.getAllowPartialResults()); + obHBaseParams.setHbaseVersion(VersionInfo.getVersion()); } obKVParams.setObParamsBase(obHBaseParams); return obKVParams; @@ -1003,6 +1005,7 @@ private ObKVParams buildOBKVParams(final Get get) { ObHBaseParams obHBaseParams = new ObHBaseParams(); obHBaseParams.setCheckExistenceOnly(get.isCheckExistenceOnly()); obHBaseParams.setCacheBlock(get.getCacheBlocks()); + obHBaseParams.setHbaseVersion(VersionInfo.getVersion()); obKVParams.setObParamsBase(obHBaseParams); return obKVParams; } From 8bc51ca9767023be164b3a120b9d02022d52dc05 Mon Sep 17 00:00:00 2001 From: miyuan-ljr Date: Mon, 25 Nov 2024 19:39:31 +0800 Subject: [PATCH 21/21] add test case (#125) --- .../com/alipay/oceanbase/hbase/OHTable.java | 276 +++++++++++------- .../hbase/HTableMultiCFTestBase.java | 85 +++--- .../hbase/OHVarPrefixPartitionTest.java | 43 +++ .../hbase/filter/HBaseFilterUtilsTest.java | 6 +- src/test/java/unit_test_db.sql | 9 + 5 files changed, 274 insertions(+), 145 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHVarPrefixPartitionTest.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index f449abd6..3a2ea2d4 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -39,7 +39,6 @@ import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult; import com.alipay.oceanbase.rpc.table.ObHBaseParams; import com.alipay.oceanbase.rpc.table.ObKVParams; -import com.alipay.sofa.common.thread.SofaThreadPoolExecutor; import com.google.protobuf.Descriptors; import com.google.protobuf.Message; @@ -55,6 +54,7 @@ import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.VersionInfo; import org.slf4j.Logger; @@ -66,7 +66,6 @@ import static com.alipay.oceanbase.hbase.constants.OHConstants.*; import static com.alipay.oceanbase.hbase.util.Preconditions.checkArgument; import static com.alipay.oceanbase.hbase.util.TableHBaseLoggerFactory.LCD; -import static com.alipay.oceanbase.hbase.util.TableHBaseLoggerFactory.TABLE_HBASE_LOGGER_SPACE; import static com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperation.getInstance; import static com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType.*; import static com.alipay.sofa.common.thread.SofaThreadPoolConstants.SOFA_THREAD_POOL_LOGGING_CAPABILITY; @@ -355,9 +354,8 @@ public static ThreadPoolExecutor createDefaultThreadPoolExecutor(int coreSize, i if (System.getProperty(SOFA_THREAD_POOL_LOGGING_CAPABILITY) == null) { System.setProperty(SOFA_THREAD_POOL_LOGGING_CAPABILITY, "false"); } - SofaThreadPoolExecutor executor = new SofaThreadPoolExecutor(coreSize, maxThreads, - keepAliveTime, SECONDS, new SynchronousQueue<>(), "OHTableDefaultExecutePool", - TABLE_HBASE_LOGGER_SPACE); + ThreadPoolExecutor executor = new ThreadPoolExecutor(coreSize, maxThreads, + keepAliveTime, SECONDS, new SynchronousQueue<>(), Threads.newDaemonThreadFactory("ohtable")); executor.allowCoreThreadTimeOut(true); return executor; } @@ -482,7 +480,113 @@ public Boolean[] exists(List gets) throws IOException { return objectResults; } - private void compatOldServerBatch(final List actions, final Object[] results) + private BatchOperation compatOldServerPut(final List actions, final Object[] results, BatchError batchError, int i, List puts) + throws Exception { + BatchOperationResult tmpResults; + List resultMapSingleOp = new LinkedList<>(); + if (((Put)actions.get(i)).getFamilyCellMap().size() == 1) { + puts.clear(); + Put put = (Put) actions.get(i++); + String family = Bytes.toString(put.getFamilyCellMap().firstKey()); + // aggregate put + puts.add(put); + while (i < actions.size()) { + if (actions.get(i) instanceof Put && ((Put)actions.get(i)).getFamilyCellMap().size() == 1 && Objects.equals( + Bytes.toString(((Put) actions.get(i)).getFamilyCellMap().firstKey()), + family)) { + puts.add(actions.get(i++)); + } else { + break; + } + } + String realTableName = getTargetTableName(tableNameString, family, + configuration); + BatchOperation batch = buildBatchOperation(realTableName, puts, false, + resultMapSingleOp); + tmpResults = batch.execute(); + int index = 0; + for (int k = 0; k < resultMapSingleOp.size(); k++) { + if (tmpResults.getResults().get(index) instanceof ObTableException) { + if (results != null) { + results[i - puts.size() + k] = tmpResults.getResults().get(index); + } + batchError.add( + (ObTableException) tmpResults.getResults().get(index), + actions.get(i - puts.size() + k), null); + } else { + if (results != null) { + results[i - puts.size() + k] = new Result(); + } + } + index += resultMapSingleOp.get(k); + } + return null; + } else { + // multi cf put + Put put = (Put) actions.get(i); + String realTableName = getTargetTableName(tableNameString); + return buildBatchOperation(realTableName, + Collections.singletonList(put), + true, resultMapSingleOp); + + } + } + + private BatchOperation compatOldServerDel(final List actions, final Object[] results, BatchError batchError, int i) + throws Exception { + Delete delete = (Delete)actions.get(i); + List resultMapSingleOp = new LinkedList<>(); + if (delete.isEmpty()) { + return buildBatchOperation(tableNameString, + Collections.singletonList(delete), true, + resultMapSingleOp); + } else if (delete.getFamilyCellMap().size() > 1) { + boolean has_delete_family = delete.getFamilyMap().entrySet().stream() + .flatMap(entry -> entry.getValue().stream()).anyMatch( + kv -> KeyValue.Type.codeToType( + kv.getType()) == KeyValue.Type.DeleteFamily || KeyValue.Type.codeToType( + kv.getType()) == KeyValue.Type.DeleteFamilyVersion); + if (!has_delete_family) { + return buildBatchOperation(tableNameString, + Collections.singletonList(delete), true, + resultMapSingleOp); + } else { + for (Map.Entry> entry : delete.getFamilyCellMap() + .entrySet()) { + byte[] family = entry.getKey(); + String realTableName = getTargetTableName(tableNameString, + Bytes.toString(family), configuration); + // split delete + List cells = entry.getValue(); + Delete del = new Delete(delete.getRow()); + del.getFamilyCellMap().put(family, cells); + BatchOperation batch = buildBatchOperation(realTableName, + Collections.singletonList(del), false, + resultMapSingleOp); + BatchOperationResult tmpResults = batch.execute(); + if (tmpResults.getResults().get(0) instanceof ObTableException) { + if (results != null) { + results[i] = tmpResults.getResults().get(0); + } + batchError.add((ObTableException) tmpResults.getResults().get(0), actions.get(i), null); + } + } + if (results != null && results[i] == null) { + results[i] = new Result(); + } + return null; + } + } else { + byte[] family = delete.getFamilyCellMap().firstKey(); + String realTableName = getTargetTableName(tableNameString, Bytes.toString(family), configuration); + return buildBatchOperation(realTableName, + Collections.singletonList(delete), + false, resultMapSingleOp); + } + + } + + private void compatOldServerBatch(final List actions, final Object[] results, BatchError batchError) throws Exception { for (Row row : actions) { if (!(row instanceof Put) && !(row instanceof Delete)) { @@ -490,89 +594,36 @@ private void compatOldServerBatch(final List actions, final Objec "not supported other type in batch yet,only support put and delete"); } } - BatchError batchError = new BatchError(); - List resultMapSingleOp = new LinkedList<>(); + List puts = new LinkedList<>(); + BatchOperation batch; for (int i = 0; i < actions.size(); i++) { - BatchOperationResult tmpResults; - BatchOperation batch = null; if (actions.get(i) instanceof Put) { - Put put = (Put) actions.get(i); - if (put.getFamilyCellMap().size() > 1) { - String realTableName = getTargetTableName(tableNameString); - batch = buildBatchOperation(realTableName, - Collections.singletonList(put), - true, resultMapSingleOp); - } else { - byte[] family = put.getFamilyCellMap().firstKey(); - String realTableName = getTargetTableName(tableNameString, Bytes.toString(family), configuration); - batch = buildBatchOperation(realTableName, - Collections.singletonList(put), - false, resultMapSingleOp); - } - } else if (actions.get(i) instanceof Delete) { - Delete delete = (Delete)actions.get(i); - if (delete.isEmpty()) { - batch = buildBatchOperation(tableNameString, - Collections.singletonList(delete), true, - resultMapSingleOp); - } else if (delete.getFamilyCellMap().size() > 1) { - boolean has_delete_family = delete.getFamilyMap().entrySet().stream() - .flatMap(entry -> entry.getValue().stream()).anyMatch( - kv -> KeyValue.Type.codeToType( - kv.getType()) == KeyValue.Type.DeleteFamily || KeyValue.Type.codeToType( - kv.getType()) == KeyValue.Type.DeleteFamilyVersion); - if (!has_delete_family) { - batch = buildBatchOperation(tableNameString, - Collections.singletonList(delete), true, - resultMapSingleOp); - } else { - for (Map.Entry> entry : delete.getFamilyCellMap() - .entrySet()) { - byte[] family = entry.getKey(); - String realTableName = getTargetTableName(tableNameString, - Bytes.toString(family), configuration); - // split delete - List cells = entry.getValue(); - Delete del = new Delete(delete.getRow()); - del.getFamilyCellMap().put(family, cells); - batch = buildBatchOperation(realTableName, - Collections.singletonList(del), false, - resultMapSingleOp); - tmpResults = batch.execute(); - if (tmpResults.getResults().get(0) instanceof ObTableException) { - results[i] = tmpResults.getResults().get(0); - } - } - if (results[i] == null) { - results[i] = new Result(); - } else { - batchError.add((ObTableException) results[i], actions.get(i), null); - } - continue; - } - } else { - byte[] family = delete.getFamilyCellMap().firstKey(); - String realTableName = getTargetTableName(tableNameString, Bytes.toString(family), configuration); - batch = buildBatchOperation(realTableName, - Collections.singletonList(delete), - false, resultMapSingleOp); + batch = compatOldServerPut(actions, results, batchError, i, puts); + if (!puts.isEmpty()) { + i = i + puts.size() - 1; } + puts.clear(); + } else { + batch = compatOldServerDel(actions, results, batchError, i); } if (batch != null) { - tmpResults = batch.execute(); + BatchOperationResult tmpResults = batch.execute(); if (tmpResults.getResults().get(0) instanceof ObTableException) { - results[i] = tmpResults.getResults().get(0); - batchError.add((ObTableException) results[i], actions.get(i), null); + if (results != null) { + results[i] = tmpResults.getResults().get(0); + } + batchError.add((ObTableException) tmpResults.getResults().get(0), actions.get(i), null); } else { - results[i] = new Result(); + if (results != null) { + results[i] = new Result(); + } } } } } @Override - public void batch(final List actions, final Object[] results) - throws IOException, FeatureNotSupportedException, IllegalArgumentException { + public void batch(final List actions, final Object[] results) throws IOException{ if (actions == null) { return; } @@ -584,33 +635,39 @@ public void batch(final List actions, final Object[] results) BatchError batchError = new BatchError(); obTableClient.setRuntimeBatchExecutor(executePool); List resultMapSingleOp = new LinkedList<>(); - try { - if (!CompatibilityUtil.isBatchSupport()) { - compatOldServerBatch(actions, results); - } else { - String realTableName = getTargetTableName(actions); - BatchOperation batch = buildBatchOperation(realTableName, actions, - tableNameString.equals(realTableName), resultMapSingleOp); - BatchOperationResult tmpResults = batch.execute(); - if (results != null) { - int index = 0; - for (int i = 0; i != results.length; ++i) { - if (tmpResults.getResults().get(index) instanceof ObTableException) { - results[i] = tmpResults.getResults().get(index); - batchError.add((ObTableException) results[i], actions.get(i), null); - } else { - results[i] = new Result(); - } - index += resultMapSingleOp.get(i); + if (!CompatibilityUtil.isBatchSupport()) { + try { + compatOldServerBatch(actions, results, batchError); + } catch (Exception e) { + throw new IOException(e); + } + } else { + String realTableName = getTargetTableName(actions); + BatchOperation batch = buildBatchOperation(realTableName, actions, + tableNameString.equals(realTableName), resultMapSingleOp); + BatchOperationResult tmpResults; + try { + tmpResults = batch.execute(); + } catch (Exception e) { + throw new IOException(e); + } + int index = 0; + for (int i = 0; i != actions.size(); ++i) { + if (tmpResults.getResults().get(index) instanceof ObTableException) { + if (results != null) { + results[i] = tmpResults.getResults().get(index); } - if (batchError.hasErrors()) { - throw batchError.makeException(); + batchError.add((ObTableException) tmpResults.getResults().get(index), actions.get(i), null); + } else { + if (results != null) { + results[i] = new Result(); } } + index += resultMapSingleOp.get(i); } - } catch (Exception e) { - logger.error(LCD.convert("01-000010"), tableNameString, actions, e); - throw new IOException("batch table " + tableNameString + " error", e); + } + if (batchError.hasErrors()) { + throw batchError.makeException(); } } @@ -821,8 +878,18 @@ public Result call() throws IOException { @Override public Result[] get(List gets) throws IOException { Result[] results = new Result[gets.size()]; + List> futures = new LinkedList<>(); + for (int i = 0; i < gets.size(); i++) { + int index = i; + Future future = executePool.submit(() -> get(gets.get(index))); + futures.add(future); + } for (int i = 0; i < gets.size(); i++) { - results[i] = get(gets.get(i)); + try { + results[i] = futures.get(i).get(); + } catch (Exception e) { + throw new RuntimeException("gets occur error. index:{" + i + "}", e); + } } return results; } @@ -1930,6 +1997,15 @@ public static ObTableOperation buildObTableOperation(KeyValue kv, boolean putToA } } + private ObTableQueryRequest buildObTableQueryRequest(ObTableQuery obTableQuery, + String targetTableName) { + ObTableQueryRequest request = new ObTableQueryRequest(); + request.setEntityType(ObTableEntityType.HKV); + request.setTableQuery(obTableQuery); + request.setTableName(targetTableName); + return request; + } + private ObTableQueryAsyncRequest buildObTableQueryAsyncRequest(ObTableQuery obTableQuery, String targetTableName) { ObTableQueryRequest request = new ObTableQueryRequest(); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java index 6297bad9..ca864ab1 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java @@ -18,7 +18,6 @@ package com.alipay.oceanbase.hbase; import org.apache.hadoop.conf.Configuration; -import com.alipay.oceanbase.rpc.mutation.result.MutationResult; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; @@ -657,9 +656,9 @@ public void testMultiColumnFamilyBatch() throws Exception { put.add(family3, family3_column1, family3_value); batchLsit.add(put); } - multiCfHTable.batchCallback(batchLsit, new Batch.Callback() { + multiCfHTable.batchCallback(batchLsit, new Batch.Callback() { @Override - public void update(byte[] region, byte[] row, MutationResult result) { + public void update(byte[] region, byte[] row, Result result) { updateCounter[0]++; } }); @@ -1301,14 +1300,14 @@ public void testFamilyFilter() throws Exception { Put putKey2Column1Value2 = new Put(toBytes(key2)); putKey2Column1Value2.add(toBytes(family3), toBytes(column1), toBytes(value2)); - multiCfHTable.delete(deleteKey1); - multiCfHTable.delete(deleteKey2); - multiCfHTable.put(putKey1Column1Value1); - multiCfHTable.put(putKey1Column1Value2); - multiCfHTable.put(putKey1Column2Value2); - multiCfHTable.put(putKey2Column2Value1); - multiCfHTable.put(putKey2Column1Value1); - multiCfHTable.put(putKey2Column1Value2); + multiCfHTable.delete(deleteKey1); + multiCfHTable.delete(deleteKey2); + multiCfHTable.put(putKey1Column1Value1); + multiCfHTable.put(putKey1Column1Value2); + multiCfHTable.put(putKey1Column2Value2); + multiCfHTable.put(putKey2Column2Value1); + multiCfHTable.put(putKey2Column1Value1); + multiCfHTable.put(putKey2Column1Value2); Scan scan; scan = new Scan(); @@ -1316,20 +1315,20 @@ public void testFamilyFilter() throws Exception { scan.addFamily(family2.getBytes()); scan.addFamily(family3.getBytes()); scan.setMaxVersions(10); - FamilyFilter f = new FamilyFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(family2))); + FamilyFilter f = new FamilyFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(family2))); scan.setFilter(f); ResultScanner scanner = multiCfHTable.getScanner(scan); int res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); Assert.assertArrayEquals(family2.getBytes(), keyValue.getFamily()); res_count += 1; } @@ -1342,20 +1341,20 @@ public void testFamilyFilter() throws Exception { scan.addFamily(family2.getBytes()); scan.addFamily(family3.getBytes()); scan.setMaxVersions(10); - f = new FamilyFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator(Bytes.toBytes(family2))); + f = new FamilyFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator( + Bytes.toBytes(family2))); scan.setFilter(f); scanner = multiCfHTable.getScanner(scan); res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); res_count += 1; } } @@ -1367,20 +1366,20 @@ public void testFamilyFilter() throws Exception { scan.addFamily(family2.getBytes()); scan.addFamily(family3.getBytes()); scan.setMaxVersions(10); - f = new FamilyFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator(Bytes.toBytes(family2))); + f = new FamilyFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator( + Bytes.toBytes(family2))); scan.setFilter(f); scanner = multiCfHTable.getScanner(scan); res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); Assert.assertArrayEquals(family3.getBytes(), keyValue.getFamily()); res_count += 1; } @@ -1393,20 +1392,20 @@ public void testFamilyFilter() throws Exception { scan.addFamily(family2.getBytes()); scan.addFamily(family3.getBytes()); scan.setMaxVersions(10); - f = new FamilyFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(Bytes.toBytes("family_with_group"))); + f = new FamilyFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator( + Bytes.toBytes("family_with_group"))); scan.setFilter(f); scanner = multiCfHTable.getScanner(scan); res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", - Bytes.toString(result.getRow()), - Bytes.toString(keyValue.getFamily()), - Bytes.toString(keyValue.getQualifier()), - keyValue.getTimestamp(), - Bytes.toString(keyValue.getValue()) - ); + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); res_count += 1; } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHVarPrefixPartitionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHVarPrefixPartitionTest.java new file mode 100644 index 00000000..2dc41621 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/OHVarPrefixPartitionTest.java @@ -0,0 +1,43 @@ +package com.alipay.oceanbase.hbase; + +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; + +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +/// Only support odp mode for now. +public class OHVarPrefixPartitionTest extends HTableTestBase { + @Before + public void before() throws Exception { + hTable = ObHTableTestUtil.newOHTableClient("test_var_prefix_partition"); + ((OHTableClient) hTable).init(); + } + + @After + public void finish() throws IOException { + hTable.close(); + } + + @After + public void after() throws IOException { + hTable.close(); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java index f35f72a2..bc567fa3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java @@ -243,8 +243,10 @@ public void testMultipleColumnPrefixFilter() throws IOException { @Test public void testFamilyFilter() throws IOException { - FamilyFilter filter = new FamilyFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator(Bytes.toBytes("cf"))); - Assert.assertArrayEquals("FamilyFilter(!=,'binary:cf')".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + FamilyFilter filter = new FamilyFilter(CompareFilter.CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("cf"))); + Assert.assertArrayEquals("FamilyFilter(!=,'binary:cf')".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test diff --git a/src/test/java/unit_test_db.sql b/src/test/java/unit_test_db.sql index 022177dc..cbb172fe 100644 --- a/src/test/java/unit_test_db.sql +++ b/src/test/java/unit_test_db.sql @@ -54,6 +54,15 @@ CREATE TABLE `test$partitionFamily1` ( PRIMARY KEY (`K`, `Q`, `T`) ) partition by key(`K`) partitions 17; +CREATE TABLE `test_var_prefix_partition$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + `K_PREFIX` varbinary(1024) generated always as (SUBSTRING_INDEX(`K`, '.', 1)), + PRIMARY KEY (`K`, `Q`, `T`) +) partition by key(`K_PREFIX`) partitions 15; + CREATE TABLEGROUP test SHARDING = 'ADAPTIVE'; CREATE TABLE `test$family_group` ( `K` varbinary(1024) NOT NULL,