diff --git a/pom.xml b/pom.xml index 0a4ddd64..e6658921 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ ${project.encoding} UTF-8 1.7.21 - 1.2.13-SNAPSHOT + 1.2.14.1-SNAPSHOT diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 85a8f26f..ec069737 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -47,9 +47,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.filter.CompareFilter; @@ -62,6 +59,7 @@ import org.apache.hadoop.hbase.util.Pair; import org.slf4j.Logger; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.*; import java.util.concurrent.*; @@ -76,6 +74,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.commons.lang.StringUtils.isBlank; import static org.apache.commons.lang.StringUtils.isNotBlank; +import static com.alipay.oceanbase.hbase.filter.HBaseFilterUtils.writeBytesWithEscape; public class OHTable implements HTableInterface { @@ -894,7 +893,7 @@ private boolean checkAndMutation(byte[] row, byte[] family, byte[] qualifier, Co checkArgument(!rowMutations.getMutations().isEmpty(), "mutation is empty"); - String filterString = buildCheckAndMutateFilterString(family, qualifier, compareOp, value); + byte[] filterString = buildCheckAndMutateFilterString(family, qualifier, compareOp, value); ObHTableFilter filter = buildObHTableFilter(filterString, null, 1, qualifier); List mutations = rowMutations.getMutations(); @@ -1415,11 +1414,11 @@ private static String getTestLoadTargetTableName(String tableNameString, String } private ObHTableFilter buildObHTableFilter(Filter filter, TimeRange timeRange, int maxVersion, - Collection columnQualifiers) { + Collection columnQualifiers) throws IOException { ObHTableFilter obHTableFilter = new ObHTableFilter(); if (filter != null) { - obHTableFilter.setFilterString(HBaseFilterUtils.toParseableString(filter)); + obHTableFilter.setFilterString(HBaseFilterUtils.toParseableByteArray(filter)); } if (timeRange != null) { @@ -1441,20 +1440,25 @@ private ObHTableFilter buildObHTableFilter(Filter filter, TimeRange timeRange, i return obHTableFilter; } - private String buildCheckAndMutateFilterString(byte[] family, byte[] qualifier, - CompareFilter.CompareOp compareOp, byte[] value) { + private byte[] buildCheckAndMutateFilterString(byte[] family, byte[] qualifier, CompareFilter.CompareOp compareOp, byte[] value) throws IOException { + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + byteStream.write("CheckAndMutateFilter(".getBytes()); + byteStream.write(HBaseFilterUtils.toParseableByteArray(compareOp)); + byteStream.write(", 'binary:".getBytes()); + writeBytesWithEscape(byteStream, value); + byteStream.write("', '".getBytes()); + writeBytesWithEscape(byteStream, family); + byteStream.write("', '".getBytes()); + writeBytesWithEscape(byteStream, qualifier); if (value != null) { - return ("CheckAndMutateFilter(" + HBaseFilterUtils.toParseableString(compareOp) - + ", 'binary:" + Bytes.toString(value) + "', '" + Bytes.toString(family) - + "', '" + (qualifier == null ? "" : Bytes.toString(qualifier)) + "', false)"); + byteStream.write("', false)".getBytes()); } else { - return ("CheckAndMutateFilter(" + HBaseFilterUtils.toParseableString(compareOp) - + ", 'binary:', '" + Bytes.toString(family) + "', '" - + (qualifier == null ? "" : Bytes.toString(qualifier)) + "', true)"); + byteStream.write("', true)".getBytes()); } + return byteStream.toByteArray(); } - private ObHTableFilter buildObHTableFilter(String filterString, TimeRange timeRange, + private ObHTableFilter buildObHTableFilter(byte[] filterString, TimeRange timeRange, int maxVersion, byte[]... columnQualifiers) { ObHTableFilter obHTableFilter = new ObHTableFilter(); @@ -1541,7 +1545,7 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, final Scan scan) { return obTableQuery; } - private ObTableQuery buildObTableQuery(final Get get, Collection columnQualifiers) { + private ObTableQuery buildObTableQuery(final Get get, Collection columnQualifiers) throws IOException { ObTableQuery obTableQuery; if (get.isClosestRowBefore()) { PageFilter pageFilter = new PageFilter(1); diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index 32238448..60570332 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -22,134 +22,175 @@ import org.apache.hadoop.hbase.util.Bytes; import java.lang.reflect.Field; + +import static org.apache.hadoop.hbase.util.Bytes.bytesToVint; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.util.List; @InterfaceAudience.Private public class HBaseFilterUtils { - public static String toParseableString(Filter filter) { + public static byte[] toParseableByteArray(Filter filter) throws IOException { + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + toParseableByteArray(byteStream, filter); + return byteStream.toByteArray(); + } + + private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filter filter) throws IOException { if (filter == null) { throw new IllegalArgumentException("Filter is null"); } else if (filter instanceof CompareFilter) { // RowFilter, ValueFilter, QualifierFilter - return toParseableString((CompareFilter) filter); + toParseableByteArray(byteStream, (CompareFilter) filter); } else if (filter instanceof SingleColumnValueFilter) { - return toParseableString((SingleColumnValueFilter) filter); + toParseableByteArray(byteStream, (SingleColumnValueFilter) filter); } else if (filter instanceof PageFilter) { - return toParseableString((PageFilter) filter); + toParseableByteArray(byteStream, (PageFilter) filter); } else if (filter instanceof ColumnCountGetFilter) { - return toParseableString((ColumnCountGetFilter) filter); + toParseableByteArray(byteStream, (ColumnCountGetFilter) filter); } else if (filter instanceof PrefixFilter) { - return toParseableString((PrefixFilter) filter); + toParseableByteArray(byteStream, (PrefixFilter) filter); } else if (filter instanceof FilterList) { - return toParseableString((FilterList) filter); + toParseableByteArray(byteStream, (FilterList) filter); } else if (filter instanceof RandomRowFilter) { - return toParseableString((RandomRowFilter) filter); + toParseableByteArray(byteStream, (RandomRowFilter) filter); } else if (filter instanceof ColumnPaginationFilter) { - return toParseableString((ColumnPaginationFilter) filter); + toParseableByteArray(byteStream, (ColumnPaginationFilter) filter); } else if (filter instanceof ColumnPrefixFilter) { - return toParseableString((ColumnPrefixFilter) filter); + toParseableByteArray(byteStream, (ColumnPrefixFilter) filter); } else if (filter instanceof FirstKeyOnlyFilter) { - return toParseableString((FirstKeyOnlyFilter) filter); + toParseableByteArray(byteStream, (FirstKeyOnlyFilter) filter); } else if (filter instanceof KeyOnlyFilter) { - return toParseableString((KeyOnlyFilter) filter); + toParseableByteArray(byteStream, (KeyOnlyFilter) filter); } else if (filter instanceof TimestampsFilter) { - return toParseableString((TimestampsFilter) filter); + toParseableByteArray(byteStream, (TimestampsFilter) filter); } else if (filter instanceof SkipFilter) { - return toParseableString((SkipFilter) filter); + toParseableByteArray(byteStream, (SkipFilter) filter); } else if (filter instanceof WhileMatchFilter) { - return toParseableString((WhileMatchFilter) filter); + toParseableByteArray(byteStream, (WhileMatchFilter) filter); } else { throw new IllegalArgumentException("Invalid filter: " + filter); } } - public static String toParseableString(CompareFilter.CompareOp op) { + public static byte[] toParseableByteArray(CompareFilter.CompareOp op) { if (op == null) { throw new IllegalArgumentException("Compare operator is null"); } switch (op) { case LESS: - return Bytes.toString(ParseConstants.LESS_THAN_ARRAY); + return ParseConstants.LESS_THAN_ARRAY; case LESS_OR_EQUAL: - return Bytes.toString(ParseConstants.LESS_THAN_OR_EQUAL_TO_ARRAY); + return ParseConstants.LESS_THAN_OR_EQUAL_TO_ARRAY; case EQUAL: - return Bytes.toString(ParseConstants.EQUAL_TO_ARRAY); + return ParseConstants.EQUAL_TO_ARRAY; case NOT_EQUAL: - return Bytes.toString(ParseConstants.NOT_EQUAL_TO_ARRAY); + return ParseConstants.NOT_EQUAL_TO_ARRAY; case GREATER_OR_EQUAL: - return Bytes.toString(ParseConstants.GREATER_THAN_OR_EQUAL_TO_ARRAY); + return ParseConstants.GREATER_THAN_OR_EQUAL_TO_ARRAY; case GREATER: - return Bytes.toString(ParseConstants.GREATER_THAN_ARRAY); + return ParseConstants.GREATER_THAN_ARRAY; default: throw new IllegalArgumentException("Invalid compare operator: " + op); } } - private static String toParseableString(ByteArrayComparable comparator) { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, ByteArrayComparable comparator) throws IOException { if (comparator == null) { throw new IllegalArgumentException("Comparator is null"); } - StringBuilder sb = new StringBuilder(); + byteStream.write('\''); if (comparator instanceof BinaryComparator) { - sb.append('\'').append(Bytes.toString(ParseConstants.binaryType)).append(':') - .append(Bytes.toString(comparator.getValue())).append('\''); + byteStream.write(ParseConstants.binaryType); } else if (comparator instanceof BinaryPrefixComparator) { - sb.append('\'').append(Bytes.toString(ParseConstants.binaryPrefixType)).append(':') - .append(Bytes.toString(comparator.getValue())).append('\''); + byteStream.write(ParseConstants.binaryPrefixType); } else if (comparator instanceof RegexStringComparator) { - sb.append('\'').append(Bytes.toString(ParseConstants.regexStringType)).append(':') - .append(Bytes.toString(comparator.getValue())).append('\''); + byteStream.write(ParseConstants.regexStringType); } else if (comparator instanceof SubstringComparator) { - sb.append('\'').append(Bytes.toString(ParseConstants.substringType)).append(':') - .append(Bytes.toString(comparator.getValue())).append('\''); + byteStream.write(ParseConstants.substringType); } else { throw new IllegalArgumentException("This comparator has not been implemented " + comparator); } - return sb.toString(); + byteStream.write(':'); + writeBytesWithEscape(byteStream, comparator.getValue()); + byteStream.write('\''); } - private static String toParseableString(CompareFilter filter) { - return filter.getClass().getSimpleName() + '(' + toParseableString(filter.getOperator()) - + ',' + toParseableString(filter.getComparator()) + ')'; + // CompareFilter(=,'binary:123') + private static void toParseableByteArray(ByteArrayOutputStream byteStream, CompareFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(')'); } - private static String toParseableString(SingleColumnValueFilter filter) { - return filter.getClass().getSimpleName() + "('" + Bytes.toString(filter.getFamily()) - + "','" + Bytes.toString(filter.getQualifier()) + "'," - + toParseableString(filter.getOperator()) + ',' - + toParseableString(filter.getComparator()) + ',' + filter.getFilterIfMissing() - + ',' + filter.getLatestVersionOnly() + ')'; + // SingleColumnValueFilter('cf1','col1',=,'binary:123',true,true) + private static void toParseableByteArray(ByteArrayOutputStream byteStream, SingleColumnValueFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getFilterIfMissing()).getBytes()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getLatestVersionOnly()).getBytes()); + byteStream.write(')'); } - private static String toParseableString(PageFilter filter) { - return filter.getClass().getSimpleName() + '(' + filter.getPageSize() + ')'; + // PageFilter(100); + private static void toParseableByteArray(ByteArrayOutputStream byteStream, PageFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + byteStream.write(Long.toString(filter.getPageSize()).getBytes()); + byteStream.write(')'); } - private static String toParseableString(RandomRowFilter filter) { - return filter.getClass().getSimpleName() + "(" + Bytes.toInt(Bytes.toBytes(filter.getChance())) + ")"; + private static void toParseableByteArray(ByteArrayOutputStream byteStream, RandomRowFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + byteStream.write(Integer.toString(Bytes.toInt(Bytes.toBytes(filter.getChance()))).getBytes()); + byteStream.write(')'); } - private static String toParseableString(ColumnPaginationFilter filter) { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnPaginationFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + byteStream.write(Long.toString(filter.getLimit()).getBytes()); + byteStream.write(','); if (filter.getColumnOffset() != null) { - return filter.getClass().getSimpleName() + '(' + filter.getLimit() + ",'" - + Bytes.toString(filter.getColumnOffset()) + "')"; + byteStream.write('\''); + writeBytesWithEscape(byteStream, filter.getColumnOffset()); + byteStream.write('\''); } else { - return filter.getClass().getSimpleName() + '(' + filter.getLimit() + ',' - + filter.getOffset() + ')'; + byteStream.write(Long.toString(filter.getOffset()).getBytes()); } + byteStream .write(')'); } - private static String toParseableString(ColumnPrefixFilter filter) { - return filter.getClass().getSimpleName() + "('" + Bytes.toString(filter.getPrefix()) + "')"; + private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnPrefixFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getPrefix()); + byteStream.write("')".getBytes()); } - private static String toParseableString(FirstKeyOnlyFilter filter) { - return filter.getClass().getSimpleName() + "()"; + private static void toParseableByteArray(ByteArrayOutputStream byteStream, FirstKeyOnlyFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + byteStream.write(')'); } - private static String toParseableString(KeyOnlyFilter filter) { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, KeyOnlyFilter filter) throws IOException { boolean lenAsVal; try { Field field = filter.getClass().getDeclaredField("lenAsVal"); @@ -158,11 +199,15 @@ private static String toParseableString(KeyOnlyFilter filter) { } catch (NoSuchFieldException | IllegalAccessException e) { throw new RuntimeException(e); } - return filter.getClass().getSimpleName() + "(" + lenAsVal + ")"; + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + byteStream.write(Boolean.toString(lenAsVal).getBytes()); + byteStream.write(')'); } - private static String toParseableString(TimestampsFilter filter) { - StringBuilder paramBuilder = new StringBuilder(); + private static void toParseableByteArray(ByteArrayOutputStream byteStream, TimestampsFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); List timestamps = filter.getTimestamps(); boolean canHint; try { @@ -174,59 +219,90 @@ private static String toParseableString(TimestampsFilter filter) { } for (int i = 0; i < timestamps.size(); i ++) { Long timestamp = timestamps.get(i); - paramBuilder.append(timestamp); - paramBuilder.append(","); + byteStream.write(Long.toString(timestamp).getBytes()); + byteStream.write(','); } - String param = paramBuilder.toString(); - return filter.getClass().getSimpleName() + "(" - + param + canHint + ")"; + byteStream.write(Boolean.toString(canHint).getBytes()); + byteStream.write(')'); } - private static String toParseableString(ColumnCountGetFilter filter) { - return filter.getClass().getSimpleName() + '(' + filter.getLimit() + ')'; + // ColumnCountGetFilter(100) + private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnCountGetFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + byteStream.write(Long.toString(filter.getLimit()).getBytes()); + byteStream .write(')'); } - private static String toParseableString(PrefixFilter filter) { - return filter.getClass().getSimpleName() + "('" + Bytes.toString(filter.getPrefix()) + "')"; + // PrefixFilter('prefix'); + private static void toParseableByteArray(ByteArrayOutputStream byteStream, PrefixFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getPrefix()); + byteStream .write("')".getBytes()); } - private static String toParseableString(SkipFilter filter) { - return "(" + Bytes.toString(ParseConstants.SKIP_ARRAY) + " " - + toParseableString(filter.getFilter()) + ")"; + // (SKIP filter) + private static void toParseableByteArray(ByteArrayOutputStream byteStream, SkipFilter filter) throws IOException { + byteStream.write('('); + byteStream.write(ParseConstants.SKIP_ARRAY); + byteStream.write(' '); + toParseableByteArray(byteStream, filter.getFilter()); + byteStream.write(')'); } - private static String toParseableString(WhileMatchFilter filter) { - return "(" + Bytes.toString(ParseConstants.WHILE_ARRAY) + " " - + toParseableString(filter.getFilter()) + ")"; + // (WHILE filter) + private static void toParseableByteArray(ByteArrayOutputStream byteStream, WhileMatchFilter filter) throws IOException { + byteStream.write('('); + byteStream.write(ParseConstants.WHILE_ARRAY); + byteStream.write(' '); + toParseableByteArray(byteStream, filter.getFilter()); + byteStream.write(')'); } - private static String toParseableString(FilterList filterList) { - StringBuilder sb = new StringBuilder(); + // (filter and filter ...) or (filter or filter ...) + // when filter list is empty, "" is generated, and empty filter list member is removed + // in result parseable byteArray + private static void toParseableByteArray(ByteArrayOutputStream byteStream, FilterList filterList) throws IOException { List filters = filterList.getFilters(); boolean isEmpty = true; + ByteArrayOutputStream oneFilterBytes = new ByteArrayOutputStream(); for (int i = 0; i < filters.size(); i++) { - String filterString = toParseableString(filters.get(i)); - if (filterString.isEmpty()) - continue; + toParseableByteArray(oneFilterBytes, filters.get(i)); + if (oneFilterBytes.size() == 0) { continue; } if (isEmpty) { - sb.append("(").append(filterString); + byteStream.write('('); isEmpty = false; } else { - sb.append(" "); + byteStream.write(' '); if (filterList.getOperator().equals(FilterList.Operator.MUST_PASS_ALL)) { - sb.append(Bytes.toString(ParseConstants.AND)); + byteStream.write(ParseConstants.AND); } else if (filterList.getOperator().equals(FilterList.Operator.MUST_PASS_ONE)) { - sb.append(Bytes.toString(ParseConstants.OR)); + byteStream.write(ParseConstants.OR); } else { throw new IllegalArgumentException("Invalid FilterList: " + filterList); } - sb.append(" ").append(filterString); + byteStream.write(' '); } + oneFilterBytes.writeTo(byteStream); + oneFilterBytes.reset(); } if (!isEmpty) { - sb.append(")"); + byteStream.write(')'); } - return sb.toString(); } + // when write family/qualifier/value/row into hbase filter, need add escape for + // special character to prevent parse error in server + public static void writeBytesWithEscape(ByteArrayOutputStream byteStream, byte[] bytes) throws IOException { + if (bytes == null) { + return; + } + for (int i = 0; i < bytes.length; i++) { + if (bytes[i] == '\'') { + byteStream.write('\''); + } + byteStream.write(bytes[i]); + } + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index cedc5842..62120ec3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -332,7 +332,7 @@ public void testMultiPut() throws IOException { Put put2 = new Put(Bytes.toBytes("testKey1")); put2.add(toBytes(family), toBytes(column2), toBytes(value1)); put2.add(toBytes(family), toBytes(column2), System.currentTimeMillis(), toBytes(value1)); - List puts = new ArrayList(); + List puts = new ArrayList<>(); puts.add(put1); puts.add(put2); hTable.put(puts); @@ -1875,7 +1875,8 @@ public void testScanSessionClean() throws Exception { Thread.sleep(61 * 1000); try { scanner.next(); - } catch (IOException e) { + fail(); + } catch (Exception e) { assertTrue(e.getCause().getMessage().contains("OB_HASH_NOT_EXIST")); } finally { scanner.close(); @@ -3773,4 +3774,61 @@ public void testIncrementConcurrency() throws Exception { public void testHtableWithIndex() throws Exception { testBasic("family_with_local_index"); } + + @Test + public void testFilterSpecialValue() throws IOException { + // 1. test checkAndMutate with special character + String specialValue = "AAEAAAGRnJiQbwAAAZGcmJwndjE="; + byte[] specialBytes = Base64.getDecoder().decode(specialValue); + String family = "family'1"; + byte[] keyBytes = specialBytes; + byte[] columnBytes = specialBytes; + byte[] valueBytes = specialBytes; + + Delete delete = new Delete(keyBytes); + delete.deleteFamily(family.getBytes()); + hTable.delete(delete); + + Put put = new Put(keyBytes); + put.add(family.getBytes(), columnBytes, valueBytes); + hTable.put(put); + + // check delete column + delete = new Delete(keyBytes); + delete.deleteColumn(family.getBytes(), columnBytes); + boolean ret = hTable.checkAndDelete(keyBytes, family.getBytes(), columnBytes, + valueBytes, delete); + Assert.assertTrue(ret); + + // 2. test normal filter with special chracter + put = new Put(keyBytes); + put.add(family.getBytes(), columnBytes, valueBytes); + hTable.put(put); + + + Get get = new Get(keyBytes); + get.addFamily(family.getBytes()); + // 2.1 test special row + RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(keyBytes)); + get.setFilter(rowFilter); + Result result = hTable.get(get); + Assert.assertEquals(1, result.raw().length); + Assert.assertArrayEquals(keyBytes, result.getRow()); + + // 2.2 test special column + SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter(family.getBytes(), + columnBytes, CompareFilter.CompareOp.EQUAL, valueBytes); + get.setFilter(singleColumnValueFilter); + result = hTable.get(get); + Assert.assertEquals(1, result.raw().length); + Assert.assertArrayEquals(keyBytes, result.getRow()); + + // 2.3 test special value + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(valueBytes)); + get.setFilter(valueFilter); + result = hTable.get(get); + Assert.assertEquals(1, result.raw().length); + Assert.assertArrayEquals(keyBytes, result.getRow()); + + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java index f6679a1a..1f979b90 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java @@ -23,6 +23,8 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; + public class HBaseFilterUtilsTest { private static final CompareFilter.CompareOp[] ops = { CompareFilter.CompareOp.LESS, CompareFilter.CompareOp.LESS_OR_EQUAL, CompareFilter.CompareOp.EQUAL, @@ -36,145 +38,146 @@ public static void beforeClass() { } @Test - public void testRowFilter() { + public void testRowFilter() throws IOException { for (int i = 0; i < ops.length; i++) { RowFilter filter = new RowFilter(ops[i], new BinaryComparator( "testRowFilter".getBytes())); String expect = String.format("RowFilter(%s,'binary:testRowFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new RowFilter(ops[i], new BinaryPrefixComparator("testRowFilter".getBytes())); expect = String.format("RowFilter(%s,'binaryprefix:testRowFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new RowFilter(ops[i], new RegexStringComparator("testRowFilter")); expect = String.format("RowFilter(%s,'regexstring:testRowFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new RowFilter(ops[i], new SubstringComparator("testRowFilter")); expect = String.format("RowFilter(%s,'substring:testrowfilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); } } @Test - public void testValueFilter() { + public void testValueFilter() throws IOException { for (int i = 0; i < ops.length; i++) { ValueFilter filter = new ValueFilter(ops[i], new BinaryComparator( "testValueFilter".getBytes())); String expect = String.format("ValueFilter(%s,'binary:testValueFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new ValueFilter(ops[i], new BinaryPrefixComparator( "testValueFilter".getBytes())); expect = String.format("ValueFilter(%s,'binaryprefix:testValueFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new ValueFilter(ops[i], new RegexStringComparator("testValueFilter")); expect = String.format("ValueFilter(%s,'regexstring:testValueFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new ValueFilter(ops[i], new SubstringComparator("testValueFilter")); expect = String.format("ValueFilter(%s,'substring:testvaluefilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); } } @Test - public void testQualifierFilter() { + public void testQualifierFilter() throws IOException { for (int i = 0; i < ops.length; i++) { QualifierFilter filter = new QualifierFilter(ops[i], new BinaryComparator( "testQualifierFilter".getBytes())); String expect = String.format("QualifierFilter(%s,'binary:testQualifierFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new QualifierFilter(ops[i], new BinaryPrefixComparator( "testQualifierFilter".getBytes())); expect = String.format("QualifierFilter(%s,'binaryprefix:testQualifierFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new QualifierFilter(ops[i], new RegexStringComparator("testQualifierFilter")); expect = String.format("QualifierFilter(%s,'regexstring:testQualifierFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new QualifierFilter(ops[i], new SubstringComparator("testQualifierFilter")); expect = String.format("QualifierFilter(%s,'substring:testqualifierfilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); } } @Test - public void testSingleColumnValueFilter() { + public void testSingleColumnValueFilter() throws IOException { for (int i = 0; i < ops.length; i++) { String expect = String.format( "SingleColumnValueFilter('family','qualifier',%s,'binary:value',false,true)", opFlags[i]); SingleColumnValueFilter filter = new SingleColumnValueFilter("family".getBytes(), "qualifier".getBytes(), ops[i], "value".getBytes()); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); } } @Test - public void testPageFilter() { + public void testPageFilter() throws IOException { PageFilter filter = new PageFilter(128); - Assert.assertEquals("PageFilter(128)", HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals("PageFilter(128)".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); } @Test - public void testColumnPaginationFilter() { + public void testColumnPaginationFilter() throws IOException { ColumnPaginationFilter filter = new ColumnPaginationFilter(2, 2); - Assert.assertEquals("ColumnPaginationFilter(2,2)", - HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals("ColumnPaginationFilter(2,2)".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + filter = new ColumnPaginationFilter(2, Bytes.toBytes("a")); - Assert.assertEquals("ColumnPaginationFilter(2,'a')", - HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals("ColumnPaginationFilter(2,'a')".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test - public void testColumnPrefixFilter() { + public void testColumnPrefixFilter() throws IOException { ColumnPrefixFilter filter = new ColumnPrefixFilter(Bytes.toBytes("pre")); - Assert - .assertEquals("ColumnPrefixFilter('pre')", HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals("ColumnPrefixFilter('pre')".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test - public void testColumnCountGetFilter() { + public void testColumnCountGetFilter() throws IOException { ColumnCountGetFilter filter = new ColumnCountGetFilter(513); - Assert - .assertEquals("ColumnCountGetFilter(513)", HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals("ColumnCountGetFilter(513)".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); } @Test - public void testPrefixFilter() { + public void testPrefixFilter() throws IOException { PrefixFilter filter = new PrefixFilter("prefix".getBytes()); - Assert.assertEquals("PrefixFilter('prefix')", HBaseFilterUtils.toParseableString(filter)); + System.out.println(new String(HBaseFilterUtils.toParseableByteArray(filter))); + Assert.assertArrayEquals("PrefixFilter('prefix')".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); } @Test - public void testSkipFilter() { + public void testSkipFilter() throws IOException { RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( "testSkipFilter".getBytes())); SkipFilter filter = new SkipFilter(rowFilter); - Assert.assertEquals("(SKIP RowFilter(=,'binary:testSkipFilter'))", - HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals("(SKIP RowFilter(=,'binary:testSkipFilter'))".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test - public void testWhileFilter() { + public void testWhileFilter() throws IOException { QualifierFilter qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.GREATER, new BinaryPrefixComparator("whileMatchFilter".getBytes())); WhileMatchFilter filter = new WhileMatchFilter(qualifierFilter); - Assert.assertEquals("(WHILE QualifierFilter(>,'binaryprefix:whileMatchFilter'))", - HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals("(WHILE QualifierFilter(>,'binaryprefix:whileMatchFilter'))".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test - public void testFilterList() { + public void testFilterList() throws IOException { RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( "testSkipFilter".getBytes())); QualifierFilter qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.GREATER, @@ -188,21 +191,21 @@ public void testFilterList() { filterList.addFilter(skipFilter); filterList.addFilter(columnPaginationFilter); - Assert - .assertEquals( - "(RowFilter(=,'binary:testSkipFilter') " - + "AND QualifierFilter(>,'binaryprefix:whileMatchFilter') AND (SKIP PageFilter(128)) AND ColumnPaginationFilter(2,2))", - HBaseFilterUtils.toParseableString(filterList)); + + System.out.println(new String(HBaseFilterUtils.toParseableByteArray(filterList))); + Assert.assertArrayEquals( + ("(RowFilter(=,'binary:testSkipFilter') " + + "AND QualifierFilter(>,'binaryprefix:whileMatchFilter') AND (SKIP PageFilter(128)) AND ColumnPaginationFilter(2,2))").getBytes(), + HBaseFilterUtils.toParseableByteArray(filterList)); filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); filterList.addFilter(rowFilter); filterList.addFilter(qualifierFilter); filterList.addFilter(columnPaginationFilter); - Assert - .assertEquals( - "(RowFilter(=,'binary:testSkipFilter') " - + "OR QualifierFilter(>,'binaryprefix:whileMatchFilter') OR ColumnPaginationFilter(2,2))", - HBaseFilterUtils.toParseableString(filterList)); + Assert.assertArrayEquals( + ("(RowFilter(=,'binary:testSkipFilter') " + + "OR QualifierFilter(>,'binaryprefix:whileMatchFilter') OR ColumnPaginationFilter(2,2))").getBytes(), + HBaseFilterUtils.toParseableByteArray(filterList)); } } diff --git a/src/test/java/unit_test_db.sql b/src/test/java/unit_test_db.sql index f838a705..44b37a1c 100644 --- a/src/test/java/unit_test_db.sql +++ b/src/test/java/unit_test_db.sql @@ -162,6 +162,21 @@ CREATE TABLE `test_t$family_with_local_index` ( PRIMARY KEY (`K`, `Q`, `T`) ); +CREATE TABLE `test$family'1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test; + +CREATE TABLE `test_t$family'1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_t; CREATE TABLEGROUP test_multi_cf SHARDING = 'ADAPTIVE';