From dac5faf51014efaed12906ed3c60f680d089f3fb Mon Sep 17 00:00:00 2001 From: Shen Yunlong <44497386+shenyunlong@users.noreply.github.com> Date: Fri, 20 Sep 2024 15:14:26 +0800 Subject: [PATCH 1/3] [Fix] checkAndMutate and get/Scan with filter return -5006 when include special character (#58) * [Fix] checkAndMutate and get/Scan with filter return -5006 when include special character * [Chore] fix review * [Fix] fix review --- .../com/alipay/oceanbase/hbase/OHTable.java | 28 ++- .../hbase/filter/HBaseFilterUtils.java | 191 ++++++++++++------ .../oceanbase/hbase/HTableTestBase.java | 58 ++++++ .../hbase/filter/HBaseFilterUtilsTest.java | 90 +++++---- src/test/java/unit_test_db.sql | 17 ++ 5 files changed, 264 insertions(+), 120 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 780957e8..a8d0d031 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -39,7 +39,6 @@ import com.alipay.oceanbase.rpc.stream.ObTableClientQueryAsyncStreamResult; import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult; import com.alipay.sofa.common.thread.SofaThreadPoolExecutor; -import com.alipay.oceanbase.hbase.exception.OperationTimeoutException; import com.google.protobuf.Descriptors; import com.google.protobuf.Message; @@ -60,6 +59,7 @@ import org.apache.hadoop.hbase.util.Pair; import org.slf4j.Logger; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.*; import java.util.concurrent.*; @@ -74,6 +74,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.commons.lang.StringUtils.isBlank; import static org.apache.commons.lang.StringUtils.isNotBlank; +import static com.alipay.oceanbase.hbase.filter.HBaseFilterUtils.writeBytesWithEscape; public class OHTable implements HTableInterface { @@ -721,7 +722,7 @@ private boolean checkAndMutation(byte[] row, byte[] family, byte[] qualifier, by checkArgument(!mutation.isEmpty(), "mutation is empty"); - String filterString = buildCheckAndMutateFilterString(family, qualifier, value); + byte[] filterString = buildCheckAndMutateFilterString(family, qualifier, value); ObHTableFilter filter = buildObHTableFilter(filterString, null, 1, qualifier); @@ -1189,11 +1190,11 @@ private String getTestLoadTargetTableName(String tableNameString, String familyS } private ObHTableFilter buildObHTableFilter(Filter filter, TimeRange timeRange, int maxVersion, - Collection columnQualifiers) { + Collection columnQualifiers) throws IOException { ObHTableFilter obHTableFilter = new ObHTableFilter(); if (filter != null) { - obHTableFilter.setFilterString(HBaseFilterUtils.toParseableString(filter)); + obHTableFilter.setFilterString(HBaseFilterUtils.toParseableByteArray(filter)); } if (timeRange != null) { @@ -1215,18 +1216,23 @@ private ObHTableFilter buildObHTableFilter(Filter filter, TimeRange timeRange, i return obHTableFilter; } - private String buildCheckAndMutateFilterString(byte[] family, byte[] qualifier, byte[] value) { + private byte[] buildCheckAndMutateFilterString(byte[] family, byte[] qualifier, byte[] value) throws IOException { + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + byteStream.write("CheckAndMutateFilter(=, 'binary:".getBytes()); + writeBytesWithEscape(byteStream, value); + byteStream.write("', '".getBytes()); + writeBytesWithEscape(byteStream, family); + byteStream.write("', '".getBytes()); + writeBytesWithEscape(byteStream, qualifier); if (value != null) { - return ("CheckAndMutateFilter(=, 'binary:" + Bytes.toString(value) + "', '" - + Bytes.toString(family) + "', '" - + (qualifier == null ? "" : Bytes.toString(qualifier)) + "', false)"); + byteStream.write("', false)".getBytes()); } else { - return ("CheckAndMutateFilter(=, 'binary:', '" + Bytes.toString(family) + "', '" - + (qualifier == null ? "" : Bytes.toString(qualifier)) + "', true)"); + byteStream.write("', true)".getBytes()); } + return byteStream.toByteArray(); } - private ObHTableFilter buildObHTableFilter(String filterString, TimeRange timeRange, + private ObHTableFilter buildObHTableFilter(byte[] filterString, TimeRange timeRange, int maxVersion, byte[]... columnQualifiers) { ObHTableFilter obHTableFilter = new ObHTableFilter(); diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index 8210e051..bfa064fc 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -18,152 +18,213 @@ package com.alipay.oceanbase.hbase.filter; import org.apache.hadoop.hbase.filter.*; -import org.apache.hadoop.hbase.util.Bytes; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.util.List; public class HBaseFilterUtils { - public static String toParseableString(Filter filter) { + public static byte[] toParseableByteArray(Filter filter) throws IOException { + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + toParseableByteArray(byteStream, filter); + return byteStream.toByteArray(); + } + + private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filter filter) throws IOException { if (filter == null) { throw new IllegalArgumentException("Filter is null"); } else if (filter instanceof CompareFilter) { // RowFilter, ValueFilter, QualifierFilter - return toParseableString((CompareFilter) filter); + toParseableByteArray(byteStream, (CompareFilter) filter); } else if (filter instanceof SingleColumnValueFilter) { - return toParseableString((SingleColumnValueFilter) filter); + toParseableByteArray(byteStream, (SingleColumnValueFilter) filter); } else if (filter instanceof PageFilter) { - return toParseableString((PageFilter) filter); + toParseableByteArray(byteStream, (PageFilter) filter); } else if (filter instanceof ColumnCountGetFilter) { - return toParseableString((ColumnCountGetFilter) filter); + toParseableByteArray(byteStream, (ColumnCountGetFilter) filter); } else if (filter instanceof PrefixFilter) { - return toParseableString((PrefixFilter) filter); + toParseableByteArray(byteStream, (PrefixFilter) filter); } else if (filter instanceof FilterList) { - return toParseableString((FilterList) filter); + toParseableByteArray(byteStream, (FilterList) filter); } else if (filter instanceof ColumnPaginationFilter) { - return toParseableString((ColumnPaginationFilter) filter); + toParseableByteArray(byteStream, (ColumnPaginationFilter) filter); } else if (filter instanceof SkipFilter) { - return toParseableString((SkipFilter) filter); + toParseableByteArray(byteStream, (SkipFilter) filter); } else if (filter instanceof WhileMatchFilter) { - return toParseableString((WhileMatchFilter) filter); + toParseableByteArray(byteStream, (WhileMatchFilter) filter); } else { throw new IllegalArgumentException("Invalid filter: " + filter); } } - private static String toParseableString(CompareFilter.CompareOp op) { + private static byte[] toParseableByteArray(CompareFilter.CompareOp op) { if (op == null) { throw new IllegalArgumentException("Compare operator is null"); } switch (op) { case LESS: - return Bytes.toString(ParseConstants.LESS_THAN_ARRAY); + return ParseConstants.LESS_THAN_ARRAY; case LESS_OR_EQUAL: - return Bytes.toString(ParseConstants.LESS_THAN_OR_EQUAL_TO_ARRAY); + return ParseConstants.LESS_THAN_OR_EQUAL_TO_ARRAY; case EQUAL: - return Bytes.toString(ParseConstants.EQUAL_TO_ARRAY); + return ParseConstants.EQUAL_TO_ARRAY; case NOT_EQUAL: - return Bytes.toString(ParseConstants.NOT_EQUAL_TO_ARRAY); + return ParseConstants.NOT_EQUAL_TO_ARRAY; case GREATER_OR_EQUAL: - return Bytes.toString(ParseConstants.GREATER_THAN_OR_EQUAL_TO_ARRAY); + return ParseConstants.GREATER_THAN_OR_EQUAL_TO_ARRAY; case GREATER: - return Bytes.toString(ParseConstants.GREATER_THAN_ARRAY); + return ParseConstants.GREATER_THAN_ARRAY; default: throw new IllegalArgumentException("Invalid compare operator: " + op); } } - private static String toParseableString(ByteArrayComparable comparator) { + private static void toParseableByteArray(ByteArrayOutputStream byteStream, ByteArrayComparable comparator) throws IOException { if (comparator == null) { throw new IllegalArgumentException("Comparator is null"); } - StringBuilder sb = new StringBuilder(); + byteStream.write('\''); if (comparator instanceof BinaryComparator) { - sb.append('\'').append(Bytes.toString(ParseConstants.binaryType)).append(':') - .append(Bytes.toString(comparator.getValue())).append('\''); + byteStream.write(ParseConstants.binaryType); } else if (comparator instanceof BinaryPrefixComparator) { - sb.append('\'').append(Bytes.toString(ParseConstants.binaryPrefixType)).append(':') - .append(Bytes.toString(comparator.getValue())).append('\''); + byteStream.write(ParseConstants.binaryPrefixType); } else if (comparator instanceof RegexStringComparator) { - sb.append('\'').append(Bytes.toString(ParseConstants.regexStringType)).append(':') - .append(Bytes.toString(comparator.getValue())).append('\''); + byteStream.write(ParseConstants.regexStringType); } else if (comparator instanceof SubstringComparator) { - sb.append('\'').append(Bytes.toString(ParseConstants.substringType)).append(':') - .append(Bytes.toString(comparator.getValue())).append('\''); + byteStream.write(ParseConstants.substringType); } else { throw new IllegalArgumentException("This comparator has not been implemented " + comparator); } - return sb.toString(); + byteStream.write(':'); + writeBytesWithEscape(byteStream, comparator.getValue()); + byteStream.write('\''); } - private static String toParseableString(CompareFilter filter) { - return filter.getClass().getSimpleName() + '(' + toParseableString(filter.getOperator()) - + ',' + toParseableString(filter.getComparator()) + ')'; + // CompareFilter(=,'binary:123') + private static void toParseableByteArray(ByteArrayOutputStream byteStream, CompareFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(')'); } - private static String toParseableString(SingleColumnValueFilter filter) { - return filter.getClass().getSimpleName() + "('" + Bytes.toString(filter.getFamily()) - + "','" + Bytes.toString(filter.getQualifier()) + "'," - + toParseableString(filter.getOperator()) + ',' - + toParseableString(filter.getComparator()) + ',' + filter.getFilterIfMissing() - + ',' + filter.getLatestVersionOnly() + ')'; + // SingleColumnValueFilter('cf1','col1',=,'binary:123',true,true) + private static void toParseableByteArray(ByteArrayOutputStream byteStream, SingleColumnValueFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getFilterIfMissing()).getBytes()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getLatestVersionOnly()).getBytes()); + byteStream.write(')'); } - private static String toParseableString(PageFilter filter) { - return filter.getClass().getSimpleName() + '(' + filter.getPageSize() + ')'; + // PageFilter(100); + private static void toParseableByteArray(ByteArrayOutputStream byteStream, PageFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + byteStream.write(Long.toString(filter.getPageSize()).getBytes()); + byteStream.write(')'); } - private static String toParseableString(ColumnPaginationFilter filter) { - return filter.getClass().getSimpleName() + '(' + filter.getLimit() + ',' - + filter.getOffset() + ')'; + // ColumnPaginationFilter(ColumnPaginationFilter(10,2) + private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnPaginationFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + byteStream.write(Long.toString(filter.getLimit()).getBytes()); + byteStream.write(','); + byteStream.write(Long.toString(filter.getOffset()).getBytes()); + byteStream .write(')'); } - private static String toParseableString(ColumnCountGetFilter filter) { - return filter.getClass().getSimpleName() + '(' + filter.getLimit() + ')'; + // ColumnCountGetFilter(100) + private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnCountGetFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + byteStream.write(Long.toString(filter.getLimit()).getBytes()); + byteStream .write(')'); } - private static String toParseableString(PrefixFilter filter) { - return filter.getClass().getSimpleName() + "('" + Bytes.toString(filter.getPrefix()) + "')"; + // PrefixFilter('prefix'); + private static void toParseableByteArray(ByteArrayOutputStream byteStream, PrefixFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getPrefix()); + byteStream .write("')".getBytes()); } - private static String toParseableString(SkipFilter filter) { - return "(" + Bytes.toString(ParseConstants.SKIP_ARRAY) + " " - + toParseableString(filter.getFilter()) + ")"; + // (SKIP filter) + private static void toParseableByteArray(ByteArrayOutputStream byteStream, SkipFilter filter) throws IOException { + byteStream.write('('); + byteStream.write(ParseConstants.SKIP_ARRAY); + byteStream.write(' '); + toParseableByteArray(byteStream, filter.getFilter()); + byteStream.write(')'); } - private static String toParseableString(WhileMatchFilter filter) { - return "(" + Bytes.toString(ParseConstants.WHILE_ARRAY) + " " - + toParseableString(filter.getFilter()) + ")"; + // (WHILE filter) + private static void toParseableByteArray(ByteArrayOutputStream byteStream, WhileMatchFilter filter) throws IOException { + byteStream.write('('); + byteStream.write(ParseConstants.WHILE_ARRAY); + byteStream.write(' '); + toParseableByteArray(byteStream, filter.getFilter()); + byteStream.write(')'); } - private static String toParseableString(FilterList filterList) { - StringBuilder sb = new StringBuilder(); + // (filter and filter ...) or (filter or filter ...) + // when filter list is empty, "" is generated, and empty filter list member is removed + // in result parseable byteArray + private static void toParseableByteArray(ByteArrayOutputStream byteStream, FilterList filterList) throws IOException { List filters = filterList.getFilters(); boolean isEmpty = true; + ByteArrayOutputStream oneFilterBytes = new ByteArrayOutputStream(); for (int i = 0; i < filters.size(); i++) { - String filterString = toParseableString(filters.get(i)); - if (filterString.isEmpty()) - continue; + toParseableByteArray(oneFilterBytes, filters.get(i)); + if (oneFilterBytes.size() == 0) { continue; } if (isEmpty) { - sb.append("(").append(filterString); + byteStream.write('('); isEmpty = false; } else { - sb.append(" "); + byteStream.write(' '); if (filterList.getOperator().equals(FilterList.Operator.MUST_PASS_ALL)) { - sb.append(Bytes.toString(ParseConstants.AND)); + byteStream.write(ParseConstants.AND); } else if (filterList.getOperator().equals(FilterList.Operator.MUST_PASS_ONE)) { - sb.append(Bytes.toString(ParseConstants.OR)); + byteStream.write(ParseConstants.OR); } else { throw new IllegalArgumentException("Invalid FilterList: " + filterList); } - sb.append(" ").append(filterString); + byteStream.write(' '); } + oneFilterBytes.writeTo(byteStream); + oneFilterBytes.reset(); } if (!isEmpty) { - sb.append(")"); + byteStream.write(')'); } - return sb.toString(); } + // when write family/qualifier/value/row into hbase filter, need add escape for + // special character to prevent parse error in server + public static void writeBytesWithEscape(ByteArrayOutputStream byteStream, byte[] bytes) throws IOException { + if (bytes == null) { + return; + } + for (int i = 0; i < bytes.length; i++) { + if (bytes[i] == '\'') { + byteStream.write('\''); + } + byteStream.write(bytes[i]); + } + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index d351de10..928d6244 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -37,6 +37,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.List; import static org.apache.hadoop.hbase.filter.FilterList.Operator.MUST_PASS_ONE; @@ -2544,4 +2545,61 @@ public void testIncrementConcurrency() throws Exception { public void testHtableWithIndex() throws Exception { testBasic("family_with_local_index"); } + + @Test + public void testFilterSpecialValue() throws IOException { + // 1. test checkAndMutate with special character + String specialValue = "AAEAAAGRnJiQbwAAAZGcmJwndjE="; + byte[] specialBytes = Base64.getDecoder().decode(specialValue); + String family = "family'1"; + byte[] keyBytes = specialBytes; + byte[] columnBytes = specialBytes; + byte[] valueBytes = specialBytes; + + Delete delete = new Delete(keyBytes); + delete.deleteFamily(family.getBytes()); + hTable.delete(delete); + + Put put = new Put(keyBytes); + put.add(family.getBytes(), columnBytes, valueBytes); + hTable.put(put); + + // check delete column + delete = new Delete(keyBytes); + delete.deleteColumn(family.getBytes(), columnBytes); + boolean ret = hTable.checkAndDelete(keyBytes, family.getBytes(), columnBytes, + valueBytes, delete); + Assert.assertTrue(ret); + + // 2. test normal filter with special chracter + put = new Put(keyBytes); + put.add(family.getBytes(), columnBytes, valueBytes); + hTable.put(put); + + + Get get = new Get(keyBytes); + get.addFamily(family.getBytes()); + // 2.1 test special row + RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(keyBytes)); + get.setFilter(rowFilter); + Result result = hTable.get(get); + Assert.assertEquals(1, result.raw().length); + Assert.assertArrayEquals(keyBytes, result.getRow()); + + // 2.2 test special column + SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter(family.getBytes(), + columnBytes, CompareFilter.CompareOp.EQUAL, valueBytes); + get.setFilter(singleColumnValueFilter); + result = hTable.get(get); + Assert.assertEquals(1, result.raw().length); + Assert.assertArrayEquals(keyBytes, result.getRow()); + + // 2.3 test special value + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(valueBytes)); + get.setFilter(valueFilter); + result = hTable.get(get); + Assert.assertEquals(1, result.raw().length); + Assert.assertArrayEquals(keyBytes, result.getRow()); + + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java index 4b8f9f21..f1cb0cab 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java @@ -22,6 +22,8 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; + public class HBaseFilterUtilsTest { private static final CompareFilter.CompareOp[] ops = { CompareFilter.CompareOp.LESS, CompareFilter.CompareOp.LESS_OR_EQUAL, CompareFilter.CompareOp.EQUAL, @@ -35,135 +37,135 @@ public static void beforeClass() { } @Test - public void testRowFilter() { + public void testRowFilter() throws IOException { for (int i = 0; i < ops.length; i++) { RowFilter filter = new RowFilter(ops[i], new BinaryComparator( "testRowFilter".getBytes())); String expect = String.format("RowFilter(%s,'binary:testRowFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new RowFilter(ops[i], new BinaryPrefixComparator("testRowFilter".getBytes())); expect = String.format("RowFilter(%s,'binaryprefix:testRowFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new RowFilter(ops[i], new RegexStringComparator("testRowFilter")); expect = String.format("RowFilter(%s,'regexstring:testRowFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new RowFilter(ops[i], new SubstringComparator("testRowFilter")); expect = String.format("RowFilter(%s,'substring:testrowfilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); } } @Test - public void testValueFilter() { + public void testValueFilter() throws IOException { for (int i = 0; i < ops.length; i++) { ValueFilter filter = new ValueFilter(ops[i], new BinaryComparator( "testValueFilter".getBytes())); String expect = String.format("ValueFilter(%s,'binary:testValueFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new ValueFilter(ops[i], new BinaryPrefixComparator( "testValueFilter".getBytes())); expect = String.format("ValueFilter(%s,'binaryprefix:testValueFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new ValueFilter(ops[i], new RegexStringComparator("testValueFilter")); expect = String.format("ValueFilter(%s,'regexstring:testValueFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new ValueFilter(ops[i], new SubstringComparator("testValueFilter")); expect = String.format("ValueFilter(%s,'substring:testvaluefilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); } } @Test - public void testQualifierFilter() { + public void testQualifierFilter() throws IOException { for (int i = 0; i < ops.length; i++) { QualifierFilter filter = new QualifierFilter(ops[i], new BinaryComparator( "testQualifierFilter".getBytes())); String expect = String.format("QualifierFilter(%s,'binary:testQualifierFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new QualifierFilter(ops[i], new BinaryPrefixComparator( "testQualifierFilter".getBytes())); expect = String.format("QualifierFilter(%s,'binaryprefix:testQualifierFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new QualifierFilter(ops[i], new RegexStringComparator("testQualifierFilter")); expect = String.format("QualifierFilter(%s,'regexstring:testQualifierFilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); filter = new QualifierFilter(ops[i], new SubstringComparator("testQualifierFilter")); expect = String.format("QualifierFilter(%s,'substring:testqualifierfilter')", opFlags[i]); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); } } @Test - public void testSingleColumnValueFilter() { + public void testSingleColumnValueFilter() throws IOException { for (int i = 0; i < ops.length; i++) { String expect = String.format( "SingleColumnValueFilter('family','qualifier',%s,'binary:value',false,true)", opFlags[i]); SingleColumnValueFilter filter = new SingleColumnValueFilter("family".getBytes(), "qualifier".getBytes(), ops[i], "value".getBytes()); - Assert.assertEquals(expect, HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); } } @Test - public void testPageFilter() { + public void testPageFilter() throws IOException { PageFilter filter = new PageFilter(128); - Assert.assertEquals("PageFilter(128)", HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals("PageFilter(128)".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); } @Test - public void testColumnPaginationFilter() { + public void testColumnPaginationFilter() throws IOException { ColumnPaginationFilter filter = new ColumnPaginationFilter(2, 2); - Assert.assertEquals("ColumnPaginationFilter(2,2)", - HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals("ColumnPaginationFilter(2,2)".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test - public void testColumnCountGetFilter() { + public void testColumnCountGetFilter() throws IOException { ColumnCountGetFilter filter = new ColumnCountGetFilter(513); - Assert - .assertEquals("ColumnCountGetFilter(513)", HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals("ColumnCountGetFilter(513)".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); } @Test - public void testPrefixFilter() { + public void testPrefixFilter() throws IOException { PrefixFilter filter = new PrefixFilter("prefix".getBytes()); - Assert.assertEquals("PrefixFilter('prefix')", HBaseFilterUtils.toParseableString(filter)); + System.out.println(new String(HBaseFilterUtils.toParseableByteArray(filter))); + Assert.assertArrayEquals("PrefixFilter('prefix')".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); } @Test - public void testSkipFilter() { + public void testSkipFilter() throws IOException { RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( "testSkipFilter".getBytes())); SkipFilter filter = new SkipFilter(rowFilter); - Assert.assertEquals("(SKIP RowFilter(=,'binary:testSkipFilter'))", - HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals("(SKIP RowFilter(=,'binary:testSkipFilter'))".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test - public void testWhileFilter() { + public void testWhileFilter() throws IOException { QualifierFilter qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.GREATER, new BinaryPrefixComparator("whileMatchFilter".getBytes())); WhileMatchFilter filter = new WhileMatchFilter(qualifierFilter); - Assert.assertEquals("(WHILE QualifierFilter(>,'binaryprefix:whileMatchFilter'))", - HBaseFilterUtils.toParseableString(filter)); + Assert.assertArrayEquals("(WHILE QualifierFilter(>,'binaryprefix:whileMatchFilter'))".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); } @Test - public void testFilterList() { + public void testFilterList() throws IOException { RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( "testSkipFilter".getBytes())); QualifierFilter qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.GREATER, @@ -177,21 +179,21 @@ public void testFilterList() { filterList.addFilter(skipFilter); filterList.addFilter(columnPaginationFilter); - Assert - .assertEquals( - "(RowFilter(=,'binary:testSkipFilter') " - + "AND QualifierFilter(>,'binaryprefix:whileMatchFilter') AND (SKIP PageFilter(128)) AND ColumnPaginationFilter(2,2))", - HBaseFilterUtils.toParseableString(filterList)); + + System.out.println(new String(HBaseFilterUtils.toParseableByteArray(filterList))); + Assert.assertArrayEquals( + ("(RowFilter(=,'binary:testSkipFilter') " + + "AND QualifierFilter(>,'binaryprefix:whileMatchFilter') AND (SKIP PageFilter(128)) AND ColumnPaginationFilter(2,2))").getBytes(), + HBaseFilterUtils.toParseableByteArray(filterList)); filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); filterList.addFilter(rowFilter); filterList.addFilter(qualifierFilter); filterList.addFilter(columnPaginationFilter); - Assert - .assertEquals( - "(RowFilter(=,'binary:testSkipFilter') " - + "OR QualifierFilter(>,'binaryprefix:whileMatchFilter') OR ColumnPaginationFilter(2,2))", - HBaseFilterUtils.toParseableString(filterList)); + Assert.assertArrayEquals( + ("(RowFilter(=,'binary:testSkipFilter') " + + "OR QualifierFilter(>,'binaryprefix:whileMatchFilter') OR ColumnPaginationFilter(2,2))").getBytes(), + HBaseFilterUtils.toParseableByteArray(filterList)); } } diff --git a/src/test/java/unit_test_db.sql b/src/test/java/unit_test_db.sql index 7ad44ed7..9b1c5f19 100644 --- a/src/test/java/unit_test_db.sql +++ b/src/test/java/unit_test_db.sql @@ -145,3 +145,20 @@ CREATE TABLE `test_t$family_with_local_index` ( key `idx1`(T) local, PRIMARY KEY (`K`, `Q`, `T`) ); + +CREATE TABLE `test$family'1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test; + +CREATE TABLE `test_t$family'1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_t; + From 3d5f8f8dad3f2a92c6736afee7a3564b92dc1c13 Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Fri, 20 Sep 2024 18:41:14 +0800 Subject: [PATCH 2/3] [Fix] fix test --- .../com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index 030526c0..60570332 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -158,7 +158,7 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, PageF private static void toParseableByteArray(ByteArrayOutputStream byteStream, RandomRowFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); - byteStream.write(Bytes.toInt(Bytes.toBytes(filter.getChance()))); + byteStream.write(Integer.toString(Bytes.toInt(Bytes.toBytes(filter.getChance()))).getBytes()); byteStream.write(')'); } From 5c96536b85d415ef10f287de147d2e754d978aec Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Fri, 20 Sep 2024 18:46:17 +0800 Subject: [PATCH 3/3] [Fix] fix test testScanSessionClean --- src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index a4ab470e..62120ec3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -1875,7 +1875,8 @@ public void testScanSessionClean() throws Exception { Thread.sleep(61 * 1000); try { scanner.next(); - } catch (IOException e) { + fail(); + } catch (Exception e) { assertTrue(e.getCause().getMessage().contains("OB_HASH_NOT_EXIST")); } finally { scanner.close();