diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 2aecad25..f6a1a4bf 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -40,6 +40,7 @@ import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate.ObTableQueryAndMutateResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.*; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncRequest; +import com.alipay.oceanbase.rpc.queryandmutate.QueryAndMutate; import com.alipay.oceanbase.rpc.stream.ObTableClientQueryAsyncStreamResult; import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult; import com.alipay.oceanbase.rpc.table.ObHBaseParams; @@ -83,6 +84,7 @@ import static org.apache.commons.lang.StringUtils.isBlank; import static org.apache.commons.lang.StringUtils.isNotBlank; import static com.alipay.oceanbase.hbase.filter.HBaseFilterUtils.writeBytesWithEscape; +import static org.apache.hadoop.hbase.KeyValue.Type.*; public class OHTable implements Table { @@ -152,7 +154,7 @@ public class OHTable implements Table { /** * the buffer of put request */ - private final ArrayList writeBuffer = new ArrayList<>(); + private final ArrayList writeBuffer = new ArrayList(); /** * when the put request reach the write buffer size the do put will * flush commits automatically @@ -1126,7 +1128,8 @@ public List call() throws IOException { List partitions = obTableClient.getPartition(phyTableName, false); for (Partition partition : partitions) { request.getObTableQueryRequest().setTableQueryPartId( - partition.getPartId()); + partition.getPartId()); + request.setAllowDistributeScan(false); clientQueryAsyncStreamResult = (ObTableClientQueryAsyncStreamResult) obTableClient .execute(request); ClientStreamScanner clientScanner = new ClientStreamScanner( @@ -1421,7 +1424,8 @@ private boolean checkAndMutation(byte[] row, byte[] family, byte[] qualifier, } byte[] filterString = buildCheckAndMutateFilterString(family, qualifier, compareOp, value); ObHTableFilter filter = buildObHTableFilter(filterString, timeRange, 1, qualifier); - ObTableQuery obTableQuery = buildObTableQuery(filter, row, true, row, true, false); + ObTableQuery obTableQuery = buildObTableQuery(filter, row, true, row, true, false, + new TimeRange()); ObTableBatchOperation batch = buildObTableBatchOperation(mutations, null); ObTableQueryAndMutateRequest request = buildObTableQueryAndMutateRequest(obTableQuery, @@ -1458,16 +1462,20 @@ public Result append(Append append) throws IOException { Collections.singletonList(append), qualifiers); // the later hbase has supported timeRange ObHTableFilter filter = buildObHTableFilter(null, null, 1, qualifiers); - ObTableQuery obTableQuery = buildObTableQuery(filter, r, true, r, true, false); + ObTableQuery obTableQuery = buildObTableQuery(filter, r, true, r, true, false, + new TimeRange()); ObTableQueryAndMutate queryAndMutate = new ObTableQueryAndMutate(); queryAndMutate.setTableQuery(obTableQuery); queryAndMutate.setMutations(batchOperation); ObTableQueryAndMutateRequest request = buildObTableQueryAndMutateRequest(obTableQuery, batchOperation, getTargetTableName(tableNameString, Bytes.toString(f), configuration)); - request.setReturningAffectedEntity(true); + request.setReturningAffectedEntity(append.isReturnResults()); ObTableQueryAndMutateResult result = (ObTableQueryAndMutateResult) obTableClient .execute(request); + if (!append.isReturnResults()) { + return null; + } ObTableQueryResult queryResult = result.getAffectedEntity(); List keyValues = new ArrayList(); for (List row : queryResult.getPropertiesRows()) { @@ -1510,13 +1518,16 @@ public Result increment(Increment increment) throws IOException { ObHTableFilter filter = buildObHTableFilter(null, increment.getTimeRange(), 1, qualifiers); - ObTableQuery obTableQuery = buildObTableQuery(filter, rowKey, true, rowKey, true, false); + ObTableQuery obTableQuery = buildObTableQuery(filter, rowKey, true, rowKey, true, false, increment.getTimeRange()); ObTableQueryAndMutateRequest request = buildObTableQueryAndMutateRequest(obTableQuery, batch, getTargetTableName(tableNameString, Bytes.toString(f), configuration)); - request.setReturningAffectedEntity(true); + request.setReturningAffectedEntity(increment.isReturnResults()); ObTableQueryAndMutateResult result = (ObTableQueryAndMutateResult) obTableClient .execute(request); + if (!increment.isReturnResults()) { + return null; + } ObTableQueryResult queryResult = result.getAffectedEntity(); List keyValues = new ArrayList(); for (List row : queryResult.getPropertiesRows()) { @@ -1525,7 +1536,6 @@ public Result increment(Increment increment) throws IOException { long t = (Long) row.get(2).getValue(); byte[] v = (byte[]) row.get(3).getValue(); KeyValue kv = new KeyValue(k, f, q, t, v); - keyValues.add(kv); } return Result.create(keyValues); @@ -1557,7 +1567,8 @@ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, lo ObHTableFilter filter = buildObHTableFilter(null, null, 1, qualifiers); - ObTableQuery obTableQuery = buildObTableQuery(filter, row, true, row, true, false); + ObTableQuery obTableQuery = buildObTableQuery(filter, row, true, row, true, false, + new TimeRange()); ObTableQueryAndMutate queryAndMutate = new ObTableQueryAndMutate(); queryAndMutate.setMutations(batch); queryAndMutate.setTableQuery(obTableQuery); @@ -1871,24 +1882,33 @@ private ObHTableFilter buildObHTableFilter(byte[] filterString, TimeRange timeRa private ObTableQuery buildObTableQuery(ObHTableFilter filter, byte[] start, boolean includeStart, byte[] stop, boolean includeStop, - boolean isReversed) { + boolean isReversed, TimeRange ts) { ObNewRange obNewRange = new ObNewRange(); + ObBorderFlag obBorderFlag = new ObBorderFlag(); + Object left_ts = ObObj.getMin(); + Object right_ts = ObObj.getMax(); + if (!ts.isAllTime()) { + left_ts = -ts.getMax(); + right_ts = -ts.getMin(); + } if (Arrays.equals(start, HConstants.EMPTY_BYTE_ARRAY)) { - obNewRange.setStartKey(ObRowKey.getInstance(ObObj.getMin(), ObObj.getMin(), - ObObj.getMin())); + obNewRange.setStartKey(ObRowKey.getInstance(ObObj.getMin(), ObObj.getMin(), left_ts)); } else if (includeStart) { - obNewRange.setStartKey(ObRowKey.getInstance(start, ObObj.getMin(), ObObj.getMin())); + obNewRange.setStartKey(ObRowKey.getInstance(start, ObObj.getMin(), left_ts)); + obBorderFlag.setInclusiveStart(); } else { - obNewRange.setStartKey(ObRowKey.getInstance(start, ObObj.getMax(), ObObj.getMax())); + obNewRange.setStartKey(ObRowKey.getInstance(start, ObObj.getMax(), left_ts)); + obBorderFlag.unsetInclusiveStart(); } if (Arrays.equals(stop, HConstants.EMPTY_BYTE_ARRAY)) { - obNewRange.setEndKey(ObRowKey.getInstance(ObObj.getMax(), ObObj.getMax(), - ObObj.getMax())); + obNewRange.setEndKey(ObRowKey.getInstance(ObObj.getMax(), ObObj.getMax(), right_ts)); } else if (includeStop) { - obNewRange.setEndKey(ObRowKey.getInstance(stop, ObObj.getMax(), ObObj.getMax())); + obNewRange.setEndKey(ObRowKey.getInstance(stop, ObObj.getMax(), right_ts)); + obBorderFlag.setInclusiveEnd(); } else { - obNewRange.setEndKey(ObRowKey.getInstance(stop, ObObj.getMin(), ObObj.getMin())); + obNewRange.setEndKey(ObRowKey.getInstance(stop, ObObj.getMin(), right_ts)); + obBorderFlag.unsetInclusiveEnd(); } ObTableQuery obTableQuery = new ObTableQuery(); if (isReversed) { @@ -1897,11 +1917,13 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, byte[] start, obTableQuery.setIndexName("PRIMARY"); obTableQuery.sethTableFilter(filter); obTableQuery.addKeyRange(obNewRange); + obTableQuery.setScanRangeColumns("K", "Q", "T"); return obTableQuery; } private ObTableQuery buildObTableQuery(ObHTableFilter filter, final Scan scan) { ObTableQuery obTableQuery; + TimeRange ts = scan.getTimeRange(); if (scan.getMaxResultsPerColumnFamily() > 0) { filter.setLimitPerRowPerCf(scan.getMaxResultsPerColumnFamily()); } @@ -1909,11 +1931,11 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, final Scan scan) { filter.setOffsetPerRowPerCf(scan.getRowOffsetPerColumnFamily()); } if (scan.isReversed()) { - obTableQuery = buildObTableQuery(filter, scan.getStopRow(), scan.includeStopRow(), - scan.getStartRow(), scan.includeStartRow(), true); + obTableQuery = buildObTableQuery(filter, scan.getStopRow(), false, scan.getStartRow(), + true, true, ts); } else { - obTableQuery = buildObTableQuery(filter, scan.getStartRow(), scan.includeStartRow(), - scan.getStopRow(), scan.includeStopRow(), false); + obTableQuery = buildObTableQuery(filter, scan.getStartRow(), true, scan.getStopRow(), + false, false, ts); } obTableQuery.setBatchSize(scan.getBatch()); obTableQuery.setLimit(scan.getLimit()); @@ -1921,6 +1943,7 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, final Scan scan) { : configuration.getLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE)); obTableQuery.setObKVParams(buildOBKVParams(scan)); + obTableQuery.setScanRangeColumns("K", "Q", "T"); return obTableQuery; } @@ -1931,12 +1954,13 @@ private ObTableQuery buildObTableQuery(final Get get, Collection columnQ get.getMaxVersions(), columnQualifiers); if (get.isClosestRowBefore()) { obTableQuery = buildObTableQuery(filter, HConstants.EMPTY_BYTE_ARRAY, true, - get.getRow(), true, true); - obTableQuery.setLimit(1); + get.getRow(), true, true, get.getTimeRange()); } else { - obTableQuery = buildObTableQuery(filter, get.getRow(), true, get.getRow(), true, false); + obTableQuery = buildObTableQuery(filter, get.getRow(), true, get.getRow(), true, false, + get.getTimeRange()); } obTableQuery.setObKVParams(buildOBKVParams(get)); + obTableQuery.setScanRangeColumns("K", "Q", "T"); return obTableQuery; } @@ -1981,6 +2005,108 @@ public static ObTableBatchOperation buildObTableBatchOperation(List ro return batch; } + private QueryAndMutate buildDeleteQueryAndMutate(KeyValue kv, + ObTableOperationType operationType, + boolean isTableGroup, byte[] family, Long TTL) { + KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getType().getCode()); + com.alipay.oceanbase.rpc.mutation.Mutation tableMutation = buildMutation(kv, operationType, + isTableGroup, family, TTL); + ObNewRange range = new ObNewRange(); + ObTableQuery tableQuery = new ObTableQuery(); + tableQuery.setObKVParams(buildOBKVParams((Scan) null)); + ObHTableFilter filter = null; + switch (kvType) { + case Delete: + if (kv.getTimestamp() == Long.MAX_VALUE) { + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); + filter = buildObHTableFilter(null, null, 1, CellUtil.cloneQualifier(kv)); + } else { + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); + filter = buildObHTableFilter(null, + new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), 1, + CellUtil.cloneQualifier(kv)); + } + break; + case DeleteColumn: + if (kv.getTimestamp() == Long.MAX_VALUE) { + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); + filter = buildObHTableFilter(null, null, Integer.MAX_VALUE, + CellUtil.cloneQualifier(kv)); + } else { + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); + filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), + Integer.MAX_VALUE, CellUtil.cloneQualifier(kv)); + } + break; + case DeleteFamily: + if (kv.getTimestamp() == Long.MAX_VALUE) { + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); + if (!isTableGroup) { + filter = buildObHTableFilter(null, null, Integer.MAX_VALUE); + } else { + filter = buildObHTableFilter(null, null, Integer.MAX_VALUE, + CellUtil.cloneQualifier(kv)); + } + } else { + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); + if (!isTableGroup) { + filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), + Integer.MAX_VALUE); + } else { + filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), + Integer.MAX_VALUE, CellUtil.cloneQualifier(kv)); + } + } + break; + case DeleteFamilyVersion: + if (kv.getTimestamp() == Long.MAX_VALUE) { + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); + filter = buildObHTableFilter(null, null, Integer.MAX_VALUE); + } else { + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); + if (!isTableGroup) { + filter = buildObHTableFilter(null, + new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), + Integer.MAX_VALUE); + } else { + filter = buildObHTableFilter(null, + new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), + Integer.MAX_VALUE, CellUtil.cloneQualifier(kv)); + } + } + break; + default: + return null; + } + tableQuery.sethTableFilter(filter); + tableQuery.addKeyRange(range); + return new QueryAndMutate(tableQuery, tableMutation); + } + private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(Cell kv, ObTableOperationType operationType, boolean isTableGroup, @@ -2086,7 +2212,7 @@ private BatchOperation buildBatchOperation(String tableName, List obTableQuery = buildObTableQuery(get, columnFilters); ObTableClientQueryImpl query = new ObTableClientQueryImpl(tableName, obTableQuery, obTableClient); try { - query.setRowKey(row(colVal("K", Bytes.toString(get.getRow())), colVal("Q", null), colVal("T", null))); + query.setRowKey(row(colVal("K", Bytes.toString(get.getRow())), colVal("Q", null), colVal("T", Integer.MAX_VALUE))); } catch (Exception e) { logger.error("unexpected error occurs when set row key", e); throw new IOException(e); @@ -2096,7 +2222,7 @@ private BatchOperation buildBatchOperation(String tableName, List Put put = (Put) row; if (put.isEmpty()) { throw new IllegalArgumentException("No columns to insert for #" - + (posInList + 1) + " item"); + + (posInList + 1) + " item"); } for (Map.Entry> entry : put.getFamilyCellMap().entrySet()) { byte[] family = entry.getKey(); @@ -2108,21 +2234,47 @@ private BatchOperation buildBatchOperation(String tableName, List } } } else if (row instanceof Delete) { + boolean disExec = obTableClient.getServerCapacity().isSupportDistributedExecute(); Delete delete = (Delete) row; if (delete.isEmpty()) { singleOpResultNum++; - KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp()); - batch.addOperation(com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), null, -kv.getTimestamp() }, - null, null)); + if (disExec) { + KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), + KeyValue.Type.DeleteFamily); + com.alipay.oceanbase.rpc.mutation.Mutation tableMutation = buildMutation(kv, DEL, isTableGroup, null, Long.MAX_VALUE); + ObNewRange range = new ObNewRange(); + ObTableQuery tableQuery = new ObTableQuery(); + ObHTableFilter filter; + tableQuery.setObKVParams(buildOBKVParams((Scan) null)); + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), ObObj.getMax())); + if (kv.getTimestamp() == Long.MAX_VALUE) { + filter = buildObHTableFilter(null, null, Integer.MAX_VALUE); + } else { + filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), Integer.MAX_VALUE); + } + tableQuery.sethTableFilter(filter); + tableQuery.addKeyRange(range); + + tableMutation.setTable(tableName); + batch.addOperation(new QueryAndMutate(tableQuery, tableMutation)); + } else { + KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp()); + batch.addOperation(com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), null, -kv.getTimestamp() }, + null, null)); + } } else { for (Map.Entry> entry : delete.getFamilyCellMap().entrySet()) { byte[] family = entry.getKey(); List keyValueList = entry.getValue(); for (Cell kv : keyValueList) { singleOpResultNum++; - batch.addOperation(buildMutation(kv, DEL, - isTableGroup, family, delete.getTTL())); + if (disExec) { + batch.addOperation(buildDeleteQueryAndMutate((KeyValue) kv, DEL, false, family, Long.MAX_VALUE)); + } else { + batch.addOperation(buildMutation(kv, DEL, isTableGroup, family, Long.MAX_VALUE)); + } } } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 6823b2be..4dd3e0f7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -2219,7 +2219,7 @@ public void testColumnValueFilter() throws Exception { scan.addFamily(family.getBytes()); scan.setMaxVersions(10); filter = new ColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column2), - CompareOperator.GREATER, Bytes.toBytes(value1)); + CompareOperator.LESS, Bytes.toBytes(value1)); scan.setFilter(filter); scanner = hTable.getScanner(scan); @@ -2890,7 +2890,7 @@ public void testGetFilter() throws Exception { get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); - valueFilter = new ValueFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator( + valueFilter = new ValueFilter(CompareFilter.CompareOp.LESS, new BinaryComparator( toBytes(value1))); get.setFilter(valueFilter); r = hTable.get(get); @@ -2908,7 +2908,7 @@ public void testGetFilter() throws Exception { get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); - valueFilter = new ValueFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator( + valueFilter = new ValueFilter(CompareFilter.CompareOp.LESS, new BinaryComparator( toBytes(value3))); get.setFilter(valueFilter); r = hTable.get(get); @@ -2948,7 +2948,7 @@ public void testGetFilter() throws Exception { get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); - qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.GREATER, + qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.LESS, new BinaryComparator(toBytes(column1))); get.setFilter(qualifierFilter); r = hTable.get(get); @@ -3073,7 +3073,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new ColumnCountGetFilter(1)); - filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.GREATER, + filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.LESS, new BinaryComparator(toBytes(column2)))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3274,7 +3274,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(7, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), CompareFilter.CompareOp.GREATER, new BinaryComparator( + Bytes.toBytes(column2), CompareFilter.CompareOp.LESS, new BinaryComparator( toBytes(value2))); singleColumnValueFilter.setLatestVersionOnly(false); get = new Get(toBytes(key1)); @@ -5014,16 +5014,16 @@ public void testCheckAndPut() throws IOException, InterruptedException { Assert.assertTrue(ret); ret = hTable.checkAndPut(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.GREATER, "value1".getBytes(), put); + CompareFilter.CompareOp.LESS, "value1".getBytes(), put); Assert.assertFalse(ret); ret = hTable.checkAndPut(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.GREATER_OR_EQUAL, "value1".getBytes(), put); + CompareFilter.CompareOp.LESS_OR_EQUAL, "value1".getBytes(), put); Assert.assertTrue(ret); ret = hTable.checkAndPut(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.LESS, "".getBytes(), put); + CompareFilter.CompareOp.GREATER, "".getBytes(), put); Assert.assertFalse(ret); ret = hTable.checkAndPut(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.LESS_OR_EQUAL, "".getBytes(), put); + CompareFilter.CompareOp.GREATER_OR_EQUAL, "".getBytes(), put); Assert.assertFalse(ret); get = new Get(key.getBytes()); @@ -5061,16 +5061,16 @@ public void testCheckAndPut() throws IOException, InterruptedException { }); ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.GREATER, toBytes("value1")).thenPut(put); + .ifMatches(CompareOperator.LESS, toBytes("value1")).thenPut(put); Assert.assertFalse(ret); ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.GREATER_OR_EQUAL, toBytes("value1")).thenPut(put); + .ifMatches(CompareOperator.LESS_OR_EQUAL, toBytes("value1")).thenPut(put); Assert.assertTrue(ret); - ret = builder.qualifier(toBytes(column)).ifMatches(CompareOperator.LESS, toBytes("")) + ret = builder.qualifier(toBytes(column)).ifMatches(CompareOperator.GREATER, toBytes("")) .thenPut(put); Assert.assertFalse(ret); ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.LESS_OR_EQUAL, toBytes("")).thenPut(put); + .ifMatches(CompareOperator.GREATER_OR_EQUAL, toBytes("")).thenPut(put); Assert.assertFalse(ret); get = new Get(key.getBytes()); @@ -5121,22 +5121,22 @@ public void testCheckAndDelete() throws IOException { put.addColumn(family.getBytes(), column.getBytes(), "value6".getBytes()); hTable.put(put); ret = hTable.checkAndDelete(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.GREATER, "value5".getBytes(), delete); + CompareFilter.CompareOp.LESS, "value5".getBytes(), delete); Assert.assertTrue(ret); put = new Put(key.getBytes()); put.addColumn(family.getBytes(), column.getBytes(), "value5".getBytes()); hTable.put(put); ret = hTable.checkAndDelete(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.GREATER_OR_EQUAL, "value5".getBytes(), delete); + CompareFilter.CompareOp.LESS_OR_EQUAL, "value5".getBytes(), delete); Assert.assertTrue(ret); put = new Put(key.getBytes()); put.addColumn(family.getBytes(), column.getBytes(), "value1".getBytes()); hTable.put(put); ret = hTable.checkAndDelete(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.LESS, "value1".getBytes(), delete); + CompareFilter.CompareOp.GREATER, "value1".getBytes(), delete); Assert.assertFalse(ret); ret = hTable.checkAndDelete(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.LESS_OR_EQUAL, "value1".getBytes(), delete); + CompareFilter.CompareOp.GREATER_OR_EQUAL, "value1".getBytes(), delete); Assert.assertTrue(ret); Get get = new Get(key.getBytes()); @@ -5209,22 +5209,22 @@ public void testCheckAndDelete() throws IOException { put.addColumn(family.getBytes(), column.getBytes(), "value6".getBytes()); hTable.put(put); ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.GREATER, toBytes("value5")).thenDelete(delete); + .ifMatches(CompareOperator.LESS, toBytes("value5")).thenDelete(delete); Assert.assertTrue(ret); put = new Put(key.getBytes()); put.addColumn(family.getBytes(), column.getBytes(), "value5".getBytes()); hTable.put(put); ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.GREATER_OR_EQUAL, toBytes("value5")).thenDelete(delete); + .ifMatches(CompareOperator.LESS_OR_EQUAL, toBytes("value5")).thenDelete(delete); Assert.assertTrue(ret); put = new Put(key.getBytes()); put.addColumn(family.getBytes(), column.getBytes(), "value1".getBytes()); hTable.put(put); - ret = builder.qualifier(toBytes(column)).ifMatches(CompareOperator.LESS, toBytes("value1")) + ret = builder.qualifier(toBytes(column)).ifMatches(CompareOperator.GREATER, toBytes("value1")) .thenDelete(delete); Assert.assertFalse(ret); ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.LESS_OR_EQUAL, toBytes("value1")).thenDelete(delete); + .ifMatches(CompareOperator.GREATER_OR_EQUAL, toBytes("value1")).thenDelete(delete); Assert.assertTrue(ret); get = new Get(key.getBytes()); @@ -5349,9 +5349,9 @@ public void testCheckAndMutate() throws IOException { rowMutations.add(put1); rowMutations.add(put2); rowMutations.add(put3); - // test greater op + // test GREATER op ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column1.getBytes(), - CompareFilter.CompareOp.LESS, value1.getBytes(), rowMutations); + CompareFilter.CompareOp.GREATER, value1.getBytes(), rowMutations); Assert.assertFalse(ret); get = new Get(key.getBytes()); get.addFamily(family.getBytes()); @@ -5361,7 +5361,7 @@ public void testCheckAndMutate() throws IOException { // test less op ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column1.getBytes(), - CompareFilter.CompareOp.LESS, value2.getBytes(), rowMutations); + CompareFilter.CompareOp.GREATER, value2.getBytes(), rowMutations); Assert.assertTrue(ret); get = new Get(key.getBytes()); get.addFamily(family.getBytes()); @@ -5487,7 +5487,7 @@ public void testCheckAndMutate() throws IOException { rowMutations.add(put1); rowMutations.add(put2); rowMutations.add(put3); - // test greater op + // test LESS op ret = builder.qualifier(toBytes(column1)).ifMatches(CompareOperator.LESS, toBytes(value1)) .thenMutate(rowMutations); Assert.assertFalse(ret); @@ -5498,7 +5498,7 @@ public void testCheckAndMutate() throws IOException { Assert.assertEquals(6, r.rawCells().length); // test less op - ret = builder.qualifier(toBytes(column1)).ifMatches(CompareOperator.LESS, toBytes(value2)) + ret = builder.qualifier(toBytes(column1)).ifMatches(CompareOperator.GREATER, toBytes(value2)) .thenMutate(rowMutations); Assert.assertTrue(ret); get = new Get(key.getBytes()); diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAbnormal.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAbnormal.java new file mode 100644 index 00000000..0bbe8bce --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAbnormal.java @@ -0,0 +1,219 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.PageFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.SERIES_TABLES; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TableType.*; +import static org.junit.Assert.*; + +public class OHTableSecondaryPartAbnormal { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); + private static byte[] ROW = Bytes.toBytes("testRow"); + private static byte[] QUALIFIER = Bytes.toBytes("testQualifier"); + private static byte[] VALUE_2 = Bytes.toBytes("abcd"); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : SERIES_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + // dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + private static void testSeriesLimit(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + Put put2 = new Put(ROW); + put2.addColumn(FAMILY, QUALIFIER, VALUE_2); + hTable.put(put2); + Get get = new Get(ROW); + get.addFamily(FAMILY); + get.setFilter(new PageFilter(10)); + try { + hTable.get(get); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with filter query not supported")); + } + get = new Get(ROW); + get.addFamily(FAMILY); + get.setCheckExistenceOnly(true); + try { + hTable.get(get); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with check existence only query not supported")); + } + Scan scan = new Scan(ROW); + scan.addFamily(FAMILY); + scan.setReversed(true); + try { + hTable.getScanner(scan); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with reverse query not supported")); + } + scan = new Scan(ROW); + scan.addFamily(FAMILY); + scan.setCaching(1); + try { + hTable.getScanner(scan); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with caching query not supported")); + } + scan = new Scan(ROW); + scan.addFamily(FAMILY); + scan.setBatch(1); + try { + hTable.getScanner(scan); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with batch query not supported")); + } + scan = new Scan(ROW); + scan.addFamily(FAMILY); + scan.setAllowPartialResults(true); + try { + hTable.getScanner(scan); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with allow partial results query not supported")); + } + scan = new Scan(ROW); + scan.addFamily(FAMILY); + scan.setMaxResultsPerColumnFamily(1); + try { + hTable.getScanner(scan); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e + .getCause() + .getMessage() + .contains( + "timeseries hbase table with max results per column family query not supported")); + } + scan = new Scan(ROW); + scan.addFamily(FAMILY); + scan.setRowOffsetPerColumnFamily(1); + try { + hTable.getScanner(scan); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with row offset query not supported")); + } + scan = new Scan(ROW); + scan.addFamily(FAMILY); + ResultScanner scanner = hTable.getScanner(scan); + assertEquals(1, scanner.next().size()); + hTable.close(); + } + + private static void testSeriesWithCellTTL(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + Put put2 = new Put(ROW); + put2.addColumn(FAMILY, QUALIFIER, VALUE_2); + try { + hTable.put(put2); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("series table not support cell ttl not supported")); + } + hTable.close(); + } + + private static void testSecondaryPartReverseScan(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + Put put2 = new Put(ROW); + put2.addColumn(FAMILY, QUALIFIER, VALUE_2); + hTable.put(put2); + Scan scan = new Scan(ROW); + scan.addFamily(FAMILY); + scan.setReversed(true); + try { + hTable.getScanners(scan); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("secondary partitioned hbase table with reverse query not supported")); + } + hTable.close(); + } + + @Test + public void testSeriesLimit() throws Throwable { + FOR_EACH(tableNames, com.alipay.oceanbase.hbase.secondary.OHTableSecondaryPartAbnormal::testSeriesLimit); + } + + @Test + public void testSeriesWithCellTTL() throws Throwable { + List tmpTable = new ArrayList<>(); + createTables(NON_PARTITIONED_TIME_CELL_TTL, tmpTable, null, true); + FOR_EACH(tmpTable, OHTableSecondaryPartAbnormal::testSeriesWithCellTTL); + dropTables(tmpTable, null); + } + + @Test + public void testSecondaryPartReverseScan() throws Throwable { + List tmpTable = new ArrayList<>(); + createTables(SECONDARY_PARTITIONED_KEY_RANGE_GEN, tmpTable, null, true); + FOR_EACH(tmpTable, OHTableSecondaryPartAbnormal::testSecondaryPartReverseScan); + dropTables(tmpTable, null); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java new file mode 100644 index 00000000..2c41fc67 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java @@ -0,0 +1,278 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTable; +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import com.google.common.base.Strings; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.getConnection; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.NORMAL_TABLES; +import static org.junit.Assert.*; + +public class OHTableSecondaryPartAppendTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + private static void assertNullResult(Result result) throws Exception { + assertTrue("expected null result but received a non-null result", result == null); + } + + private static void testAppend(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + try { + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + byte[] ROW = "appendKey".getBytes(); + byte[] v1 = Bytes.toBytes("42"); + byte[] v2 = Bytes.toBytes("23"); + byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("a"), + Bytes.toBytes("c") }; + Append a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[0], v1); + a.add(FAMILY, QUALIFIERS[1], v2); + a.setReturnResults(false); + assertNullResult(hTable.append(a)); + + a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[0], v2); + a.add(FAMILY, QUALIFIERS[1], v1); + a.add(FAMILY, QUALIFIERS[2], v2); + Result r = hTable.append(a); + assertEquals(0, Bytes.compareTo(Bytes.add(v1, v2), r.getValue(FAMILY, QUALIFIERS[0]))); + assertEquals(0, Bytes.compareTo(Bytes.add(v2, v1), r.getValue(FAMILY, QUALIFIERS[1]))); + // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct + assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2]))); + assertEquals(r.getColumnLatestCell(FAMILY, QUALIFIERS[0]).getTimestamp(), r + .getColumnLatestCell(FAMILY, QUALIFIERS[2]).getTimestamp()); + + Get get = new Get(ROW); + get.setMaxVersions(10); + get.addFamily(FAMILY); + Result result = hTable.get(get); + assertEquals(2, result.getColumnCells(FAMILY, QUALIFIERS[0]).size()); + assertEquals(2, result.getColumnCells(FAMILY, QUALIFIERS[1]).size()); + assertEquals(1, result.getColumnCells(FAMILY, QUALIFIERS[2]).size()); + assertEquals( + 0, + Bytes.compareTo(Bytes.add(v1, v2), + CellUtil.cloneValue(result.getColumnCells(FAMILY, QUALIFIERS[0]).get(0)))); + assertEquals( + 0, + Bytes.compareTo(v2, + CellUtil.cloneValue(result.getColumnCells(FAMILY, QUALIFIERS[2]).get(0)))); + } finally { + hTable.close(); + } + } + + private static void testAppendBorder(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + try { + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + byte[] ROW = "appendKey".getBytes(); + byte[] v1 = Bytes.toBytes("ab"); + byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("a"), + Bytes.toBytes("c") }; + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIERS[1], v1); + hTable.put(put); + Append a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[1], v1); + a.add(FAMILY, QUALIFIERS[2], "".getBytes()); + hTable.append(a); + Get get = new Get(ROW); + get.setMaxVersions(10); + get.addFamily(FAMILY); + Result result = hTable.get(get); + assertEquals(3, result.size()); + + a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[2], v1); + a.add(FAMILY, QUALIFIERS[2], "".getBytes()); + hTable.append(a); + get = new Get(ROW); + get.setMaxVersions(10); + get.addFamily(FAMILY); + result = hTable.get(get); + assertEquals(4, result.size()); + + byte[] randomBytes = new byte[1025]; + Random random = new Random(); + random.nextBytes(randomBytes); + a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[2], randomBytes); + try { + hTable.append(a); + fail("unexpect error, too long data should fail"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage().contains("Data too long for column 'V'")); + } + } finally { + hTable.close(); + } + + } + + private static void testAppendCon(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + try { + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + String column = "appColumn"; + byte[] ROW = "appendKey".getBytes(); + byte[] v = "a".getBytes(); + ThreadPoolExecutor threadPoolExecutor = OHTable.createDefaultThreadPoolExecutor(1, 100, 100); + AtomicInteger atomicInteger = new AtomicInteger(0); + CountDownLatch countDownLatch = new CountDownLatch(100); + + for (int i = 0; i < 100; i++) { + Append append = new Append(ROW); + append.add(FAMILY, column.getBytes(), v); + threadPoolExecutor.submit(() -> { + try { + hTable.append(append); + } catch (Exception e) { + if (!e.getCause().getMessage().contains("OB_TRY_LOCK_ROW_CONFLICT") + && !e.getCause().getMessage().contains("OB_TIMEOUT")) { + throw new RuntimeException(e); + } + } finally { + atomicInteger.incrementAndGet(); + countDownLatch.countDown(); + } + }); + } + threadPoolExecutor.shutdown(); + countDownLatch.await(100000, TimeUnit.MILLISECONDS); + final byte[] expect = Strings.repeat("a", atomicInteger.get()).getBytes(); + System.out.println("atomicInteger: " + atomicInteger.get()); + Get get = new Get(ROW); + get.setMaxVersions(1); + get.addColumn(FAMILY, column.getBytes()); + Result result = hTable.get(get); + ObHTableTestUtil.Assert(tableName, ()-> assertTrue(0 <= Bytes.compareTo(expect, CellUtil.cloneValue(result.getColumnCells(FAMILY, column.getBytes()).get(0))))); + } finally { + hTable.close(); + } + } + + private static void testAppendMultiCF(Map.Entry> entry) throws Exception { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + List tableNames = entry.getValue(); + String column = "appColumn"; + byte[] ROW = "appendKey".getBytes(); + byte[] v = "a".getBytes(); + Append append = new Append(ROW); + for (String tableName : tableNames) { + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + append.add(FAMILY, column.getBytes(), v); + } + try { + hTable.append(append); + fail("unexpect error, append should not support multi cf"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("multi family is not supported")); + } + hTable.close(); + } + + private static void testAppendSeires(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + String column = "appColumn"; + byte[] ROW = "appendKey".getBytes(); + byte[] v = "a".getBytes(); + Append append = new Append(ROW); + append.add(FAMILY, column.getBytes(), v); + try { + hTable.append(append); + fail("unexpect error, append should not support series table"); + } catch (Exception e) { + assertTrue(e.getCause().getMessage() + .contains("query and mutate with hbase series type not supported")); + } + hTable.close(); + } + + @Test + public void testAppend() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartAppendTest::testAppend); + } + + @Test + public void testBorderAppend() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartAppendTest::testAppendBorder); + } + + @Test + public void testAppendConcurrency() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartAppendTest::testAppendCon); + } + + @Test + public void testAppendMultiCF() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartAppendTest::testAppendMultiCF); + } + + @Test + public void testAppendSeires() throws Throwable { + List series_tables = new LinkedList(); + createTables(TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, series_tables, null, true); + FOR_EACH(series_tables, OHTableSecondaryPartAppendTest::testAppendSeires); + dropTables(series_tables, null); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java new file mode 100644 index 00000000..0445b6a3 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java @@ -0,0 +1,103 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + +public class OHTableSecondaryPartBatchGetTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testBatchGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); + put.addColumn(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); + hTable.put(put); + + List gets = new ArrayList<>(); + Get get1 = new Get(key.getBytes()); + get1.addFamily(family.getBytes()); + gets.add(get1); + + Get get2 = new Get(key.getBytes()); + get2.addColumn(family.getBytes(), column1.getBytes()); + gets.add(get2); + + Get get3 = new Get(key.getBytes()); + get3.addColumn(family.getBytes(), column2.getBytes()); + gets.add(get3); + + + Result[] results = hTable.get(gets); + for (Result result : results) { + for (Cell cell : result.listCells()) { + String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); + String V = Bytes.toString(CellUtil.cloneValue(cell)); + System.out.println("Column: " + Q + ", Value: " + V); + } + } + hTable.close(); + } + + @Test + public void testBatchGet() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartBatchGetTest::testBatchGetImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java new file mode 100644 index 00000000..4e345d28 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java @@ -0,0 +1,985 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; + +import java.io.IOException; +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + +public class OHTableSecondaryPartBatchTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testBatchPutImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String family = getColumnFamilyName(tableName); + List puts = new ArrayList<>(); + long timestamp = System.currentTimeMillis() - 100; + int batchSize = 10; + for (int i = 0; i < batchSize; i++) { + Put put = new Put(toBytes("row-" + i)); + put.addColumn(toBytes(family), toBytes("col1"), timestamp ,toBytes("value-1-" + i)); + put.addColumn(toBytes(family), toBytes("col2"), timestamp, toBytes("value-2-" + i)); + put.addColumn(toBytes(family), toBytes("col3") ,toBytes("value-3-" + i)); + put.addColumn(toBytes(family), toBytes("col4"), toBytes("value-4-" + i)); + puts.add(put); + } + hTable.put(puts); + + // verify result + long timestamp2 = System.currentTimeMillis(); + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes("row-" + i)); + get.addColumn(toBytes(family), toBytes("col1")); + get.addColumn(toBytes(family), toBytes("col2")); + get.addColumn(toBytes(family), toBytes("col3")); + get.addColumn(toBytes(family), toBytes("col4")); + + get.setMaxVersions(1); + Result result = hTable.get(get); + ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(4, result.size())); + for (Cell cell: result.listCells()) { + Assert.assertEquals(family, Bytes.toString(CellUtil.cloneFamily(cell))); + String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); + if (Q.equals("col1")) { + Assert.assertEquals("value-1-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertEquals(timestamp, cell.getTimestamp()); + } else if (Q.equals("col2")) { + Assert.assertEquals("value-2-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertEquals(timestamp, cell.getTimestamp()); + } else if (Q.equals("col3")) { + Assert.assertEquals("value-3-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertTrue(timestamp < cell.getTimestamp()); + timestamp2 = cell.getTimestamp(); + } else if (Q.equals("col4")) { + Assert.assertEquals("value-4-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertTrue(timestamp < cell.getTimestamp()); + } else { + Assert.fail(); + } + } + } + + // put exist key + puts.clear(); + for (int i = 0; i < batchSize; i++) { + Put put = new Put(toBytes("row-" + i)); + // update + put.addColumn(toBytes(family), toBytes("col1"), timestamp ,toBytes("update-value-1-" + i)); + put.addColumn(toBytes(family), toBytes("col2"), timestamp, toBytes("update-value-2-" + i)); + // insert + put.addColumn(toBytes(family), toBytes("col3") ,toBytes("update-value-3-" + i)); + put.addColumn(toBytes(family), toBytes("col4"), toBytes("update-value-4-" + i)); + puts.add(put); + } + hTable.put(puts); + + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes("row-" + i)); + get.addColumn(toBytes(family), toBytes("col1")); + get.addColumn(toBytes(family), toBytes("col2")); + get.addColumn(toBytes(family), toBytes("col3")); + get.addColumn(toBytes(family), toBytes("col4")); + get.setMaxVersions(); + Result result = hTable.get(get); + ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(6, result.size())); + for (Cell cell: result.listCells()) { + Assert.assertEquals(family, Bytes.toString(CellUtil.cloneFamily(cell))); + String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); + if (Q.equals("col1")) { + Assert.assertEquals("update-value-1-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertEquals(timestamp, cell.getTimestamp()); + } else if (Q.equals("col2")) { + Assert.assertEquals("update-value-2-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertEquals(timestamp, cell.getTimestamp()); + } else if (Q.equals("col3")) { + if (timestamp2 == cell.getTimestamp()) { + Assert.assertEquals("value-3-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + } else { + Assert.assertEquals("update-value-3-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertTrue(timestamp2 < cell.getTimestamp()); + } + } else if (Q.equals("col4")) { + if (timestamp2 == cell.getTimestamp()) { + Assert.assertEquals("value-4-" + i, Bytes.toString(CellUtil.cloneValue(cell))); + } else { + Assert.assertEquals("update-value-4-" + i, Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertTrue(timestamp2 < cell.getTimestamp()); + } + } else { + Assert.fail(); + } + } + } + hTable.close(); + } + + public static void testMultiCFBatchPutImpl(Map.Entry> entry) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + int batchSize = 10; + List puts = new ArrayList<>(); + String[] qualifier = new String[] {"col-1", "col-2", "col-3", "col-4"}; + String[] value = new String[] {"value-1", "value-2", "value-3", "value-4"}; + long timestamp = System.currentTimeMillis(); + for (int i = 0; i < batchSize; i++) { + Put put = new Put(toBytes("row-" + i)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(toBytes(family), toBytes(qualifier[0]), timestamp, toBytes(value[0])); + put.addColumn(toBytes(family), toBytes(qualifier[1]), timestamp, toBytes(value[1])); + put.addColumn(toBytes(family), toBytes(qualifier[2]), toBytes(value[2])); + put.addColumn(toBytes(family), toBytes(qualifier[3]), toBytes(value[3])); + } + puts.add(put); + } + hTable.put(puts); + // verify initial put result + for (int i = 0; i < batchSize; i++) { + for (String tableName : entry.getValue()) { + Get get = new Get(toBytes("row-" + i)); + get.setMaxVersions(1); + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(qualifier[0])); + get.addColumn(toBytes(family), toBytes(qualifier[1])); + get.addColumn(toBytes(family), toBytes(qualifier[2])); + get.addColumn(toBytes(family), toBytes(qualifier[3])); + + Result result = hTable.get(get); + ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(4, result.size())); + Assert.assertEquals(value[0], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[0])))); + Assert.assertEquals(timestamp, result.getColumnLatestCell(toBytes(family), toBytes(qualifier[0])).getTimestamp()); + Assert.assertEquals(value[1], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[1])))); + Assert.assertEquals(timestamp, result.getColumnLatestCell(toBytes(family), toBytes(qualifier[1])).getTimestamp()); + Assert.assertEquals(value[2], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[2])))); + Assert.assertTrue(timestamp < result.getColumnLatestCell(toBytes(family), toBytes(qualifier[2])).getTimestamp()); + Assert.assertEquals(value[3], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[3])))); + Assert.assertTrue(timestamp < result.getColumnLatestCell(toBytes(family), toBytes(qualifier[3])).getTimestamp()); + } + } + + // put exist key + puts.clear(); + String[] updateValue = new String[] {"update-value-1", "update-value-2", "update-value-3", "update-value-4"}; + for (int i = 0; i < batchSize; i++) { + Put put = new Put(toBytes("row-" + i)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(toBytes(family), toBytes(qualifier[0]), timestamp, toBytes(updateValue[0])); + put.addColumn(toBytes(family), toBytes(qualifier[1]), timestamp, toBytes(updateValue[1])); + put.addColumn(toBytes(family), toBytes(qualifier[2]), toBytes(updateValue[2])); + put.addColumn(toBytes(family), toBytes(qualifier[3]), toBytes(updateValue[3])); + } + puts.add(put); + } + hTable.put(puts); + + // verify update result + for (int i = 0; i < batchSize; i++) { + for (String tableName : entry.getValue()) { + Get get = new Get(toBytes("row-" + i)); + get.setMaxVersions(); // Get both versions + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(qualifier[0])); + get.addColumn(toBytes(family), toBytes(qualifier[1])); + get.addColumn(toBytes(family), toBytes(qualifier[2])); + get.addColumn(toBytes(family), toBytes(qualifier[3])); + + // Verify values + Result result = hTable.get(get); + ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(6, result.size())); + // qualifier[0] - 1 versions + List col0Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[0])); + Assert.assertEquals(1, col0Cells.size()); + Assert.assertEquals(updateValue[0], Bytes.toString(CellUtil.cloneValue(col0Cells.get(0)))); + Assert.assertEquals(timestamp, col0Cells.get(0).getTimestamp()); + + // qualifier[1] - 1 versions + List col1Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[1])); + Assert.assertEquals(1, col1Cells.size()); + Assert.assertEquals(updateValue[1], Bytes.toString(CellUtil.cloneValue(col1Cells.get(0)))); + Assert.assertEquals(timestamp, col1Cells.get(0).getTimestamp()); + + // qualifier[2] - 2 version + List col2Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[2])); + Assert.assertEquals(2, col2Cells.size()); + Assert.assertEquals(updateValue[2], Bytes.toString(CellUtil.cloneValue(col2Cells.get(0)))); + Assert.assertTrue(timestamp < col2Cells.get(0).getTimestamp()); + Assert.assertEquals(value[2], Bytes.toString(CellUtil.cloneValue(col2Cells.get(1)))); + + // qualifier[3] - 2 version + List col3Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[3])); + Assert.assertEquals(2, col3Cells.size()); + Assert.assertEquals(updateValue[3], Bytes.toString(CellUtil.cloneValue(col3Cells.get(0)))); + Assert.assertTrue(timestamp < col3Cells.get(0).getTimestamp()); + Assert.assertEquals(value[3], Bytes.toString(CellUtil.cloneValue(col3Cells.get(1)))); + + } + } + + hTable.close(); + } + + public static void testBatchGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String keyPrefix = "putKey_"; + String family = getColumnFamilyName(tableName); + String[] columns = {"putColumn1", "putColumn2", "putColumn3"}; + long curTs = System.currentTimeMillis(); + long[] ts = {curTs, curTs - 100}; + String valuePrefix = "value_"; + int batchSize = 10; + // prepare data + List puts = new ArrayList<>(); + for (int i = 0; i < batchSize; i++) { + Put put = new Put(toBytes(keyPrefix + i)); + for (int j = 0; j < columns.length; j++) { + put.addColumn(toBytes(family), toBytes(columns[j]), ts[0], toBytes(valuePrefix + "1_" +i)); + put.addColumn(toBytes(family), toBytes(columns[j]), ts[1], toBytes(valuePrefix + "2_" +i)); + } + puts.add(put); + } + hTable.put(puts); + // start to batch get test + List gets = new ArrayList<>(); + // 1. get specify column + { + int index = 0; + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes(keyPrefix + i)); + get.addColumn(family.getBytes(), columns[index].getBytes()); + get.setMaxVersions(); + gets.add(get); + } + Result[] batchResults = hTable.get(gets); + Assert.assertEquals(batchSize, batchResults.length); + for (int i = 0; i < batchSize; i++) { + Result r = batchResults[i]; + Assert.assertEquals(2, r.size()); + List cells = r.listCells(); + for (int j = 0; j < r.size(); j++) { + Cell cell = cells.get(j); + Assert.assertEquals(keyPrefix+i, Bytes.toString(CellUtil.cloneRow(cell))); + Assert.assertEquals(columns[index], Bytes.toString(CellUtil.cloneQualifier(cell))); + Assert.assertEquals(ts[j], cell.getTimestamp()); + if (j == 0) { + Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(CellUtil.cloneValue(cell))); + } else { + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(CellUtil.cloneValue(cell))); + } + } + } + } + gets.clear(); + // 2. get do not specify column + { + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes(keyPrefix + i)); + get.setMaxVersions(); + get.addFamily(family.getBytes()); + gets.add(get); + } + Result[] batchResults = hTable.get(gets); + Assert.assertEquals(batchSize, batchResults.length); + for (int i = 0; i < batchSize; i++) { + Result r = batchResults[i]; + Assert.assertEquals(6, r.size()); + for (int j = 0; j < columns.length; j++) { + List cells = r.getColumnCells(toBytes(family), toBytes(columns[j])); + Assert.assertEquals(2, cells.size()); + for (int k = 0; k < cells.size(); k++) { + Assert.assertEquals(ts[k], cells.get(k).getTimestamp()); + if (k == 0) { + Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(CellUtil.cloneValue(cells.get(k)))); + } else { + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(CellUtil.cloneValue(cells.get(k)))); + } + } + } + } + } + // 3. get specify version + gets.clear(); + { + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes(keyPrefix + i)); + get.addFamily(family.getBytes()); + get.setMaxVersions(2); + gets.add(get); + } + Result[] batchResults = hTable.get(gets); + Assert.assertEquals(batchSize, batchResults.length); + for (int i = 0; i < batchSize; i++) { + Result r = batchResults[i]; + Assert.assertEquals(6, r.size()); + for (int j = 0; j < columns.length; j++) { + List cells = r.getColumnCells(family.getBytes(), toBytes(columns[j])); + Assert.assertEquals(2, cells.size()); + for (int k = 0; k < cells.size(); k++) { + Assert.assertEquals(ts[k], cells.get(k).getTimestamp()); + if (k == 0) { + Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(CellUtil.cloneValue(cells.get(k)))); + } else { + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(CellUtil.cloneValue(cells.get(k)))); + } + } + } + } + } + // 4. get specify time range + gets.clear(); + { + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes(keyPrefix + i)); + get.addFamily(family.getBytes()); + get.setMaxVersions(2); + get.setTimeStamp(ts[1]); + gets.add(get); + } + Result[] batchResults = hTable.get(gets); + Assert.assertEquals(batchSize, batchResults.length); + for (int i = 0; i < batchSize; i++) { + Result r = batchResults[i]; + Assert.assertEquals(3, r.size()); + for (int j = 0; j < columns.length; j++) { + List cells = r.getColumnCells(family.getBytes(), toBytes(columns[j])); + Assert.assertEquals(1, cells.size()); + Assert.assertEquals(ts[1], cells.get(0).getTimestamp()); + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(CellUtil.cloneValue(cells.get(0)))); + } + } + } + // 5. get specify filter + gets.clear(); + { + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes(keyPrefix+i)); + get.addFamily(family.getBytes()); + get.setMaxVersions(2); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(valuePrefix+"2_"+i))); + get.setFilter(valueFilter); + gets.add(get); + } + Result[] batchResults = hTable.get(gets); + Assert.assertEquals(batchSize, batchResults.length); + for (int i = 0; i < batchSize; i++) { + Result r = batchResults[i]; + Assert.assertEquals(3, r.size()); + for (int j = 0; j < columns.length; j++) { + List cells = r.getColumnCells(family.getBytes(), toBytes(columns[j])); + Assert.assertEquals(1, cells.size()); + Assert.assertEquals(ts[1], cells.get(0).getTimestamp()); + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(CellUtil.cloneValue(cells.get(0)))); + } + } + } + hTable.close(); + } + + public static void testMixBatchImpl(String tableName) throws Exception { + byte[] family = getColumnFamilyName(tableName).getBytes(); + byte[][] keys = new byte[][]{"key1".getBytes(), "key2".getBytes()}; + byte[][] qualifiers = new byte[][]{"col1".getBytes(), "col2".getBytes()}; + byte[][] values = new byte[][]{"value1".getBytes(), "value2".getBytes()}; + long curTs = System.currentTimeMillis(); + long[] ts = {curTs, curTs - 1000}; + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + List batchList = new LinkedList<>(); + // 0. load data + for (int i = 0; i < keys.length; i++) { + Put put = new Put(keys[i]); + for (int j = 0; j < qualifiers.length; j++) { + for (int k = 0; k < values.length; k++) { + put.addColumn(family, qualifiers[j], ts[k], values[k]); + } + } + batchList.add(put); + } + Result[] results = new Result[batchList.size()]; + hTable.batch(batchList, results); + // 1. get + put + get + batchList.clear(); + { + // get old result + Get get1 = new Get(keys[0]); + get1.setMaxVersions(); + get1.addFamily(family); + // get old result + Get get2 = new Get(keys[1]); + get2.setMaxVersions(1); + get2.addColumn(family, qualifiers[1]); + + // put new value + byte[] newValue = "new_value".getBytes(); + Put put1 = new Put(keys[0]); + put1.addColumn(family, qualifiers[0], newValue); + // put new value + Put put2 = new Put(keys[1]); + put2.addColumn(family, qualifiers[1], newValue); + // get new result + Get get3 = new Get(keys[0]); + get3.setMaxVersions(1); + get3.addColumn(family, qualifiers[0]); + + // get new result + Get get4 = new Get(keys[1]); + get4.setMaxVersions(1); + get4.addColumn(family, qualifiers[1]); + + // execute + batchList.addAll(Arrays.asList(get1 ,get2, put1, put2, get3, get4)); + results = new Result[batchList.size()]; + hTable.batch(batchList, results); + // verify result + Assert.assertEquals(6, results.length); + // get1 + Result get1Result = (Result) results[0]; + Assert.assertEquals(4, get1Result.size()); + List k0cq0 = get1Result.getColumnCells(family, qualifiers[0]); + Assert.assertEquals(2, k0cq0.size()); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(qualifiers[0]), ts[0], Bytes.toString(values[0]), k0cq0.get(0)); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(qualifiers[0]), ts[1], Bytes.toString(values[1]), k0cq0.get(1)); + List k0cq1 = get1Result.getColumnCells(family, qualifiers[1]); + Assert.assertEquals(2, k0cq1.size()); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(qualifiers[1]), ts[0], Bytes.toString(values[0]), k0cq1.get(0)); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(qualifiers[1]), ts[1], Bytes.toString(values[1]), k0cq1.get(1)); + // get2 + Result get2Result = (Result) results[1]; + Assert.assertEquals(1, get2Result.size()); + List k1cq1 = get2Result.getColumnCells(family, qualifiers[1]); + Assert.assertEquals(1, k1cq1.size()); + AssertKeyValue(Bytes.toString(keys[1]), Bytes.toString(qualifiers[1]), ts[0], Bytes.toString(values[0]), k1cq1.get(0)); + // get3 + Result get3Result = (Result) results[4]; + Assert.assertEquals(1, get3Result.size()); + k0cq0 = get3Result.getColumnCells(family, qualifiers[0]); + Assert.assertEquals(1, k0cq0.size()); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(qualifiers[0]), Bytes.toString(newValue), k0cq0.get(0)); + // get4 + Result get4Result = (Result) results[5]; + Assert.assertEquals(1, get4Result.size()); + k1cq1 = get4Result.getColumnCells(family, qualifiers[1]); + Assert.assertEquals(1, k1cq1.size()); + AssertKeyValue(Bytes.toString(keys[1]), Bytes.toString(qualifiers[1]), Bytes.toString(newValue), k1cq1.get(0)); + } + // 2. delete + get + delete + get + batchList.clear(); + { + // delete all version keys[0]-qualifiers[0] + Delete delete1 = new Delete(keys[0]); + delete1.addColumns(family, qualifiers[0]); + + // get family: keys[0]-qualifier[1] may have two versions and keys[0]-qualifier[0] have been deleted + Get get1 = new Get(keys[0]); + get1.addFamily(family); + get1.setMaxVersions(); + // delete all version keys[0]-qualifier[1] + Delete delete2 = new Delete(keys[0]); + delete2.addColumns(family, qualifiers[1]); + // keys[0] may not have results + Get get2 = new Get(keys[0]); + get2.addFamily(family); + get2.setMaxVersions(); + // execute + batchList.addAll(Arrays.asList(delete1, get1, delete2, get2)); + results = new Result[batchList.size()]; + hTable.batch(batchList, results); + // verify result + Assert.assertEquals(4, results.length); + Result get1Result = (Result) results[1]; + ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(2, get1Result.size())); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(family), Bytes.toString(qualifiers[1]), + ts[0], Bytes.toString(values[0]), get1Result.listCells().get(0)); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(family), Bytes.toString(qualifiers[1]), + ts[1], Bytes.toString(values[1]), get1Result.listCells().get(1)); + Result get2Result = (Result) results[3]; + Assert.assertEquals(0, get2Result.size()); + } + // 3. put + delete + get + batchList.clear(); + { + Put put1 = new Put(keys[0]); + put1.addColumn(family, qualifiers[0], "new_new_value".getBytes()); + put1.addColumn(family, qualifiers[1], "new_new_value".getBytes()); + Put put2 = new Put(keys[1]); + put2.addColumn(family, qualifiers[0], "new_new_value".getBytes()); + put2.addColumn(family, qualifiers[1], "new_new_value".getBytes()); + Delete delete = new Delete(keys[0]); + delete.addColumns(family, qualifiers[0]); + Get get1 = new Get(keys[0]); + get1.setMaxVersions(1); + get1.addFamily(family); + + Get get2 = new Get(keys[1]); + get2.setMaxVersions(1); + get2.addFamily(family); + + batchList.addAll(Arrays.asList(put1, put2, delete, get1, get2)); + results = new Result[batchList.size()]; + hTable.batch(batchList, results); + Assert.assertEquals(5, results.length); + Result getResult1 = (Result) results[3]; + Assert.assertEquals(1, getResult1.size()); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(qualifiers[1]), + "new_new_value", getResult1.listCells().get(0)); + Result getResult2 = (Result) results[4]; + Assert.assertEquals(2, getResult2.size()); + AssertKeyValue(Bytes.toString(keys[1]), Bytes.toString(qualifiers[0]), + "new_new_value", getResult2.listCells().get(0)); + AssertKeyValue(Bytes.toString(keys[1]), Bytes.toString(qualifiers[1]), + "new_new_value", getResult2.listCells().get(1)); + } + hTable.close(); + } + + public static void testMultiCFBatchGetImpl(Map.Entry> entry) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(entry.getKey())); + hTable.init(); + String keyPrefix = "putKey_"; + String[] columns = {"putColumn1", "putColumn2", "putColumn3"}; + String[] values = {"version1_", "version2_"}; // each column have two versions + String latestValue = values[1]; + List tableNames = entry.getValue(); + long timestamp = System.currentTimeMillis(); + long[] ts = {timestamp, timestamp+1}; + long lastTs = ts[1]; + int batchSize = 10; + // 0. prepare data + List puts = new ArrayList<>(); + for (int k = 0; k < batchSize; k++) { + String key = keyPrefix+k; + Put put = new Put(toBytes(key)); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < columns.length; j++) { + put.addColumn(family.getBytes(), columns[j].getBytes(), ts[i], toBytes(values[i]+k)); + } + } + } + puts.add(put); + } + hTable.put(puts); + // 1. get specify column + { + List gets = new ArrayList<>(); + int columnIndex = 0; + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + for (String tableName : tableNames) { + Get get = new Get(key.getBytes()); + String family = getColumnFamilyName(tableName); + get.addColumn(family.getBytes(), columns[columnIndex].getBytes()); + gets.add(get); + } + } + Result[] results = hTable.get(gets); + Assert.assertEquals(gets.size(), results.length); + for (int i = 0; i < gets.size(); i++) { + int idx = i / tableNames.size(); + String key = keyPrefix + idx; + Result res = results[i]; + Assert.assertEquals(1, res.size()); + AssertKeyValue(key, columns[columnIndex], lastTs, latestValue + idx, res.rawCells()[0]); + } + } + // 2. get do not specify column + { + List gets = new ArrayList<>(); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Get get = new Get(key.getBytes()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + get.addFamily(family.getBytes()); + } + gets.add(get); + } + Result[] results = hTable.get(gets); + Assert.assertEquals(batchSize, results.length); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Result result = results[i]; + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int j = 0; j < columns.length; j++) { + Cell cell = result.getColumnCells(family.getBytes(), columns[j].getBytes()).get(0); + AssertKeyValue(key, columns[j], lastTs, latestValue + i, cell); + } + } + } + } + // 3. get do not specify column family + { + List gets = new ArrayList<>(); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Get get = new Get(key.getBytes()); + gets.add(get); + } + Result[] results = hTable.get(gets); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Result result = results[i]; + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int j = 0; j < columns.length; j++) { + Cell cell = result.getColumnCells(family.getBytes(), columns[j].getBytes()).get(0); + AssertKeyValue(key, columns[j], lastTs, latestValue + i, cell); + } + } + } + } + // 4. get specify multi cf and column + { + List gets = new ArrayList<>(); + int columnIndex = 0; + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Get get = new Get(key.getBytes()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + get.addColumn(family.getBytes(), columns[columnIndex].getBytes()); + } + gets.add(get); + } + Result[] results = hTable.get(gets); + Assert.assertEquals(batchSize, results.length); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Result res = results[i]; + Assert.assertEquals(tableNames.size(), res.size()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + AssertKeyValue(key, columns[columnIndex], lastTs, latestValue + i, + res.getColumnCells(family.getBytes(), columns[columnIndex].getBytes()).get(0)); + + } + } + } + // 5. get specify multi cf and versions + { + List gets = new ArrayList<>(); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Get get = new Get(key.getBytes()); + get.setMaxVersions(2); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + get.addFamily(family.getBytes()); + } + gets.add(get); + } + Result[] results = hTable.get(gets); + Assert.assertEquals(batchSize, results.length); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Result res = results[i]; + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int j = 0; j < columns.length; j++) { + List cells = res.getColumnCells(family.getBytes(), columns[j].getBytes()); + Assert.assertEquals(2, cells.size()); + for (int k = ts.length - 1; k >= 0; k--) { + AssertKeyValue(key, family, columns[j], ts[k], values[k] + i, cells.get(ts.length - k - 1)); + } + } + } + } + } + // 6. get specify multi cf and set timestamp + { + List gets = new ArrayList<>(); + int columnIndex = 0; + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Get get = new Get(key.getBytes()); + get.setTimeStamp(ts[1]); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + get.addColumn(family.getBytes(), columns[columnIndex].getBytes()); + } + gets.add(get); + } + Result[] results = hTable.get(gets); + Assert.assertEquals(batchSize, results.length); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Result res = results[i]; + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + AssertKeyValue(key, columns[columnIndex], ts[1], latestValue + i, + res.getColumnCells(family.getBytes(), columns[columnIndex].getBytes()).get(0)); + } + } + } + // 7. get specify multi cf and filter + { + List gets = new ArrayList<>(); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Get get = new Get(key.getBytes()); + get.setMaxVersions(2); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(values[0] + i))); + get.setFilter(valueFilter); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + get.addFamily(family.getBytes()); + } + gets.add(get); + } + Result[] results = hTable.get(gets); + Assert.assertEquals(batchSize, results.length); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Result res = results[i]; + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int j = 0; j < columns.length; j++) { + AssertKeyValue(key, columns[j], ts[0], values[0] + i, + res.getColumnCells(family.getBytes(), columns[j].getBytes()).get(0)); + } + } + } + } + } + + public static void testMultiCFMixBatchImpl(Map.Entry> entry) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(entry.getKey())); + hTable.init(); + String keyPrefix = "Key_"; + String[] columns = {"Column1", "Column2"}; + String[] values = {"version1_", "version2_"}; // each column have two versions + String latestValue = values[1]; + long timestamp = System.currentTimeMillis(); + long[] ts = {timestamp, timestamp+1}; + long lastTs = ts[1]; + int batchSize = 10; + List tableNames = entry.getValue(); + + // 0. prepare data + { + List puts = new ArrayList<>(); + for (int k = 0; k < batchSize; k++) { + String key = keyPrefix + k; + Put put = new Put(toBytes(key)); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < columns.length; j++) { + put.addColumn(family.getBytes(), columns[j].getBytes(), ts[i], toBytes(values[i] + k)); + } + } + } + puts.add(put); + } + hTable.put(puts); + } + String[] families = getAllFamilies(tableNames); + // 1. get + put + get + { + List batchList = new LinkedList<>(); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + batchList.add(createGetOp(key, families, columns[0], ts[1])); + batchList.add(createPutOp(key, families, columns[0], ts[1], "new_value_"+i)); + batchList.add(createGetOp(key, families, columns[0], ts[1])); + } + Result[] results = new Result[batchList.size()]; + hTable.batch(batchList, results); + Assert.assertEquals(batchSize*3, results.length); + for (int i = 0; i < batchSize; i++) { + Result get1Result = (Result) results[3*i+0]; + checkResult(keyPrefix+i, families, columns[0], ts[1], values[1]+i, get1Result); + Result get2Result = (Result) results[3*i+2]; + checkResult(keyPrefix+i, families, columns[0], ts[1], "new_value_"+i, get2Result); + } + } + + // 2. delete + get + delete + get + { + List batchList = new LinkedList<>(); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + batchList.add(createDeleteOp(key, families, columns[1], ts[0])); + batchList.add(createGetOp(key, families, columns[1], ts[0])); + batchList.add(createGetOp(key, families, columns[0], ts[1])); + batchList.add(createDeleteOp(key, families, columns[0], ts[1])); + batchList.add(createGetOp(key, families, columns[0], ts[1])); + } + Result[] results = new Result[batchList.size()]; + hTable.batch(batchList, results); + Assert.assertEquals(batchSize*5, results.length); + for (int i = 0; i < batchSize; i++) { + Result get1Result = (Result) results[5*i+1]; + Assert.assertEquals(0, get1Result.size()); + Result get2Result = (Result) results[5*i+2]; + checkResult(keyPrefix+i, families, columns[0], ts[1], "new_value_"+i, get2Result); + Result get3Result = (Result) results[5*i+4]; + Assert.assertEquals(0, get3Result.size()); + } + } + + // 3. put + delete + get + { + List batchList = new LinkedList<>(); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + batchList.add(createPutOp(key, families, columns[0], ts[1], "new_new_value"+i)); + batchList.add(createGetOp(key, families, columns[0], ts[1])); + batchList.add(createDeleteOp(key, families, columns[0], ts[1])); + batchList.add(createGetOp(key, families, columns[0], ts[1])); + } + Result[] results = new Result[batchList.size()]; + hTable.batch(batchList, results); + Assert.assertEquals(batchSize*4, results.length); + for (int i = 0; i < batchSize; i++) { + Result get1Result = (Result) results[4*i+1]; + checkResult(keyPrefix+i, families, columns[0], ts[1], "new_new_value"+i, get1Result); + Result get2Result = (Result) results[4*i+3]; + Assert.assertEquals(0, get2Result.size()); + } + } + hTable.close(); + } + + private static String[] getAllFamilies(List tableNames) throws Exception { + String[] families = new String[tableNames.size()]; + for (int i = 0; i < tableNames.size(); i++) { + families[i] = getColumnFamilyName(tableNames.get(i)); + } + return families; + } + + private static void checkResult(String key, String[] families, String qualifier, long ts, + String value, Result result) { + Assert.assertEquals(families.length, result.size()); + for (String family : families) { + Cell cell = result.getColumnCells(family.getBytes(), qualifier.getBytes()).get(0); + AssertKeyValue(key, family, qualifier, ts, value, cell); + } + } + + private static Put createPutOp(String key, String[] families, String qualifier, long ts, + String value) { + if (families == null || qualifier == null || value == null) { + throw new IllegalArgumentException("input parameters is illegal"); + } + Put put = new Put(key.getBytes()); + for (int i = 0; i < families.length; i++) { + put.addColumn(families[i].getBytes(), qualifier.getBytes(), ts, value.getBytes()); + } + return put; + } + + private static Delete createDeleteOp(String key, String[] families, String qualifier, long ts) { + if (families == null || qualifier == null) { + throw new IllegalArgumentException("input parameters is illegal"); + } + Delete delete = new Delete(key.getBytes()); + for (int i = 0; i < families.length; i++) { + delete.addColumn(families[i].getBytes(), qualifier.getBytes(), ts); + } + return delete; + } + + private static Get createGetOp(String key, String[] families, String qualifier, long ts) + throws IOException { + if (families == null || qualifier == null) { + throw new IllegalArgumentException("input parameters is illegal"); + } + Get get = new Get(key.getBytes()); + for (int i = 0; i < families.length; i++) { + get.addColumn(families[i].getBytes(), qualifier.getBytes()); + get.setTimeStamp(ts); + } + return get; + } + + @Test + public void testBatchPut() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartBatchTest::testBatchPutImpl); + } + + @Test + public void testMixBatch() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartBatchTest::testMixBatchImpl); + truncateTables(ObHTableTestUtil.getConnection(), tableNames); + } + + @Test + public void testMultiCFMixBatch() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartBatchTest::testMultiCFMixBatchImpl); + truncateTables(ObHTableTestUtil.getConnection(), group2tableNames); + } + + @Test + public void testMultiCFPut() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartBatchTest::testMultiCFBatchPutImpl); + } + + @Test + public void testBatchGet() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartBatchTest::testBatchGetImpl); + } + + @Test + public void testMultiCFGet() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartBatchTest::testMultiCFBatchGetImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java new file mode 100644 index 00000000..95811d8d --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java @@ -0,0 +1,483 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.*; +import static org.junit.Assert.assertTrue; + +public class OHTableSecondaryPartCellTTLTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TableTemplateManager.CELL_TTL_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testCellTTL(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String key1 = "key1"; + String column1 = "cf1"; + String column2 = "cf2"; + String column3 = "cf3"; + String family = getColumnFamilyName(tableName); + String value1 = "v1"; + String value2 = "v2"; + String app = "app"; + + Result r; + Put put1 = new Put(key1.getBytes()); + put1.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put1.setTTL(5000); + Put put2 = new Put(key1.getBytes()); + put2.addColumn(family.getBytes(), column1.getBytes(), toBytes(22L)); + put2.addColumn(family.getBytes(), column2.getBytes(), toBytes(33L)); + put2.setTTL(10000); + Put put3 = new Put(key1.getBytes()); + put3.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put3.setTTL(-3000); + Put put4 = new Put(key1.getBytes()); + put4.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put4.setTTL(0); + + Get get = new Get(key1.getBytes()); + get.addFamily(family.getBytes()); + get.setMaxVersions(10); + + // test put and get + hTable.put(put1); + hTable.put(put2); + hTable.put(put3); + hTable.put(put4); + r = hTable.get(get); + assertEquals(3, r.size()); + Thread.sleep(5000); + r = hTable.get(get); + assertEquals(2, r.size()); + Thread.sleep(5000); + r = hTable.get(get); + assertEquals(0, r.size()); + + // test increment + hTable.put(put1); + hTable.put(put2); + Thread.sleep(1000); + Increment increment = new Increment(key1.getBytes()); + increment.addColumn(family.getBytes(), column1.getBytes(), 1L); + increment.addColumn(family.getBytes(), column2.getBytes(), 2L); + increment.addColumn(family.getBytes(), column3.getBytes(), 5L); + increment.setTTL(-5000); + hTable.increment(increment); + increment.setTTL(5000); + hTable.increment(increment); + get.setMaxVersions(1); + r = hTable.get(get); + + assertEquals(3, r.size()); + assertEquals(23L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + assertEquals(35L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column2.getBytes()).get(0)))); + assertEquals(5L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column3.getBytes()).get(0)))); + + Thread.sleep(10000); + r = hTable.get(get); + assertEquals(0, r.size()); + + increment = new Increment(key1.getBytes()); + increment.addColumn(family.getBytes(), column1.getBytes(), 1L); + increment.addColumn(family.getBytes(), column2.getBytes(), 2L); + increment.setTTL(5000); + hTable.increment(increment); + r = hTable.get(get); + + assertEquals(2, r.size()); + assertEquals(1L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + assertEquals(2L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column2.getBytes()).get(0)))); + + Thread.sleep(5000); + r = hTable.get(get); + assertEquals(0, r.size()); + + hTable.put(put1); + hTable.put(put2); + increment.addColumn(family.getBytes(), column1.getBytes(), 4L); + hTable.increment(increment); + r = hTable.get(get); + assertEquals(2, r.size()); + assertEquals(26L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + assertEquals(35L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column2.getBytes()).get(0)))); + + // test append + Thread.sleep(10000); + r = hTable.get(get); + assertEquals(0, r.size()); + + put3 = new Put(key1.getBytes()); + put3.addColumn(family.getBytes(), column1.getBytes(), toBytes(value1)); + put3.addColumn(family.getBytes(), column2.getBytes(), toBytes(value2)); + put3.setTTL(10000); + hTable.put(put3); + Append append = new Append(key1.getBytes()); + KeyValue kv = new KeyValue(key1.getBytes(), family.getBytes(), column1.getBytes(), + app.getBytes()); + append.add(kv); + append.setTTL(-3000); + hTable.append(append); + append.setTTL(3000); + hTable.append(append); + + r = hTable.get(get); + assertEquals(2, r.size()); + assertEquals( + value1 + app, + Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + + Thread.sleep(3000); + r = hTable.get(get); + assertEquals(2, r.size()); + assertEquals( + value1, + Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + + Thread.sleep(7000); + r = hTable.get(get); + assertEquals(0, r.size()); + + append.add(family.getBytes(), column1.getBytes(), app.getBytes()); + hTable.append(append); + r = hTable.get(get); + assertEquals(1, r.size()); + assertEquals( + app, + Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + + Thread.sleep(3000); + append.add(family.getBytes(), column2.getBytes(), app.getBytes()); + hTable.append(append); + r = hTable.get(get); + assertEquals(2, r.size()); + assertEquals( + app, + Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + assertEquals( + app, + Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column2.getBytes()).get(0)))); + + // test checkAndMutate + Thread.sleep(3000); + r = hTable.get(get); + assertEquals(0, r.size()); + hTable.put(put1); + RowMutations rowMutations = new RowMutations(key1.getBytes()); + rowMutations.add(put2); + Delete delete = new Delete(key1.getBytes()); + delete.addColumn(family.getBytes(), column1.getBytes()); + rowMutations.add(delete); + boolean succ = hTable.checkAndMutate(key1.getBytes(), family.getBytes(), + column1.getBytes(), CompareFilter.CompareOp.EQUAL, toBytes(11L), rowMutations); + assertTrue(succ); + r = hTable.get(get); + assertEquals(r.size(), 2); + assertEquals(11L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + assertEquals(33L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column2.getBytes()).get(0)))); + + Thread.sleep(10000); + r = hTable.get(get); + assertEquals(r.size(), 0); + + hTable.put(put1); + rowMutations = new RowMutations(key1.getBytes()); + put4 = new Put(key1.getBytes()); + put4.addColumn(family.getBytes(), column1.getBytes(), toBytes(22L)); + put4.addColumn(family.getBytes(), column2.getBytes(), toBytes(33L)); + put4.setTTL(10000); + rowMutations.add(put4); + succ = hTable.checkAndMutate(key1.getBytes(), family.getBytes(), column1.getBytes(), + CompareFilter.CompareOp.EQUAL, toBytes(1L), rowMutations); + assertFalse(succ); + succ = hTable.checkAndMutate(key1.getBytes(), family.getBytes(), column1.getBytes(), + CompareFilter.CompareOp.EQUAL, toBytes(11L), rowMutations); + assertTrue(succ); + + r = hTable.get(get); + assertEquals(r.size(), 2); + assertEquals(22L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + assertEquals(33L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column2.getBytes()).get(0)))); + + Thread.sleep(5000); + r = hTable.get(get); + assertEquals(2, r.size()); + assertEquals(22L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + assertEquals(33L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column2.getBytes()).get(0)))); + + Thread.sleep(5000); + r = hTable.get(get); + assertEquals(r.size(), 0); + put1 = new Put(key1.getBytes()); + put1.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + hTable.put(put1); + + increment = new Increment(key1.getBytes()); + increment.addColumn(family.getBytes(), column1.getBytes(), 1L); + hTable.increment(increment); + r = hTable.get(get); + assertEquals(r.size(), 1); + assertEquals(Bytes.toLong(CellUtil.cloneValue(r.rawCells()[0])), 12L); + get.setMaxVersions(10); + r = hTable.get(get); + assertEquals(r.size(), 2); + + put1.setTTL(-100); + hTable.put(put1); + Delete delete1 = new Delete(key1.getBytes()); + delete1.addColumn(family.getBytes(), column1.getBytes()); + hTable.delete(delete1); + get.setMaxVersions(10); + r = hTable.get(get); + assertEquals(r.size(), 1); + assertEquals(Bytes.toLong(CellUtil.cloneValue(r.rawCells()[0])), 11L); + hTable.close(); + } + + public static void testCellTTLWithRowkeyTTL(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String key1 = "key1"; + String column1 = "cf1"; + String column2 = "cf2"; + String family = getColumnFamilyName(tableName); + Put put1 = new Put(key1.getBytes()); + put1.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put1.setTTL(5000); + Put put2 = new Put(key1.getBytes()); + put2.addColumn(family.getBytes(), column1.getBytes(), toBytes(22L)); + put2.addColumn(family.getBytes(), column2.getBytes(), toBytes(33L)); + put2.setTTL(10000); + Put put3 = new Put(key1.getBytes()); + put3.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put3.setTTL(-3000); + Put put4 = new Put(key1.getBytes()); + put4.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put4.setTTL(0); + Put errorPut = new Put(key1.getBytes()); + errorPut.addColumn("family1".getBytes(), column1.getBytes(), toBytes(11L)); + errorPut.setTTL(10); + + // test put and get + hTable.put(put1); + hTable.put(put2); + hTable.put(put3); + hTable.put(put4); + + Assert.assertEquals(5, getSQLTableRowCnt(tableName)); + Thread.sleep(10000); + enableTTL(); + Get get = new Get(key1.getBytes()); + get.addFamily(family.getBytes()); + hTable.get(get); + Thread.sleep(10000); + + Assert.assertEquals(0, getSQLTableRowCnt(tableName)); + + // 6. close ttl knob + disableTTL(); + hTable.close(); + } + + public static void testCellTTLSQL(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String key1 = "key1"; + String column1 = "cf1"; + String column2 = "cf2"; + String family = getColumnFamilyName(tableName); + + Result r; + Put put1 = new Put(key1.getBytes()); + put1.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put1.setTTL(5000); + Put put2 = new Put(key1.getBytes()); + put2.addColumn(family.getBytes(), column1.getBytes(), toBytes(22L)); + put2.addColumn(family.getBytes(), column2.getBytes(), toBytes(33L)); + put2.setTTL(10000); + Put put3 = new Put(key1.getBytes()); + put3.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put3.setTTL(-3000); + Put put4 = new Put(key1.getBytes()); + put4.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put4.setTTL(0); + Put errorPut = new Put(key1.getBytes()); + errorPut.addColumn("family1".getBytes(), column1.getBytes(), toBytes(11L)); + errorPut.setTTL(10); + + // test put and get + hTable.put(put1); + hTable.put(put2); + hTable.put(put3); + hTable.put(put4); + + Assert.assertEquals(5, getSQLTableRowCnt(tableName)); + Thread.sleep(10000); + enableTTL(); + triggerTTL(); + List tableNames = new ArrayList<>(); + tableNames.add(tableName); + checkUtilTimeout(tableNames, ()-> { + try { + return getRunningNormalTTLTaskCnt() == 0; + } catch (Exception e) { + throw new RuntimeException(e); + } + }, 50000, 3000); + + Assert.assertEquals(0, getSQLTableRowCnt(tableName)); + + // 6. close ttl knob + disableTTL(); + hTable.close(); + } + + public static void testMultiCFCellTTL(Map.Entry> entry) throws Exception { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + List tableNames = entry.getValue(); + int familySize = tableNames.size(); + int ttl; + String key1 = "key1"; + String column1 = "cf1"; + for (int i = 0; i < familySize; i++) { + String tableName = tableNames.get(i); + String family = getColumnFamilyName(tableName); + Put put = new Put(key1.getBytes()); + put.addColumn(family.getBytes(), column1.getBytes(), toBytes("value")); + ttl = i * 5000; + put.setTTL(ttl); + hTable.put(put); + } + for (int i = 0; i < familySize; i++) { + int index = 0; + for (String tableName : tableNames) { + Get get = new Get(key1.getBytes()); + String family = getColumnFamilyName(tableName); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + int num = index <= i ? 0 : 1; + index++; + assertEquals(num, result.size()); + } + Thread.sleep(5000); + } + + hTable.close(); + } + + public static void testCellTTLWithNonTTLTable(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String family = getColumnFamilyName(tableName); + String key1 = "key1"; + String column1 = "cf1"; + Put errorPut = new Put(key1.getBytes()); + errorPut.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + errorPut.setTTL(10); + try { + hTable.put(errorPut); + fail("unexpect error, put with ttl field and table not have ttl column should occur error"); + } catch (Exception e) { + assertTrue(e.getCause().getMessage().contains("Unknown column 'TTL'")); + } + hTable.close(); + } + + @Test + public void testCellTTL() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartCellTTLTest::testCellTTL); + } + + @Test + public void testMultiCFCellTTL() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartCellTTLTest::testMultiCFCellTTL); + } + + @Test + public void testCellTTLSQL() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartCellTTLTest::testCellTTLSQL); + } + + @Test + public void testCellTTLWithRowkeyTTL() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartCellTTLTest::testCellTTLWithRowkeyTTL); + } + + @Test + public void testCellTTLWithNonTTLTable() throws Throwable { + List NonTTLTable = new LinkedList(); + createTables(TableTemplateManager.TableType.NON_PARTITIONED_REGULAR, NonTTLTable, null, true); + FOR_EACH(NonTTLTable, OHTableSecondaryPartCellTTLTest::testCellTTLWithNonTTLTable); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java new file mode 100644 index 00000000..14383ba2 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java @@ -0,0 +1,444 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.NORMAL_TABLES; +import static org.junit.Assert.*; + +public class OHTableSecondaryPartCheckAndMutateTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); + private static byte[] ROW = Bytes.toBytes("testRow"); + private static byte[] QUALIFIER = Bytes.toBytes("testQualifier"); + private static byte[] VALUE_1 = Bytes.toBytes("testValue"); + private static byte[] ROW_1 = Bytes.toBytes("testRow1"); + private static byte[] VALUE_2 = Bytes.toBytes("abcd"); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testCheckAndMutate(String tableName) throws Throwable { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + final byte[] ROW = Bytes.toBytes("12345"); + final byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + Put put = new Put(ROW); + put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); + put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); + put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")); + hTable.put(put); + // get row back and assert the values + Get get = new Get(ROW); + get.addFamily(FAMILY); + Result result = hTable.get(get); + assertTrue("Column A value should be a", + Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("A"))).equals("a")); + assertTrue("Column B value should be b", + Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))).equals("b")); + assertTrue("Column C value should be c", + Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("C"))).equals("c")); + + // put the same row again with C column deleted + RowMutations rm = new RowMutations(ROW); + put = new Put(ROW); + put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); + put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); + rm.add(put); + Delete del = new Delete(ROW); + del.addColumn(FAMILY, Bytes.toBytes("C")); + rm.add(del); + boolean res = hTable.checkAndMutate(ROW, FAMILY, Bytes.toBytes("A"), + CompareFilter.CompareOp.EQUAL, Bytes.toBytes("a"), rm); + assertTrue(res); + + // get row back and assert the values + get = new Get(ROW); + get.addFamily(FAMILY); + result = hTable.get(get); + assertTrue("Column A value should be a", + Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("A"))).equals("a")); + assertTrue("Column B value should be b", + Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))).equals("b")); + assertTrue("Column C should not exist", result.getValue(FAMILY, Bytes.toBytes("C")) == null); + + //Test that we get a hTable level exception + try { + Put p = new Put(ROW); + p.addColumn(new byte[] { 'b', 'o', 'g', 'u', 's' }, new byte[] { 'A' }, new byte[0]); + rm = new RowMutations(ROW); + rm.add(p); + hTable.checkAndMutate(ROW, FAMILY, Bytes.toBytes("A"), CompareFilter.CompareOp.EQUAL, + Bytes.toBytes("a"), rm); + fail("Expected NoSuchColumnFamilyException"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("mutation family is not equal check family")); + } + hTable.close(); + } + + public static void testCheckAndPut(String tableName) throws Throwable { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + + Put put1 = new Put(ROW); + put1.addColumn(FAMILY, QUALIFIER, VALUE_1); + + // row doesn't exist, so using non-null value should be considered "not match". + boolean ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE_1, put1); + assertEquals(ok, false); + + // row doesn't exist, so using "null" to check for existence should be considered "match". + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1); + assertEquals(ok, true); + + // row now exists, so using "null" to check for existence should be considered "not match". + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1); + assertEquals(ok, false); + + Put put2 = new Put(ROW); + put2.addColumn(FAMILY, QUALIFIER, VALUE_2); + + // row now exists, use the matching value to check + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE_1, put2); + assertEquals(ok, true); + + Put put3 = new Put(ROW_1); + put3.addColumn(FAMILY, QUALIFIER, VALUE_1); + + // try to do CheckAndPut on different rows + try { + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE_2, put3); + fail("trying to check and modify different rows should have failed."); + } catch (Exception e) { + } + hTable.close(); + } + + public static void testCheckAndPutWithCompareOp(String tableName) throws Throwable { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + final byte[] value1 = Bytes.toBytes("aaaa"); + final byte[] value2 = Bytes.toBytes("bbbb"); + final byte[] value3 = Bytes.toBytes("cccc"); + final byte[] value4 = Bytes.toBytes("dddd"); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + + Put put2 = new Put(ROW); + put2.addColumn(FAMILY, QUALIFIER, value2); + + Put put3 = new Put(ROW); + put3.addColumn(FAMILY, QUALIFIER, value3); + + // row doesn't exist, so using "null" to check for existence should be considered "match". + boolean ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, null, put2); + assertEquals(ok, true); + + // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL + // turns out "match" + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value1, + put2); + assertEquals(ok, false); + ok = hTable + .checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value1, put2); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, + value1, put2); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value1, put2); + assertEquals(ok, true); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, + value1, put2); + assertEquals(ok, true); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value1, + put3); + assertEquals(ok, true); + + // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL + // turns out "match" + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value4, put3); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, + value4, put3); + assertEquals(ok, false); + ok = hTable + .checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value4, put3); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value4, + put3); + assertEquals(ok, true); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, + value4, put3); + assertEquals(ok, true); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value4, + put2); + assertEquals(ok, true); + + // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL + // turns out "match" + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value2, + put2); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value2, + put2); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value2, put2); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, + value2, put2); + assertEquals(ok, true); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, + value2, put2); + assertEquals(ok, true); + ok = hTable + .checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value2, put3); + assertEquals(ok, true); + hTable.close(); + } + + public static void testCheckAndDeleteWithCompareOp(String tableName) throws Throwable { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + final byte[] value1 = Bytes.toBytes("aaaa"); + final byte[] value2 = Bytes.toBytes("bbbb"); + final byte[] value3 = Bytes.toBytes("cccc"); + final byte[] value4 = Bytes.toBytes("dddd"); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + + Put put2 = new Put(ROW); + put2.addColumn(FAMILY, QUALIFIER, value2); + hTable.put(put2); + + Put put3 = new Put(ROW); + put3.addColumn(FAMILY, QUALIFIER, value3); + + Delete delete = new Delete(ROW); + delete.addColumns(FAMILY, QUALIFIER); + + // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL + // turns out "match" + boolean ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, + value1, delete); + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value1, + delete); + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, + CompareFilter.CompareOp.GREATER_OR_EQUAL, value1, delete); + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value1, + delete); + assertEquals(ok, true); + hTable.put(put2); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, + value1, delete); + assertEquals(ok, true); + hTable.put(put2); + + assertEquals(ok, true); + + // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL + // turns out "match" + hTable.put(put3); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value4, + delete); + + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, + value4, delete); + + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value4, + delete); + + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value4, + delete); + + assertEquals(ok, true); + hTable.put(put3); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, + CompareFilter.CompareOp.GREATER_OR_EQUAL, value4, delete); + assertEquals(ok, true); + hTable.put(put3); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, + value4, delete); + + assertEquals(ok, true); + + // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL + // turns out "match" + hTable.put(put2); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value2, + delete); + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, + value2, delete); + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value2, + delete); + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, + CompareFilter.CompareOp.GREATER_OR_EQUAL, value2, delete); + assertEquals(ok, true); + hTable.put(put2); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, + value2, delete); + assertEquals(ok, true); + hTable.put(put2); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value2, + delete); + assertEquals(ok, true); + hTable.close(); + } + + private static void testCheckAndMutateMultiCF(Map.Entry> entry) + throws Exception { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + assertTrue(entry.getValue().size() > 1); + List tableNames = entry.getValue(); + byte[] FAMILY_1 = getColumnFamilyName(tableNames.get(0)).getBytes(); + Put put2 = new Put(ROW); + put2.addColumn(FAMILY_1, QUALIFIER, VALUE_2); + hTable.put(put2); + RowMutations mutations = new RowMutations(ROW); + + Put put = new Put(ROW); + for (String tableName : tableNames) { + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + put.addColumn(FAMILY, QUALIFIER, VALUE_1); + + } + mutations.add(put); + try { + hTable.checkAndMutate(ROW, FAMILY_1, QUALIFIER, CompareFilter.CompareOp.GREATER, + VALUE_2, mutations); + fail("unexpect error, check and mutate should not support multi cf"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("multi family is not supported")); + } + + mutations = new RowMutations(ROW); + byte[] FAMILY_2 = tableNames.get(1).getBytes(); + Put put3 = new Put(ROW); + put3.addColumn(FAMILY_2, QUALIFIER, VALUE_1); + mutations.add(put2); + mutations.add(put3); + try { + hTable.checkAndMutate(ROW, FAMILY_1, QUALIFIER, CompareFilter.CompareOp.GREATER, + VALUE_2, mutations); + fail("unexpect error, check and mutate should not support multi cf"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("mutation family is not equal check family")); + } + hTable.close(); + } + + private static void testCheckAndMutateSeires(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + String column = "appColumn"; + byte[] ROW = "appendKey".getBytes(); + RowMutations mutations = new RowMutations(ROW); + Put put2 = new Put(ROW); + put2.addColumn(FAMILY, QUALIFIER, VALUE_2); + hTable.put(put2); + Put put3 = new Put(ROW); + put3.addColumn(FAMILY, QUALIFIER, VALUE_1); + mutations.add(put3); + try { + hTable.checkAndMutate(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, VALUE_2, + mutations); + fail("unexpect error, check and mutate should not support series table"); + } catch (Exception e) { + assertTrue(e.getMessage().contains( + "query and mutate with hbase series type not supported")); + } + hTable.close(); + } + + @Test + public void testCheckAndMutate() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndMutate); + } + + @Test + public void testCheckAndDelete() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndDeleteWithCompareOp); + } + + @Test + public void testCheckAndPut() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndPut); + } + + @Test + public void testCheckAndPutWithCompareOp() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndPutWithCompareOp); + } + + @Test + public void testCheckAndMutateMultiCF() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndMutateMultiCF); + } + + @Test + public void testCheckAndMutateSeires() throws Throwable { + List series_tables = new LinkedList(); + createTables(TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, series_tables, null, true); + FOR_EACH(series_tables, OHTableSecondaryPartCheckAndMutateTest::testCheckAndMutateSeires); + dropTables(series_tables, null); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java new file mode 100644 index 00000000..f1b574d1 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java @@ -0,0 +1,542 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.junit.*; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.*; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + +public class OHTableSecondaryPartDeleteTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testDeleteLastVersionImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + + Long ts1 = System.currentTimeMillis(); + Long ts2 = ts1 + 1000; + Long ts3 = ts2 + 1000; + Long ts4 = ts3 + 1000; + { // delete last version + Put put = new Put(toBytes(key)); + put.addColumn(toBytes(family), toBytes(column), ts1, toBytes(value)); + put.addColumn(toBytes(family), toBytes(column), ts2, toBytes(value)); + put.addColumn(toBytes(family), toBytes(column), ts3, toBytes(value)); + put.addColumn(toBytes(family), toBytes(column), ts4, toBytes(value)); + hTable.put(put); + + Delete delete = new Delete(toBytes(key)); + delete.addColumn(toBytes(family), toBytes(column)); + hTable.delete(delete); + + Get get = new Get(toBytes(key)); + get.addColumn(toBytes(family), toBytes(column)); + get.setMaxVersions(); + Result result = hTable.get(get); + Assert.assertEquals(3, result.size()); // ts4 is deleted + Assert(tableName, () -> Assert.assertTrue(secureCompare(value.getBytes(), result.getValue(family.getBytes(), column.getBytes())))); + for (Cell cell : result.rawCells()) { + Assert.assertTrue(cell.getTimestamp() != ts4); + } + } + hTable.close(); + } + + public static void testDeleteSpecifiedImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + + Long ts1 = System.currentTimeMillis(); + Long ts2 = ts1 + 1000; + Long ts3 = ts2 + 1000; + Long ts4 = ts3 + 1000; + + { + Put put = new Put(toBytes(key)); + put.addColumn(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); + hTable.put(put); + + Delete delete = new Delete(toBytes(key)); + delete.addColumn(toBytes(family), toBytes(column), ts1); + hTable.delete(delete); + + Get get = new Get(toBytes(key)); + get.addColumn(toBytes(family), toBytes(column)); + get.setMaxVersions(); + Result result = hTable.get(get); + Assert(tableName, ()->Assert.assertEquals(3, result.size())); // ts1 is deleted + Assert(tableName, ()->Assert.assertTrue(secureCompare((value + ts4).getBytes(), result.getValue(family.getBytes(), column.getBytes())))); + for (Cell cell : result.rawCells()) { + Assert(tableName, ()->Assert.assertTrue(cell.getTimestamp() != ts1)); + } + } + hTable.close(); + } + + public static void testDeleteColumnImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + + Long ts1 = System.currentTimeMillis(); + Long ts2 = ts1 + 1000; + Long ts3 = ts2 + 1000; + Long ts4 = ts3 + 1000; + + { + Put put = new Put(toBytes(key)); + put.addColumn(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); + hTable.put(put); + + Delete delete = new Delete(toBytes(key)); + delete.addColumns(toBytes(family), toBytes(column), ts3); + hTable.delete(delete); + + Get get = new Get(toBytes(key)); + get.addColumn(toBytes(family), toBytes(column)); + get.setMaxVersions(); + Result result = hTable.get(get); + Assert(tableName, ()->Assert.assertEquals(1, result.size())); // ts1, ts2, ts3 is deleted + Assert.assertEquals(Arrays.toString((value + ts4).getBytes()), Arrays.toString(result.getValue(family.getBytes(), column.getBytes()))); + for (Cell cell : result.rawCells()) { + Assert.assertTrue(cell.getTimestamp() != ts1); + } + } + hTable.close(); + } + + public static void testDeleteFamilyImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + + Long ts1 = System.currentTimeMillis(); + Long ts2 = ts1 + 1000; + Long ts3 = ts2 + 1000; + Long ts4 = ts3 + 1000; + + { + Put put = new Put(toBytes(key)); + put.addColumn(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); + hTable.put(put); + + Delete delete = new Delete(toBytes(key)); + delete.addFamily(toBytes(family)); + hTable.delete(delete); + + Get get = new Get(toBytes(key)); + get.addColumn(toBytes(family), toBytes(column)); + get.setMaxVersions(); + Result result = hTable.get(get); + Assert.assertEquals(0, result.size()); + } + hTable.close(); + } + + public static void testDeleteFamilyVersionImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + + Long ts1 = System.currentTimeMillis(); + Long ts2 = ts1 + 1000; + Long ts3 = ts2 + 1000; + Long ts4 = ts3 + 1000; + + { + Put put = new Put(toBytes(key)); + put.addColumn(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); + hTable.put(put); + + Delete delete = new Delete(toBytes(key)); + delete.addFamily(toBytes(family), ts2); + hTable.delete(delete); + + Get get = new Get(toBytes(key)); + get.addColumn(toBytes(family), toBytes(column)); + get.setMaxVersions(); + Result result = hTable.get(get); + Assert.assertEquals(2, result.size()); + for (Cell cell : result.rawCells()) { + Assert(tableName, ()->Assert.assertTrue(cell.getTimestamp() != ts1)); + Assert(tableName, ()->Assert.assertTrue(cell.getTimestamp() != ts2)); + } + } + hTable.close(); + } + + public static void testMultiCFDeleteLastVersionImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String value = "value"; + String column = "putColumn"; + + long ts1 = System.currentTimeMillis(); + long ts2 = ts1 + 1000; + long ts3 = ts2 + 1000; + long ts4 = ts3 + 1000; + System.out.println("ts1:" + ts1 + ", ts2:" + ts2 + ", ts3:" + ts3 + ", ts4:" + ts4); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + Put put = new Put(toBytes(key)); + Delete delete = new Delete(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(toBytes(family), toBytes(column + family), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column + family), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column + family), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column + family), ts4, toBytes(value + ts4)); + delete.addColumn(toBytes(family), toBytes(column + family)); + } + hTable.put(put); + hTable.delete(delete); + + + Get get = new Get(toBytes(key)); + get.setMaxVersions(); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(column + family)); + } + Result result = hTable.get(get); + + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + // last version ts4 is deleted, so the last version is ts3 + Assert(tableName, ()->Assert.assertTrue(secureCompare((value + ts3).getBytes(), + result.getValue(family.getBytes(), (column + family).getBytes())))); + for (Cell cell : result.rawCells()) { + Assert(tableName, ()->Assert.assertTrue("should not found last version ts4", cell.getTimestamp() != ts4)); + } + } + hTable.close(); + } + + public static void testMultiCFDeleteSpecifiedImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String value = "value"; + String column = "putColumn"; + + long ts1 = System.currentTimeMillis(); + long ts2 = ts1 + 1000; + long ts3 = ts2 + 1000; + long ts4 = ts3 + 1000; + System.out.println("ts1:" + ts1 + ", ts2:" + ts2 + ", ts3:" + ts3 + ", ts4:" + ts4); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + Put put = new Put(toBytes(key)); + Delete delete = new Delete(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(toBytes(family), toBytes(column + family), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column + family), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column + family), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column + family), ts4, toBytes(value + ts4)); + delete.addColumn(toBytes(family), toBytes(column + family), ts2); + } + hTable.put(put); + hTable.delete(delete); + + + Get get = new Get(toBytes(key)); + get.setMaxVersions(); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(column + family)); + } + Result result = hTable.get(get); + + for (String tableName : entry.getValue()) { + for (Cell cell : result.rawCells()) { + Assert(tableName, ()->Assert.assertTrue("should not found last version ts2",cell.getTimestamp() != ts2)); + } + } + hTable.close(); + } + + public static void testMultiCFDeleteColumnImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String value = "value"; + String column = "putColumn"; + + long ts1 = System.currentTimeMillis(); + long ts2 = ts1 + 1000; + long ts3 = ts2 + 1000; + long ts4 = ts3 + 1000; + System.out.println("ts1:" + ts1 + ", ts2:" + ts2 + ", ts3:" + ts3 + ", ts4:" + ts4); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + Put put = new Put(toBytes(key)); + Delete delete = new Delete(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(toBytes(family), toBytes(column + family), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column + family), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column + family), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column + family), ts4, toBytes(value + ts4)); + delete.addColumns(toBytes(family), toBytes(column + family), ts2); + } + hTable.put(put); + hTable.delete(delete); + + + Get get = new Get(toBytes(key)); + get.setMaxVersions(); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(column + family)); + } + Result result = hTable.get(get); + + for (String tableName : entry.getValue()) { + for (Cell cell : result.rawCells()) { + Assert(tableName, ()->Assert.assertTrue("should not found last version ts2",cell.getTimestamp() != ts2)); + } + } + hTable.close(); + } + + public static void testMultiCFDeleteFamilyImpl(Map.Entry> entry) + throws Exception { + String key = "putKey"; + String value = "value"; + String column = "putColumn"; + + long ts1 = System.currentTimeMillis(); + long ts2 = ts1 + 1000; + long ts3 = ts2 + 1000; + long ts4 = ts3 + 1000; + System.out.println("ts1:" + ts1 + ", ts2:" + ts2 + ", ts3:" + ts3 + ", ts4:" + ts4); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + Put put = new Put(toBytes(key)); + Delete delete = new Delete(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(toBytes(family), toBytes(column + family), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column + family), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column + family), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column + family), ts4, toBytes(value + ts4)); + delete.addFamily(toBytes(family)); + } + hTable.put(put); + hTable.delete(delete); + + Get get = new Get(toBytes(key)); + get.setMaxVersions(); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(column + family)); + } + Result result = hTable.get(get); + Assert.assertEquals(0, result.size()); + hTable.close(); + } + + public static void testMultiCFDeleteFamilyVersionImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String value = "value"; + String column = "putColumn"; + + long ts1 = System.currentTimeMillis(); + long ts2 = ts1 + 1000; + long ts3 = ts2 + 1000; + long ts4 = ts3 + 1000; + System.out.println("ts1:" + ts1 + ", ts2:" + ts2 + ", ts3:" + ts3 + ", ts4:" + ts4); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + Put put = new Put(toBytes(key)); + Delete delete = new Delete(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(toBytes(family), toBytes(column + family), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column + family), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column + family), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column + family), ts4, toBytes(value + ts4)); + delete.addFamily(toBytes(family), ts2); + } + hTable.put(put); + hTable.delete(delete); + + + Get get = new Get(toBytes(key)); + get.setMaxVersions(); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(column + family)); + } + Result result = hTable.get(get); + Assert(entry.getValue(), ()->Assert.assertEquals(6, result.size())); + hTable.close(); + } + + public static void testDeleteAllImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String value = "value"; + String column = "putColumn"; + + long ts1 = System.currentTimeMillis(); + long ts2 = ts1 + 1000; + long ts3 = ts2 + 1000; + long ts4 = ts3 + 1000; + System.out.println("ts1:" + ts1 + ", ts2:" + ts2 + ", ts3:" + ts3 + ", ts4:" + ts4); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + Put put = new Put(toBytes(key)); + Delete delete = new Delete(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(toBytes(family), toBytes(column + family), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column + family), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column + family), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column + family), ts4, toBytes(value + ts4)); + } + hTable.put(put); + hTable.delete(delete); + + + Get get = new Get(toBytes(key)); + get.setMaxVersions(); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(column + family)); + } + Result result = hTable.get(get); + Assert(entry.getValue(), ()->Assert.assertEquals(0, result.size())); + hTable.close(); + } + + @Test + public void testDelete() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteSpecifiedImpl); + } + + @Test + public void testDeleteLastVersion() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteLastVersionImpl); + } + + @Test + public void testDeleteColumn() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteColumnImpl); + } + + @Test + public void testDeleteFamily() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteFamilyImpl); + } + + @Test + public void testDeleteFamilyVersion() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteFamilyVersionImpl); + } + + @Test + public void testMultiCFDelete() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteLastVersionImpl); + } + + @Test + public void testMultiCFDeleteSpecified() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteSpecifiedImpl); + } + + @Test + public void testMultiCFDeleteColumn() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteColumnImpl); + } + + @Test + public void testMultiCFDeleteFamily() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteFamilyImpl); + } + + @Test + public void testMultiCFDeleteFamilyVersion() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteFamilyVersionImpl); + } + + @Test + public void testDeleteAll() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testDeleteAllImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java new file mode 100644 index 00000000..cae33d1e --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java @@ -0,0 +1,304 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.junit.*; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TableType.*; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + +public class OHTableSecondaryPartGetTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String[] columns = { "putColumn1", "putColumn2", "putColumn3" }; + String[] values = { "version1", "version2" }; // each column have two versions + long curTs = System.currentTimeMillis(); + long[] ts = { curTs, curTs + 1 }; // each column have two versions + String latestValue = values[1]; + long lastTs = ts[1]; + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < columns.length; j++) { + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), columns[j].getBytes(), ts[i], toBytes(values[i])); + hTable.put(put); + } + } + + // 1. get specify column + { + int index = 0; + Get get = new Get(key.getBytes()); + get.addColumn(family.getBytes(), columns[index].getBytes()); + Result r = hTable.get(get); + Assert.assertEquals(1, r.size()); + AssertKeyValue(key, columns[index], lastTs, latestValue, r.rawCells()[0]); + } + + // 2. get do not specify column + { + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Cell[] cells = result.rawCells(); + assertEquals(columns.length, cells.length); + for (int i = 0; i < columns.length; i++) { + ObHTableSecondaryPartUtil.AssertKeyValue(key, columns[i], lastTs, latestValue, + cells[i]); + } + } + + // 3. get specify versions + { + int index = 0; + Get get = new Get(key.getBytes()); + get.addColumn(family.getBytes(), columns[index].getBytes()); + get.setMaxVersions(2); + Result r = hTable.get(get); + Assert.assertEquals(2, r.size()); + AssertKeyValue(key, columns[index], ts[1], values[1], r.rawCells()[0]); + AssertKeyValue(key, columns[index], ts[0], values[0], r.rawCells()[1]); + } + + // 4. get specify time range + { + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + get.setMaxVersions(2); + get.setTimeStamp(ts[1]); + Result r = hTable.get(get); + Assert.assertEquals(columns.length, r.size()); + for (int i = 0; i < columns.length; i++) { + AssertKeyValue(key, columns[i], values[1], r.rawCells()[i]); + } + } + + // 5. get specify filter + { + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + get.setMaxVersions(2); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(values[0]))); + get.setFilter(valueFilter); + Result r = hTable.get(get); + Assert.assertEquals(columns.length, r.size()); + for (int i = 0; i < columns.length; i++) { + AssertKeyValue(key, columns[i], values[0], r.rawCells()[i]); + } + } + + hTable.close(); + } + + public static void testMultiCFGetImpl(Map.Entry> entry) throws Exception { + + // 0. prepare data + String key = "putKey"; + String[] columns = { "putColumn1", "putColumn2", "putColumn3" }; + String groupName = getTableName(entry.getKey()); + String[] values = { "version1", "version2" }; // each column have two versions + String latestValue = values[1]; + List tableNames = entry.getValue(); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + long timestamp = System.currentTimeMillis(); + long[] ts = { timestamp, timestamp + 1 }; + long lastTs = ts[1]; + hTable.init(); + + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < columns.length; j++) { + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), columns[j].getBytes(), ts[i], + toBytes(values[i])); + hTable.put(put); + } + } + } + + // 1. get specify column + { + int columnIndex = 0; + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + Get get = new Get(key.getBytes()); + get.addColumn(family.getBytes(), columns[columnIndex].getBytes()); + Result r = hTable.get(get); + Assert.assertEquals(1, r.size()); + AssertKeyValue(key, columns[columnIndex], lastTs, latestValue, r.rawCells()[0]); + } + } + + // 2. get do not specify column + { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Cell[] cells = result.rawCells(); + assertEquals(columns.length, cells.length); + for (int i = 0; i < columns.length; i++) { + ObHTableSecondaryPartUtil.AssertKeyValue(key, columns[i], lastTs, latestValue, + cells[i]); + } + } + } + + // 3. get do not specify column family + { + Get get = new Get(key.getBytes()); + Result r = hTable.get(get); + Assert.assertEquals(tableNames.size() * columns.length, r.size()); + int cur = 0; + for (String tableName : tableNames) { + for (int i = 0; i < columns.length; i++) { + AssertKeyValue(key, columns[i], lastTs, latestValue, r.rawCells()[cur]); + cur++; + } + } + } + + // 4. get specify multi cf and column + { + int columnIndex = 0; + Get get = new Get(key.getBytes()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + get.addColumn(family.getBytes(), columns[columnIndex].getBytes()); + } + Result r = hTable.get(get); + Assert.assertEquals(tableNames.size(), r.rawCells().length); + for (int i = 0; i < tableNames.size(); i++) { + AssertKeyValue(key, columns[columnIndex], lastTs, latestValue, r.rawCells()[i]); + } + } + + // 5. get specify multi cf and versions + { + Get get = new Get(key.getBytes()); + get.setMaxVersions(2); + Result r = hTable.get(get); + Assert.assertEquals(tableNames.size() * columns.length * ts.length, r.size()); + int cur = 0; + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < columns.length; i++) { + for (int k = ts.length - 1; k >= 0; k--) { + AssertKeyValue(key, family, columns[i], ts[k], values[k], r.rawCells()[cur]); + cur++; + } + } + } + } + + // 6. get specify multi cf and time range + { + Get get = new Get(key.getBytes()); + get.setMaxVersions(2); + get.setTimeStamp(ts[1]); + Result r = hTable.get(get); + Assert.assertEquals(tableNames.size() * columns.length, r.size()); + int cur = 0; + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < columns.length; i++) { + AssertKeyValue(key, family, columns[i], ts[1], values[1], r.rawCells()[cur]); + cur++; + } + } + } + + // 7. get specify multi cf and filter + { + Get get = new Get(key.getBytes()); + get.setMaxVersions(2); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(values[0]))); + get.setFilter(valueFilter); + Result r = hTable.get(get); + Assert.assertEquals(tableNames.size() * columns.length, r.size()); + int cur = 0; + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < columns.length; i++) { + AssertKeyValue(key, family, columns[i], ts[0], values[0], r.rawCells()[cur]); + cur++; + } + } + } + hTable.close(); + } + + @Test + public void testGet() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartGetTest::testGetImpl); + } + + @Test + public void testMultiCFGet() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartGetTest::testMultiCFGetImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java new file mode 100644 index 00000000..8770b1d3 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java @@ -0,0 +1,268 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTable; +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.NORMAL_TABLES; +import static org.junit.Assert.*; + +public class OHTableSecondaryPartIncrementTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testIncrement(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] cf = getColumnFamilyName(tableName).getBytes(); + byte[] row = Bytes.toBytes("rk"); + byte[] qualifier = Bytes.toBytes("qual"); + byte[] qualifier2 = Bytes.toBytes("qual2"); + byte[] val = Bytes.toBytes(0L); + Put p = new Put(row); + p.addColumn(cf, qualifier, val); + hTable.put(p); + + for (int count = 0; count < 13; count++) { + Increment inc = new Increment(row); + inc.addColumn(cf, qualifier, 100L); + hTable.increment(inc); + } + Get get = new Get(row); + get.setMaxVersions(1); + get.addFamily(cf); + Result result = hTable.get(get); + assertEquals(1300L, Bytes.toLong(CellUtil.cloneValue(result.rawCells()[0]))); + get.setMaxVersions(100); + result = hTable.get(get); + assertEquals(14, result.size()); + + Increment inc = new Increment(row); + inc.addColumn(cf, qualifier, -100L); + inc.addColumn(cf, qualifier2, -100L); + hTable.increment(inc); + get.setMaxVersions(1); + result = hTable.get(get); + assertEquals(1200L, + Bytes.toLong(CellUtil.cloneValue(result.getColumnCells(cf, qualifier).get(0)))); + assertEquals(-100L, + Bytes.toLong(CellUtil.cloneValue(result.getColumnCells(cf, qualifier2).get(0)))); + hTable.close(); + } + + private static void testIncBorder(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + try { + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + byte[] ROW = "incKey".getBytes(); + byte[] v1 = Bytes.toBytes("ab"); + byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("a"), + Bytes.toBytes("c") }; + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIERS[1], v1); + hTable.put(put); + Increment inc = new Increment(ROW); + inc.addColumn(FAMILY, QUALIFIERS[1], 2L); + try { + hTable.increment(inc); + fail("unexpect error, increment only support long value type"); + } catch (Exception e) { + assertTrue(e.getCause().getMessage().contains("OB_KV_HBASE_INCR_FIELD_IS_NOT_LONG")); + } + Get get = new Get(ROW); + get.setMaxVersions(10); + get.addFamily(FAMILY); + Result result = hTable.get(get); + assertEquals(1, result.size()); + byte[] ROW1 = "incKey1".getBytes(); + inc = new Increment(ROW1); + inc.addColumn(FAMILY, QUALIFIERS[1], 2L); + hTable.increment(inc); + get = new Get(ROW1); + get.setMaxVersions(10); + get.addFamily(FAMILY); + result = hTable.get(get); + assertEquals(1, result.size()); + assertEquals(2L, Bytes.toLong(CellUtil.cloneValue(result.rawCells()[0]))); + inc.addColumn(FAMILY, QUALIFIERS[0], 2L); + hTable.increment(inc); + get.setMaxVersions(10); + get.addFamily(FAMILY); + result = hTable.get(get); + assertEquals(3, result.size()); + assertEquals(4L, Bytes.toLong(CellUtil.cloneValue(result.getColumnCells(FAMILY, + QUALIFIERS[1]).get(0)))); + assertEquals(2L, Bytes.toLong(CellUtil.cloneValue(result.getColumnCells(FAMILY, + QUALIFIERS[1]).get(1)))); + assertEquals(2L, Bytes.toLong(CellUtil.cloneValue(result.getColumnCells(FAMILY, + QUALIFIERS[0]).get(0)))); + } finally { + hTable.close(); + } + } + + private static void testIncCon(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + try { + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + String column = "incColumn"; + byte[] ROW = "incKey".getBytes(); + long expect = 0; + ThreadPoolExecutor threadPoolExecutor = OHTable.createDefaultThreadPoolExecutor(1, 100, 100); + AtomicInteger atomicInteger = new AtomicInteger(0); + CountDownLatch countDownLatch = new CountDownLatch(100); + for (int i = 0; i < 100; i++) { + Increment inc = new Increment(ROW); + inc.addColumn(FAMILY, column.getBytes(), 2L); + threadPoolExecutor.submit(() -> { + try { + hTable.increment(inc); + atomicInteger.incrementAndGet(); + } catch (Exception e) { + if (!e.getCause().getMessage().contains("OB_TRY_LOCK_ROW_CONFLICT") && !e.getCause().getMessage().contains("OB_TIMEOUT")) { + throw new RuntimeException(e); + } + } finally { + countDownLatch.countDown(); + } + }); + } + countDownLatch.await(100000, TimeUnit.MILLISECONDS); + for (int i = 0; i < atomicInteger.get(); i++) { + expect += 2; + } + Get get = new Get(ROW); + get.setMaxVersions(1); + get.addColumn(FAMILY, column.getBytes()); + Result result = hTable.get(get); + assertEquals(expect, Bytes.toLong(CellUtil.cloneValue(result.getColumnCells(FAMILY, column.getBytes()).get(0)))); + } finally { + hTable.close(); + } + } + + private static void testIncrementMultiCF(Map.Entry> entry) + throws Exception { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + try { + List tableNames = entry.getValue(); + String column = "appColumn"; + byte[] ROW = "appendKey".getBytes(); + Long v = 11L; + Increment increment = new Increment(ROW); + for (String tableName : tableNames) { + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + increment.addColumn(FAMILY, column.getBytes(), v); + } + try { + hTable.increment(increment); + fail("unexpect error, increment should not support multi cf"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("multi family is not supported")); + } + } finally { + hTable.close(); + } + } + + private static void testIncrementSeires(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + String column = "appColumn"; + byte[] ROW = "appendKey".getBytes(); + Long v = 11L; + Increment increment = new Increment(ROW); + increment.addColumn(FAMILY, column.getBytes(), v); + try { + hTable.increment(increment); + fail("unexpect error, increment should not support series table"); + } catch (Exception e) { + assertTrue(e.getCause().getMessage() + .contains("query and mutate with hbase series type not supported")); + } + hTable.close(); + } + + @Test + public void testIncrement() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartIncrementTest::testIncrement); + } + + @Test + public void testBorderInc() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartIncrementTest::testIncBorder); + } + + @Test + public void testIncConcurrency() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartIncrementTest::testIncCon); + } + + @Test + public void testIncrementMultiCF() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartIncrementTest::testIncrementMultiCF); + } + + @Test + public void testIncrementSeires() throws Throwable { + List series_tables = new LinkedList(); + createTables(TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, series_tables, null, true); + FOR_EACH(series_tables, OHTableSecondaryPartIncrementTest::testIncrementSeires); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java new file mode 100644 index 00000000..fef4dce0 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java @@ -0,0 +1,333 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.*; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.*; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + +public class OHTableSecondaryPartPutTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testPutImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + String value = "value"; + + { // put new key and get + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value)); + put.addColumn(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value)); + hTable.put(put); + + Get get = new Get(toBytes(key)); + get.addColumn(family.getBytes(), column1.getBytes()); + get.addColumn(family.getBytes(), column2.getBytes()); + Result r = hTable.get(get); + Assert(tableName, ()->Assert.assertEquals(2, r.size())); + Assert(tableName, ()->Assert.assertTrue(ObHTableTestUtil.secureCompare((column1 + value).getBytes(), r.getValue(family.getBytes(), column1.getBytes())))); + Assert(tableName, ()->Assert.assertTrue(ObHTableTestUtil.secureCompare((column1 + value).getBytes(), r.getValue(family.getBytes(), column1.getBytes())))); + } + { // put exist key and get + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); + hTable.put(put); + + Get get = new Get(toBytes(key)); + get.addColumn(family.getBytes(), column1.getBytes()); + Result r = hTable.get(get); + Assert(tableName, ()->Assert.assertEquals(1, r.size())); + Assert(tableName, ()->Assert.assertTrue(ObHTableTestUtil.secureCompare(toBytes(column1 + value + timestamp), r.getValue(family.getBytes(), column1.getBytes())))); + } + + { // test timestamp update + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); + hTable.put(put); + + Get get = new Get(toBytes(key)); + get.addColumn(family.getBytes(), column1.getBytes()); + Result r = hTable.get(get); + Assert(tableName, ()->Assert.assertEquals(1, r.size())); + Assert(tableName, ()->Assert.assertTrue(ObHTableTestUtil.secureCompare(toBytes(column1 + value + timestamp), r.getValue(family.getBytes(), column1.getBytes())))); + Assert(tableName, ()->Assert.assertEquals(timestamp, r.rawCells()[0].getTimestamp())); + + Put put1 = new Put(toBytes(key)); + put1.addColumn(family.getBytes(), column1.getBytes(), timestamp + 100, toBytes(column1 + value)); + hTable.put(put1); + + Result r2 = hTable.get(get); + Assert(tableName, ()->Assert.assertEquals(1, r2.size())); + Assert(tableName, ()->Assert.assertTrue(ObHTableTestUtil.secureCompare(toBytes(column1 + value), r2.getValue(family.getBytes(), column1.getBytes())))); + Assert(tableName, ()->Assert.assertTrue(timestamp < r2.rawCells()[0].getTimestamp())); + } + + + + hTable.close(); + } + + public static void testBatchPutImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + String value = "value"; + { + long timestamp = System.currentTimeMillis(); + List puts = new ArrayList<>(); + Get get = new Get(toBytes(key)); + for (int i = 0; i < 10; ++i) { + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp + i)); + put.addColumn(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp + i)); + puts.add(put); + get.addColumn(family.getBytes(), column1.getBytes()); + get.addColumn(family.getBytes(), column2.getBytes()); + } + Result[] results = new Result[puts.size()]; + hTable.batch(puts, results); + Result result = hTable.get(get); + Assert(tableName, ()->Assert.assertEquals(2, result.size())); + for (Cell cell : result.rawCells()) { + Assert(tableName, ()->Assert.assertEquals(timestamp, cell.getTimestamp())); + } + Assert(tableName, () -> Assert.assertTrue(secureCompare((column1 + value + timestamp + 9).getBytes(), result.getValue(family.getBytes(), column1.getBytes())))); + } + { + long timestamp = System.currentTimeMillis(); + List puts = new ArrayList<>(); + List gets = new ArrayList<>(); + + for (int i = 0; i < 10; ++i) { + Put put = new Put(toBytes(key + i)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp + i)); + put.addColumn(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp + i)); + puts.add(put); + Get get = new Get(toBytes(key + i)); + get.addColumn(family.getBytes(), column1.getBytes()); + get.addColumn(family.getBytes(), column2.getBytes()); + gets.add(get); + } + + Result[] results = new Result[puts.size()]; + hTable.batch(puts, results); + + for (int i = 0; i < 10; ++i) { + Result result = hTable.get(gets.get(i)); + Assert(tableName, ()->Assert.assertEquals(2, result.size())); + for (Cell cell : result.rawCells()) { + Assert(tableName, ()->Assert.assertEquals(timestamp, cell.getTimestamp())); + } + int finalI = i; + Assert(tableName, () -> Assert.assertTrue(secureCompare((column1 + value + timestamp + finalI).getBytes(), result.getValue(family.getBytes(), column1.getBytes())))); + } + } + + + hTable.close(); + } + + public static void testMultiCFPutImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + String value = "value"; + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(entry.getKey())); + hTable.init(); + { + long currentTime = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + Get get = new Get(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(family.getBytes(), column1.getBytes(), currentTime, toBytes(column1 + value)); + put.addColumn(family.getBytes(), column2.getBytes(), currentTime, toBytes(column2 + value)); + get.addColumn(family.getBytes(), column1.getBytes()); + get.addColumn(family.getBytes(), column2.getBytes()); + } + hTable.put(put); + Result r = hTable.get(get); + Assert(entry.getValue(), ()->Assert.assertEquals(entry.getValue().size() * 2, r.size())); + long remoteTimestamp = r.rawCells()[0].getTimestamp(); + Assert(entry.getValue(), ()->Assert.assertTrue(remoteTimestamp >= currentTime)); + for (Cell cell : r.rawCells()) { + Assert(entry.getValue(), ()->Assert.assertEquals(remoteTimestamp, cell.getTimestamp())); + } + } + + { + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + Get get = new Get(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value)); + put.addColumn(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value)); + get.addColumn(family.getBytes(), column1.getBytes()); + get.addColumn(family.getBytes(), column2.getBytes()); + } + + hTable.put(put); + Result r = hTable.get(get); + Assert(entry.getValue(), ()->Assert.assertEquals(entry.getValue().size() * 2, r.size())); + for (Cell cell : r.rawCells()) { + Assert(entry.getValue(), ()->Assert.assertEquals(timestamp, cell.getTimestamp())); + } + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column1 + value).getBytes(), r.getValue(family.getBytes(), column1.getBytes())))); + Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column2 + value).getBytes(), r.getValue(family.getBytes(), column2.getBytes())))); + } + } + + hTable.close(); + } + + public static void testMltiCFPutBatchImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + String value = "value"; + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(entry.getKey())); + hTable.init(); + + { + long timestamp = System.currentTimeMillis(); + List puts = new ArrayList<>(); + Get get = new Get(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); + put.addColumn(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp)); + puts.add(put); + get.addColumn(family.getBytes(), column1.getBytes()); + get.addColumn(family.getBytes(), column2.getBytes()); + } + Result[] results = new Result[puts.size()]; + hTable.batch(puts, results); + Result result = hTable.get(get); + Assert(entry.getValue(), ()->Assert.assertEquals(entry.getValue().size() * 2, result.size())); + for (Cell cell : result.rawCells()) { + Assert(entry.getValue(), ()->Assert.assertEquals(timestamp, cell.getTimestamp())); + } + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + // TODO: Get/Scan返回的结果Q 带了cf, 这里预期跑不过 + Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column1 + value + timestamp).getBytes(), result.getValue(family.getBytes(), column1.getBytes())))); + Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column2 + value + timestamp).getBytes(), result.getValue(family.getBytes(), column2.getBytes())))); + } + } + + { + long timestamp = System.currentTimeMillis(); + List puts = new ArrayList<>(); + List> gets = new ArrayList<>(); + + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < 10; ++i) { + Put put = new Put(toBytes(key + i)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); + put.addColumn(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp)); + puts.add(put); + Get get = new Get(toBytes(key + i)); + get.addColumn(family.getBytes(), column1.getBytes()); + get.addColumn(family.getBytes(), column2.getBytes()); + gets.add(new Pair<>(get, family)); + } + } + Result[] results = new Result[puts.size()]; + hTable.batch(puts, results); + for (int i = 0; i < 10; ++i) { + Result result = hTable.get(gets.get(i).getFirst()); + Assert(entry.getValue(), () -> Assert.assertEquals(2, result.size())); + for (Cell cell : result.rawCells()) { + Assert(entry.getValue(), () -> Assert.assertEquals(timestamp, cell.getTimestamp())); + } + int finalI = i; + Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column1 + value + timestamp).getBytes(), result.getValue(gets.get(finalI).getSecond().getBytes(), column1.getBytes())))); + Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column2 + value + timestamp).getBytes(), result.getValue(gets.get(finalI).getSecond().getBytes(), column2.getBytes())))); + } + + } + hTable.close(); + } + + @Test + public void testPut() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartPutTest::testPutImpl); + } + + @Test + public void testBatchPut() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartPutTest::testBatchPutImpl); + } + + @Test + public void testMultiCFPut() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartPutTest::testMultiCFPutImpl); + } + + @Test + public void testMultiCFPutBatch() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartPutTest::testMltiCFPutBatchImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java new file mode 100644 index 00000000..d267f39c --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java @@ -0,0 +1,505 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.junit.*; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + +public class OHTableSecondaryPartScanTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testScanImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data + // putKey1 putColumn1 putValue1,ts + // putKey1 putColumn1 putValue2,ts+1 + // putKey1 putColumn2 putValue1,ts + // putKey1 putColumn2 putValue2,ts+1 + // ... + String family = getColumnFamilyName(tableName); + long ts = System.currentTimeMillis(); + + String keys[] = { "putKey1", "putKey2", "putKey3" }; + String columns[] = { "putColumn1", "putColumn2" }; + String values[] = { "putValue1", "putValue2" }; + long tss[] = { ts, ts + 1 }; + long lastTs = tss[1]; + String latestValue = values[1]; + + for (String key : keys) { + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), column.getBytes(), tss[i], + values[i].getBytes()); + hTable.put(put); + } + } + } + + // 1. scan specify column + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.addColumn(family.getBytes(), columns[0].getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + AssertKeyValue(keys[count], columns[0], lastTs, latestValue, cell); + count++; + } + } + assertEquals(2, count); + } + + // 2. scan do not specify column + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String column : columns) { + AssertKeyValue(keys[i], column, lastTs, latestValue, cells.get(cellIndex)); + cellIndex++; + } + } + assertEquals(columns.length * 2, cells.size()); + } + + // 3. scan specify versions + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.setMaxVersions(2); + scan.addColumn(family.getBytes(), columns[0].getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + + assertEquals(tss.length * 2, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (int k = tss.length - 1; k >= 0; k--) { + AssertKeyValue(keys[i], columns[0], tss[k], values[k], cells.get(cellIndex)); + cellIndex++; + } + } + } + + // 4. scan specify time range + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.setMaxVersions(2); + scan.setTimeStamp(tss[1]); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(columns.length * 2, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String column : columns) { + AssertKeyValue(keys[i], column, lastTs, latestValue, cells.get(cellIndex)); + cellIndex++; + } + } + } + + // 5. scan specify filter + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.setMaxVersions(2); + scan.addFamily(family.getBytes()); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(values[0]))); + scan.setFilter(valueFilter); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + Assert.assertEquals(columns.length * 2, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String column : columns) { + AssertKeyValue(keys[i], column, values[0], cells.get(cellIndex)); + cellIndex++; + } + } + } + + // 6. scan using setStartRow/setEndRow + { + Scan scan = new Scan(); + scan.setStartRow(keys[0].getBytes()); + scan.setStopRow(keys[2].getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String column : columns) { + AssertKeyValue(keys[i], column, lastTs, latestValue, cells.get(cellIndex)); + cellIndex++; + } + } + assertEquals(columns.length * 2, cells.size()); + } + + // 7. scan using batch + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.addFamily(family.getBytes()); + scan.setBatch(2); + ResultScanner scanner = hTable.getScanner(scan); + Result result = scanner.next(); + Assert.assertEquals(2, result.size()); + result = scanner.next(); + Assert.assertEquals(2, result.size()); + result = scanner.next(); + Assert.assertEquals(null, result); + } + + // 7. scan using setAllowPartialResults/setAllowPartialResults + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.addFamily(family.getBytes()); + scan.setMaxResultSize(10); + scan.setAllowPartialResults(true); + ResultScanner scanner = hTable.getScanner(scan); + for (int i = 0; i < 4; i++) { + Result result = scanner.next(); + Assert.assertEquals(1, result.size()); + } + Result result = scanner.next(); + Assert.assertEquals(null, result); + } + + // 8. scan in reverse + { + Scan scan = new Scan(keys[1].getBytes(), "putKey".getBytes()); + scan.addFamily(family.getBytes()); + scan.setReversed(true); + try { + hTable.getScanner(scan); + } catch (Exception e) { + Assert + .assertTrue(e + .getCause() + .getMessage() + .contains( + "secondary partitioned hbase table with reverse query not supported")); + } + } + hTable.close(); + } + + public static void testMultiCFScanImpl(Map.Entry> entry) throws Exception { + // 0. prepare data + // putKey1 putColumn1 putValue1,ts + // putKey1 putColumn1 putValue2,ts+1 + // putKey1 putColumn2 putValue1,ts + // putKey1 putColumn2 putValue2,ts+1 + // ... + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + long ts = System.currentTimeMillis(); + + String keys[] = { "putKey1", "putKey2", "putKey3" }; + String columns[] = { "putColumn1", "putColumn2" }; + String values[] = { "putValue1", "putValue2" }; + long tss[] = { ts, ts + 1 }; + long lastTs = tss[1]; + String latestValue = values[1]; + List tableNames = entry.getValue(); + + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String key : keys) { + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), column.getBytes(), tss[i], + values[i].getBytes()); + hTable.put(put); + } + } + } + } + + // 1. multi cf scan specify one cf and one column + { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.addColumn(family.getBytes(), columns[0].getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + Assert.assertEquals(2, cells.size()); + for (int i = 0; i < 2; i++) { + AssertKeyValue(keys[i], family, columns[0], lastTs, latestValue, cells.get(i)); + } + } + } + + // 2. multi cf scan specify one cf without specify column + { + for (String tableName : tableNames) { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + String family = getColumnFamilyName(tableName); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String column : columns) { + AssertKeyValue(keys[i], family, column, lastTs, latestValue, + cells.get(cellIndex)); + cellIndex++; + } + } + assertEquals(columns.length * 2, cells.size()); + } + } + + // 3. multi cf scan do not specify cf + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + + assertEquals(tableNames.size() * 2 * columns.length, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + AssertKeyValue(keys[i], family, column, lastTs, latestValue, + cells.get(cellIndex)); + cellIndex++; + } + } + } + } + + // 4. multi cf scan specify multi cf and multi column + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + scan.addColumn(family.getBytes(), column.getBytes()); + } + } + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + + assertEquals(tableNames.size() * 2 * columns.length, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + AssertKeyValue(keys[i], family, column, lastTs, latestValue, + cells.get(cellIndex)); + cellIndex++; + } + } + } + } + + // 5. multi cf scan specify versions + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.setMaxVersions(2); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + + assertEquals(tableNames.size() * 2 * columns.length * tss.length, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + for (int j = values.length - 1; j >= 0; j--) { + AssertKeyValue(keys[i], family, column, tss[j], values[j], + cells.get(cellIndex)); + cellIndex++; + } + } + } + } + } + + // 6. multi cf scan specify time range + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.setMaxVersions(2); + scan.setTimeStamp(tss[1]); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(tableNames.size() * columns.length * 2, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + AssertKeyValue(keys[i], family, column, lastTs, latestValue, + cells.get(cellIndex)); + cellIndex++; + } + } + } + } + + // 7. multi cf scan specify filter + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.setMaxVersions(2); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(values[0]))); + scan.setFilter(valueFilter); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + Assert.assertEquals(tableNames.size() * columns.length * 2, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + AssertKeyValue(keys[i], family, column, tss[0], values[0], + cells.get(cellIndex)); + cellIndex++; + } + } + } + } + + // 8. multi cf scan using setStartRow/setEndRow + { + Scan scan = new Scan(); + scan.setStartRow(keys[0].getBytes()); + scan.setStopRow(keys[2].getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(tableNames.size() * columns.length * 2, cells.size()); + + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + AssertKeyValue(keys[i], family, column, lastTs, latestValue, + cells.get(cellIndex)); + cellIndex++; + } + } + } + } + + // 9. multi cf scan using batch + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.setBatch(2); + ResultScanner scanner = hTable.getScanner(scan); + Result result = null; + for (String tableName : tableNames) { + result = scanner.next(); + Assert.assertEquals(2, result.size()); + result = scanner.next(); + Assert.assertEquals(2, result.size()); + } + result = scanner.next(); + Assert.assertEquals(null, result); + } + + // 10. multi cf scan with family scan and column-specific scan + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + for (int i = 0; i < tableNames.size(); i++) { + String family = getColumnFamilyName(tableNames.get(i)); + if (i % 2 == 0) { + scan.addFamily(family.getBytes()); + } else { + for (String column : columns) { + scan.addColumn(family.getBytes(), column.getBytes()); + } + } + } + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(tableNames.size() * 2 * columns.length, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + AssertKeyValue(keys[i], family, column, lastTs, latestValue, + cells.get(cellIndex)); + cellIndex++; + } + } + } + } + hTable.close(); + } + + @Test + public void testScan() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartScanTest::testScanImpl); + } + + @Test + public void testMultiCFScan() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartScanTest::testMultiCFScanImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java new file mode 100644 index 00000000..5a392586 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java @@ -0,0 +1,385 @@ +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.*; +import org.junit.*; + +import java.util.*; +import java.util.stream.Collectors; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static java.lang.Thread.sleep; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + +public class OHTableSecondaryPartTTLTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + alterTableTimeToLive(tableNames, true, 10); + for (List groupTableNames : group2tableNames.values()) { + alterTableTimeToLive(groupTableNames, true, 10); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testTTLImpl(List tableNames) throws Exception { + disableTTL(); + // 0. prepare data + String keys[] = {"putKey1", "putKey2", "putKey3"}; + String endKey = "putKey4"; + String columns[] = {"putColumn1", "putColumn2"}; + String values[] = {"putValue1", "putValue2", "putValue3", "putValue4"}; + for (String tableName : tableNames) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String family = getColumnFamilyName(tableName); + for (String key : keys) { + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), column.getBytes(), values[i].getBytes()); + hTable.put(put); + } + } + } + hTable.close(); + } + + // 1. sleep util data expired + sleep(12000); + + // 2. hbase scan/get cannot not return expired data + for (String tableName : tableNames) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String family = getColumnFamilyName(tableName); + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(0, cells.size()); + + Get get = new Get(keys[0].getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Assert.assertEquals(0, result.rawCells().length); + hTable.close(); + } + + // 3. using sql to scan expired but not delete yet hbase data + for (String tableName : tableNames) { + Assert.assertEquals(keys.length * columns.length * values.length, + getSQLTableRowCnt(tableName)); + } + + // 4. open ttl knob to delete expired hbase data + enableTTL(); + triggerTTL(); + + // 5. check util expired hbase data is deleted by ttl tasks + checkUtilTimeout(tableNames, ()-> { + try { + return getRunningNormalTTLTaskCnt() == 0; + } catch (Exception e) { + throw new RuntimeException(e); + } + }, 50000, 3000); + + for (String tableName : tableNames) { + Assert.assertEquals(0, getSQLTableRowCnt(tableName)); + } + + // 6. close ttl knob + disableTTL(); + } + + public static void testMultiCFTTLImpl(Map> group2tableNames) throws Exception { + disableTTL(); + List allTableNames = group2tableNames.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); + // 0. prepare data + String keys[] = {"putKey1", "putKey2", "putKey3"}; + String endKey = "putKey4"; + String columns[] = {"putColumn1", "putColumn2"}; + String values[] = {"putValue1", "putValue2", "putValue3", "putValue4"}; + + for (Map.Entry> entry : group2tableNames.entrySet()) { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + List tableNames = entry.getValue(); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String key : keys) { + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), column.getBytes(), values[i].getBytes()); + hTable.put(put); + } + } + } + } + hTable.close(); + } + + // 1. sleep util data expired + sleep(12000); + + // 2. hbase multi cf scan cannot not return expired data + for (Map.Entry> entry : group2tableNames.entrySet()) { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(0, cells.size()); + + Get get = new Get(keys[0].getBytes()); + Result result = hTable.get(get); + Assert.assertEquals(0, result.rawCells().length); + hTable.close(); + } + + // 3. using sql to scan expired but not delete yet hbase data + for (String tableName : allTableNames) { + Assert.assertEquals(keys.length * columns.length * values.length, + getSQLTableRowCnt(tableName)); + } + + // 4. open ttl knob to delete expired hbase data + enableTTL(); + triggerTTL(); + + // 5. check util expired hbase data is deleted by ttl tasks + checkUtilTimeout(allTableNames, ()-> { + try { + return getRunningNormalTTLTaskCnt() == 0; + } catch (Exception e) { + throw new RuntimeException(e); + } + }, 50000, 3000); + + for (String tableName : allTableNames) { + Assert.assertEquals(0, getSQLTableRowCnt(tableName)); + } + + // 6. close ttl knob + disableTTL(); + } + + void testRowkeyTTL(List tableNames, Boolean useScan, Boolean isReversed) throws Exception { + disableTTL(); + // 0. prepare data + String keys[] = {"putKey1", "putKey2", "putKey3"}; + String endKey = "putKey4"; + String reversedEndKey = "putKey"; + String columns[] = {"putColumn1", "putColumn2"}; + String values[] = {"putValue1", "putValue2", "putValue3", "putValue4"}; + for (String tableName : tableNames) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String family = getColumnFamilyName(tableName); + for (String key : keys) { + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), column.getBytes(), values[i].getBytes()); + hTable.put(put); + } + } + } + hTable.close(); + } + + // 1. sleep util data expired + sleep(12000); + + // 2. enable kv ttl + enableTTL(); + + // 3. SQL can scan expired but not delete yet hbase data + for (String tableName : tableNames) { + ObHTableTestUtil.Assert(tableName, ()-> { + try { + Assert.assertEquals(keys.length * columns.length * values.length, + getSQLTableRowCnt(tableName)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + // 4. use Hbase scan/get expired data to trigger ttl + for (String tableName : tableNames) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String family = getColumnFamilyName(tableName); + if (useScan) { + Scan scan = new Scan(); + if (isReversed) { + scan.setReversed(true); + scan.setStartRow(keys[2].getBytes()); + scan.setStopRow(reversedEndKey.getBytes()); + } else { + scan.setStartRow(keys[0].getBytes()); + scan.setStopRow(endKey.getBytes()); + } + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(0, cells.size()); + } else { + for (String key : keys) { + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + assertEquals(0, result.rawCells().length); + } + } + hTable.close(); + } + + // 5. wait to disable + checkUtilTimeout(tableNames, ()-> { + try { + Boolean passed = true; + for (int i = 0; passed && i < tableNames.size(); i++) { + if (getSQLTableRowCnt(tableNames.get(i)) > 0) { + passed = false; + } + } + return passed; + } catch (Exception e) { + throw new RuntimeException(e); + } + }, 50000, 3000); + + // 6. disable ttl + disableTTL(); + } + + void testMultiCFRowkeyTTL(Map> group2tableNames, Boolean useScan, Boolean isReversed) throws Exception { + disableTTL(); + List allTableNames = group2tableNames.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); + // 0. prepare data + String keys[] = {"putKey1", "putKey2", "putKey3"}; + String endKey = "putKey4"; + String columns[] = {"putColumn1", "putColumn2"}; + String values[] = {"putValue1", "putValue2", "putValue3", "putValue4"}; + for (Map.Entry> entry : group2tableNames.entrySet()) { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + for (String key : keys) { + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), column.getBytes(), values[i].getBytes()); + hTable.put(put); + } + } + } + } + hTable.close(); + } + + // 1. sleep util data expired + sleep(12000); + + // 2. enable kv ttl + enableTTL(); + + // 3. SQL can scan expired but not delete yet hbase data + for (String tableName : allTableNames) { + Assert.assertEquals(keys.length * columns.length * values.length, + getSQLTableRowCnt(tableName)); + } + + // 4. use Hbase scan expired data to trigger ttl + for (Map.Entry> entry : group2tableNames.entrySet()) { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + if (useScan) { + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + scan.setReversed(isReversed); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(0, cells.size()); + } else { + for (String key : keys) { + Get get = new Get(key.getBytes()); + Result result = hTable.get(get); + assertEquals(0, result.rawCells().length); + } + } + hTable.close(); + } + + // 5. wait to disable + checkUtilTimeout(allTableNames, ()-> { + try { + Boolean passed = true; + for (int i = 0; passed && i < allTableNames.size(); i++) { + if (getSQLTableRowCnt(allTableNames.get(i)) > 0) { + passed = false; + } + } + return passed; + } catch (Exception e) { + throw new RuntimeException(e); + } + }, 70000, 5000); + + // 6. disable ttl + disableTTL(); + } + + @Test + public void testTTL() throws Throwable { + testTTLImpl(tableNames); + } + + @Test + public void testMultiCFTTL() throws Throwable { + testMultiCFTTLImpl(group2tableNames); + } + + @Test + public void testRowkeyTTL() throws Exception { + testRowkeyTTL(tableNames, true, false); + testRowkeyTTL(tableNames, false, false); + // TODO: open the test after reverse scan is ok + // testRowkeyTTL(tableNames, true, true); + } + + @Test + public void testMultiCFRowkeyTTL() throws Exception { + testMultiCFRowkeyTTL(group2tableNames, true, false); + testMultiCFRowkeyTTL(group2tableNames, false, false); + // TODO: open the test after reverse scan is ok + // testMultiCFRowkeyTTL(group2tableNames, true, true); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java new file mode 100644 index 00000000..1e99f130 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java @@ -0,0 +1,148 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.junit.*; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TIMESERIES_TABLES; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + + +public class OHTableTimeSeriesGetTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TIMESERIES_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + + public static void testGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String[] columns = {"putColumn1", "putColumn2", "putColumn3"}; + String[] columns1 = {"putColumn1", "putColumn2"}; + String[] columns2 = {"putColumn3"}; + String[] values = {"version1", "version2"}; // each column have two versions + long curTs = System.currentTimeMillis(); + long[] ts = {curTs, curTs+1}; // each column have two versions + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < columns1.length; j++) { + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), columns1[j].getBytes(), ts[i], toBytes(values[i])); + hTable.put(put); + } + } + + Put put = new Put(toBytes(key)); + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < columns2.length; j++) { + put.addColumn(family.getBytes(), columns2[j].getBytes(), ts[i], toBytes(values[i])); + } + } + hTable.put(put); + + + // 1. get specify column + { + int index = 0; + Get get = new Get(key.getBytes()); + get.addColumn(family.getBytes(), columns[index].getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + assertEquals(2, cells.length); + sortCells(cells); + for (int i = values.length - 1; i >= 0; i--) { + AssertKeyValue(key, columns[index], ts[i], values[i], cells[values.length - 1-i]); + } + } + + // 2. get do not specify column + { + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Cell[] cells = result.rawCells(); + assertEquals(columns.length * values.length, cells.length); + sortCells(cells); + int idx = 0; + for (int i = 0; i < columns.length; i++) { + for (int j = values.length - 1; j >= 0; j--) { + ObHTableSecondaryPartUtil.AssertKeyValue(key, columns[i], ts[j], values[j], cells[idx]); + idx++; + } + } + } + + // 3. get specify time range + { + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + get.setTimeStamp(ts[1]); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + assertEquals(columns.length, cells.length); + sortCells(cells); + for (int i = 0; i < columns.length; i++) { + AssertKeyValue(key, columns[i], values[1],cells[i]); + } + } + + hTable.close(); + } + + @Test + public void testGet() throws Throwable { + FOR_EACH(tableNames, OHTableTimeSeriesGetTest::testGetImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java new file mode 100644 index 00000000..28c5a84f --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java @@ -0,0 +1,218 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.*; +import org.junit.*; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TIMESERIES_TABLES; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + +public class OHTableTimeSeriesScanTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TIMESERIES_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testScanImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data + // putKey1 putColumn1 putValue1,ts + // putKey1 putColumn1 putValue2,ts+1 + // putKey1 putColumn2 putValue1,ts + // putKey1 putColumn2 putValue2,ts+1 + // ... + String family = getColumnFamilyName(tableName); + long ts = System.currentTimeMillis(); + + String keys[] = {"putKey1", "putKey2", "putKey3"}; + String keys1[] = {"putKey1", "putKey2"}; + String keys2[] = {"putKey3"}; + String endKey = "putKey4"; + String columns[] = {"putColumn1", "putColumn2"}; + String values[] = {"putValue1", "putValue2"}; + long tss[] = {ts, ts + 1}; + + for (String key : keys1) { + Put put = new Put(toBytes(key)); + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + put.addColumn(family.getBytes(), column.getBytes(), tss[i], values[i].getBytes()); + } + } + hTable.put(put); + } + + for (String key : keys2) { + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + Put put = new Put(toBytes(key)); + put.addColumn(family.getBytes(), column.getBytes(), tss[i], values[i].getBytes()); + hTable.put(put); + } + } + } + + // 1. scan specify column + { + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + scan.addColumn(family.getBytes(), columns[0].getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + Cell cells[] = getCellsFromScanner(scanner).toArray(new Cell[0]); + Assert.assertEquals(keys.length*values.length, cells.length); + sortCells(cells); + int idx = 0; + for (String key : keys) { + for (int i = values.length-1; i >= 0; i--) { + AssertKeyValue(key, columns[0], tss[i], values[i], cells[idx++]); + } + } + } + + // 2. scan do not specify column + { + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + Cell cells[] = getCellsFromScanner(scanner).toArray(new Cell[0]); + Assert.assertEquals(keys.length*columns.length*values.length, cells.length); + sortCells(cells); + int idx = 0; + for (String key : keys) { + for (String column : columns) { + for (int i = values.length-1; i >= 0; i--) { + AssertKeyValue(key, column, tss[i], values[i], cells[idx++]); + } + } + } + } + + // 3. scan specify time range + { + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + scan.setTimeStamp(tss[1]); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + Cell cells[] = getCellsFromScanner(scanner).toArray(new Cell[0]); + assertEquals(keys.length * columns.length, cells.length); + sortCells(cells); + int idx = 0; + for (String key : keys) { + for (String column : columns) { + AssertKeyValue(key, column, tss[1], values[1], cells[idx++]); + } + } + } + + + // 4. scan using setStartRow/setEndRow + { + Scan scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setStartRow(keys[0].getBytes()); + scan.setStopRow(endKey.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + Cell cells[] = getCellsFromScanner(scanner).toArray(new Cell[0]); + assertEquals(keys.length * columns.length * values.length, cells.length); + sortCells(cells); + int idx = 0; + for (String key : keys) { + for (String column : columns) { + for (int i = values.length-1; i >= 0; i--) { + AssertKeyValue(key, column, tss[i], values[i], cells[idx++]); + } + } + } + } + + // 5. scan using batch + { + int batchSize = 2; + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + scan.addFamily(family.getBytes()); + scan.setBatch(batchSize); + try { + ResultScanner scanner = hTable.getScanner(scan); + } catch (Exception e) { + Assert.assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with batch query not supported")); + } + + } + + // 7. scan using setAllowPartialResults/setAllowPartialResults + { + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + scan.addFamily(family.getBytes()); + scan.setMaxResultSize(10); + scan.setAllowPartialResults(true); + try { + ResultScanner scanner = hTable.getScanner(scan); + } catch (Exception e) { + Assert.assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with allow partial results query not supported")); + } + } + + // 8. scan in reverse + { + Scan scan = new Scan(keys[2].getBytes(), keys[0].getBytes()); + scan.addFamily(family.getBytes()); + scan.setReversed(true); + try { + ResultScanner scanner = hTable.getScanner(scan); + } catch (Exception e) { + Assert.assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with reverse query not supported")); + } + } + } + + @Test + public void testScan() throws Throwable { + FOR_EACH(tableNames, OHTableTimeSeriesScanTest::testScanImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java new file mode 100644 index 00000000..03b1691b --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -0,0 +1,387 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Assert; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLSyntaxErrorException; +import java.util.*; +import java.util.function.Supplier; + +public class ObHTableSecondaryPartUtil { + public static void openDistributedExecute() throws Exception { + Connection conn = ObHTableTestUtil.getSysConnection(); + String stmt = "ALTER SYSTEM SET _obkv_feature_mode = 'distributed_execute=on';"; + conn.createStatement().execute(stmt); + } + + public static void closeDistributedExecute() throws Exception { + Connection conn = ObHTableTestUtil.getSysConnection(); + String stmt = "ALTER SYSTEM SET _obkv_feature_mode = 'distributed_execute=off';"; + conn.createStatement().execute(stmt); + } + + public static void createTables(TableTemplateManager.TableType type, List tableNames, + Map> group2tableNames, boolean printSql) + throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + // single cf table + if (tableNames != null) { + createTables(conn, type, tableNames, printSql); + } + // multi cf table + if (group2tableNames != null) { + createTables(conn, type, group2tableNames, printSql); + } + } + + public static void createTables(Connection conn, TableTemplateManager.TableType type, + List tableNames, boolean printSql) throws Exception { + // create single cf table + if (tableNames != null) { + TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); + String tableGroup = TableTemplateManager.getTableGroupName(type, false); + String tableGroupSql = TableTemplateManager.generateTableGroupSQL(tableGroup); + conn.createStatement().execute(tableGroupSql); + String tableName = TableTemplateManager.generateTableName(tableGroup, false, 1); + String sql = TableTemplateManager.getCreateTableSQL(type, tableName, timeRange); + try { + conn.createStatement().execute(sql); + System.out.println("============= create table: " + tableName + " table_group: " + + getTableName(tableName) + " =============\n" + + (printSql ? sql : "") + + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + if (!e.getMessage().contains("already exists")) { + throw e; + } else { + System.out.println("============= table: " + tableName + " table_group: " + + getTableName(tableName) + " already exist ============="); + } + } + tableNames.add(tableName); + } + } + + public static void createTables(Connection conn, TableTemplateManager.TableType type, Map> group2tableNames, boolean printSql) throws Exception { + if (group2tableNames != null) { + TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); + String tableGroup = TableTemplateManager.getTableGroupName(type, true); + String tableGroupSql = TableTemplateManager.generateTableGroupSQL(tableGroup); + try { + conn.createStatement().execute(tableGroupSql); + System.out.println("============= create table_group: " + getTableName(tableGroup) + " =============\n" + (printSql ? tableGroupSql : "") + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + if (!e.getMessage().contains("already exists")) { + throw e; + } else { + System.out.println("============= table_group: " + getTableName(tableGroup) + " already exist ============="); + } + } + group2tableNames.put(tableGroup, new LinkedList<>()); + for (int i = 1; i <= 3; ++i) { + String tableName = TableTemplateManager.generateTableName(tableGroup, true, i); + String sql = TableTemplateManager.getCreateTableSQL(type, tableName, timeRange); + try { + conn.createStatement().execute(sql); + System.out.println("============= create table: " + tableName + + " table_group: " + getTableName(tableName) + " =============\n" + + (printSql ? sql : "") + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + if (!e.getMessage().contains("already exists")) { + throw e; + } else { + System.out.println("============= table: " + tableName + " table_group: " + getTableName(tableName) + " already exist ============="); + } + } + group2tableNames.get(tableGroup).add(tableName); + } + } + } + + public static void truncateTables(List tableNames, + Map> group2tableNames) throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + // truncate single cf table + truncateTables(conn, tableNames); + // truncate multi cf table + truncateTables(conn, group2tableNames); + } + + public static void truncateTables(Connection conn, List tableNames) throws Exception { + if (tableNames != null) { + for (int i = 0; i < tableNames.size(); i++) { + String stmt = "TRUNCATE TABLE " + tableNames.get(i) + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= truncate table " + tableNames.get(i) + + " done ============="); + } + } + } + + public static void truncateTables(Connection conn, Map> group2tableNames) + throws Exception { + if (group2tableNames != null) { + for (Map.Entry> entry : group2tableNames.entrySet()) { + for (String tableName : entry.getValue()) { + String stmt = "TRUNCATE TABLE " + tableName + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= truncate table " + tableName + + " done ============="); + } + } + } + } + + public static void dropTables(List tableNames, + Map> group2tableNames) throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + // drop single cf table + dropTables(conn, tableNames); + // drop multi cf table + dropTables(conn, group2tableNames); + } + + public static void dropTables(Connection conn, List tableNames) throws Exception { + if (tableNames != null) { + for (String tableName : tableNames) { + String stmt = "DROP TABLE IF EXISTS " + tableName + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= drop table " + tableName + " done ============="); + } + } + } + + public static void dropTables(Connection conn, Map> group2tableNames) + throws Exception { + if (group2tableNames != null) { + for (Map.Entry> entry : group2tableNames.entrySet()) { + for (String tableName : entry.getValue()) { + String stmt = "DROP TABLE IF EXISTS " + tableName + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= drop table " + tableName + + " done ============="); + } + String stmt = "DROP TABLEGROUP IF EXISTS " + entry.getKey() + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= drop tablegroup " + entry.getKey() + + " done ============="); + } + } + } + + public static String getTableName(String input) throws Exception { + // 查找 '$' 的索引 + int index = input.indexOf('$'); + // 如果找到了 '$'，提取其前面的部分 + String result; + if (index != -1) { + result = input.substring(0, index); // 提取从开始到 '$' 的部分 + } else { + result = input; // 如果没有 '$' 则返回原字符串 + } + return result; + } + + public static String getColumnFamilyName(String input) throws Exception { + // 查找 '$' 的索引 + int index = input.indexOf('$'); + // 如果找到了 '$'，提取其后面的部分 + String result; + if (index != -1 && index + 1 < input.length()) { + result = input.substring(index + 1); // 提取从 '$' 后一个字符到结束的部分 + } else { + result = ""; // 如果没有 '$' 或 '$' 是最后一个字符，则返回空字符串 + } + return result; + } + + public static void alterTableTimeToLive(List tableNames, boolean printSql, + long timeToLive) throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + if (tableNames != null) { + for (String tableName : tableNames) { + String alterTableTTLSQL = "ALTER TABLE " + + tableName + + String + .format( + " kv_attributes ='{\"Hbase\": {\"TimeToLive\": %d}}';", + timeToLive); + try { + conn.createStatement().execute(alterTableTTLSQL); + System.out.println("============= alter table ttl: " + tableName + + " table_group: " + getTableName(tableName) + + " =============\n" + (printSql ? alterTableTTLSQL : "") + + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + throw e; + } + } + } + } + + public static void alterTableMaxVersion(List tableNames, boolean printSql, + long maxVersion) throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + if (tableNames != null) { + for (String tableName : tableNames) { + String alterTableTTLSQL = "ALTER TABLE " + + tableName + + String + .format( + " kv_attributes ='{\"Hbase\": {\"MaxVersions\": %d}}';", + maxVersion); + try { + conn.createStatement().execute(alterTableTTLSQL); + System.out.println("============= alter table ttl: " + tableName + + " table_group: " + getTableName(tableName) + + " =============\n" + (printSql ? alterTableTTLSQL : "") + + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + throw e; + } + } + } + } + + public static int getSQLTableRowCnt(String tableName) throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + String RowCountSQL = "SELECT COUNT(*) FROM " + tableName + ";"; + ResultSet resultSet = conn.createStatement().executeQuery(RowCountSQL); + int rowCnt = 0; + if (resultSet.next()) { + rowCnt = resultSet.getInt(1); + } + System.out.println("============= rowCnt: " + rowCnt + " ============="); + return rowCnt; + } + + public static int getRunningNormalTTLTaskCnt() throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + String RowCountSQL = "SELECT COUNT(*) FROM " + + "OCEANBASE.DBA_OB_KV_TTL_TASKS where TASK_TYPE = 'NORMAL'"; + ResultSet resultSet = conn.createStatement().executeQuery(RowCountSQL); + int rowCnt = 0; + if (resultSet.next()) { + rowCnt = resultSet.getInt(1); + } + return rowCnt; + } + + public static void enableTTL() throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + String stmt = "ALTER SYSTEM set enable_kv_ttl = true;"; + conn.createStatement().execute(stmt); + } + + public static void triggerTTL() throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + String stmt = "ALTER SYSTEM trigger TTL;"; + conn.createStatement().execute(stmt); + } + + public static void disableTTL() throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + String stmt = "ALTER SYSTEM set enable_kv_ttl = false;"; + conn.createStatement().execute(stmt); + } + + public static void AssertKeyValue(String key, String qualifier, long timestamp, String value, + Cell cell) { + Assert.assertEquals(key, Bytes.toString(CellUtil.cloneRow(cell))); + Assert.assertEquals(qualifier, Bytes.toString(CellUtil.cloneQualifier(cell))); + Assert.assertEquals(timestamp, cell.getTimestamp()); + Assert.assertEquals(value, Bytes.toString(CellUtil.cloneValue(cell))); + } + + public static void AssertKeyValue(String key, String qualifier, String value, Cell cell) { + Assert.assertEquals(key, Bytes.toString(CellUtil.cloneRow(cell))); + Assert.assertEquals(qualifier, Bytes.toString(CellUtil.cloneQualifier(cell))); + Assert.assertEquals(value, Bytes.toString(CellUtil.cloneValue(cell))); + } + + public static void AssertKeyValue(String key, String family, String qualifier, long timestamp, + String value, Cell cell) { + Assert.assertEquals(key, Bytes.toString(CellUtil.cloneRow(cell))); + Assert.assertEquals(family, Bytes.toString(CellUtil.cloneFamily(cell))); + Assert.assertEquals(qualifier, Bytes.toString(CellUtil.cloneQualifier(cell))); + Assert.assertEquals(timestamp, cell.getTimestamp()); + Assert.assertEquals(value, Bytes.toString(CellUtil.cloneValue(cell))); + } + + public static List getCellsFromScanner(ResultScanner scanner) { + List cells = new ArrayList(); + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + cells.add(cell); + } + } + return cells; + } + + public static void checkUtilTimeout(List tableNames, Supplier function, long timeout, long interval) throws Exception { + long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() - startTime < timeout) { + if (function.get()) { + return; + } + Thread.sleep(interval); + } + ObHTableTestUtil.Assert(tableNames, ()-> Assert.fail("Timeout while waiting for the function to return expected result")); + } + + public static void sortCells(Cell[] cells) { + if (cells == null || cells.length <= 1) { + return; + } + + Arrays.sort(cells, new Comparator() { + @Override + public int compare(Cell c1, Cell c2) { + if (c1 == null) + return 1; + if (c2 == null) + return -1; + int cmpRet = Bytes.compareTo(CellUtil.cloneRow(c1), CellUtil.cloneRow(c2)); + if (cmpRet != 0) { + return cmpRet; + } + + cmpRet = Bytes.compareTo(CellUtil.cloneFamily(c1), CellUtil.cloneFamily(c2)); + if (cmpRet != 0) { + return cmpRet; + } + + cmpRet = Bytes.compareTo(CellUtil.cloneQualifier(c1), CellUtil.cloneQualifier(c2)); + if (cmpRet != 0) { + return cmpRet; + } + + cmpRet = Long.compare(c2.getTimestamp(), c1.getTimestamp()); + return cmpRet; + } + }); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index 44583f74..81fd3688 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -20,7 +20,6 @@ import com.alipay.oceanbase.hbase.OHTableClient; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HTableDescriptor; import java.sql.Connection; @@ -52,14 +51,17 @@ public class ObHTableTestUtil { + JDBC_DATABASE + "?" + "useUnicode=TRUE&" + "characterEncoding=utf-8&" + "socketTimeout=3000000&" + "connectTimeout=60000"; + public static String SYS_JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " + + "oceanbase?" + "useUnicode=TRUE&" + + "characterEncoding=utf-8&" + + "socketTimeout=3000000&" + "connectTimeout=60000"; public static String SQL_FORMAT = "truncate %s"; - public static List tableNameList; + public static List tableNameList = new LinkedList(); public static Connection conn; public static Statement stmt; static { - tableNameList = new LinkedList<>(); conn = getConnection(); try { stmt = conn.createStatement(); @@ -121,29 +123,28 @@ public static OHTableClient newOHTableClient(String tableName) { } static public List getOHTableNameList(String tableGroup) throws IOException { - // 读取建表语句 - List res = new LinkedList<>(); - String sql = new String(Files.readAllBytes(Paths.get(NativeHBaseUtil.SQL_PATH))); - String[] sqlList = sql.split(";"); - Map tableMap = new LinkedHashMap<>(); - for (String singleSql : sqlList) { - String realTableName; - if (singleSql.contains("CREATE TABLE ")) { - singleSql.trim(); - String[] splits = singleSql.split(" "); - String tableGroupName = splits[2].substring(1, splits[2].length() - 1); - if (tableGroupName.contains(":")) { - String[] tmpStr = tableGroupName.split(":", 2); - tableGroupName = tmpStr[1]; - } - realTableName = tableGroupName.split("\\$", 2)[0]; - if (realTableName.equals(tableGroup)) { - res.add(tableGroupName); + // 读取建表语句 + List res = new LinkedList<>(); + String sql = new String(Files.readAllBytes(Paths.get(NativeHBaseUtil.SQL_PATH))); + String[] sqlList = sql.split(";"); + for (String singleSql : sqlList) { + String realTableName; + if (singleSql.contains("CREATE TABLE ")) { + singleSql.trim(); + String[] splits = singleSql.split(" "); + String tableGroupName = splits[2].substring(1, splits[2].length() - 1); + if (tableGroupName.contains(":")) { + String[] tmpStr = tableGroupName.split(":", 2); + tableGroupName = tmpStr[1]; + } + realTableName = tableGroupName.split("\\$", 2)[0]; + if (realTableName.equals(tableGroup)) { + res.add(tableGroupName); + } } } + return res; } - return res; - } static public Connection getConnection() { try { @@ -156,4 +157,74 @@ static public Connection getConnection() { throw new RuntimeException(e); } } + + static public Connection getSysConnection() { + try { + Class.forName("com.mysql.cj.jdbc.Driver"); + Connection conn = DriverManager + .getConnection(SYS_JDBC_URL, SYS_USER_NAME, SYS_PASSWORD); + + return conn; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @FunctionalInterface + public interface CheckedConsumer { + void accept(T t) throws Throwable; + } + + public static void FOR_EACH(List tableNames, CheckedConsumer consumer) + throws Throwable { + for (String tableName : tableNames) { + System.out.println("============================= table::{" + tableName + + "} ============================="); + consumer.accept(tableName); + } + } + + public static void FOR_EACH(Map> group2Tables, + CheckedConsumer>> consumer) + throws Throwable { + for (Map.Entry> entry : group2Tables.entrySet()) { + consumer.accept(entry); + } + } + + public static void Assert(String tableName, Runnable assertMethod) throws SQLException { + try { + assertMethod.run(); + } catch (AssertionError e) { + Connection conn = ObHTableTestUtil.getConnection(); + String selectSql = "select * from " + tableName; + System.out.println("assert fail, execute sql: " + selectSql); + java.sql.ResultSet resultSet = conn.createStatement().executeQuery(selectSql); + ResultSetPrinter.print(resultSet); + throw e; + } + } + + public static void Assert(List tableNames, Runnable assertMethod) throws SQLException { + try { + assertMethod.run(); + } catch (AssertionError e) { + for (String tableName : tableNames) { + Connection conn = ObHTableTestUtil.getConnection(); + String selectSql = "select * from " + tableName; + System.out.println("assert fail, execute sql: " + selectSql); + java.sql.ResultSet resultSet = conn.createStatement().executeQuery(selectSql); + ResultSetPrinter.print(resultSet); + } + throw e; + } + } + + public static boolean secureCompare(byte[] a, byte[] b) { + int diff = a.length ^ b.length; + for (int i = 0; i < a.length && i < b.length; i++) { + diff |= a[i] ^ b[i]; + } + return diff == 0; + } } \ No newline at end of file diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ResultSetPrinter.java b/src/test/java/com/alipay/oceanbase/hbase/util/ResultSetPrinter.java new file mode 100644 index 00000000..0ae65bb2 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ResultSetPrinter.java @@ -0,0 +1,191 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import java.sql.*; +import java.sql.Date; +import java.text.SimpleDateFormat; +import java.util.*; + +public class ResultSetPrinter { + private static final int MAX_COL_WIDTH = 30; + private static final String VERTICAL = "│"; + private static final String HORIZONTAL = "─"; + private static final String TOP_LEFT = "┌"; + private static final String TOP_RIGHT = "┐"; + private static final String MID_LEFT = "├"; + private static final String MID_RIGHT = "┤"; + private static final String BOTTOM_LEFT = "└"; + private static final String BOTTOM_RIGHT = "┘"; + private static final String CROSS = "┼"; + private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + + public static void print(ResultSet rs) throws SQLException { + ResultSetMetaData meta = rs.getMetaData(); + int colCount = meta.getColumnCount(); + + List columns = new ArrayList(); + for (int i = 1; i <= colCount; i++) { + columns.add(new ColumnMeta(meta.getColumnLabel(i), meta.getColumnType(i), meta + .getPrecision(i))); + } + + // 计算初始列宽 + List widths = new ArrayList(); + for (ColumnMeta col : columns) { + int width = Math.min(Math.max(col.name.length(), getTypeWidth(col)), MAX_COL_WIDTH); + widths.add(width); + } + + List> rows = new ArrayList>(); + while (rs.next()) { + List row = new ArrayList(); + for (int i = 0; i < colCount; i++) { + Object value = rs.getObject(i + 1); + String str = formatValue(value, widths.get(i)); + + int actualWidth = str.length(); + if (actualWidth > widths.get(i)) { + widths.set(i, Math.min(actualWidth, MAX_COL_WIDTH)); + } + row.add(str); + } + rows.add(row); + } + + StringBuilder fmt = new StringBuilder(VERTICAL); + for (int w : widths) { + fmt.append(" %-").append(w).append("s ").append(VERTICAL); + } + + printDivider(widths, TOP_LEFT, CROSS, TOP_RIGHT); + System.out.printf(fmt.toString() + "%n", getHeaders(columns)); + printDivider(widths, MID_LEFT, CROSS, MID_RIGHT); + for (List row : rows) { + System.out.printf(fmt.toString() + "%n", row.toArray()); + } + printDivider(widths, BOTTOM_LEFT, CROSS, BOTTOM_RIGHT); + } + + private static class ColumnMeta { + final String name; + final int type; + final int precision; + + ColumnMeta(String name, int type, int precision) { + this.name = name; + this.type = type; + this.precision = precision; + } + } + + private static int getTypeWidth(ColumnMeta col) { + switch (col.type) { + case Types.DATE: + return 10; + case Types.TIMESTAMP: + return 19; + case Types.INTEGER: + return 11; + case Types.DECIMAL: + case Types.NUMERIC: + return col.precision + 2; + default: + return Math.min(col.precision, MAX_COL_WIDTH); + } + } + + private static String formatValue(Object value, int maxWidth) { + if (value == null) + return "NULL"; + + if (value instanceof byte[]) { + return formatByteArray((byte[]) value, maxWidth); + } + if (value instanceof Date) { + return DATE_FORMAT.format((Date) value); + } + return truncateString(value.toString(), maxWidth); + } + + private static String formatByteArray(byte[] bytes, int maxWidth) { + if (bytes.length == 0) + return "[NULL]"; + if (isPrintable(bytes)) { + return truncateString(new String(bytes), maxWidth); + } + + if (bytes.length <= 4) { + return hexFormat(bytes, maxWidth); + } + + return base64Format(bytes, maxWidth); + } + + private static boolean isPrintable(byte[] bytes) { + for (byte b : bytes) { + if (b < 0x20 || b > 0x7E) + return false; + } + return true; + } + + private static String hexFormat(byte[] bytes, int maxWidth) { + int maxBytes = (maxWidth - 3) / 3; // [XX XX...] + maxBytes = Math.max(1, maxBytes); + + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < Math.min(bytes.length, maxBytes); i++) { + sb.append(String.format("%02X ", bytes[i])); + } + if (bytes.length > maxBytes) + sb.append("..."); + sb.setLength(sb.length() - 1); + return sb.append("]").toString(); + } + + private static String base64Format(byte[] bytes, int maxWidth) { + String base64 = Base64.getEncoder().encodeToString(bytes); + return truncateString("b64:" + base64, maxWidth); + } + + private static String truncateString(String str, int maxWidth) { + if (str.length() <= maxWidth) + return str; + return str.substring(0, maxWidth - 3) + "..."; + } + + private static Object[] getHeaders(List columns) { + String[] headers = new String[columns.size()]; + for (int i = 0; i < headers.length; i++) { + headers[i] = columns.get(i).name; + } + return headers; + } + + private static void printDivider(List widths, String left, String mid, String right) { + StringBuilder sb = new StringBuilder(left); + for (int i = 0; i < widths.size(); i++) { + sb.append(String.join("", Collections.nCopies(widths.get(i) + 2, HORIZONTAL))); + sb.append(i < widths.size() - 1 ? mid : right); + } + System.out.println(sb); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java new file mode 100644 index 00000000..75a3e32d --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -0,0 +1,378 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import java.util.Arrays; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TableType.*; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN; + +public class TableTemplateManager { + public static final long PART_NUM = 3; + public static final String TABLE_GROUP_PREFIX = "test_group_"; + public static final String COLUMN_FAMILY = "cf"; + + public enum TableType { + NON_PARTITIONED_REGULAR, NON_PARTITIONED_TIME_SERIES, SINGLE_PARTITIONED_REGULAR, SINGLE_PARTITIONED_TIME_SERIES, SECONDARY_PARTITIONED_RANGE_KEY, // RANGE-KEY分区（使用K） + SECONDARY_PARTITIONED_RANGE_KEY_GEN, // RANGE-KEY分区（使用生成列） + SECONDARY_PARTITIONED_KEY_RANGE, // KEY-RANGE分区（使用K） + SECONDARY_PARTITIONED_KEY_RANGE_GEN, // KEY-RANGE分区（使用生成列） + SECONDARY_PARTITIONED_TIME_RANGE_KEY, // 时序表RANGE-KEY + SECONDARY_PARTITIONED_TIME_KEY_RANGE, // 时序表KEY-RANGE + + /* ------------------ CELL TTL ----------------*/ + NON_PARTITIONED_REGULAR_CELL_TTL, SINGLE_PARTITIONED_REGULAR_CELL_TTL, SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, // RANGE-KEY分区（使用K） + SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, // RANGE-KEY分区（使用生成列） + SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, // KEY-RANGE分区（使用K） + SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL, // KEY-RANGE分区（使用生成列） + NON_PARTITIONED_TIME_CELL_TTL, // 时序表带CELL TTL列 + } + + public static List NORMAL_AND_SERIES_TABLES = Arrays + .asList( + NON_PARTITIONED_REGULAR, + NON_PARTITIONED_TIME_SERIES, + SINGLE_PARTITIONED_REGULAR, + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_RANGE_KEY, + SECONDARY_PARTITIONED_RANGE_KEY_GEN, + SECONDARY_PARTITIONED_KEY_RANGE, + SECONDARY_PARTITIONED_KEY_RANGE_GEN, + SECONDARY_PARTITIONED_TIME_RANGE_KEY, + SECONDARY_PARTITIONED_TIME_KEY_RANGE); + + public static List SERIES_TABLES = Arrays + .asList( + NON_PARTITIONED_TIME_SERIES, + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_TIME_RANGE_KEY, + SECONDARY_PARTITIONED_TIME_KEY_RANGE); + + public static List NORMAL_TABLES = Arrays + .asList( + NON_PARTITIONED_REGULAR, + SINGLE_PARTITIONED_REGULAR, + SECONDARY_PARTITIONED_RANGE_KEY, + SECONDARY_PARTITIONED_RANGE_KEY_GEN, + SECONDARY_PARTITIONED_KEY_RANGE, + SECONDARY_PARTITIONED_KEY_RANGE_GEN); + + public static List CELL_TTL_TABLES = Arrays + .asList( + NON_PARTITIONED_REGULAR_CELL_TTL, + SINGLE_PARTITIONED_REGULAR_CELL_TTL, + SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, + SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, + SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, + SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL); + + public static List TIMESERIES_TABLES = Arrays + .asList( + NON_PARTITIONED_TIME_SERIES, + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_TIME_RANGE_KEY, + SECONDARY_PARTITIONED_TIME_KEY_RANGE); + + private static final Map SQL_TEMPLATES = new EnumMap( + TableType.class); + + static { + // 普通表非分区表模版 + SQL_TEMPLATES.put(TableType.NON_PARTITIONED_REGULAR, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s"); + // 时序表非分区表模版 + SQL_TEMPLATES.put(TableType.NON_PARTITIONED_TIME_SERIES, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s"); + // 普通表一级分区模板 + SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); + // 时序表一级分区模板 + SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_TIME_SERIES, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); + // 普通表RANGE-KEY分区（使用K） + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE)"); + + // 合并GEN类型的注释处理 + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + // 普通表KEY-RANGE分区（使用K） + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + // 普通表KEY-RANGE分区（使用生成列） + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + // 时序表RANGE-KEY分区 + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + // 时序表KEY-RANGE分区 + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_KEY_RANGE, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE)"); + + /* ------------------ CELL TTL ----------------*/ + SQL_TEMPLATES.put(TableType.NON_PARTITIONED_REGULAR_CELL_TTL, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s"); + + SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR_CELL_TTL, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); + + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE)"); + + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + SQL_TEMPLATES.put(NON_PARTITIONED_TIME_CELL_TTL, "CREATE TABLE IF NOT EXISTS `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s"); + } + + public static String getCreateTableSQL(TableType type, String tableName, + TimeGenerator.TimeRange timeRange) { + System.out.println(tableName); + String template = SQL_TEMPLATES.get(type); + Object[] params; + String tableGroup = extractTableGroup(tableName); + + switch (type) { + case NON_PARTITIONED_REGULAR: + case NON_PARTITIONED_TIME_SERIES: + case NON_PARTITIONED_REGULAR_CELL_TTL: + case NON_PARTITIONED_TIME_CELL_TTL: + params = new Object[] { tableName, tableGroup }; + break; + case SINGLE_PARTITIONED_REGULAR: + case SINGLE_PARTITIONED_TIME_SERIES: // 合并相同处理逻辑 + case SINGLE_PARTITIONED_REGULAR_CELL_TTL: + params = new Object[] { tableName, tableGroup, PART_NUM }; + break; + case SECONDARY_PARTITIONED_RANGE_KEY: + case SECONDARY_PARTITIONED_RANGE_KEY_GEN: + case SECONDARY_PARTITIONED_KEY_RANGE: + case SECONDARY_PARTITIONED_KEY_RANGE_GEN: + case SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL: + case SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL: + case SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL: + case SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL: + boolean isGen = type.name().contains("GEN"); + params = new Object[] { tableName, getGeneratedColumn(type), tableGroup, + isGen ? "K_PREFIX" : "K", PART_NUM, timeRange.lowerBound1(), + timeRange.lowerBound1() + 86400000, timeRange.lowerBound1() + 172800000 }; + break; + case SECONDARY_PARTITIONED_TIME_RANGE_KEY: + case SECONDARY_PARTITIONED_TIME_KEY_RANGE: // 合并时序表处理 + params = new Object[] { tableName, "", tableGroup, "K", PART_NUM, + timeRange.lowerBound1(), timeRange.lowerBound1() + 86400000, + timeRange.lowerBound1() + 172800000 }; + break; + default: + throw new IllegalArgumentException("Unsupported table type"); + } + + return String.format(template, params); + } + + private static String getGeneratedColumn(TableType type) { + StringBuilder sb = new StringBuilder(); + boolean needsKPrefix = type.name().startsWith("SECONDARY_PARTITIONED") + && !type.name().contains("TIME") && type.name().contains("GEN"); + + if (needsKPrefix) { + sb.append(",\n K_PREFIX varbinary(1024) GENERATED ALWAYS AS (substring(`K`, 1, 4))"); + } + return sb.toString(); + } + + private static String getPartitionStrategy(TableType type) { + if (type.name().contains("RANGE_KEY")) { + return type.name().contains("GEN") ? "RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS " + + PART_NUM + : "RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS " + PART_NUM; + } + if (type.name().contains("KEY_RANGE")) { + return type.name().contains("GEN") ? "KEY(`K_PREFIX`) PARTITIONS " + PART_NUM + + " SUBPARTITION BY RANGE COLUMNS(`G`)" + : "KEY(`K`) PARTITIONS " + PART_NUM + " SUBPARTITION BY RANGE COLUMNS(`G`)"; + } + return ""; + } + + public static String generateTableGroupSQL(String tableGroup) { + return String + .format("CREATE TABLEGROUP IF NOT EXISTS %s SHARDING = 'ADAPTIVE'", tableGroup); + } + + public static String getTableGroupName(TableTemplateManager.TableType type, boolean multiCf) { + return TABLE_GROUP_PREFIX + type.name().toLowerCase() + (multiCf ? "_mcf" : ""); + } + + public static String generateTableName(String tableGroup, boolean multiCf, int cfIndex) { + return String + .format("%s$%s", tableGroup, multiCf ? COLUMN_FAMILY + cfIndex : COLUMN_FAMILY); + } + + public static String extractTableGroup(String tableName) { + int dollarIndex = tableName.indexOf('$'); + if (dollarIndex > 0) { + return tableName.substring(0, dollarIndex); + } + throw new IllegalArgumentException("Invalid table name: " + tableName); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java b/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java new file mode 100644 index 00000000..ae526872 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java @@ -0,0 +1,58 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +public class TimeGenerator { + public static class TimeRange { + private final long lowerBound1; + private final long lowerBound2; + private final long upperBound1; + private final long upperBound2; + + public TimeRange(long lowerBound1, long lowerBound2, long upperBound1, long upperBound2) { + this.lowerBound1 = lowerBound1; + this.lowerBound2 = lowerBound2; + this.upperBound1 = upperBound1; + this.upperBound2 = upperBound2; + } + + public long lowerBound1() { + return lowerBound1; + } + + public long lowerBound2() { + return lowerBound2; + } + + public long upperBound1() { + return upperBound1; + } + + public long upperBound2() { + return upperBound2; + } + } + + public static TimeRange generateTestTimeRange() { + long current = System.currentTimeMillis(); + return new TimeRange(current - 86400000, // 24小时前 + current, current + 86400000, // 24小时后 + current + 172800000 // 48小时后 + ); + } +}