diff --git a/pom.xml b/pom.xml index 25aee488..ac53a096 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ ${project.encoding} UTF-8 1.7.21 - 1.2.14.1-SNAPSHOT + 1.2.14.4-SNAPSHOT diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 66af86e1..3a2ea2d4 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -24,6 +24,7 @@ import com.alipay.oceanbase.hbase.result.ClientStreamScanner; import com.alipay.oceanbase.hbase.util.*; import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.mutation.BatchOperation; import com.alipay.oceanbase.rpc.mutation.result.BatchOperationResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; @@ -38,7 +39,6 @@ import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult; import com.alipay.oceanbase.rpc.table.ObHBaseParams; import com.alipay.oceanbase.rpc.table.ObKVParams; -import com.alipay.sofa.common.thread.SofaThreadPoolExecutor; import com.google.protobuf.Descriptors; import com.google.protobuf.Message; @@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.VersionInfo; import org.slf4j.Logger; @@ -65,7 +66,6 @@ import static com.alipay.oceanbase.hbase.constants.OHConstants.*; import static com.alipay.oceanbase.hbase.util.Preconditions.checkArgument; import static com.alipay.oceanbase.hbase.util.TableHBaseLoggerFactory.LCD; -import static com.alipay.oceanbase.hbase.util.TableHBaseLoggerFactory.TABLE_HBASE_LOGGER_SPACE; import static com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperation.getInstance; import static com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType.*; import static com.alipay.sofa.common.thread.SofaThreadPoolConstants.SOFA_THREAD_POOL_LOGGING_CAPABILITY; @@ -132,7 +132,7 @@ public class OHTable implements HTableInterface { /** * the buffer of put request */ - private final ArrayList writeBuffer = new ArrayList(); + private final ArrayList writeBuffer = new ArrayList<>(); /** * when the put request reach the write buffer size the do put will * flush commits automatically @@ -197,8 +197,14 @@ public OHTable(Configuration configuration, String tableName) throws IOException long keepAliveTime = configuration.getLong(HBASE_HTABLE_THREAD_KEEP_ALIVE_TIME, DEFAULT_HBASE_HTABLE_THREAD_KEEP_ALIVE_TIME); this.executePool = createDefaultThreadPoolExecutor(1, maxThreads, keepAliveTime); - this.obTableClient = ObTableClientManager - .getOrCreateObTableClient(new OHConnectionConfiguration(configuration)); + OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); + int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( + this.tableNameString, ohConnectionConf)); + this.obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); + this.obTableClient.setRuntimeRetryTimes(numRetries); + setOperationTimeout(ohConnectionConf.getOperationTimeout()); finishSetUp(); } @@ -244,8 +250,14 @@ public OHTable(Configuration configuration, final byte[] tableName, this.tableNameString = Bytes.toString(tableName); this.executePool = executePool; this.cleanupPoolOnClose = false; - this.obTableClient = ObTableClientManager - .getOrCreateObTableClient(new OHConnectionConfiguration(configuration)); + OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); + int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( + this.tableNameString, ohConnectionConf)); + this.obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); + this.obTableClient.setRuntimeRetryTimes(numRetries); + setOperationTimeout(ohConnectionConf.getOperationTimeout()); finishSetUp(); } @@ -262,6 +274,7 @@ public OHTable(Configuration configuration, final byte[] tableName, * @param executePool ExecutorService to be used. * @throws IllegalArgumentException if the param error */ + @InterfaceAudience.Private public OHTable(final byte[] tableName, final ObTableClient obTableClient, final ExecutorService executePool) { checkArgument(tableName != null, "tableNameString is blank."); @@ -298,6 +311,7 @@ public OHTable(TableName tableName, Connection connection, } else { this.cleanupPoolOnClose = false; } + this.rpcTimeout = connectionConfig.getRpcTimeout(); this.operationTimeout = connectionConfig.getOperationTimeout(); this.operationExecuteInPool = this.configuration.getBoolean( @@ -308,7 +322,15 @@ public OHTable(TableName tableName, Connection connection, DEFAULT_HBASE_HTABLE_PUT_WRITE_BUFFER_CHECK); this.writeBufferSize = connectionConfig.getWriteBufferSize(); this.tableName = tableName.getName(); - this.obTableClient = ObTableClientManager.getOrCreateObTableClient(connectionConfig); + int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( + this.tableNameString, connectionConfig)); + this.obTableClient.setRpcExecuteTimeout(rpcTimeout); + this.obTableClient.setRuntimeRetryTimes(numRetries); + setOperationTimeout(operationTimeout); + + finishSetUp(); } /** @@ -332,9 +354,8 @@ public static ThreadPoolExecutor createDefaultThreadPoolExecutor(int coreSize, i if (System.getProperty(SOFA_THREAD_POOL_LOGGING_CAPABILITY) == null) { System.setProperty(SOFA_THREAD_POOL_LOGGING_CAPABILITY, "false"); } - SofaThreadPoolExecutor executor = new SofaThreadPoolExecutor(coreSize, maxThreads, - keepAliveTime, SECONDS, new SynchronousQueue(), "OHTableDefaultExecutePool", - TABLE_HBASE_LOGGER_SPACE); + ThreadPoolExecutor executor = new ThreadPoolExecutor(coreSize, maxThreads, + keepAliveTime, SECONDS, new SynchronousQueue<>(), Threads.newDaemonThreadFactory("ohtable")); executor.allowCoreThreadTimeOut(true); return executor; } @@ -366,6 +387,36 @@ private void finishSetUp() { WRITE_BUFFER_SIZE_DEFAULT); } + private OHConnectionConfiguration setUserDefinedNamespace(String tableNameString, + OHConnectionConfiguration ohConnectionConf) + throws IOException { + if (tableNameString.indexOf(':') != -1) { + String[] params = tableNameString.split(":"); + if (params.length != 2) { + throw new IllegalArgumentException("Please check the format of self-defined " + + "namespace and qualifier: { " + + tableNameString + " }"); + } + String database = params[0]; + checkArgument(isNotBlank(database), "self-defined namespace cannot be blank or null { " + + tableNameString + " }"); + if (ohConnectionConf.isOdpMode()) { + ohConnectionConf.setDatabase(database); + } else { + String databaseSuffix = "database=" + database; + String paramUrl = ohConnectionConf.getParamUrl(); + int databasePos = paramUrl.indexOf("database"); + if (databasePos == -1) { + paramUrl += "&" + databaseSuffix; + } else { + paramUrl = paramUrl.substring(0, databasePos) + databaseSuffix; + } + ohConnectionConf.setParamUrl(paramUrl); + } + } + return ohConnectionConf; + } + @Override public byte[] getTableName() { return tableName; @@ -429,28 +480,250 @@ public Boolean[] exists(List gets) throws IOException { return objectResults; } + private BatchOperation compatOldServerPut(final List actions, final Object[] results, BatchError batchError, int i, List puts) + throws Exception { + BatchOperationResult tmpResults; + List resultMapSingleOp = new LinkedList<>(); + if (((Put)actions.get(i)).getFamilyCellMap().size() == 1) { + puts.clear(); + Put put = (Put) actions.get(i++); + String family = Bytes.toString(put.getFamilyCellMap().firstKey()); + // aggregate put + puts.add(put); + while (i < actions.size()) { + if (actions.get(i) instanceof Put && ((Put)actions.get(i)).getFamilyCellMap().size() == 1 && Objects.equals( + Bytes.toString(((Put) actions.get(i)).getFamilyCellMap().firstKey()), + family)) { + puts.add(actions.get(i++)); + } else { + break; + } + } + String realTableName = getTargetTableName(tableNameString, family, + configuration); + BatchOperation batch = buildBatchOperation(realTableName, puts, false, + resultMapSingleOp); + tmpResults = batch.execute(); + int index = 0; + for (int k = 0; k < resultMapSingleOp.size(); k++) { + if (tmpResults.getResults().get(index) instanceof ObTableException) { + if (results != null) { + results[i - puts.size() + k] = tmpResults.getResults().get(index); + } + batchError.add( + (ObTableException) tmpResults.getResults().get(index), + actions.get(i - puts.size() + k), null); + } else { + if (results != null) { + results[i - puts.size() + k] = new Result(); + } + } + index += resultMapSingleOp.get(k); + } + return null; + } else { + // multi cf put + Put put = (Put) actions.get(i); + String realTableName = getTargetTableName(tableNameString); + return buildBatchOperation(realTableName, + Collections.singletonList(put), + true, resultMapSingleOp); + + } + } + + private BatchOperation compatOldServerDel(final List actions, final Object[] results, BatchError batchError, int i) + throws Exception { + Delete delete = (Delete)actions.get(i); + List resultMapSingleOp = new LinkedList<>(); + if (delete.isEmpty()) { + return buildBatchOperation(tableNameString, + Collections.singletonList(delete), true, + resultMapSingleOp); + } else if (delete.getFamilyCellMap().size() > 1) { + boolean has_delete_family = delete.getFamilyMap().entrySet().stream() + .flatMap(entry -> entry.getValue().stream()).anyMatch( + kv -> KeyValue.Type.codeToType( + kv.getType()) == KeyValue.Type.DeleteFamily || KeyValue.Type.codeToType( + kv.getType()) == KeyValue.Type.DeleteFamilyVersion); + if (!has_delete_family) { + return buildBatchOperation(tableNameString, + Collections.singletonList(delete), true, + resultMapSingleOp); + } else { + for (Map.Entry> entry : delete.getFamilyCellMap() + .entrySet()) { + byte[] family = entry.getKey(); + String realTableName = getTargetTableName(tableNameString, + Bytes.toString(family), configuration); + // split delete + List cells = entry.getValue(); + Delete del = new Delete(delete.getRow()); + del.getFamilyCellMap().put(family, cells); + BatchOperation batch = buildBatchOperation(realTableName, + Collections.singletonList(del), false, + resultMapSingleOp); + BatchOperationResult tmpResults = batch.execute(); + if (tmpResults.getResults().get(0) instanceof ObTableException) { + if (results != null) { + results[i] = tmpResults.getResults().get(0); + } + batchError.add((ObTableException) tmpResults.getResults().get(0), actions.get(i), null); + } + } + if (results != null && results[i] == null) { + results[i] = new Result(); + } + return null; + } + } else { + byte[] family = delete.getFamilyCellMap().firstKey(); + String realTableName = getTargetTableName(tableNameString, Bytes.toString(family), configuration); + return buildBatchOperation(realTableName, + Collections.singletonList(delete), + false, resultMapSingleOp); + } + + } + + private void compatOldServerBatch(final List actions, final Object[] results, BatchError batchError) + throws Exception { + for (Row row : actions) { + if (!(row instanceof Put) && !(row instanceof Delete)) { + throw new FeatureNotSupportedException( + "not supported other type in batch yet,only support put and delete"); + } + } + List puts = new LinkedList<>(); + BatchOperation batch; + for (int i = 0; i < actions.size(); i++) { + if (actions.get(i) instanceof Put) { + batch = compatOldServerPut(actions, results, batchError, i, puts); + if (!puts.isEmpty()) { + i = i + puts.size() - 1; + } + puts.clear(); + } else { + batch = compatOldServerDel(actions, results, batchError, i); + } + if (batch != null) { + BatchOperationResult tmpResults = batch.execute(); + if (tmpResults.getResults().get(0) instanceof ObTableException) { + if (results != null) { + results[i] = tmpResults.getResults().get(0); + } + batchError.add((ObTableException) tmpResults.getResults().get(0), actions.get(i), null); + } else { + if (results != null) { + results[i] = new Result(); + } + } + } + } + } + @Override - public void batch(List actions, Object[] results) { - throw new FeatureNotSupportedException("not supported yet."); + public void batch(final List actions, final Object[] results) throws IOException{ + if (actions == null) { + return; + } + if (results != null) { + if (results.length != actions.size()) { + throw new AssertionError("results.length"); + } + } + BatchError batchError = new BatchError(); + obTableClient.setRuntimeBatchExecutor(executePool); + List resultMapSingleOp = new LinkedList<>(); + if (!CompatibilityUtil.isBatchSupport()) { + try { + compatOldServerBatch(actions, results, batchError); + } catch (Exception e) { + throw new IOException(e); + } + } else { + String realTableName = getTargetTableName(actions); + BatchOperation batch = buildBatchOperation(realTableName, actions, + tableNameString.equals(realTableName), resultMapSingleOp); + BatchOperationResult tmpResults; + try { + tmpResults = batch.execute(); + } catch (Exception e) { + throw new IOException(e); + } + int index = 0; + for (int i = 0; i != actions.size(); ++i) { + if (tmpResults.getResults().get(index) instanceof ObTableException) { + if (results != null) { + results[i] = tmpResults.getResults().get(index); + } + batchError.add((ObTableException) tmpResults.getResults().get(index), actions.get(i), null); + } else { + if (results != null) { + results[i] = new Result(); + } + } + index += resultMapSingleOp.get(i); + } + } + if (batchError.hasErrors()) { + throw batchError.makeException(); + } + } + + private String getTargetTableName(List actions) { + byte[] family = null; + for (Row action : actions) { + if (action instanceof RowMutations || action instanceof RegionCoprocessorServiceExec) { + throw new FeatureNotSupportedException("not supported yet'"); + } else { + Mutation mutation = (Mutation) action; + if (mutation.getFamilyCellMap().size() != 1) { + return getTargetTableName(tableNameString); + } else { + byte[] nextFamily = mutation.getFamilyCellMap().keySet().iterator().next(); + if (family != null && !Arrays.equals(family, nextFamily)) { + return getTargetTableName(tableNameString); + } else if (family == null) { + family = nextFamily; + } + } + } + } + return getTargetTableName(tableNameString, Bytes.toString(family), configuration); } @Override - public Object[] batch(List actions) { - throw new FeatureNotSupportedException("not supported yet."); + public Object[] batch(List actions) throws IOException { + Object[] results = new Object[actions.size()]; + batch(actions, results); + return results; } @Override public void batchCallback(List actions, Object[] results, Batch.Callback callback) throws IOException, InterruptedException { - throw new FeatureNotSupportedException("not supported yet'"); + try { + batch(actions, results); + } finally { + if (results != null) { + for (int i = 0; i < results.length; i++) { + if (!(results[i] instanceof ObTableException)) { + callback.update(null, actions.get(i).getRow(), (R) results[i]); + } + } + } + } } @Override - public Object[] batchCallback(List actions, Batch.Callback callback) - throws IOException, - InterruptedException { - throw new FeatureNotSupportedException("not supported yet'"); + public Object[] batchCallback(final List actions, + final Batch.Callback callback) throws IOException, + InterruptedException { + Object[] results = new Object[actions.size()]; + batchCallback(actions, results, callback); + return results; } public static int compareByteArray(byte[] bt1, byte[] bt2) { @@ -540,7 +813,7 @@ public Result get(final Get get) throws IOException { ServerCallable serverCallable = new ServerCallable(configuration, obTableClient, tableNameString, get.getRow(), get.getRow(), operationTimeout) { public Result call() throws IOException { - List keyValueList = new ArrayList(); + List keyValueList = new ArrayList<>(); byte[] family = new byte[] {}; ObTableQuery obTableQuery; try { @@ -550,6 +823,9 @@ public Result call() throws IOException { // In a Get operation where the family map is greater than 1 or equal to 0, // we handle this by appending the column family to the qualifier on the client side. // The server can then use this information to filter the appropriate column families and qualifiers. + if (!get.getColumnFamilyTimeRange().isEmpty()) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } NavigableSet columnFilters = new TreeSet<>(Bytes.BYTES_COMPARATOR); processColumnFilters(columnFilters, get.getFamilyMap()); obTableQuery = buildObTableQuery(get, columnFilters); @@ -563,6 +839,17 @@ public Result call() throws IOException { for (Map.Entry> entry : get.getFamilyMap() .entrySet()) { family = entry.getKey(); + if (!get.getColumnFamilyTimeRange().isEmpty()) { + Map colFamTimeRangeMap = get.getColumnFamilyTimeRange(); + if (colFamTimeRangeMap.size() > 1) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } else if (colFamTimeRangeMap.get(family) == null) { + throw new IllegalArgumentException("Get family is not matched in ColumnFamilyTimeRange"); + } else { + TimeRange tr = colFamTimeRangeMap.get(family); + get.setTimeRange(tr.getMin(), tr.getMax()); + } + } obTableQuery = buildObTableQuery(get, entry.getValue()); ObTableQueryRequest request = buildObTableQueryRequest(obTableQuery, getTargetTableName(tableNameString, Bytes.toString(family), @@ -591,8 +878,18 @@ public Result call() throws IOException { @Override public Result[] get(List gets) throws IOException { Result[] results = new Result[gets.size()]; + List> futures = new LinkedList<>(); + for (int i = 0; i < gets.size(); i++) { + int index = i; + Future future = executePool.submit(() -> get(gets.get(index))); + futures.add(future); + } for (int i = 0; i < gets.size(); i++) { - results[i] = get(gets.get(i)); + try { + results[i] = futures.get(i).get(); + } catch (Exception e) { + throw new RuntimeException("gets occur error. index:{" + i + "}", e); + } } return results; } @@ -633,6 +930,9 @@ public ResultScanner call() throws IOException { // In a Scan operation where the family map is greater than 1 or equal to 0, // we handle this by appending the column family to the qualifier on the client side. // The server can then use this information to filter the appropriate column families and qualifiers. + if (!scan.getColumnFamilyTimeRange().isEmpty()) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } NavigableSet columnFilters = new TreeSet<>(Bytes.BYTES_COMPARATOR); processColumnFilters(columnFilters, scan.getFamilyMap()); filter = buildObHTableFilter(scan.getFilter(), scan.getTimeRange(), @@ -649,6 +949,17 @@ public ResultScanner call() throws IOException { for (Map.Entry> entry : scan.getFamilyMap() .entrySet()) { family = entry.getKey(); + if (!scan.getColumnFamilyTimeRange().isEmpty()) { + Map colFamTimeRangeMap = scan.getColumnFamilyTimeRange(); + if (colFamTimeRangeMap.size() > 1) { + throw new FeatureNotSupportedException("setColumnFamilyTimeRange is only supported in single column family for now"); + } else if (colFamTimeRangeMap.get(family) == null) { + throw new IllegalArgumentException("Scan family is not matched in ColumnFamilyTimeRange"); + } else { + TimeRange tr = colFamTimeRangeMap.get(family); + scan.setTimeRange(tr.getMin(), tr.getMax()); + } + } filter = buildObHTableFilter(scan.getFilter(), scan.getTimeRange(), scan.getMaxVersions(), entry.getValue()); obTableQuery = buildObTableQuery(filter, scan); @@ -799,67 +1110,11 @@ public boolean checkAndPut(byte[] row, byte[] family, byte[] qualifier, private void innerDelete(Delete delete) throws IOException { checkArgument(delete.getRow() != null, "row is null"); - List errorCodeList = new ArrayList(); - BatchOperationResult results = null; - try { - checkFamilyViolation(delete.getFamilyMap().keySet(), false); - if (delete.getFamilyMap().isEmpty()) { - // For a Delete operation without any qualifiers, we construct a DeleteFamily request. - // The server then performs the operation on all column families. - KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), - KeyValue.Type.DeleteFamily); - - BatchOperation batch = buildBatchOperation(tableNameString, Arrays.asList(kv), - false, null); - results = batch.execute(); - } else if (delete.getFamilyMap().size() > 1) { - // Currently, the Delete Family operation type cannot transmit qualifiers to the server. - // As a result, the server cannot identify which families need to be deleted. - // Therefore, this process is handled sequentially. - boolean hasDeleteFamily = false; - for (Map.Entry> entry : delete.getFamilyMap().entrySet()) { - for (KeyValue kv : entry.getValue()) { - if (KeyValue.Type.codeToType(kv.getType()) == KeyValue.Type.DeleteFamily) { - hasDeleteFamily = true; - break; - } - } - if (hasDeleteFamily) { - break; - } - } - if (!hasDeleteFamily) { - BatchOperation batch = buildBatchOperation(tableNameString, - delete.getFamilyMap(), false, null); - results = batch.execute(); - } else { - for (Map.Entry> entry : delete.getFamilyMap().entrySet()) { - BatchOperation batch = buildBatchOperation( - getTargetTableName(tableNameString, Bytes.toString(entry.getKey()), - configuration), entry.getValue(), false, null); - results = batch.execute(); - } - } - } else { - Map.Entry> entry = delete.getFamilyMap().entrySet() - .iterator().next(); - - BatchOperation batch = buildBatchOperation( - getTargetTableName(tableNameString, Bytes.toString(entry.getKey()), - configuration), entry.getValue(), false, null); - results = batch.execute(); - } - - errorCodeList = results.getErrorCodeList(); - boolean hasError = results.hasError(); - if (hasError) { - throw results.getFirstException(); - } + batch(Collections.singletonList(delete)); } catch (Exception e) { - logger.error(LCD.convert("01-00004"), tableNameString, errorCodeList, e); - throw new IOException("delete table " + tableNameString + " error codes " - + errorCodeList, e); + logger.error(LCD.convert("01-00004"), tableNameString, e); + throw new IOException("delete table " + tableNameString + " error.", e); } } @@ -1136,77 +1391,34 @@ public boolean isAutoFlush() { public void flushCommits() throws IOException { try { + if (writeBuffer.isEmpty()){ + return; + } + Map exceptionRowMap = new LinkedHashMap(); boolean[] resultSuccess = new boolean[writeBuffer.size()]; try { - Map, List>> familyMap = new HashMap, List>>(); - for (int i = 0; i < writeBuffer.size(); i++) { - Put aPut = writeBuffer.get(i); - Map> innerFamilyMap = aPut.getFamilyMap(); - if (innerFamilyMap.size() > 1) { - // Bypass logic: directly construct BatchOperation for puts with family map size > 1 - try { - BatchOperation batch = buildBatchOperation(this.tableNameString, - innerFamilyMap, false, null); - BatchOperationResult results = batch.execute(); - - boolean hasError = results.hasError(); - resultSuccess[i] = !hasError; - if (hasError) { - throw results.getFirstException(); - } - } catch (Exception e) { - logger.error(LCD.convert("01-00008"), tableNameString, null, autoFlush, - writeBuffer.size(), e); - throw new IOException("put table " + tableNameString + " error codes " - + null + "auto flush " + autoFlush - + " current buffer size " + writeBuffer.size(), e); - } - } else { - // Existing logic for puts with family map size = 1 - for (Map.Entry> entry : innerFamilyMap.entrySet()) { - String family = Bytes.toString(entry.getKey()); - Pair, List> keyValueWithIndex = familyMap - .get(family); - if (keyValueWithIndex == null) { - keyValueWithIndex = new Pair, List>( - new ArrayList(), new ArrayList()); - familyMap.put(family, keyValueWithIndex); - } - keyValueWithIndex.getFirst().add(i); - keyValueWithIndex.getSecond().addAll(entry.getValue()); - } - } - } - for (Map.Entry, List>> entry : familyMap - .entrySet()) { - List errorCodeList = new ArrayList(entry.getValue() - .getSecond().size()); - try { - String targetTableName = getTargetTableName(this.tableNameString, - entry.getKey(), configuration); - - BatchOperation batch = buildBatchOperation(targetTableName, entry - .getValue().getSecond(), false, null); - BatchOperationResult results = batch.execute(); - - errorCodeList = results.getErrorCodeList(); - boolean hasError = results.hasError(); - for (Integer index : entry.getValue().getFirst()) { - resultSuccess[index] = !hasError; + String realTableName = getTargetTableName(writeBuffer); + List resultMapSingleOp = new LinkedList<>(); + BatchOperation batch = buildBatchOperation(realTableName, writeBuffer, tableNameString.equals(realTableName), resultMapSingleOp); + BatchOperationResult results = batch.execute(); + if (results != null) { + int index = 0; + for (int i = 0; i != resultSuccess.length; ++i) { + if (results.getResults().get(index) instanceof ObTableException) { + resultSuccess[i] = false; + exceptionRowMap.put((ObTableException)results.getResults().get(index), writeBuffer.get(i)); + } else { + resultSuccess[i] = true; } - if (hasError) { - throw results.getFirstException(); - } - } catch (Exception e) { - logger.error(LCD.convert("01-00008"), tableNameString, errorCodeList, - autoFlush, writeBuffer.size(), e); - throw new IOException("put table " + tableNameString + " error codes " - + errorCodeList + "auto flush " + autoFlush - + " current buffer size " + writeBuffer.size(), e); + index += resultMapSingleOp.get(i); } - } - + } catch (Exception e) { + logger.error(LCD.convert("01-00008"), tableNameString, null, autoFlush, + writeBuffer.size(), e); + throw new IOException("put table " + tableNameString + " error codes " + null + + "auto flush " + autoFlush + " current buffer size " + + writeBuffer.size(), e); } finally { // mutate list so that it is empty for complete success, or contains // only failed records results are returned in the same order as the @@ -1218,6 +1430,12 @@ public void flushCommits() throws IOException { writeBuffer.remove(i); } } + if (!exceptionRowMap.isEmpty()) { + exceptionRowMap.forEach((e, row)->{ + logger.error(LCD.convert("01-00008"), row, tableNameString, autoFlush, + writeBuffer.size(), e); + }); + } } } finally { if (clearBufferOnFail) { @@ -1528,14 +1746,16 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, byte[] start, boolean includeStart, byte[] stop, boolean includeStop, boolean isReversed) { ObNewRange obNewRange = new ObNewRange(); - + ObBorderFlag obBorderFlag = new ObBorderFlag(); if (Arrays.equals(start, HConstants.EMPTY_BYTE_ARRAY)) { obNewRange.setStartKey(ObRowKey.getInstance(ObObj.getMin(), ObObj.getMin(), ObObj.getMin())); } else if (includeStart) { obNewRange.setStartKey(ObRowKey.getInstance(start, ObObj.getMin(), ObObj.getMin())); + obBorderFlag.setInclusiveStart(); } else { obNewRange.setStartKey(ObRowKey.getInstance(start, ObObj.getMax(), ObObj.getMax())); + obBorderFlag.unsetInclusiveStart(); } if (Arrays.equals(stop, HConstants.EMPTY_BYTE_ARRAY)) { @@ -1543,10 +1763,13 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, byte[] start, ObObj.getMax())); } else if (includeStop) { obNewRange.setEndKey(ObRowKey.getInstance(stop, ObObj.getMax(), ObObj.getMax())); + obBorderFlag.setInclusiveEnd(); } else { obNewRange.setEndKey(ObRowKey.getInstance(stop, ObObj.getMin(), ObObj.getMin())); + obBorderFlag.unsetInclusiveEnd(); } ObTableQuery obTableQuery = new ObTableQuery(); + obNewRange.setBorderFlag(obBorderFlag); if (isReversed) { obTableQuery.setScanOrder(ObScanOrder.Reverse); } @@ -1623,62 +1846,54 @@ public static ObTableBatchOperation buildObTableBatchOperation(List ke return batch; } - private ObTableBatchOperation buildObTableBatchOperation(Map> familyMap, - boolean putToAppend, - List qualifiers) { - ObTableBatchOperation batch = new ObTableBatchOperation(); - for (Map.Entry> entry : familyMap.entrySet()) { - byte[] family = entry.getKey(); - List keyValueList = entry.getValue(); - for (KeyValue kv : keyValueList) { - if (qualifiers != null) { - qualifiers - .add((Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())) - .getBytes()); - } - KeyValue new_kv = modifyQualifier(kv, - (Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())).getBytes()); - batch.addTableOperation(buildObTableOperation(new_kv, putToAppend)); - } - } - batch.setSameType(true); - batch.setSamePropertiesNames(true); - return batch; - } - private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(KeyValue kv, - boolean putToAppend) { + ObTableOperationType operationType, + boolean isTableGroup) { KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getType()); - switch (kvType) { - case Put: - ObTableOperationType operationType; - if (putToAppend) { - operationType = APPEND; - } else { - operationType = INSERT_OR_UPDATE; - } + switch (operationType) { + case INSERT_OR_UPDATE: + case APPEND: return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(operationType, ROW_KEY_COLUMNS, new Object[] { kv.getRow(), kv.getQualifier(), kv.getTimestamp() }, V_COLUMNS, new Object[] { kv.getValue() }); - case Delete: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), kv.getQualifier(), kv.getTimestamp() }, null, null); - case DeleteColumn: - return com.alipay.oceanbase.rpc.mutation.Mutation - .getInstance(DEL, ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), kv.getQualifier(), -kv.getTimestamp() }, null, - null); - case DeleteFamily: - return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, - new Object[] { kv.getRow(), null, -kv.getTimestamp() }, null, null); + case DEL: + switch (kvType) { + case Delete: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), kv.getQualifier(), kv.getTimestamp() }, + null, null); + case Maximum: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), null, -kv.getTimestamp() }, null, null); + case DeleteColumn: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), kv.getQualifier(), -kv.getTimestamp() }, + null, null); + case DeleteFamily: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), isTableGroup ? kv.getQualifier() : null, + -kv.getTimestamp() }, null, null); + case DeleteFamilyVersion: + return com.alipay.oceanbase.rpc.mutation.Mutation.getInstance( + DEL, + ROW_KEY_COLUMNS, + new Object[] { kv.getRow(), isTableGroup ? kv.getQualifier() : null, + kv.getTimestamp() }, null, null); + default: + throw new IllegalArgumentException("illegal mutation type " + kvType); + } default: - throw new IllegalArgumentException("illegal mutation type " + kvType); + throw new IllegalArgumentException("illegal mutation type " + operationType); } } private KeyValue modifyQualifier(KeyValue original, byte[] newQualifier) { - // Extract existing components + // Extract existing components byte[] row = original.getRow(); byte[] family = original.getFamily(); byte[] value = original.getValue(); @@ -1689,36 +1904,67 @@ private KeyValue modifyQualifier(KeyValue original, byte[] newQualifier) { value); } - private BatchOperation buildBatchOperation(String tableName, - Map> familyMap, - boolean putToAppend, List qualifiers) { + private BatchOperation buildBatchOperation(String tableName, List actions, + boolean isTableGroup, List resultMapSingleOp) + throws FeatureNotSupportedException, + IllegalArgumentException { BatchOperation batch = obTableClient.batchOperation(tableName); - - for (Map.Entry> entry : familyMap.entrySet()) { - byte[] family = entry.getKey(); - List keyValueList = entry.getValue(); - for (KeyValue kv : keyValueList) { - if (qualifiers != null) { - qualifiers.add(kv.getQualifier()); + int posInList = -1; + int singleOpResultNum; + for (Row row : actions) { + singleOpResultNum = 0; + posInList++; + if (row instanceof Put) { + Put put = (Put) row; + if (put.isEmpty()) { + throw new IllegalArgumentException("No columns to insert for #" + + (posInList + 1) + " item"); } - KeyValue new_kv = modifyQualifier(kv, - (Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())).getBytes()); - batch.addOperation(buildMutation(new_kv, putToAppend)); - } - } - - batch.setEntityType(ObTableEntityType.HKV); - return batch; - } - - private BatchOperation buildBatchOperation(String tableName, List keyValueList, - boolean putToAppend, List qualifiers) { - BatchOperation batch = obTableClient.batchOperation(tableName); - for (KeyValue kv : keyValueList) { - if (qualifiers != null) { - qualifiers.add(kv.getQualifier()); + for (Map.Entry> entry : put.getFamilyMap().entrySet()) { + byte[] family = entry.getKey(); + List keyValueList = entry.getValue(); + for (KeyValue kv : keyValueList) { + singleOpResultNum++; + if (isTableGroup) { + KeyValue new_kv = modifyQualifier(kv, + (Bytes.toString(family) + "." + Bytes.toString(kv.getQualifier())) + .getBytes()); + batch + .addOperation(buildMutation(new_kv, INSERT_OR_UPDATE, isTableGroup)); + } else { + batch.addOperation(buildMutation(kv, INSERT_OR_UPDATE, isTableGroup)); + } + } + } + } else if (row instanceof Delete) { + Delete delete = (Delete) row; + if (delete.isEmpty()) { + singleOpResultNum++; + KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), + KeyValue.Type.Maximum); + batch.addOperation(buildMutation(kv, DEL, isTableGroup)); + } else { + for (Map.Entry> entry : delete.getFamilyMap().entrySet()) { + byte[] family = entry.getKey(); + List keyValueList = entry.getValue(); + for (KeyValue kv : keyValueList) { + singleOpResultNum++; + if (isTableGroup) { + KeyValue new_kv = modifyQualifier(kv, + (Bytes.toString(family) + "." + Bytes.toString(kv + .getQualifier())).getBytes()); + batch.addOperation(buildMutation(new_kv, DEL, true)); + } else { + batch.addOperation(buildMutation(kv, DEL, false)); + } + } + } + } + } else { + throw new FeatureNotSupportedException( + "not supported other type in batch yet,only support put and delete"); } - batch.addOperation(buildMutation(kv, putToAppend)); + resultMapSingleOp.add(singleOpResultNum); } batch.setEntityType(ObTableEntityType.HKV); return batch; diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java b/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java index d4b92d08..507a09b1 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTablePool.java @@ -49,6 +49,7 @@ public class OHTablePool implements Closeable { + private String originTabelName = null; private final PoolMap tables; private final int maxSize; private final PoolMap.PoolType poolType; @@ -314,6 +315,9 @@ int getCurrentPoolSize(String tableName) { * @param paramUrl the table root server http url */ public void setParamUrl(final String tableName, String paramUrl) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_PARAM_URL, Bytes.toBytes(paramUrl)); } @@ -334,6 +338,9 @@ public String getParamUrl(final String tableName) { * @param fullUserName the table login username */ public void setFullUserName(final String tableName, final String fullUserName) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_FULL_USER_NAME, Bytes.toBytes(fullUserName)); } @@ -354,6 +361,9 @@ public String getFullUserName(final String tableName) { * @param password the table login password */ public void setPassword(final String tableName, final String password) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_PASSWORD, Bytes.toBytes(password)); } @@ -374,6 +384,9 @@ public String getPassword(final String tableName) { * @param sysUserName the sys username */ public void setSysUserName(final String tableName, final String sysUserName) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_SYS_USER_NAME, Bytes.toBytes(sysUserName)); } @@ -394,6 +407,9 @@ public String getSysUserName(final String tableName) { * @param sysPassword the sys user password */ public void setSysPassword(final String tableName, final String sysPassword) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_SYS_PASSWORD, Bytes.toBytes(sysPassword)); } @@ -481,6 +497,10 @@ public long getWriteBufferSize(String tableName) { .toLong(attr); } + public String getOriginTableName() { + return this.originTabelName; + } + /** * Sets the operation timeout for the specified tables in this pool. * @@ -520,6 +540,9 @@ public int getOperationTimeout(String tableName) { * @param odpAddr ODP address */ public void setOdpAddr(final String tableName, String odpAddr) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_ODP_ADDR, Bytes.toBytes(odpAddr)); } @@ -545,6 +568,9 @@ public String getOdpAddr(String tableName) { * @param odpPort ODP port */ public void setOdpPort(final String tableName, int odpPort) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_ODP_PORT, Bytes.toBytes(odpPort)); } @@ -564,6 +590,9 @@ public int getOdpPort(String tableName) { * @param odpMode ODP mode */ public void setOdpMode(final String tableName, boolean odpMode) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_ODP_MODE, Bytes.toBytes(odpMode)); } @@ -583,6 +612,9 @@ public boolean getOdpMode(String tableName) { * @param database ODP database name */ public void setDatabase(final String tableName, String database) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableAttribute(tableName, HBASE_OCEANBASE_DATABASE, Bytes.toBytes(database)); } @@ -653,6 +685,9 @@ public Object getTableExtendAttribute(String tableName, String attributeName) { } public void setRuntimeBatchExecutor(String tableName, ExecutorService runtimeBatchExecutor) { + if (originTabelName == null) { + originTabelName = tableName; + } setTableExtendAttribute(tableName, HBASE_OCEANBASE_BATCH_EXECUTOR, runtimeBatchExecutor); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java index bda64927..e10ef0b1 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java +++ b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java @@ -159,4 +159,8 @@ public final class OHConstants { public static final int MAX_KEYVALUE_SIZE_DEFAULT = -1; + public static final String SOCKET_TIMEOUT = "ipc.socket.timeout"; + + public static final int DEFAULT_SOCKET_TIMEOUT = 20000; // 20 seconds + } diff --git a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java index e58d2590..296f70bc 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java +++ b/src/main/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtils.java @@ -20,12 +20,14 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.filter.*; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import java.lang.reflect.Field; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.List; +import java.util.Set; @InterfaceAudience.Private public class HBaseFilterUtils { @@ -40,15 +42,21 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte throws IOException { if (filter == null) { throw new IllegalArgumentException("Filter is null"); + } else if (filter instanceof DependentColumnFilter) { + toParseableByteArray(byteStream, (DependentColumnFilter) filter); } else if (filter instanceof CompareFilter) { - // RowFilter, ValueFilter, QualifierFilter + // RowFilter, ValueFilter, QualifierFilter, FamilyFilter toParseableByteArray(byteStream, (CompareFilter) filter); + } else if (filter instanceof SingleColumnValueExcludeFilter) { + toParseableByteArray(byteStream, (SingleColumnValueExcludeFilter) filter); } else if (filter instanceof SingleColumnValueFilter) { toParseableByteArray(byteStream, (SingleColumnValueFilter) filter); } else if (filter instanceof PageFilter) { toParseableByteArray(byteStream, (PageFilter) filter); } else if (filter instanceof ColumnCountGetFilter) { toParseableByteArray(byteStream, (ColumnCountGetFilter) filter); + } else if (filter instanceof FirstKeyValueMatchingQualifiersFilter) { + toParseableByteArray(byteStream, (FirstKeyValueMatchingQualifiersFilter) filter); } else if (filter instanceof PrefixFilter) { toParseableByteArray(byteStream, (PrefixFilter) filter); } else if (filter instanceof FilterList) { @@ -63,8 +71,18 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Filte toParseableByteArray(byteStream, (FirstKeyOnlyFilter) filter); } else if (filter instanceof KeyOnlyFilter) { toParseableByteArray(byteStream, (KeyOnlyFilter) filter); + } else if (filter instanceof FuzzyRowFilter) { + toParseableByteArray(byteStream, (FuzzyRowFilter) filter); } else if (filter instanceof TimestampsFilter) { toParseableByteArray(byteStream, (TimestampsFilter) filter); + } else if (filter instanceof MultiRowRangeFilter) { + toParseableByteArray(byteStream, (MultiRowRangeFilter) filter); + } else if (filter instanceof InclusiveStopFilter) { + toParseableByteArray(byteStream, (InclusiveStopFilter) filter); + } else if (filter instanceof ColumnRangeFilter) { + toParseableByteArray(byteStream, (ColumnRangeFilter) filter); + } else if (filter instanceof MultipleColumnPrefixFilter) { + toParseableByteArray(byteStream, (MultipleColumnPrefixFilter) filter); } else if (filter instanceof SkipFilter) { toParseableByteArray(byteStream, (SkipFilter) filter); } else if (filter instanceof WhileMatchFilter) { @@ -149,6 +167,26 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write(')'); } + // SingleColumnValueExcludeFilter('cf1','col1',=,'binary:123',true,true) + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + SingleColumnValueExcludeFilter filter) + throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getFilterIfMissing()).getBytes()); + byteStream.write(','); + byteStream.write(Boolean.toString(filter.getLatestVersionOnly()).getBytes()); + byteStream.write(')'); + } + // PageFilter(100); private static void toParseableByteArray(ByteArrayOutputStream byteStream, PageFilter filter) throws IOException { @@ -167,6 +205,34 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write(')'); } + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + DependentColumnFilter filter) throws IOException { + // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ')' + if (filter.getComparator() == null) { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(filter.getDropDependentColumn()).getBytes()); + byteStream.write(')'); + } else { // DependentColumnFilter '(' family ',' qualifier ',' BOOL_VALUE ',' compare_op ',' comparator ')' + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write("('".getBytes()); + writeBytesWithEscape(byteStream, filter.getFamily()); + byteStream.write("','".getBytes()); + writeBytesWithEscape(byteStream, filter.getQualifier()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(filter.getDropDependentColumn()).getBytes()); + byteStream.write(','); + byteStream.write(toParseableByteArray(filter.getOperator())); + byteStream.write(','); + toParseableByteArray(byteStream, filter.getComparator()); + byteStream.write(')'); + } + } + private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnPaginationFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); @@ -213,6 +279,35 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, KeyOn byteStream.write(')'); } + // FuzzyRowFilter('abc','101','ddd','010'); + private static void toParseableByteArray(ByteArrayOutputStream byteStream, FuzzyRowFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + + List> fuzzyKeysData; + try { + Field field = filter.getClass().getDeclaredField("fuzzyKeysData"); + field.setAccessible(true); + fuzzyKeysData = (List>)field.get(filter); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + for (int i = 0; i < fuzzyKeysData.size(); i ++) { + Pair data = fuzzyKeysData.get(i); + byteStream.write("'".getBytes()); + writeBytesWithEscape(byteStream, data.getFirst()); + byteStream.write("'".getBytes()); + byteStream.write(','); + byteStream.write("'".getBytes()); + writeBytesWithEscape(byteStream, data.getSecond()); + byteStream.write("'".getBytes()); + if (i < fuzzyKeysData.size() - 1) { + byteStream.write(','); + } + } + byteStream.write(')'); + } + private static void toParseableByteArray(ByteArrayOutputStream byteStream, TimestampsFilter filter) throws IOException { byteStream.write(filter.getClass().getSimpleName().getBytes()); byteStream.write('('); @@ -234,6 +329,81 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, Times byteStream.write(')'); } + // MultiRowRangeFilter('a',true,'b',false,'c',true,'d',false); + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + MultiRowRangeFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + + List ranges = filter.getRowRanges(); + for (int i = 0; i < ranges.size(); i++) { + MultiRowRangeFilter.RowRange range = ranges.get(i); + byteStream.write("'".getBytes()); + writeBytesWithEscape(byteStream, range.getStartRow()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(range.isStartRowInclusive()).getBytes()); + byteStream.write(','); + + byteStream.write("'".getBytes()); + writeBytesWithEscape(byteStream, range.getStopRow()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(range.isStopRowInclusive()).getBytes()); + if (i < ranges.size() - 1) { + byteStream.write(','); + } + } + byteStream.write(')'); + } + + // InclusiveStopFilter('aaa'); + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + InclusiveStopFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + byteStream.write('\''); + writeBytesWithEscape(byteStream, filter.getStopRowKey()); + byteStream.write('\''); + byteStream.write(')'); + } + + // ColumnRangeFilter('a',true,'b',false); + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + ColumnRangeFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + + byteStream.write("'".getBytes()); + writeBytesWithEscape(byteStream, filter.getMinColumn()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(filter.getMinColumnInclusive()).getBytes()); + byteStream.write(','); + + byteStream.write("'".getBytes()); + writeBytesWithEscape(byteStream, filter.getMaxColumn()); + byteStream.write("',".getBytes()); + byteStream.write(Boolean.toString(filter.getMaxColumnInclusive()).getBytes()); + byteStream.write(')'); + } + + // MultipleColumnPrefixFilter('a','b','d'); + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + MultipleColumnPrefixFilter filter) throws IOException { + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + + byte[][] ranges = filter.getPrefix(); + for (int i = 0; i < ranges.length; i++) { + byte[] range = ranges[i]; + byteStream.write("'".getBytes()); + writeBytesWithEscape(byteStream, range); + byteStream.write("'".getBytes()); + if (i < ranges.length - 1) { + byteStream.write(','); + } + } + byteStream.write(')'); + } + // ColumnCountGetFilter(100) private static void toParseableByteArray(ByteArrayOutputStream byteStream, ColumnCountGetFilter filter) throws IOException { @@ -243,6 +413,32 @@ private static void toParseableByteArray(ByteArrayOutputStream byteStream, byteStream.write(')'); } + // FirstKeyValueMatchingQualifiersFilter('q1','q2') + private static void toParseableByteArray(ByteArrayOutputStream byteStream, + FirstKeyValueMatchingQualifiersFilter filter) throws IOException { + Set qualifiers; + try { + Field field = filter.getClass().getDeclaredField("qualifiers"); + field.setAccessible(true); + qualifiers = (Set)field.get(filter); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + byteStream.write(filter.getClass().getSimpleName().getBytes()); + byteStream.write('('); + int i = 0; + for (byte[] qualifier: qualifiers) { + byteStream.write("'".getBytes()); + writeBytesWithEscape(byteStream, qualifier); + byteStream.write("'".getBytes()); + if (i < qualifiers.size() - 1) { + byteStream.write(','); + } + i++; + } + byteStream.write(')'); + } + // PrefixFilter('prefix'); private static void toParseableByteArray(ByteArrayOutputStream byteStream, PrefixFilter filter) throws IOException { diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java b/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java new file mode 100644 index 00000000..49fe17e9 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/BatchError.java @@ -0,0 +1,39 @@ +package com.alipay.oceanbase.hbase.util; + +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; +import org.apache.hadoop.hbase.client.Row; + +import java.util.ArrayList; +import java.util.List; + +public class BatchError { + private final List throwables = new ArrayList(); + private final List actions = new ArrayList(); + private final List addresses = new ArrayList(); + + public synchronized void add(Throwable ex, Row row, ServerName serverName) { + if (row == null) { + throw new IllegalArgumentException("row cannot be null. location=" + serverName); + } + + throwables.add(ex); + actions.add(row); + addresses.add(serverName != null ? serverName.toString() : "null"); + } + + public boolean hasErrors() { + return !throwables.isEmpty(); + } + + public synchronized RetriesExhaustedWithDetailsException makeException() { + return new RetriesExhaustedWithDetailsException(new ArrayList(throwables), + new ArrayList(actions), new ArrayList(addresses)); + } + + public synchronized void clear() { + throwables.clear(); + actions.clear(); + addresses.clear(); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/CompatibilityUtil.java b/src/main/java/com/alipay/oceanbase/hbase/util/CompatibilityUtil.java new file mode 100644 index 00000000..39db637c --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/CompatibilityUtil.java @@ -0,0 +1,9 @@ +package com.alipay.oceanbase.hbase.util; + +import com.alipay.oceanbase.rpc.ObGlobal; + +public class CompatibilityUtil { + public static boolean isBatchSupport() { + return ObGlobal.OB_VERSION > ObGlobal.OB_VERSION_4_3_4_0; + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index 0cef7e62..42e44737 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -18,14 +18,11 @@ package com.alipay.oceanbase.hbase.util; import com.alipay.oceanbase.hbase.OHTable; -import com.alipay.oceanbase.rpc.ObTableClient; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.*; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger; import java.io.IOException; @@ -34,54 +31,53 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.LCD; @InterfaceAudience.Private public class OHBufferedMutatorImpl implements BufferedMutator { - private static final Logger LOGGER = TableHBaseLoggerFactory - .getLogger(OHBufferedMutatorImpl.class); + private static final Logger LOGGER = TableHBaseLoggerFactory + .getLogger(OHBufferedMutatorImpl.class); - private final ExceptionListener listener; + private final ExceptionListener listener; - protected final ObTableClient obTableClient; - private final TableName tableName; - private volatile Configuration conf; - private final OHConnectionConfiguration connectionConfig; + private final OHTable ohTable; + private final TableName tableName; + private volatile Configuration conf; - final ConcurrentLinkedQueue asyncWriteBuffer = new ConcurrentLinkedQueue(); - AtomicLong currentAsyncBufferSize = new AtomicLong(0); + @VisibleForTesting + final ConcurrentLinkedQueue asyncWriteBuffer = new ConcurrentLinkedQueue(); + @VisibleForTesting + AtomicLong currentAsyncBufferSize = new AtomicLong(0); - private AtomicReference> type = new AtomicReference<>(null); - private final long writeBufferSize; - private final int maxKeyValueSize; - private boolean closed = false; - private final ExecutorService pool; - private final int rpcTimeout; + private long writeBufferSize; + private final int maxKeyValueSize; + private boolean closed = false; + private final ExecutorService pool; + private int rpcTimeout; + private int operationTimeout; public OHBufferedMutatorImpl(OHConnectionImpl ohConnection, BufferedMutatorParams params) throws IOException { if (ohConnection == null || ohConnection.isClosed()) { throw new IllegalArgumentException("Connection is null or closed."); } - // create a ObTableClient to do rpc operations - this.obTableClient = ObTableClientManager.getOrCreateObTableClient(ohConnection - .getOHConnectionConfiguration()); - // init params in OHBufferedMutatorImpl this.tableName = params.getTableName(); this.conf = ohConnection.getConfiguration(); - this.connectionConfig = ohConnection.getOHConnectionConfiguration(); this.listener = params.getListener(); + + OHConnectionConfiguration connectionConfig = ohConnection.getOHConnectionConfiguration(); this.pool = params.getPool(); - this.obTableClient.setRuntimeBatchExecutor(pool); + this.rpcTimeout = connectionConfig.getRpcTimeout(); + this.operationTimeout = connectionConfig.getOperationTimeout(); this.writeBufferSize = params.getWriteBufferSize() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params .getWriteBufferSize() : connectionConfig.getWriteBufferSize(); this.maxKeyValueSize = params.getMaxKeyValueSize() != OHConnectionImpl.BUFFERED_PARAM_UNSET ? params .getMaxKeyValueSize() : connectionConfig.getMaxKeyValueSize(); - this.rpcTimeout = connectionConfig.getRpcTimeout(); - this.obTableClient.setRpcExecuteTimeout(rpcTimeout); + + // create an OHTable object to do batch work + this.ohTable = new OHTable(tableName, ohConnection, connectionConfig, pool); } @Override @@ -119,38 +115,37 @@ public void mutate(List mutations) throws IOException { } long toAddSize = 0; - // check if every mutation's family is the same - // check if mutations are the same type for (Mutation m : mutations) { - OHTable.checkFamilyViolation(m.getFamilyMap().keySet(), true); - validateInsUpAndDelete(m); - Class curType = m.getClass(); - // set the type of this BufferedMutator - if (type.get() == null) { - type.compareAndSet(null, mutations.get(0).getClass()); - } - if (!type.get().equals(curType)) { - throw new IllegalArgumentException("Not support different type in one batch."); - } + validateOperation(m); toAddSize += m.heapSize(); } currentAsyncBufferSize.addAndGet(toAddSize); asyncWriteBuffer.addAll(mutations); - asyncExecute(false); + if (currentAsyncBufferSize.get() > writeBufferSize) { + execute(false); + } + } /** * Check whether the mutation is Put or Delete in 1.x * @param mt - mutation operation */ - private void validateInsUpAndDelete(Mutation mt) throws IllegalArgumentException { + private void validateOperation(Mutation mt) throws IllegalArgumentException { + if (mt == null) { + throw new IllegalArgumentException("Mutation operation cannot be null"); + } if (!(mt instanceof Put) && !(mt instanceof Delete)) { throw new IllegalArgumentException("Only support for Put and Delete for now."); } if (mt instanceof Put) { + // family empty check is in validatePut HTable.validatePut((Put) mt, maxKeyValueSize); + OHTable.checkFamilyViolation(mt.getFamilyMap().keySet(), true); + } else { + OHTable.checkFamilyViolation(mt.getFamilyMap().keySet(), false); } } @@ -161,91 +156,49 @@ private void validateInsUpAndDelete(Mutation mt) throws IllegalArgumentException * @param flushAll - if true, sends all the writes and wait for all of them to finish before * returning. */ - private void asyncExecute(boolean flushAll) throws IOException { + private void execute(boolean flushAll) throws IOException { LinkedList execBuffer = new LinkedList<>(); - ObTableBatchOperationRequest request = null; - // namespace n1, n1:table_name - // namespace default, table_name - String tableNameString = tableName.getNameAsString(); + long dequeuedSize = 0L; try { - while (true) { - try{ - if (!flushAll || asyncWriteBuffer.isEmpty()) { - if (currentAsyncBufferSize.get() <= writeBufferSize) { - break; - } - } - Mutation m; - while ((m = asyncWriteBuffer.poll()) != null) { - execBuffer.add(m); - long size = m.heapSize(); - currentAsyncBufferSize.addAndGet(-size); - } - // in concurrent situation, asyncWriteBuffer may be empty here - // for other threads flush all buffer - if (execBuffer.isEmpty()) { - break; - } - // for now, operations' family is the same - byte[] family = execBuffer.getFirst().getFamilyMap().firstKey(); - ObTableBatchOperation batch = buildObTableBatchOperation(execBuffer); - // table_name$cf_name - String targetTableName = OHTable.getTargetTableName(tableNameString, Bytes.toString(family), conf); - request = OHTable.buildObTableBatchOperationRequest(batch, targetTableName); - } catch (Exception ex) { - LOGGER.error("Errors occur before mutation operation", ex); - throw new IllegalArgumentException("Errors occur before mutation operation", ex); - } - try { - ObTableBatchOperationResult result = (ObTableBatchOperationResult) obTableClient.execute(request); - } catch (Exception ex) { - LOGGER.debug("Errors occur during mutation operation", ex); - Mutation m = null; - try { - // retry every single operation - while (!execBuffer.isEmpty()) { - // poll elements from execBuffer to recollect remaining operations - m = execBuffer.poll(); - byte[] family = m.getFamilyMap().firstKey(); - ObTableBatchOperation batch = buildObTableBatchOperation(Collections.singletonList(m)); - String targetTableName = OHTable.getTargetTableName(tableNameString, Bytes.toString(family), conf); - request = OHTable.buildObTableBatchOperationRequest(batch, targetTableName); - ObTableBatchOperationResult result = (ObTableBatchOperationResult) obTableClient.execute(request); - } - } catch (Exception newEx) { - if (m != null) { - execBuffer.addFirst(m); - } - // if retry fails, only recollect remaining operations - while(!execBuffer.isEmpty()) { - m = execBuffer.poll(); - long size = m.heapSize(); - asyncWriteBuffer.add(m); - currentAsyncBufferSize.addAndGet(size); - } - throw newEx; - } - } + Mutation m; + while ((writeBufferSize <= 0 || dequeuedSize < (writeBufferSize * 2) || flushAll) + && (m = asyncWriteBuffer.poll()) != null) { + execBuffer.add(m); + long size = m.heapSize(); + currentAsyncBufferSize.addAndGet(-size); + dequeuedSize += size; + } + + if (execBuffer.isEmpty()) { + return; } + ohTable.batch(execBuffer); + // if commit all successfully, clean execBuffer + execBuffer.clear(); } catch (Exception ex) { LOGGER.error(LCD.convert("01-00026"), ex); - // if the cause is illegal argument, directly throw to user - if (ex instanceof IllegalArgumentException) { - throw (IllegalArgumentException) ex; - } - // TODO: need to collect error information and actions during batch operations - // TODO: maybe keep in ObTableBatchOperationResult - List throwables = new ArrayList(); - List actions = new ArrayList(); - List addresses = new ArrayList(); - throwables.add(ex); - RetriesExhaustedWithDetailsException error = new RetriesExhaustedWithDetailsException( - new ArrayList(throwables), - new ArrayList(actions), new ArrayList(addresses)); - if (listener == null) { - throw error; + if (ex.getCause() instanceof RetriesExhaustedWithDetailsException) { + LOGGER.error(tableName + ": One or more of the operations have failed after retries."); + RetriesExhaustedWithDetailsException retryException = (RetriesExhaustedWithDetailsException) ex.getCause(); + // recollect mutations + execBuffer.clear(); + for (int i = 0; i < retryException.getNumExceptions(); ++i) { + execBuffer.add((Mutation) retryException.getRow(i)); + } + if (listener != null) { + listener.onException(retryException, this); + } else { + throw retryException; + } } else { - listener.onException(error, this); + LOGGER.error("Errors unrelated to operations occur during mutation operation", ex); + throw ex; + } + } finally { + for (Mutation mutation : execBuffer) { + long size = mutation.heapSize(); + currentAsyncBufferSize.addAndGet(size); + asyncWriteBuffer.add(mutation); } } } @@ -256,7 +209,7 @@ public void close() throws IOException { return; } try { - asyncExecute(true); + execute(true); } finally { // the pool in ObTableClient will be shut down too this.pool.shutdown(); @@ -273,13 +226,21 @@ public void close() throws IOException { } } + @Deprecated + public void setWriteBufferSize(long writeBufferSize) throws IOException { + this.writeBufferSize = writeBufferSize; + if (currentAsyncBufferSize.get() > writeBufferSize) { + flush(); + } + } + /** * Force to commit all operations * do not care whether the pool is shut down or this BufferedMutator is closed */ @Override public void flush() throws IOException { - asyncExecute(true); + execute(true); } @Override @@ -287,13 +248,18 @@ public long getWriteBufferSize() { return this.writeBufferSize; } - private ObTableBatchOperation buildObTableBatchOperation(List execBuffer) { - List keyValueList = new LinkedList<>(); - for (Mutation mutation : execBuffer) { - for (Map.Entry> entry : mutation.getFamilyMap().entrySet()) { - keyValueList.addAll(entry.getValue()); - } - } - return OHTable.buildObTableBatchOperation(keyValueList, false, null); + public void setRpcTimeout(int rpcTimeout) { + this.rpcTimeout = rpcTimeout; + this.ohTable.setRpcTimeout(rpcTimeout); + } + + public void setOperationTimeout(int operationTimeout) { + this.operationTimeout = operationTimeout; + this.ohTable.setOperationTimeout(operationTimeout); + } + + @Deprecated + public List getWriteBuffer() { + return Arrays.asList(asyncWriteBuffer.toArray(new Row[0])); } } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java index e4789df3..f602fdda 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java @@ -25,11 +25,15 @@ import java.util.Properties; import static com.alipay.oceanbase.hbase.constants.OHConstants.*; +import static org.apache.commons.lang.StringUtils.isBlank; +import static org.apache.hadoop.hbase.ipc.RpcClient.DEFAULT_SOCKET_TIMEOUT_CONNECT; +import static org.apache.hadoop.hbase.ipc.RpcClient.SOCKET_TIMEOUT_CONNECT; @InterfaceAudience.Private public class OHConnectionConfiguration { + private String paramUrl; + private String database; private final Properties properties; - private final String paramUrl; private final String fullUsername; private final String password; private final String sysUsername; @@ -37,13 +41,13 @@ public class OHConnectionConfiguration { private final String odpAddr; private final int odpPort; private final boolean odpMode; - private final String database; private final long writeBufferSize; private final int operationTimeout; private final int scannerCaching; private final long scannerMaxResultSize; private final int maxKeyValueSize; private final int rpcTimeout; + private final int rpcConnectTimeout; public OHConnectionConfiguration(Configuration conf) { this.paramUrl = conf.get(HBASE_OCEANBASE_PARAM_URL); @@ -54,11 +58,27 @@ public OHConnectionConfiguration(Configuration conf) { this.odpAddr = conf.get(HBASE_OCEANBASE_ODP_ADDR); this.odpPort = conf.getInt(HBASE_OCEANBASE_ODP_PORT, -1); this.odpMode = conf.getBoolean(HBASE_OCEANBASE_ODP_MODE, false); - this.database = conf.get(HBASE_OCEANBASE_DATABASE); + String database = conf.get(HBASE_OCEANBASE_DATABASE); + if (isBlank(database)) { + database = "default"; + } + this.database = database; this.writeBufferSize = conf.getLong(WRITE_BUFFER_SIZE_KEY, WRITE_BUFFER_SIZE_DEFAULT); this.operationTimeout = conf.getInt("hbase.client.operation.timeout", 1200000); this.rpcTimeout = conf.getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT); + int rpcConnectTimeout = -1; + if (conf.get(SOCKET_TIMEOUT_CONNECT) != null) { + rpcConnectTimeout = conf.getInt(SOCKET_TIMEOUT_CONNECT, DEFAULT_SOCKET_TIMEOUT_CONNECT); + } else { + if (conf.get(SOCKET_TIMEOUT) != null) { + rpcConnectTimeout = conf.getInt(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT); + } else { + rpcConnectTimeout = conf.getInt(SOCKET_TIMEOUT_CONNECT, + DEFAULT_SOCKET_TIMEOUT_CONNECT); + } + } + this.rpcConnectTimeout = rpcConnectTimeout; this.scannerCaching = conf.getInt("hbase.client.scanner.caching", Integer.MAX_VALUE); this.scannerMaxResultSize = conf.getLong("hbase.client.scanner.max.result.size", WRITE_BUFFER_SIZE_DEFAULT); @@ -72,6 +92,14 @@ public OHConnectionConfiguration(Configuration conf) { } } + public void setParamUrl(String paramUrl) { + this.paramUrl = paramUrl; + } + + public void setDatabase(String database) { + this.database = database; + } + public long getWriteBufferSize() { return this.writeBufferSize; } @@ -92,6 +120,10 @@ public int getRpcTimeout() { return this.rpcTimeout; } + public int getRpcConnectTimeout() { + return this.rpcConnectTimeout; + } + public long getScannerMaxResultSize() { return this.scannerMaxResultSize; } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableFactory.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableFactory.java index 06c88892..8d42daa4 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableFactory.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableFactory.java @@ -65,6 +65,8 @@ public OHTableFactory(Configuration conf, OHTablePool tablePool, public HTableInterface createHTableInterface(Configuration config, byte[] tableName) { try { String tableNameStr = Bytes.toString(tableName); + tableNameStr = tableNameStr.equals(this.tablePool.getOriginTableName()) ? tableNameStr + : this.tablePool.getOriginTableName(); OHTable ht = new OHTable(adjustConfiguration(copyConfiguration(config), tableNameStr), tableName, this.threadPool); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java index 9b0c8182..036ce265 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java @@ -21,6 +21,8 @@ import com.alipay.oceanbase.rpc.constant.Constants; import com.google.common.base.Objects; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.ConnectionConfiguration; import java.io.IOException; import java.util.Map; @@ -58,7 +60,11 @@ public static ObTableClient getOrCreateObTableClient(OHConnectionConfiguration c checkArgument(isNotBlank(connectionConfig.getParamUrl()), HBASE_OCEANBASE_PARAM_URL + " is blank"); obTableClientKey = new ObTableClientKey(); - obTableClientKey.setParamUrl(connectionConfig.getParamUrl()); + String paramUrl = connectionConfig.getParamUrl(); + if (!paramUrl.contains("database")) { + paramUrl += "&database=default"; + } + obTableClientKey.setParamUrl(paramUrl); obTableClientKey.setSysUserName(connectionConfig.getSysUsername()); if (connectionConfig.getSysPassword() == null) { obTableClientKey.setSysPassword(Constants.EMPTY_STRING); @@ -80,11 +86,11 @@ public static ObTableClient getOrCreateObTableClient(OHConnectionConfiguration c obTableClientKey.getProperties().put(property.getKey(), property.getValue()); } - return getOrCreateObTableClient(obTableClientKey); + return getOrCreateObTableClient(obTableClientKey, connectionConfig.getRpcConnectTimeout()); } - public static ObTableClient getOrCreateObTableClient(ObTableClientKey obTableClientKey) - throws IOException { + public static ObTableClient getOrCreateObTableClient(ObTableClientKey obTableClientKey, + int rpcConnectTimeout) throws IOException { if (OB_TABLE_CLIENT_INSTANCE.get(obTableClientKey) == null) { ReentrantLock tmp = new ReentrantLock(); ReentrantLock lock = OB_TABLE_CLIENT_LOCK.putIfAbsent(obTableClientKey, tmp); @@ -109,6 +115,7 @@ public static ObTableClient getOrCreateObTableClient(ObTableClientKey obTableCli } obTableClient.setFullUserName(obTableClientKey.getFullUserName()); obTableClient.setPassword(obTableClientKey.getPassword()); + obTableClient.setRpcConnectTimeout(rpcConnectTimeout); obTableClient.init(); OB_TABLE_CLIENT_INSTANCE.put(obTableClientKey, obTableClient); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java new file mode 100644 index 00000000..ca864ab1 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java @@ -0,0 +1,1415 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2024 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase; + +import org.apache.hadoop.conf.Configuration; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org.apache.hadoop.hbase.filter.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; +import org.junit.rules.ExpectedException; + +import java.util.*; + +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.*; + +public abstract class HTableMultiCFTestBase { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + protected static Table multiCfHTable; + + public void tryPut(Table multiCfHTable, Put put) throws Exception { + multiCfHTable.put(put); + Thread.sleep(1); + } + + @Test + public void testDeleteFamilyVerison() throws Exception { + String key1 = "scanKey1x"; + String key2 = "scanKey2x"; + String key3 = "scanKey3x"; + String column1 = "column1"; + String column2 = "column2"; + String column3 = "column3"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + String family1 = "family_with_group1"; + String family2 = "family_with_group2"; + // delete previous data + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family1)); + deleteKey1Family.deleteFamily(toBytes(family2)); + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family1)); + deleteKey2Family.deleteFamily(toBytes(family2)); + Delete deleteKey3Family = new Delete(toBytes(key3)); + deleteKey3Family.deleteFamily(toBytes(family1)); + deleteKey3Family.deleteFamily(toBytes(family2)); + + multiCfHTable.delete(deleteKey1Family); + multiCfHTable.delete(deleteKey2Family); + multiCfHTable.delete(deleteKey3Family); + + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + + Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); + putKey1Fam1Column1MinTs.add(toBytes(family1), toBytes(column1), minTimeStamp, + toBytes(value1)); + + Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp1, toBytes(value2)); + + Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); + putKey1Fam1Column2MinTs.add(toBytes(family1), toBytes(column2), minTimeStamp, + toBytes(value1)); + + Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column2Ts4 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts4.add(toBytes(family1), toBytes(column2), timeStamp4, toBytes(value1)); + + Put putKey2Fam1Column3Ts3 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts3.add(toBytes(family1), toBytes(column3), timeStamp3, toBytes(value1)); + + tryPut(multiCfHTable, putKey1Fam1Column1MinTs); + tryPut(multiCfHTable, putKey3Fam1Column1Ts1); + tryPut(multiCfHTable, putKey1Fam1Column2MinTs); + tryPut(multiCfHTable, putKey1Fam1Column2Ts3); + tryPut(multiCfHTable, putKey2Fam1Column2Ts3); + tryPut(multiCfHTable, putKey2Fam1Column3Ts1); + tryPut(multiCfHTable, putKey3Fam1Column3Ts1); + tryPut(multiCfHTable, putKey3Fam1Column2Ts4); + tryPut(multiCfHTable, putKey2Fam1Column3Ts3); + + // test DeleteFamilyVersion single cf + Get get = new Get(toBytes(key1)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + Result r = multiCfHTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(2, r.raw().length); + + Delete delKey1MinTs = new Delete(toBytes(key1)); + delKey1MinTs.deleteFamilyVersion(toBytes(family1), minTimeStamp); + multiCfHTable.delete(delKey1MinTs); + + get = new Get(toBytes(key1)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts1 = new Delete(toBytes(key3)); + delKey3Ts1.deleteFamilyVersion(toBytes(family1), timeStamp1); + multiCfHTable.delete(delKey3Ts1); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey2Ts3 = new Delete(toBytes(key2)); + delKey2Ts3.deleteFamilyVersion(family1.getBytes(), timeStamp3); + multiCfHTable.delete(delKey2Ts3); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Scan scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(toBytes(family1)); + scan.setMaxVersions(10); + ResultScanner scanner = multiCfHTable.getScanner(scan); + int key1Cnt = 0, key2Cnt = 0, key3Cnt = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (key1.equals(Bytes.toString(kv.getRow()))) { + ++key1Cnt; + } else if (key2.equals(Bytes.toString(kv.getRow()))) { + ++key2Cnt; + } else { + ++key3Cnt; + } + } + } + Assert.assertEquals(1, key1Cnt); + Assert.assertEquals(1, key2Cnt); + Assert.assertEquals(1, key3Cnt); + + multiCfHTable.delete(deleteKey1Family); + multiCfHTable.delete(deleteKey2Family); + multiCfHTable.delete(deleteKey3Family); + + // test DeleteFamilyVersion multiple cf + Put putKey1Fam1Column3Ts4 = new Put(toBytes(key1)); + putKey1Fam1Column3Ts4.add(toBytes(family1), toBytes(column3), timeStamp4, toBytes(value3)); + + Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column2Ts2.add(toBytes(family2), toBytes(column2), timeStamp2, toBytes(value1)); + + Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column3Ts2.add(toBytes(family2), toBytes(column3), timeStamp2, toBytes(value1)); + + Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts1.add(toBytes(family1), toBytes(column2), timeStamp1, toBytes(value2)); + + Put putKey2Fam1Column2Ts5 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts5.add(toBytes(family1), toBytes(column2), timeStamp5, toBytes(value2)); + + Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam2Column3Ts1.add(toBytes(family2), toBytes(column3), timeStamp3, toBytes(value3)); + + Put putKey2Fam1Column1Ts5 = new Put(toBytes(key2)); + putKey2Fam1Column1Ts5.add(toBytes(family1), toBytes(column1), timeStamp5, toBytes(value1)); + + Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); + putKey2Fam2Column1Ts3.add(toBytes(family2), toBytes(column1), timeStamp3, toBytes(value2)); + + Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts6.add(toBytes(family1), toBytes(column2), timeStamp6, toBytes(value2)); + + Put putKey3Fam2Column3Ts7 = new Put(toBytes(key3)); + putKey3Fam2Column3Ts7.add(toBytes(family2), toBytes(column3), timeStamp7, toBytes(value1)); + + Put putKey3Fam2Column1Ts7 = new Put(toBytes(key3)); + putKey3Fam2Column1Ts7.add(toBytes(family2), toBytes(column1), timeStamp7, toBytes(value2)); + + Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts2.add(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); + + tryPut(multiCfHTable, putKey1Fam1Column3Ts4); + tryPut(multiCfHTable, putKey1Fam2Column2Ts2); + tryPut(multiCfHTable, putKey1Fam2Column3Ts2); + tryPut(multiCfHTable, putKey1Fam1Column2Ts1); + tryPut(multiCfHTable, putKey2Fam1Column2Ts5); + tryPut(multiCfHTable, putKey2Fam2Column3Ts1); + tryPut(multiCfHTable, putKey2Fam1Column1Ts5); + tryPut(multiCfHTable, putKey2Fam2Column1Ts3); + tryPut(multiCfHTable, putKey3Fam1Column2Ts6); + tryPut(multiCfHTable, putKey3Fam2Column3Ts7); + tryPut(multiCfHTable, putKey3Fam2Column1Ts7); + tryPut(multiCfHTable, putKey3Fam1Column2Ts2); + + Get getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = multiCfHTable.get(getKey1); + Assert.assertEquals(4, r.raw().length); + + Get getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = multiCfHTable.get(getKey2); + Assert.assertEquals(4, r.raw().length); + + Get getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setMaxVersions(10); + r = multiCfHTable.get(getKey3); + Assert.assertEquals(4, r.raw().length); + + Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); + delKey1Ts_6_2.deleteFamilyVersion(toBytes(family1), timeStamp4); + delKey1Ts_6_2.deleteFamilyVersion(toBytes(family2), timeStamp2); + multiCfHTable.delete(delKey1Ts_6_2); + + getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = multiCfHTable.get(getKey1); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp1, kv.getTimestamp()); + } + + Delete delKey2Ts_5_3 = new Delete(toBytes(key2)); + delKey2Ts_5_3.deleteFamilyVersion(toBytes(family1), timeStamp5); + delKey2Ts_5_3.deleteFamilyVersion(toBytes(family2), timeStamp3); + multiCfHTable.delete(delKey2Ts_5_3); + + getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = multiCfHTable.get(getKey2); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts_2_7 = new Delete(toBytes(key3)); + delKey3Ts_2_7.deleteFamilyVersion(toBytes(family1), timeStamp2); + delKey3Ts_2_7.deleteFamilyVersion(toBytes(family2), timeStamp7); + multiCfHTable.delete(delKey3Ts_2_7); + + getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setMaxVersions(10); + r = multiCfHTable.get(getKey3); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp6, kv.getTimestamp()); + } + + scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(toBytes(family1)); + scan.addFamily(toBytes(family2)); + scan.setMaxVersions(10); + scanner = multiCfHTable.getScanner(scan); + int ts1Cnt = 0, ts9Cnt = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (kv.getTimestamp() == timeStamp1) { + ++ts1Cnt; + } else if (kv.getTimestamp() == timeStamp6) { + ++ts9Cnt; + } + } + } + Assert.assertEquals(1, ts1Cnt); + Assert.assertEquals(1, ts9Cnt); + + multiCfHTable.delete(deleteKey1Family); + multiCfHTable.delete(deleteKey2Family); + multiCfHTable.delete(deleteKey3Family); + } + + @Test + public void testMulfiColumnFamilyBufferedMutator() throws Exception { + byte[] family1 = "family_with_group1".getBytes(); + byte[] family2 = "family_with_group2".getBytes(); + byte[] family3 = "family_with_group3".getBytes(); + + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family3_column2 = "family3_column2".getBytes(); + byte[] family1_value = "VVV1".getBytes(); + byte[] family2_value = "VVV2".getBytes(); + byte[] family3_value = "VVV3".getBytes(); + + Configuration conf = ObHTableTestUtil.newConfiguration(); + TableName tableName = TableName.valueOf("test_multi_cf"); + Connection connection = ConnectionFactory.createConnection(conf); + BufferedMutator mutator = connection.getBufferedMutator(tableName); + + int rows = 10; + List keys = new ArrayList<>(); + List mutations = new ArrayList<>(); + for (int i = 0; i < rows; ++i) { + String key = "Key" + i; + keys.add(key); + Delete delete = new Delete(toBytes(key)); + mutations.add(delete); + Put put = new Put(toBytes(key)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + mutations.add(put); + } + mutator.mutate(mutations); + + // test force flush + mutator.flush(); + Get get = new Get(toBytes("Key2")); + get.addFamily(family1); + get.addFamily(family2); + Result result = multiCfHTable.get(get); + Assert.assertEquals(5, result.raw().length); + + mutations.clear(); + for (int i = 0; i < rows; ++i) { + if (i % 5 == 0) { // 0, 5 + Delete delete = new Delete(toBytes("Key" + i)); + delete.deleteFamily(family2); + delete.deleteFamily(family3); + mutations.add(delete); + } + } + mutator.mutate(mutations); + mutator.flush(); + + get = new Get(toBytes("Key0")); + result = multiCfHTable.get(get); + Assert.assertEquals(3, result.raw().length); + Assert.assertFalse(result.containsColumn(family2, family2_column1)); + Assert.assertFalse(result.containsColumn(family2, family2_column2)); + Assert.assertFalse(result.containsColumn(family3, family3_column1)); + + get = new Get(toBytes("Key5")); + result = multiCfHTable.get(get); + Assert.assertEquals(3, result.raw().length); + Assert.assertFalse(result.containsColumn(family2, family2_column1)); + Assert.assertFalse(result.containsColumn(family2, family2_column2)); + Assert.assertFalse(result.containsColumn(family3, family3_column1)); + + mutations.clear(); + for (String key : keys) { + Delete delete = new Delete(toBytes(key)); + mutations.add(delete); + } + mutator.mutate(mutations); + mutator.flush(); + + Scan scan = new Scan(); + scan.setStartRow(toBytes("Key0")); + scan.setStopRow(toBytes("Key10")); + scan.addFamily(family1); + scan.addFamily(family2); + scan.addFamily(family3); + ResultScanner scanner = multiCfHTable.getScanner(scan); + int count = 0; + for (Result r : scanner) { + count += r.raw().length; + } + Assert.assertEquals(0, count); + + // test auto flush + long bufferSize = 45000L; + BufferedMutatorParams params = new BufferedMutatorParams(tableName); + params.writeBufferSize(bufferSize); + mutator = connection.getBufferedMutator(params); + + while (true) { + for (int i = 0; i < rows; ++i) { + mutations.clear(); + Put put = new Put(toBytes(keys.get(i))); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family3, family3_column1, family2_value); + put.add(family3, family3_column2, family3_value); + mutations.add(put); + if (i % 3 == 0) { // 0, 3, 6, 9 + Delete delete = new Delete(toBytes(keys.get(i))); + delete.deleteFamily(family1); + delete.deleteFamily(family2); + mutations.add(delete); + } + mutator.mutate(mutations); + } + + get = new Get(toBytes("Key0")); + result = multiCfHTable.get(get); + if (!result.isEmpty()) { + break; + } + } + get = new Get(toBytes("Key2")); + result = multiCfHTable.get(get); + Assert.assertEquals(6 , result.raw().length); + Assert.assertTrue(result.containsColumn(family1, family1_column1)); + Assert.assertTrue(result.containsColumn(family1, family1_column2)); + Assert.assertTrue(result.containsColumn(family1, family1_column3)); + Assert.assertTrue(result.containsColumn(family2, family2_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column2)); + + get = new Get(toBytes("Key3")); + result = multiCfHTable.get(get); + if (result.containsColumn(family1, family1_column1) || result.containsColumn(family2, family2_column1)) { + mutator.flush(); + } + get = new Get(toBytes("Key3")); + result = multiCfHTable.get(get); + Assert.assertEquals(2, result.raw().length); + Assert.assertFalse(result.containsColumn(family1, family1_column1)); + Assert.assertFalse(result.containsColumn(family1, family1_column2)); + Assert.assertFalse(result.containsColumn(family1, family1_column3)); + Assert.assertFalse(result.containsColumn(family2, family2_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column2)); + + get = new Get(toBytes("Key9")); + result = multiCfHTable.get(get); + if (result.containsColumn(family1, family1_column1) || result.containsColumn(family2, family2_column1)) { + mutator.flush(); + } + get = new Get(toBytes("Key9")); + result = multiCfHTable.get(get); + Assert.assertEquals(2, result.raw().length); + Assert.assertFalse(result.containsColumn(family1, family1_column1)); + Assert.assertFalse(result.containsColumn(family1, family1_column2)); + Assert.assertFalse(result.containsColumn(family1, family1_column3)); + Assert.assertFalse(result.containsColumn(family2, family2_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column1)); + Assert.assertTrue(result.containsColumn(family3, family3_column2)); + + // clean data + mutations.clear(); + for (String key : keys) { + Delete delete = new Delete(toBytes(key)); + mutations.add(delete); + } + mutator.mutate(mutations); + mutator.flush(); + + scan = new Scan(); + scan.setStartRow(toBytes("Key0")); + scan.setStopRow(toBytes("Key10")); + scan.addFamily(family1); + scan.addFamily(family2); + scan.addFamily(family3); + scanner = multiCfHTable.getScanner(scan); + count = 0; + for (Result r : scanner) { + count += r.raw().length; + } + Assert.assertEquals(0, count); + } + + @Test + public void testMultiColumnFamilyBatch() throws Exception { + byte[] family1 = "family_with_group1".getBytes(); + byte[] family2 = "family_with_group2".getBytes(); + byte[] family3 = "family_with_group3".getBytes(); + + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family1_value = "VVV1".getBytes(); + byte[] family2_value = "VVV2".getBytes(); + byte[] family3_value = "VVV3".getBytes(); + + int rows = 10; + List batchLsit = new LinkedList<>(); + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + Delete delete = new Delete(toBytes("Key" + i)); + batchLsit.add(delete); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + batchLsit.add(put); + } + + // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 + Delete delete = new Delete(toBytes("Key1")); + delete.deleteColumns(family1, family1_column1); + delete.deleteColumns(family2, family2_column1); + batchLsit.add(delete); + multiCfHTable.batch(batchLsit); + // f1c2 f1c3 f2c2 f3c1 + Get get = new Get(toBytes("Key1")); + Result result = multiCfHTable.get(get); + KeyValue[] keyValues = result.raw(); + assertEquals(4, keyValues.length); + assertFalse(result.containsColumn(family1, family1_column1)); + assertFalse(result.containsColumn(family2, family2_column1)); + + assertTrue(result.containsColumn(family1, family1_column2)); + assertArrayEquals(result.getValue(family1, family1_column2), family1_value); + assertTrue(result.containsColumn(family1, family1_column3)); + assertArrayEquals(result.getValue(family1, family1_column3), family1_value); + assertTrue(result.containsColumn(family2, family2_column2)); + assertArrayEquals(result.getValue(family2, family2_column2), family2_value); + assertTrue(result.containsColumn(family3, family3_column1)); + assertArrayEquals(result.getValue(family3, family3_column1), family3_value); + + // f1c1 f2c1 f2c2 + delete = new Delete(toBytes("Key2")); + delete.deleteColumns(family1, family1_column2); + delete.deleteColumns(family1, family1_column3); + delete.deleteColumns(family3, family3_column1); + batchLsit.add(delete); + // null + multiCfHTable.batch(batchLsit); + get = new Get(toBytes("Key2")); + result = multiCfHTable.get(get); + keyValues = result.raw(); + assertEquals(3, keyValues.length); + batchLsit.clear(); + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + batchLsit.add(put); + } + + delete = new Delete(toBytes("Key3")); + delete.deleteColumn(family1, family1_column2); + delete.deleteColumn(family2, family2_column1); + batchLsit.add(delete); + multiCfHTable.batch(batchLsit); + get = new Get(toBytes("Key3")); + result = multiCfHTable.get(get); + keyValues = result.raw(); + assertEquals(6, keyValues.length); + + batchLsit.clear(); + delete = new Delete(toBytes("Key4")); + delete.deleteColumns(family1, family1_column2); + delete.deleteColumns(family2, family2_column1); + delete.deleteFamily(family3); + batchLsit.add(delete); + multiCfHTable.batch(batchLsit); + get = new Get(toBytes("Key4")); + get.setMaxVersions(10); + result = multiCfHTable.get(get); + keyValues = result.raw(); + assertEquals(6, keyValues.length); + + batchLsit.clear(); + final long[] updateCounter = new long[] { 0L }; + delete = new Delete(toBytes("Key5")); + delete.deleteColumns(family1, family1_column2); + delete.deleteColumns(family2, family2_column1); + delete.deleteFamily(family3); + batchLsit.add(delete); + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + batchLsit.add(put); + } + multiCfHTable.batchCallback(batchLsit, new Batch.Callback() { + @Override + public void update(byte[] region, byte[] row, Result result) { + updateCounter[0]++; + } + }); + assertEquals(11, updateCounter[0]); + + } + + @Test + public void testMultiColumnFamilyPut() throws Exception { + byte[] family1 = "family_with_group1".getBytes(); + byte[] family2 = "family_with_group2".getBytes(); + byte[] family3 = "family_with_group3".getBytes(); + + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family1_value = "VVV1".getBytes(); + byte[] family2_value = "VVV2".getBytes(); + byte[] family3_value = "VVV3".getBytes(); + + Map expectedValues = new HashMap<>(); + expectedValues.put(family1_column1, family1_value); + expectedValues.put(family1_column2, family1_value); + expectedValues.put(family1_column3, family1_value); + expectedValues.put(family2_column1, family2_value); + expectedValues.put(family2_column2, family2_value); + expectedValues.put(family3_column1, family3_value); + + int rows = 30; + + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + multiCfHTable.put(put); + } + + Scan scan = new Scan(); + scan.setStartRow(toBytes("Key")); + scan.setStopRow(toBytes("Kf")); + ResultScanner scanner = multiCfHTable.getScanner(scan); + int count = 0; + + for (Result result : scanner) { + KeyValue[] keyValues = result.raw(); + long timestamp = keyValues[0].getTimestamp(); + for (int i = 1; i < keyValues.length; ++i) { + assertEquals(timestamp, keyValues[i].getTimestamp()); + byte[] qualifier = keyValues[i].getQualifier(); + byte[] expectedValue = expectedValues.get(qualifier); + if (expectedValue != null) { + assertEquals(expectedValue, keyValues[i].getValue()); + } + } + count++; + } + assertEquals(count, rows); + } + + @Ignore + public void testMultiColumnFamilyAppend() throws Exception { + byte[] family1 = "family_with_group1".getBytes(); + byte[] family2 = "family_with_group2".getBytes(); + byte[] family3 = "family_with_group3".getBytes(); + + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family1_value = "VVV1".getBytes(); + byte[] family2_value = "VVV2".getBytes(); + byte[] family3_value = "VVV3".getBytes(); + + Map expectedValues = new HashMap<>(); + expectedValues.put(family1_column1, family1_value); + expectedValues.put(family1_column2, family1_value); + expectedValues.put(family1_column3, family1_value); + expectedValues.put(family2_column1, family2_value); + expectedValues.put(family2_column2, family2_value); + expectedValues.put(family3_column1, family3_value); + + int rows = 30; + + for (int i = 0; i < rows; ++i) { + Append append = new Append(toBytes("Key" + i)); + append.add(family1, family1_column1, family1_value); + append.add(family1, family1_column2, family1_value); + append.add(family1, family1_column3, family1_value); + append.add(family2, family2_column1, family2_value); + append.add(family2, family2_column2, family2_value); + append.add(family3, family3_column1, family3_value); + multiCfHTable.append(append); + } + + Scan scan = new Scan(); + scan.setStartRow(toBytes("Key")); + scan.setStopRow(toBytes("Kf")); + ResultScanner scanner = multiCfHTable.getScanner(scan); + int count = 0; + + for (Result result : scanner) { + KeyValue[] keyValues = result.raw(); + long timestamp = keyValues[0].getTimestamp(); + for (int i = 1; i < keyValues.length; ++i) { + assertEquals(timestamp, keyValues[i].getTimestamp()); + byte[] qualifier = keyValues[i].getQualifier(); + byte[] expectedValue = expectedValues.get(qualifier); + if (expectedValue != null) { + assertEquals(expectedValue, keyValues[i].getValue()); + } + } + count++; + } + assertEquals(count, rows); + } + + @Test + public void testMultiColumnFamilyReverseScan() throws Exception { + byte[] family1 = "family_with_group1".getBytes(); + byte[] family2 = "family_with_group2".getBytes(); + byte[] family3 = "family_with_group3".getBytes(); + + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family1_value = "VVV1".getBytes(); + byte[] family2_value = "VVV2".getBytes(); + byte[] family3_value = "VVV3".getBytes(); + + Map expectedValues = new HashMap<>(); + expectedValues.put(family1_column1, family1_value); + expectedValues.put(family1_column2, family1_value); + expectedValues.put(family1_column3, family1_value); + expectedValues.put(family2_column1, family2_value); + expectedValues.put(family2_column2, family2_value); + expectedValues.put(family3_column1, family3_value); + + int rows = 30; + + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + multiCfHTable.put(put); + } + + Scan scan = new Scan(); + scan.addFamily(family1); + scan.addFamily(family2); + scan.setReversed(true); + ResultScanner scanner2 = multiCfHTable.getScanner(scan); + + for (Result result : scanner2) { + KeyValue[] keyValues = result.raw(); + long timestamp = keyValues[0].getTimestamp(); + for (int i = 1; i < keyValues.length; ++i) { + assertEquals(timestamp, keyValues[i].getTimestamp()); + byte[] qualifier = keyValues[i].getQualifier(); + byte[] expectedValue = expectedValues.get(qualifier); + if (expectedValue != null) { + assertEquals(expectedValue, keyValues[i].getValue()); + } + } + } + } + + @Test + public void testMultiColumnFamilyScanWithColumns() throws Exception { + byte[] family1 = "family_with_group1".getBytes(); + byte[] family2 = "family_with_group2".getBytes(); + byte[] family3 = "family_with_group3".getBytes(); + + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family1_value = "VVV1".getBytes(); + byte[] family2_value = "VVV2".getBytes(); + byte[] family3_value = "VVV3".getBytes(); + + Map expectedValues = new HashMap<>(); + expectedValues.put(family1_column1, family1_value); + expectedValues.put(family1_column2, family1_value); + expectedValues.put(family1_column3, family1_value); + expectedValues.put(family2_column1, family2_value); + expectedValues.put(family2_column2, family2_value); + expectedValues.put(family3_column1, family3_value); + + int rows = 30; + + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + multiCfHTable.put(put); + } + + Scan scan = new Scan(); + scan.setStartRow(toBytes("Key")); + scan.setStopRow(toBytes("Kf")); + scan.addColumn(family1, family1_column1); + scan.addColumn(family2, family2_column1); + ResultScanner scanner = multiCfHTable.getScanner(scan); + + for (Result result : scanner) { + KeyValue[] keyValues = result.raw(); + long timestamp = keyValues[0].getTimestamp(); + for (int i = 1; i < keyValues.length; ++i) { + assertEquals(timestamp, keyValues[i].getTimestamp()); + byte[] qualifier = keyValues[i].getQualifier(); + byte[] expectedValue = expectedValues.get(qualifier); + if (expectedValue != null) { + assertEquals(expectedValue, keyValues[i].getValue()); + } + } + assertEquals(2, keyValues.length); + } + scanner.close(); + scan = new Scan(); + scan.setStartRow(toBytes("Key")); + scan.setStopRow(toBytes("Kf")); + scan.addColumn(family1, family1_column1); + scan.addColumn(family1, family1_column2); + scan.addColumn(family1, family1_column3); + scan.addColumn(family2, family2_column1); + scan.addColumn(family2, family2_column2); + scanner = multiCfHTable.getScanner(scan); + + for (Result result : scanner) { + KeyValue[] keyValues = result.raw(); + long timestamp = keyValues[0].getTimestamp(); + for (int i = 1; i < keyValues.length; ++i) { + assertEquals(timestamp, keyValues[i].getTimestamp()); + byte[] qualifier = keyValues[i].getQualifier(); + byte[] expectedValue = expectedValues.get(qualifier); + if (expectedValue != null) { + assertEquals(expectedValue, keyValues[i].getValue()); + } + } + assertEquals(5, keyValues.length); + } + scanner.close(); + scan = new Scan(); + scan.setStartRow(toBytes("Key")); + scan.setStopRow(toBytes("Kf")); + scan.addFamily(family1); + scan.addFamily(family2); + + scanner = multiCfHTable.getScanner(scan); + + for (Result result : scanner) { + KeyValue[] keyValues = result.raw(); + long timestamp = keyValues[0].getTimestamp(); + for (int i = 1; i < keyValues.length; ++i) { + assertEquals(timestamp, keyValues[i].getTimestamp()); + byte[] qualifier = keyValues[i].getQualifier(); + byte[] expectedValue = expectedValues.get(qualifier); + if (expectedValue != null) { + assertEquals(expectedValue, keyValues[i].getValue()); + } + } + assertEquals(5, keyValues.length); + } + scanner.close(); + scan = new Scan(); + scan.setStartRow(toBytes("Key")); + scan.setStopRow(toBytes("Kf")); + scan.addFamily(family1); + scan.addFamily(family3); + + scanner = multiCfHTable.getScanner(scan); + + for (Result result : scanner) { + KeyValue[] keyValues = result.raw(); + long timestamp = keyValues[0].getTimestamp(); + for (int i = 1; i < keyValues.length; ++i) { + assertEquals(timestamp, keyValues[i].getTimestamp()); + byte[] qualifier = keyValues[i].getQualifier(); + byte[] expectedValue = expectedValues.get(qualifier); + if (expectedValue != null) { + assertEquals(expectedValue, keyValues[i].getValue()); + } + } + // f1c1 f1c2 f1c3 f3c1 + assertEquals(4, keyValues.length); + } + scanner.close(); + } + + @Test + public void testMultiColumnFamilyScanWithFilter() throws Exception { + byte[] family1 = "family_with_group1".getBytes(); + byte[] family2 = "family_with_group2".getBytes(); + byte[] family3 = "family_with_group3".getBytes(); + + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family1_value = "VVV1".getBytes(); + byte[] family2_value = "VVV2".getBytes(); + byte[] family3_value = "VVV3".getBytes(); + + Map expectedValues = new HashMap<>(); + expectedValues.put(family1_column1, family1_value); + expectedValues.put(family1_column2, family1_value); + expectedValues.put(family1_column3, family1_value); + expectedValues.put(family2_column1, family2_value); + expectedValues.put(family2_column2, family2_value); + expectedValues.put(family3_column1, family3_value); + + int rows = 30; + + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + multiCfHTable.put(put); + } + + PrefixFilter filter = new PrefixFilter(toBytes("Key1")); + Scan scan = new Scan(); + scan.setStartRow(toBytes("Key")); + scan.setStopRow(toBytes("Kf")); + scan.setFilter(filter); + ResultScanner scanner = multiCfHTable.getScanner(scan); + + // Key1, Key10, Key11, Key12, Key13, Key14, Key15, Key16, Key17, Key18, Key19 + int count = 0; + for (Result result : scanner) { + KeyValue[] keyValues = result.raw(); + long timestamp = keyValues[0].getTimestamp(); + for (int i = 1; i < keyValues.length; ++i) { + assertEquals(timestamp, keyValues[i].getTimestamp()); + byte[] qualifier = keyValues[i].getQualifier(); + byte[] expectedValue = expectedValues.get(qualifier); + if (expectedValue != null) { + assertEquals(expectedValue, keyValues[i].getValue()); + } + } + assertEquals(6, keyValues.length); + count++; + } + assertEquals(11, count); + } + + @Test + public void testMultiColumnFamilyGet() throws Exception { + byte[] family1 = "family_with_group1".getBytes(); + byte[] family2 = "family_with_group2".getBytes(); + byte[] family3 = "family_with_group3".getBytes(); + + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family1_value = "VVV1".getBytes(); + byte[] family2_value = "VVV2".getBytes(); + byte[] family3_value = "VVV3".getBytes(); + + Map expectedValues = new HashMap<>(); + expectedValues.put(family1_column1, family1_value); + expectedValues.put(family1_column2, family1_value); + expectedValues.put(family1_column3, family1_value); + expectedValues.put(family2_column1, family2_value); + expectedValues.put(family2_column2, family2_value); + expectedValues.put(family3_column1, family3_value); + + int rows = 3; + + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + multiCfHTable.put(put); + } + + // get with empty family + // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 + Get get = new Get(toBytes("Key1")); + Result result = multiCfHTable.get(get); + KeyValue[] keyValues = result.raw(); + long timestamp = keyValues[0].getTimestamp(); + for (int i = 1; i < keyValues.length; ++i) { + assertEquals(timestamp, keyValues[i].getTimestamp()); + byte[] qualifier = keyValues[i].getQualifier(); + byte[] expectedValue = expectedValues.get(qualifier); + if (expectedValue != null) { + assertEquals(expectedValue, keyValues[i].getValue()); + } + } + assertEquals(6, keyValues.length); + + // f1c1 f2c1 f2c2 + Get get2 = new Get(toBytes("Key1")); + get2.addColumn(family1, family1_column1); + get2.addColumn(family2, family2_column1); + get2.addColumn(family2, family2_column2); + Result result2 = multiCfHTable.get(get2); + keyValues = result2.raw(); + timestamp = keyValues[0].getTimestamp(); + for (int i = 1; i < keyValues.length; ++i) { + assertEquals(timestamp, keyValues[i].getTimestamp()); + byte[] qualifier = keyValues[i].getQualifier(); + byte[] expectedValue = expectedValues.get(qualifier); + if (expectedValue != null) { + assertEquals(expectedValue, keyValues[i].getValue()); + } + } + assertEquals(3, keyValues.length); + + //f2c1 f2c2 + Get get3 = new Get(toBytes("Key1")); + get3.addFamily(family1); + get3.addColumn(family2, family2_column1); + get3.addColumn(family2, family2_column2); + Result result3 = multiCfHTable.get(get3); + keyValues = result3.raw(); + timestamp = keyValues[0].getTimestamp(); + for (int i = 1; i < keyValues.length; ++i) { + assertEquals(timestamp, keyValues[i].getTimestamp()); + byte[] qualifier = keyValues[i].getQualifier(); + byte[] expectedValue = expectedValues.get(qualifier); + if (expectedValue != null) { + assertEquals(expectedValue, keyValues[i].getValue()); + } + } + assertEquals(5, keyValues.length); + } + + @Test + public void testMultiColumnFamilyDelete() throws Exception { + String key1 = "scanKey1x"; + String key2 = "scanKey2x"; + String key3 = "scanKey3x"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + + byte[] family1 = "family_with_group1".getBytes(); + byte[] family2 = "family_with_group2".getBytes(); + byte[] family3 = "family_with_group3".getBytes(); + + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family2_column3 = "family2_column3".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family1_value = "VVV1".getBytes(); + byte[] family2_value = "VVV2".getBytes(); + byte[] family3_value = "VVV3".getBytes(); + + int rows = 10; + + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + Delete delete = new Delete(toBytes("Key" + i)); + multiCfHTable.delete(delete); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + multiCfHTable.put(put); + } + + // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 + Delete delete = new Delete(toBytes("Key1")); + delete.deleteColumns(family1, family1_column1); + delete.deleteColumns(family2, family2_column1); + multiCfHTable.delete(delete); + // f1c2 f1c3 f2c2 f3c1 + Get get = new Get(toBytes("Key1")); + Result result = multiCfHTable.get(get); + KeyValue[] keyValues = result.raw(); + assertEquals(4, keyValues.length); + assertFalse(result.containsColumn(family1, family1_column1)); + assertFalse(result.containsColumn(family2, family2_column1)); + + assertTrue(result.containsColumn(family1, family1_column2)); + assertArrayEquals(result.getValue(family1, family1_column2), family1_value); + assertTrue(result.containsColumn(family1, family1_column3)); + assertArrayEquals(result.getValue(family1, family1_column3), family1_value); + assertTrue(result.containsColumn(family2, family2_column2)); + assertArrayEquals(result.getValue(family2, family2_column2), family2_value); + assertTrue(result.containsColumn(family3, family3_column1)); + assertArrayEquals(result.getValue(family3, family3_column1), family3_value); + + // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 + delete = new Delete(toBytes("Key2")); + delete.deleteFamily(family1); + delete.deleteFamily(family2); + // f3c1 + multiCfHTable.delete(delete); + get = new Get(toBytes("Key2")); + result = multiCfHTable.get(get); + keyValues = result.raw(); + assertEquals(1, keyValues.length); + + // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 + delete = new Delete(toBytes("Key3")); + delete.deleteFamily(family1); + delete.deleteColumns(family2, family2_column1); + multiCfHTable.delete(delete); + // f2c2 f3c1 + get = new Get(toBytes("Key3")); + result = multiCfHTable.get(get); + keyValues = result.raw(); + assertEquals(2, keyValues.length); + + // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 + delete = new Delete(toBytes("Key4")); + multiCfHTable.delete(delete); + // null + get = new Get(toBytes("Key4")); + result = multiCfHTable.get(get); + keyValues = result.raw(); + assertEquals(0, keyValues.length); + + // f1c1 f2c1 f2c2 + delete = new Delete(toBytes("Key5")); + delete.deleteColumns(family1, family1_column2); + delete.deleteColumns(family1, family1_column3); + delete.deleteColumns(family3, family3_column1); + multiCfHTable.delete(delete); + // null + get = new Get(toBytes("Key5")); + result = multiCfHTable.get(get); + keyValues = result.raw(); + assertEquals(3, keyValues.length); + + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.add(family1, family1_column1, family1_value); + put.add(family1, family1_column2, family1_value); + put.add(family1, family1_column3, family1_value); + put.add(family2, family2_column1, family2_value); + put.add(family2, family2_column2, family2_value); + put.add(family3, family3_column1, family3_value); + multiCfHTable.put(put); + } + + delete = new Delete(toBytes("Key6")); + delete.deleteColumn(family1, family1_column2); + delete.deleteColumn(family2, family2_column1); + multiCfHTable.delete(delete); + get = new Get(toBytes("Key6")); + result = multiCfHTable.get(get); + keyValues = result.raw(); + assertEquals(6, keyValues.length); + + long lastTimestamp = result.getColumnCells(family1, family1_column1).get(0).getTimestamp(); + assertEquals(lastTimestamp, result.getColumnCells(family1, family1_column3).get(0) + .getTimestamp()); + assertEquals(lastTimestamp, result.getColumnCells(family2, family2_column2).get(0) + .getTimestamp()); + assertEquals(lastTimestamp, result.getColumnCells(family3, family3_column1).get(0) + .getTimestamp()); + + long oldTimestamp = result.getColumnCells(family1, family1_column2).get(0).getTimestamp(); + assertEquals(oldTimestamp, result.getColumnCells(family2, family2_column1).get(0) + .getTimestamp()); + assertTrue(lastTimestamp > oldTimestamp); + } + + @Test + public void testFamilyFilter() throws Exception { + String key1 = "getKey1"; + String key2 = "getKey2"; + String column1 = "abc"; + String column2 = "def"; + String value1 = "value1"; + String value2 = "value2"; + String family1 = "family_with_group1"; + String family2 = "family_with_group2"; + String family3 = "family_with_group3"; + + Delete deleteKey1 = new Delete(toBytes(key1)); + deleteKey1.deleteFamily(toBytes(family1)); + deleteKey1.deleteFamily(toBytes(family2)); + deleteKey1.deleteFamily(toBytes(family3)); + Delete deleteKey2 = new Delete(toBytes(key2)); + deleteKey2.deleteFamily(toBytes(family1)); + deleteKey2.deleteFamily(toBytes(family2)); + deleteKey2.deleteFamily(toBytes(family3)); + + Put putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family1), toBytes(column1), toBytes(value1)); + + Put putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family2), toBytes(column1), toBytes(value2)); + + Put putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family2), toBytes(column2), toBytes(value2)); + + Put putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family3), toBytes(column2), toBytes(value1)); + + Put putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family3), toBytes(column1), toBytes(value1)); + + Put putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family3), toBytes(column1), toBytes(value2)); + + multiCfHTable.delete(deleteKey1); + multiCfHTable.delete(deleteKey2); + multiCfHTable.put(putKey1Column1Value1); + multiCfHTable.put(putKey1Column1Value2); + multiCfHTable.put(putKey1Column2Value2); + multiCfHTable.put(putKey2Column2Value1); + multiCfHTable.put(putKey2Column1Value1); + multiCfHTable.put(putKey2Column1Value2); + + Scan scan; + scan = new Scan(); + scan.addFamily(family1.getBytes()); + scan.addFamily(family2.getBytes()); + scan.addFamily(family3.getBytes()); + scan.setMaxVersions(10); + FamilyFilter f = new FamilyFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator( + Bytes.toBytes(family2))); + scan.setFilter(f); + ResultScanner scanner = multiCfHTable.getScanner(scan); + + int res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); + Assert.assertArrayEquals(family2.getBytes(), keyValue.getFamily()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 2); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family1.getBytes()); + scan.addFamily(family2.getBytes()); + scan.addFamily(family3.getBytes()); + scan.setMaxVersions(10); + f = new FamilyFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator( + Bytes.toBytes(family2))); + scan.setFilter(f); + scanner = multiCfHTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); + res_count += 1; + } + } + Assert.assertEquals(res_count, 4); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family1.getBytes()); + scan.addFamily(family2.getBytes()); + scan.addFamily(family3.getBytes()); + scan.setMaxVersions(10); + f = new FamilyFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator( + Bytes.toBytes(family2))); + scan.setFilter(f); + scanner = multiCfHTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); + Assert.assertArrayEquals(family3.getBytes(), keyValue.getFamily()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 3); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family1.getBytes()); + scan.addFamily(family2.getBytes()); + scan.addFamily(family3.getBytes()); + scan.setMaxVersions(10); + f = new FamilyFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator( + Bytes.toBytes("family_with_group"))); + scan.setFilter(f); + scanner = multiCfHTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); + res_count += 1; + } + } + Assert.assertEquals(res_count, 6); + scanner.close(); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 4c93efde..9d32921d 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -24,7 +24,9 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.*; +import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.junit.Assert; import org.junit.Ignore; import org.junit.Rule; @@ -42,12 +44,12 @@ import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.*; -public abstract class HTableTestBase { +public abstract class HTableTestBase extends HTableMultiCFTestBase { @Rule public ExpectedException expectedException = ExpectedException.none(); - protected Table hTable; + protected static Table hTable; @Test public void testTableGroup() throws IOError, IOException { @@ -103,7 +105,7 @@ PRIMARY KEY (`K`, `Q`, `T`) } // test scan with empty family - Scan scan = new Scan(); + Scan scan = new Scan(toBytes(key)); ResultScanner scanner = hTable.getScanner(scan); for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { @@ -131,9 +133,6 @@ private void testBasic(String family) throws Exception { String column2 = "putColumn2"; String value = "value"; long timestamp = System.currentTimeMillis(); - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); Put put = new Put(toBytes(key)); put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(value)); @@ -167,7 +166,7 @@ private void testBasic(String family) throws Exception { r = hTable.get(get); Assert.assertEquals(1, r.raw().length); - delete = new Delete(key.getBytes()); + Delete delete = new Delete(key.getBytes()); delete.deleteFamily(family.getBytes()); hTable.delete(delete); @@ -405,27 +404,7 @@ public void testMultiPartitionDel() throws IOException { String column1 = "column1"; String column2 = "column2"; String column3 = "column3"; - String value = "value"; String family = "familyPartition"; - // delete - { - List deletes = new ArrayList(); - for (String key : keys) { - Delete del = new Delete(Bytes.toBytes(key)); - del.deleteColumns(toBytes(family), toBytes(column1)); - del.deleteColumns(toBytes(family), toBytes(column2), System.currentTimeMillis()); - deletes.add(del); - } - - for (String key : keys) { - // del same k, q, t - Delete del = new Delete(Bytes.toBytes(key)); - del.deleteColumn(toBytes(family), toBytes(column3), 100L); - del.deleteColumn(toBytes(family), toBytes(column3), 100L); - deletes.add(del); - } - hTable.delete(deletes); - } // get { List gets = new ArrayList(); @@ -458,7 +437,6 @@ public void testFilter() throws Exception { String column2 = "def"; String value1 = "value1"; String value2 = "value2"; - String value3 = "value3"; String family = "family1"; Delete deleteKey1Family = new Delete(toBytes(key1)); deleteKey1Family.deleteFamily(toBytes(family)); @@ -466,9 +444,6 @@ public void testFilter() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -497,8 +472,6 @@ public void testFilter() throws Exception { Result r; ColumnPrefixFilter filter; - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); tryPut(hTable, putKey1Column1Value1); tryPut(hTable, putKey1Column1Value2); tryPut(hTable, putKey1Column1Value1); @@ -509,6 +482,81 @@ public void testFilter() throws Exception { tryPut(hTable, putKey2Column2Value1); tryPut(hTable, putKey2Column2Value2); + // time may be different + // +---------+-----+----------------+--------+ + // | K | Q | T | V | + // +---------+-----+----------------+--------+ + // | getKey1 | abc | -1728834971469 | value1 | + // | getKey1 | abc | -1728834971399 | value2 | + // | getKey1 | abc | -1728834971330 | value1 | + // | getKey1 | def | -1728834971748 | value2 | + // | getKey1 | def | -1728834971679 | value1 | + // | getKey1 | def | -1728834971609 | value2 | + // | getKey1 | def | -1728834971540 | value1 | + // | getKey2 | def | -1728834971887 | value2 | + // | getKey2 | def | -1728834971818 | value1 | + // +---------+-----+----------------+--------+ + + SingleColumnValueFilter singleColumnValueFilter; + singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueFilter); + r = hTable.get(get); + Assert.assertEquals(7, r.raw().length); + + SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; + singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueExcludeFilter); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter( + Bytes.toBytes(family), Bytes.toBytes(column1), false); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), true); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + filter = new ColumnPrefixFilter(Bytes.toBytes("e")); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -817,28 +865,84 @@ public void testFilter() throws Exception { } Assert.assertEquals(res_count, 12); scanner.close(); + + long timestamp = System.currentTimeMillis(); + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), timestamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timestamp, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + // putKey1Column1Value1 and putKey2Column1Value1 have the same timestamp + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column2Value2); + + dependentColumnFilter = new DependentColumnFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setStartRow("getKey1".getBytes()); + scan.setStopRow("getKey3".getBytes()); + scan.setMaxVersions(10); + scan.setFilter(dependentColumnFilter); + scanner = hTable.getScanner(scan); + + long prevTimestamp = -1; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + if (prevTimestamp == -1) { + prevTimestamp = keyValue.getTimestamp(); + } else { + Assert.assertEquals(prevTimestamp, keyValue.getTimestamp()); + } + } + } + scanner.close(); } @Test - public void testFilter2() throws Exception { - String key1 = "getKey1"; - String key2 = "getKey2"; + public void testRowRangeFilter() throws Exception { + String key1 = "ka1"; + String key2 = "kb2"; String column1 = "abc"; String column2 = "def"; String value1 = "value1"; String value2 = "value2"; String value3 = "value3"; String family = "family1"; - long ts; Delete deleteKey1Family = new Delete(toBytes(key1)); deleteKey1Family.deleteFamily(toBytes(family)); Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -863,9 +967,6 @@ public void testFilter2() throws Exception { Put putKey2Column2Value1 = new Put(toBytes(key2)); putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); - Get get; - Result r; - hTable.delete(deleteKey1Family); hTable.delete(deleteKey2Family); tryPut(hTable, putKey1Column1Value1); @@ -878,90 +979,56 @@ public void testFilter2() throws Exception { tryPut(hTable, putKey2Column2Value1); tryPut(hTable, putKey2Column2Value2); - FirstKeyOnlyFilter filter = new FirstKeyOnlyFilter(); - get = new Get(toBytes(key1)); - get.setMaxVersions(10); - get.addFamily(toBytes(family)); - get.setFilter(filter); - r = hTable.get(get); - Assert.assertEquals(1, r.raw().length); - - KeyOnlyFilter kFilter = new KeyOnlyFilter(); - get = new Get(toBytes(key1)); - get.setMaxVersions(10); - get.addFamily(toBytes(family)); - get.setFilter(kFilter); - r = hTable.get(get); - Assert.assertEquals(7, r.raw().length); - - ts = r.rawCells()[0].getTimestamp(); - - List tss = new ArrayList<>(); - tss.add(ts - 1); - tss.add(ts); - tss.add(ts + 1); - TimestampsFilter tFilter = new TimestampsFilter(tss); - get = new Get(toBytes(key1)); - get.setMaxVersions(10); - get.addFamily(toBytes(family)); - get.setFilter(tFilter); - r = hTable.get(get); - Assert.assertEquals(1, r.raw().length); - - tss = new ArrayList<>(); - tss.add(ts - 1); - tss.add(ts + 1); - tFilter = new TimestampsFilter(tss); - get = new Get(toBytes(key1)); - get.setMaxVersions(10); - get.addFamily(toBytes(family)); - get.setFilter(tFilter); - r = hTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Put putKey1Column3Value1 = new Put(toBytes(key1)); - putKey1Column3Value1.add(toBytes(family), toBytes("ggg"), toBytes(value1)); - tryPut(hTable, putKey1Column3Value1); - Scan scan; scan = new Scan(); scan.addFamily(family.getBytes()); - scan.setStartRow("getKey1".getBytes()); - scan.setStopRow("getKey3".getBytes()); scan.setMaxVersions(10); + List ranges = new ArrayList<>(); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("ka3"), true, Bytes.toBytes("kd4"), false)); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("c"), true, Bytes.toBytes("d$%%"), false)); + MultiRowRangeFilter filter = new MultiRowRangeFilter(ranges); + scan.setFilter(filter); ResultScanner scanner = hTable.getScanner(scan); int res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - if (res_count < 8) { - Assert.assertArrayEquals(key1.getBytes(), keyValue.getRow()); - } else { - Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); - } + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); res_count += 1; } } - Assert.assertEquals(res_count, 10); + Assert.assertEquals(res_count, 2); scanner.close(); scan = new Scan(); scan.addFamily(family.getBytes()); - scan.setStartRow("getKey1".getBytes()); - scan.setStopRow("getKey3".getBytes()); scan.setMaxVersions(10); - filter = new FirstKeyOnlyFilter(); + scan.setReversed(true); + ranges = new ArrayList<>(); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("ka3"), true, Bytes.toBytes("kd4"), false)); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("c"), true, Bytes.toBytes("d$%%"), false)); + filter = new MultiRowRangeFilter(ranges); scan.setFilter(filter); scanner = hTable.getScanner(scan); res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - if (res_count < 1) { - Assert.assertArrayEquals(key1.getBytes(), keyValue.getRow()); - } else { - Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); - } + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); res_count += 1; } } @@ -970,157 +1037,1134 @@ public void testFilter2() throws Exception { scan = new Scan(); scan.addFamily(family.getBytes()); - scan.setStartRow("getKey1".getBytes()); - scan.setStopRow("getKey3".getBytes()); scan.setMaxVersions(10); - kFilter = new KeyOnlyFilter(); - scan.setFilter(kFilter); + scan.setReversed(true); + ranges = new ArrayList<>(); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("kb2"), false, Bytes.toBytes("kd4"), false)); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("kb11"), true, Bytes.toBytes("kb2"), true)); + filter = new MultiRowRangeFilter(ranges); + scan.setFilter(filter); scanner = hTable.getScanner(scan); res_count = 0; for (Result result : scanner) { for (KeyValue keyValue : result.raw()) { - if (res_count < 8) { - Assert.assertArrayEquals(key1.getBytes(), keyValue.getRow()); - } else { - Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); - } - Assert.assertEquals(0, keyValue.getValueLength()); + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); res_count += 1; } } - Assert.assertEquals(res_count, 10); + Assert.assertEquals(res_count, 2); scanner.close(); scan = new Scan(); scan.addFamily(family.getBytes()); - scan.setStartRow("getKey1".getBytes()); - scan.setStopRow("getKey3".getBytes()); scan.setMaxVersions(10); - kFilter = new KeyOnlyFilter(true); - scan.setFilter(kFilter); + scan.setReversed(true); + ranges = new ArrayList<>(); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("z4"), true, Bytes.toBytes("zzzsad"), true)); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("kb2"), true, Bytes.toBytes("kd4"), false)); + filter = new MultiRowRangeFilter(ranges); + scan.setFilter(filter); scanner = hTable.getScanner(scan); res_count = 0; for (Result result : scanner) { - for (Cell cell : result.rawCells()) { - if (res_count < 8) { - Assert.assertArrayEquals(key1.getBytes(), cell.getRow()); - } else { - Assert.assertArrayEquals(key2.getBytes(), cell.getRow()); - } - Assert.assertEquals(4, cell.getValueLength()); - Assert.assertEquals(6, Bytes.toInt(CellUtil.cloneValue(cell))); + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); res_count += 1; } } - Assert.assertEquals(res_count, 10); + Assert.assertEquals(res_count, 2); scanner.close(); - FilterList filterList = new FilterList(MUST_PASS_ONE); - filterList.addFilter(new KeyOnlyFilter(false)); - filterList.addFilter(new KeyOnlyFilter(true)); + // Inclusive stop filter scan = new Scan(); scan.addFamily(family.getBytes()); - scan.setStartRow("getKey1".getBytes()); - scan.setStopRow("getKey3".getBytes()); scan.setMaxVersions(10); - scan.setFilter(filterList); + InclusiveStopFilter iFilter = new InclusiveStopFilter(Bytes.toBytes("ka1")); + scan.setFilter(iFilter); scanner = hTable.getScanner(scan); res_count = 0; for (Result result : scanner) { - for (Cell cell : result.rawCells()) { - if (res_count < 8) { - Assert.assertArrayEquals(key1.getBytes(), cell.getRow()); - } else { - Assert.assertArrayEquals(key2.getBytes(), cell.getRow()); - } - Assert.assertEquals(4, cell.getValueLength()); - Assert.assertEquals(0, Bytes.toInt(CellUtil.cloneValue(cell))); + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + Assert.assertArrayEquals(key1.getBytes(), keyValue.getRow()); res_count += 1; } } - Assert.assertEquals(res_count, 10); + Assert.assertEquals(res_count, 7); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setReversed(true); + scan.setMaxVersions(10); + iFilter = new InclusiveStopFilter(Bytes.toBytes("ka1")); + scan.setFilter(iFilter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + Assert.assertArrayEquals(key1.getBytes(), keyValue.getRow()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 7); scanner.close(); - filterList = new FilterList(MUST_PASS_ALL); - filterList.addFilter(new KeyOnlyFilter(false)); - filterList.addFilter(new KeyOnlyFilter(true)); scan = new Scan(); scan.addFamily(family.getBytes()); - scan.setStartRow("getKey1".getBytes()); - scan.setStopRow("getKey3".getBytes()); scan.setMaxVersions(10); - scan.setFilter(filterList); - scanner = hTable.getScanner(scan); + iFilter = new InclusiveStopFilter(Bytes.toBytes("kb1")); + scan.setFilter(iFilter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 7); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + iFilter = new InclusiveStopFilter(Bytes.toBytes("kb2")); + scan.setFilter(iFilter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 9); + scanner.close(); + } + + @Test + public void testColumnRangeFilter() throws Exception { + String key1 = "ka1"; + String key2 = "kb2"; + String column1 = "abc"; + String column2 = "def"; + String value1 = "value1"; + String value2 = "value2"; + String family = "family1"; + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family)); + + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family)); + + Put putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + Put putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + + Scan scan; + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, + Bytes.toBytes("b"), false); + scan.setFilter(filter); + ResultScanner scanner = hTable.getScanner(scan); + + int res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); + res_count += 1; + } + } + Assert.assertEquals(3, res_count); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + filter = new ColumnRangeFilter(Bytes.toBytes("abc"), true, Bytes.toBytes("def"), false); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); + res_count += 1; + } + } + Assert.assertEquals(3, res_count); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + filter = new ColumnRangeFilter(Bytes.toBytes("abc"), false, Bytes.toBytes("def"), true); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); + res_count += 1; + } + } + Assert.assertEquals(6, res_count); + scanner.close(); + + // MultipleColumnPrefixFilter + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + byte[][] range = { Bytes.toBytes("g"), Bytes.toBytes("3"), Bytes.toBytes("d"), }; + MultipleColumnPrefixFilter iFilter = new MultipleColumnPrefixFilter(range); + scan.setFilter(iFilter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); + res_count += 1; + } + } + Assert.assertEquals(6, res_count); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + // 和原生hbase不一致，已知 + range = new byte[][] { Bytes.toBytes("de"), Bytes.toBytes("bg"), Bytes.toBytes("nc"), + Bytes.toBytes("aa"), Bytes.toBytes("abcd"), Bytes.toBytes("dea"), }; + iFilter = new MultipleColumnPrefixFilter(range); + scan.setFilter(iFilter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); + res_count += 1; + } + } + Assert.assertEquals(6, res_count); + scanner.close(); + } + + @Test + public void testFilterNullRange() throws Exception { + String key1 = "ka1"; + String key2 = "kb2"; + String column1 = "abc"; + String column2 = "def"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + String family = "family1"; + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family)); + + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family)); + + Put putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + Put putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + + Scan scan; + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + byte[][] r = {}; + MultipleColumnPrefixFilter iFilter = new MultipleColumnPrefixFilter(r); + scan.setFilter(iFilter); + ResultScanner scanner = hTable.getScanner(scan); + + int res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out + .printf( + "Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue())); + Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 0); + scanner.close(); + } + + @Test + public void testFilter2() throws Exception { + String key1 = "getKey1"; + String key2 = "getKey2"; + String column1 = "abc"; + String column2 = "def"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + String family = "family1"; + long ts; + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family)); + + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family)); + + Put putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + Put putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + Get get; + Result r; + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + + FirstKeyOnlyFilter filter = new FirstKeyOnlyFilter(); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filter); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + + KeyOnlyFilter kFilter = new KeyOnlyFilter(); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(kFilter); + r = hTable.get(get); + Assert.assertEquals(7, r.raw().length); + + ts = r.rawCells()[0].getTimestamp(); + + List tss = new ArrayList<>(); + tss.add(ts - 1); + tss.add(ts); + tss.add(ts + 1); + TimestampsFilter tFilter = new TimestampsFilter(tss); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(tFilter); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + + tss = new ArrayList<>(); + tss.add(ts - 1); + tss.add(ts + 1); + tFilter = new TimestampsFilter(tss); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(tFilter); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Put putKey1Column3Value1 = new Put(toBytes(key1)); + putKey1Column3Value1.add(toBytes(family), toBytes("ggg"), toBytes(value1)); + tryPut(hTable, putKey1Column3Value1); + + Scan scan; + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setStartRow("getKey1".getBytes()); + scan.setStopRow("getKey3".getBytes()); + scan.setMaxVersions(10); + ResultScanner scanner = hTable.getScanner(scan); + + int res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + if (res_count < 8) { + Assert.assertArrayEquals(key1.getBytes(), keyValue.getRow()); + } else { + Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); + } + res_count += 1; + } + } + Assert.assertEquals(res_count, 10); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setStartRow("getKey1".getBytes()); + scan.setStopRow("getKey3".getBytes()); + scan.setMaxVersions(10); + filter = new FirstKeyOnlyFilter(); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + if (res_count < 1) { + Assert.assertArrayEquals(key1.getBytes(), keyValue.getRow()); + } else { + Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); + } + res_count += 1; + } + } + Assert.assertEquals(res_count, 2); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setStartRow("getKey1".getBytes()); + scan.setStopRow("getKey3".getBytes()); + scan.setMaxVersions(10); + kFilter = new KeyOnlyFilter(); + scan.setFilter(kFilter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + if (res_count < 8) { + Assert.assertArrayEquals(key1.getBytes(), keyValue.getRow()); + } else { + Assert.assertArrayEquals(key2.getBytes(), keyValue.getRow()); + } + Assert.assertEquals(0, keyValue.getValueLength()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 10); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setStartRow("getKey1".getBytes()); + scan.setStopRow("getKey3".getBytes()); + scan.setMaxVersions(10); + kFilter = new KeyOnlyFilter(true); + scan.setFilter(kFilter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + if (res_count < 8) { + Assert.assertArrayEquals(key1.getBytes(), cell.getRow()); + } else { + Assert.assertArrayEquals(key2.getBytes(), cell.getRow()); + } + Assert.assertEquals(4, cell.getValueLength()); + Assert.assertEquals(6, Bytes.toInt(CellUtil.cloneValue(cell))); + res_count += 1; + } + } + Assert.assertEquals(res_count, 10); + scanner.close(); + + FilterList filterList = new FilterList(MUST_PASS_ONE); + filterList.addFilter(new KeyOnlyFilter(false)); + filterList.addFilter(new KeyOnlyFilter(true)); + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setStartRow("getKey1".getBytes()); + scan.setStopRow("getKey3".getBytes()); + scan.setMaxVersions(10); + scan.setFilter(filterList); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + if (res_count < 8) { + Assert.assertArrayEquals(key1.getBytes(), cell.getRow()); + } else { + Assert.assertArrayEquals(key2.getBytes(), cell.getRow()); + } + Assert.assertEquals(4, cell.getValueLength()); + Assert.assertEquals(0, Bytes.toInt(CellUtil.cloneValue(cell))); + res_count += 1; + } + } + Assert.assertEquals(res_count, 10); + scanner.close(); + + filterList = new FilterList(MUST_PASS_ALL); + filterList.addFilter(new KeyOnlyFilter(false)); + filterList.addFilter(new KeyOnlyFilter(true)); + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setStartRow("getKey1".getBytes()); + scan.setStopRow("getKey3".getBytes()); + scan.setMaxVersions(10); + scan.setFilter(filterList); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + if (res_count < 8) { + Assert.assertArrayEquals(key1.getBytes(), cell.getRow()); + } else { + Assert.assertArrayEquals(key2.getBytes(), cell.getRow()); + } + Assert.assertEquals(4, cell.getValueLength()); + Assert.assertEquals(0, Bytes.toInt(CellUtil.cloneValue(cell))); + res_count += 1; + } + } + Assert.assertEquals(res_count, 10); + scanner.close(); + + filterList = new FilterList(MUST_PASS_ONE); + filterList.addFilter(new KeyOnlyFilter(true)); + filterList.addFilter(new KeyOnlyFilter(false)); + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setStartRow("getKey1".getBytes()); + scan.setStopRow("getKey3".getBytes()); + scan.setMaxVersions(10); + scan.setFilter(filterList); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + if (res_count < 8) { + Assert.assertArrayEquals(key1.getBytes(), cell.getRow()); + } else { + Assert.assertArrayEquals(key2.getBytes(), cell.getRow()); + } + Assert.assertEquals(0, cell.getValueLength()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 10); + scanner.close(); + + filterList = new FilterList(MUST_PASS_ALL); + filterList.addFilter(new KeyOnlyFilter(true)); + filterList.addFilter(new KeyOnlyFilter(false)); + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setStartRow("getKey1".getBytes()); + scan.setStopRow("getKey3".getBytes()); + scan.setMaxVersions(10); + scan.setFilter(filterList); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + if (res_count < 8) { + Assert.assertArrayEquals(key1.getBytes(), cell.getRow()); + } else { + Assert.assertArrayEquals(key2.getBytes(), cell.getRow()); + } + Assert.assertEquals(0, cell.getValueLength()); + res_count += 1; + } + } + Assert.assertEquals(res_count, 10); + scanner.close(); + } + + @Test + public void testFuzzyRowFilter() throws Exception { + String key1 = "abab"; + String key2 = "abcc"; + String column1 = "c1"; + String column2 = "c2"; + String column3 = "c3"; + String column4 = "c4"; + String column5 = "c5"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + String family = "family1"; + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family)); + + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family)); + + Put putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + Put putKey1Column3Value1 = new Put(toBytes(key1)); + putKey1Column3Value1.add(toBytes(family), toBytes(column3), toBytes(value1)); + + Put putKey1Column4Value1 = new Put(toBytes(key1)); + putKey1Column4Value1.add(toBytes(family), toBytes(column4), toBytes(value1)); + + Put putKey1Column5Value1 = new Put(toBytes(key1)); + putKey1Column5Value1.add(toBytes(family), toBytes(column5), toBytes(value1)); + + Put putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column3Value1); + tryPut(hTable, putKey1Column4Value1); + tryPut(hTable, putKey1Column5Value1); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + + Scan scan; + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + List> fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("abab"), Bytes.toBytes("0000"))); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + FuzzyRowFilter filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + ResultScanner scanner = hTable.getScanner(scan); + + int res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 10); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + fuzzyKey.add(new Pair(Bytes.toBytes("abcc"), Bytes.toBytes("0000"))); + filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 2); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("ccab"), Bytes.toBytes("1100"))); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 10); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("cccc"), Bytes.toBytes("1100"))); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 2); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("ab##"), Bytes.toBytes("0011"))); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 12); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("azc"), Bytes.toBytes("010"))); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 2); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("azccd"), Bytes.toBytes("01001"))); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 2); + scanner.close(); + + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + scan.setReversed(true); + fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes(""), Bytes.toBytes(""))); + fuzzyKey.add(new Pair(Bytes.toBytes("dddd"), Bytes.toBytes("0000"))); + filter = new FuzzyRowFilter(fuzzyKey); + scan.setFilter(filter); + scanner = hTable.getScanner(scan); + + res_count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); + res_count += 1; + } + } + Assert.assertEquals(res_count, 12); + scanner.close(); + } + + @Test + public void testFirstKeyValueMatchingQualifiersFilter() throws Exception { + String key1 = "getKey1"; + String key2 = "getKey2"; + String column1 = "c1"; + String column2 = "c2"; + String column3 = "c3"; + String column4 = "c4"; + String column5 = "c5"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + String family = "family1"; + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family)); + + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family)); + + Put putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + Put putKey1Column3Value1 = new Put(toBytes(key1)); + putKey1Column3Value1.add(toBytes(family), toBytes(column3), toBytes(value1)); + + Put putKey1Column4Value1 = new Put(toBytes(key1)); + putKey1Column4Value1.add(toBytes(family), toBytes(column4), toBytes(value1)); + + Put putKey1Column5Value1 = new Put(toBytes(key1)); + putKey1Column5Value1.add(toBytes(family), toBytes(column5), toBytes(value1)); + + Put putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); + + Put putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), toBytes(value2)); + + Put putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), toBytes(value2)); + + Put putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey1Column3Value1); + tryPut(hTable, putKey1Column4Value1); + tryPut(hTable, putKey1Column5Value1); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + + Scan scan; + scan = new Scan(); + scan.addFamily(family.getBytes()); + scan.setMaxVersions(10); + TreeSet qualifiers = new TreeSet<>(Bytes.BYTES_COMPARATOR); + qualifiers.add(Bytes.toBytes("c11")); + qualifiers.add(Bytes.toBytes("c2")); + FirstKeyValueMatchingQualifiersFilter filter = new FirstKeyValueMatchingQualifiersFilter(qualifiers); + scan.setFilter(filter); + ResultScanner scanner = hTable.getScanner(scan); - res_count = 0; + int res_count = 0; for (Result result : scanner) { - for (Cell cell : result.rawCells()) { - if (res_count < 8) { - Assert.assertArrayEquals(key1.getBytes(), cell.getRow()); - } else { - Assert.assertArrayEquals(key2.getBytes(), cell.getRow()); - } - Assert.assertEquals(4, cell.getValueLength()); - Assert.assertEquals(0, Bytes.toInt(CellUtil.cloneValue(cell))); + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); res_count += 1; } } - Assert.assertEquals(res_count, 10); + Assert.assertEquals(res_count, 5); scanner.close(); - filterList = new FilterList(MUST_PASS_ONE); - filterList.addFilter(new KeyOnlyFilter(true)); - filterList.addFilter(new KeyOnlyFilter(false)); scan = new Scan(); scan.addFamily(family.getBytes()); - scan.setStartRow("getKey1".getBytes()); - scan.setStopRow("getKey3".getBytes()); scan.setMaxVersions(10); - scan.setFilter(filterList); + scan.setReversed(true); + qualifiers = new TreeSet<>(Bytes.BYTES_COMPARATOR); + qualifiers.add(Bytes.toBytes("c22")); + qualifiers.add(Bytes.toBytes("c4")); + filter = new FirstKeyValueMatchingQualifiersFilter(qualifiers); + scan.setFilter(filter); scanner = hTable.getScanner(scan); res_count = 0; for (Result result : scanner) { - for (Cell cell : result.rawCells()) { - if (res_count < 8) { - Assert.assertArrayEquals(key1.getBytes(), cell.getRow()); - } else { - Assert.assertArrayEquals(key2.getBytes(), cell.getRow()); - } - Assert.assertEquals(0, cell.getValueLength()); + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); res_count += 1; } } - Assert.assertEquals(res_count, 10); + Assert.assertEquals(res_count, 11); scanner.close(); - filterList = new FilterList(MUST_PASS_ALL); - filterList.addFilter(new KeyOnlyFilter(true)); - filterList.addFilter(new KeyOnlyFilter(false)); scan = new Scan(); scan.addFamily(family.getBytes()); - scan.setStartRow("getKey1".getBytes()); - scan.setStopRow("getKey3".getBytes()); scan.setMaxVersions(10); - scan.setFilter(filterList); + scan.setReversed(true); + qualifiers = new TreeSet<>(Bytes.BYTES_COMPARATOR); + qualifiers.add(Bytes.toBytes("c22")); + qualifiers.add(Bytes.toBytes("a")); + filter = new FirstKeyValueMatchingQualifiersFilter(qualifiers); + scan.setFilter(filter); scanner = hTable.getScanner(scan); res_count = 0; for (Result result : scanner) { - for (Cell cell : result.rawCells()) { - if (res_count < 8) { - Assert.assertArrayEquals(key1.getBytes(), cell.getRow()); - } else { - Assert.assertArrayEquals(key2.getBytes(), cell.getRow()); - } - Assert.assertEquals(0, cell.getValueLength()); + for (KeyValue keyValue : result.raw()) { + System.out.printf("Rowkey: %s, Column Family: %s, Column Qualifier: %s, Timestamp: %d, Value: %s%n", + Bytes.toString(result.getRow()), + Bytes.toString(keyValue.getFamily()), + Bytes.toString(keyValue.getQualifier()), + keyValue.getTimestamp(), + Bytes.toString(keyValue.getValue()) + ); res_count += 1; } } - Assert.assertEquals(res_count, 10); + Assert.assertEquals(res_count, 12); scanner.close(); } @@ -1140,9 +2184,6 @@ public void testGetFilter() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -1242,6 +2283,15 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(1, r.raw().length); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + DependentColumnFilter dependentColumnFilter = new DependentColumnFilter( + Bytes.toBytes(family), Bytes.toBytes(column1)); + get.setFilter(dependentColumnFilter); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + // columnCountGetFilter filter 2 get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -1422,6 +2472,67 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(7, r.raw().length); + filterList = new FilterList(); + filterList.addFilter(new SingleColumnValueExcludeFilter(Bytes.toBytes(family), Bytes + .toBytes(column1), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column1), false)); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), false)); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), true)); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(0, r.raw().length); + + filterList = new FilterList(); + filterList.addFilter(new DependentColumnFilter(Bytes.toBytes(family), Bytes + .toBytes(column2), false, CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value2)))); + get = new Get(toBytes(key2)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(filterList); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + filterList = new FilterList(); filterList.addFilter(new ColumnCountGetFilter(1)); filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.GREATER, @@ -1547,6 +2658,17 @@ public void testGetFilter() throws Exception { r = hTable.get(get); Assert.assertEquals(7, r.raw().length); + SingleColumnValueExcludeFilter singleColumnValueExcludeFilter; + singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(family), + Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( + toBytes(value1))); + get = new Get(toBytes(key1)); + get.setMaxVersions(10); + get.addFamily(toBytes(family)); + get.setFilter(singleColumnValueExcludeFilter); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column1), CompareFilter.CompareOp.EQUAL, new BinaryComparator( toBytes(value2))); @@ -1721,11 +2843,6 @@ public void testScanWithObParams() throws Exception { Delete deleteKey4Family = new Delete(toBytes(key4)); deleteKey4Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteKey4Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -1825,12 +2942,6 @@ public void testScanSessionClean() throws Exception { Delete deleteKey5Family = new Delete(toBytes(key5)); deleteKey5Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteKey4Family); - hTable.delete(deleteKey5Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -1894,7 +3005,6 @@ public void testGet() throws Exception { String column2 = "column2"; String value1 = "value1"; String value2 = "value2"; - String value3 = "value3"; String family = "family1"; // delete previous data @@ -1909,12 +3019,6 @@ public void testGet() throws Exception { Delete deleteZKey2Family = new Delete(toBytes(zKey2)); deleteZKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteZKey1Family); - hTable.delete(deleteZKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -1978,54 +3082,206 @@ public void testGet() throws Exception { tryPut(hTable, putzKey2Column1Value1); // show table (time maybe different) - //+-----------+---------+----------------+--------+ - //| K | Q | T | V | - //+-----------+---------+----------------+--------+ - //| scanKey1x | column1 | -1709714409669 | value1 | - //| scanKey1x | column1 | -1709714409637 | value2 | - //| scanKey1x | column1 | -1709714409603 | value1 | - //| scanKey1x | column2 | -1709714409802 | value2 | - //| scanKey1x | column2 | -1709714409768 | value1 | - //| scanKey1x | column2 | -1709714409735 | value2 | - //| scanKey1x | column2 | -1709714409702 | value1 | - //| scanKey2x | column2 | -1709714409869 | value2 | - //| scanKey2x | column2 | -1709714409836 | value1 | - //| scanKey3x | column1 | -1709714409940 | value2 | - //| scanKey3x | column1 | -1709714409904 | value1 | - //| scanKey3x | column2 | -1709714410010 | value2 | - //| scanKey3x | column2 | -1709714409977 | value1 | - //+-----------+---------+----------------+--------+ + // +-----------+---------+----------------+--------+ + // | K | Q | T | V | + // +-----------+---------+----------------+--------+ + // | scanKey1x | column1 | -1729223351579 | value1 | + // | scanKey1x | column1 | -1729223351504 | value2 | + // | scanKey1x | column1 | -1729223351431 | value1 | + // | scanKey1x | column2 | -1729223351867 | value2 | + // | scanKey1x | column2 | -1729223351796 | value1 | + // | scanKey1x | column2 | -1729223351724 | value2 | + // | scanKey1x | column2 | -1729223351651 | value1 | + // | scanKey2x | column2 | -1729223352015 | value2 | + // | scanKey2x | column2 | -1729223351941 | value1 | + // | scanKey3x | column1 | -1729223352159 | value2 | + // | scanKey3x | column1 | -1729223352088 | value1 | + // | scanKey3x | column2 | -1729223352304 | value2 | + // | scanKey3x | column2 | -1729223352232 | value1 | + // | zScanKey1 | column1 | -1729223352378 | value1 | + // | zScanKey2 | column1 | -1729223352450 | value1 | + // +-----------+---------+----------------+--------+ + + // test closestRowBefore + get = new Get("scanKey2x2".getBytes()); + get.addFamily(family.getBytes()); + get.setClosestRowBefore(true); + r = hTable.get(get); + assertEquals(key2, Bytes.toString(r.getRow())); + + // test exists + LinkedList gets = new LinkedList<>(); + Get get1 = new Get(key1.getBytes()); + get1.addFamily(family.getBytes()); + Get get2 = new Get(key3.getBytes()); + get2.addFamily(family.getBytes()); + Get get3 = new Get(key1.getBytes()); + get3.addFamily(family.getBytes()); + Get get4 = new Get("scanKey2x2".getBytes()); + get4.addFamily(family.getBytes()); + Get get5 = new Get(key2.getBytes()); + get5.addFamily(family.getBytes()); + gets.add(get1); + gets.add(get2); + gets.add(get3); + gets.add(get4); + gets.add(get5); + boolean[] booleans = hTable.existsAll(gets); + assertTrue(booleans[0]); + assertTrue(booleans[1]); + assertTrue(booleans[2]); + assertFalse(booleans[3]); + assertTrue(booleans[4]); + + // test single cf setColumnFamilyTimeRange + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + hTable.delete(deleteZKey1Family); + hTable.delete(deleteZKey2Family); + + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), timeStamp2, toBytes(value1)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timeStamp4, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), timeStamp5, toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), timeStamp7, toBytes(value2)); + + putKey3Column1Value1 = new Put(toBytes(key3)); + putKey3Column1Value1.add(toBytes(family), toBytes(column1), timeStamp8, toBytes(value1)); + + putKey3Column1Value2 = new Put(toBytes(key3)); + putKey3Column1Value2.add(toBytes(family), toBytes(column1), timeStamp9, toBytes(value2)); + + putKey3Column2Value1 = new Put(toBytes(key3)); + putKey3Column2Value1.add(toBytes(family), toBytes(column2), timeStamp10, toBytes(value1)); + + putKey3Column2Value2 = new Put(toBytes(key3)); + putKey3Column2Value2.add(toBytes(family), toBytes(column2), timeStamp11, toBytes(value2)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column1Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + tryPut(hTable, putKey3Column1Value1); + tryPut(hTable, putKey3Column1Value2); + tryPut(hTable, putKey3Column2Value1); + tryPut(hTable, putKey3Column2Value2); + + + get = new Get(toBytes(key1)); + get.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, maxTimeStamp); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(4, r.raw().length); + + get = new Get(toBytes(key1)); + get.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, timeStamp2 + 1); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + get = new Get(toBytes(key2)); + // set invalid timeRange + get.setTimeRange(minTimeStamp, maxTimeStamp); + get.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, timeStamp5 + 1); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key2)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp5, maxTimeStamp); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); - // test closestRowBefore - get = new Get("scanKey2x2".getBytes()); - get.addFamily(family.getBytes()); - get.setClosestRowBefore(true); + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp8, timeStamp8); + get.addFamily(toBytes(family)); + get.setMaxVersions(); r = hTable.get(get); - assertEquals(key2, Bytes.toString(r.getRow())); + Assert.assertEquals(0, r.raw().length); - // test exists - LinkedList gets = new LinkedList<>(); - Get get1 = new Get(key1.getBytes()); - get1.addFamily(family.getBytes()); - Get get2 = new Get(key3.getBytes()); - get2.addFamily(family.getBytes()); - Get get3 = new Get(key1.getBytes()); - get3.addFamily(family.getBytes()); - Get get4 = new Get("scanKey2x2".getBytes()); - get4.addFamily(family.getBytes()); - Get get5 = new Get(key2.getBytes()); - get5.addFamily(family.getBytes()); - gets.add(get1); - gets.add(get2); - gets.add(get3); - gets.add(get4); - gets.add(get5); - boolean[] booleans = hTable.existsAll(gets); - assertTrue(booleans[0]); - assertTrue(booleans[1]); - assertTrue(booleans[2]); - assertFalse(booleans[3]); - assertTrue(booleans[4]); + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp8, timeStamp9); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes(family), timeStamp8, timeStamp9); + get.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp8, timeStamp9); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + final Get multiFamGet = get; + Assert.assertThrows(IOException.class, () -> { + hTable.get(multiFamGet); + }); + + get = new Get(toBytes(key3)); + get.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp8, timeStamp9); + get.addFamily(toBytes(family)); + get.setMaxVersions(); + final Get missFamGet = get; + Assert.assertThrows(IOException.class, () -> { + hTable.get(missFamGet); + }); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); } @Test @@ -2039,7 +3295,6 @@ public void testScan() throws Exception { String column2 = "column2"; String value1 = "value1"; String value2 = "value2"; - String value3 = "value3"; String family = "family1"; // delete previous data @@ -2054,12 +3309,6 @@ public void testScan() throws Exception { Delete deleteZKey2Family = new Delete(toBytes(zKey2)); deleteZKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteZKey1Family); - hTable.delete(deleteZKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -2125,23 +3374,25 @@ public void testScan() throws Exception { tryPut(hTable, putzKey2Column1Value1); // show table (time maybe different) - //+-----------+---------+----------------+--------+ - //| K | Q | T | V | - //+-----------+---------+----------------+--------+ - //| scanKey1x | column1 | -1709714409669 | value1 | - //| scanKey1x | column1 | -1709714409637 | value2 | - //| scanKey1x | column1 | -1709714409603 | value1 | - //| scanKey1x | column2 | -1709714409802 | value2 | - //| scanKey1x | column2 | -1709714409768 | value1 | - //| scanKey1x | column2 | -1709714409735 | value2 | - //| scanKey1x | column2 | -1709714409702 | value1 | - //| scanKey2x | column2 | -1709714409869 | value2 | - //| scanKey2x | column2 | -1709714409836 | value1 | - //| scanKey3x | column1 | -1709714409940 | value2 | - //| scanKey3x | column1 | -1709714409904 | value1 | - //| scanKey3x | column2 | -1709714410010 | value2 | - //| scanKey3x | column2 | -1709714409977 | value1 | - //+-----------+---------+----------------+--------+ + // +-----------+---------+----------------+--------+ + // | K | Q | T | V | + // +-----------+---------+----------------+--------+ + // | scanKey1x | column1 | -1729236392149 | value1 | + // | scanKey1x | column1 | -1729236392078 | value2 | + // | scanKey1x | column1 | -1729236392008 | value1 | + // | scanKey1x | column2 | -1729236392436 | value2 | + // | scanKey1x | column2 | -1729236392364 | value1 | + // | scanKey1x | column2 | -1729236392291 | value2 | + // | scanKey1x | column2 | -1729236392220 | value1 | + // | scanKey2x | column2 | -1729236392576 | value2 | + // | scanKey2x | column2 | -1729236392506 | value1 | + // | scanKey3x | column1 | -1729236392720 | value2 | + // | scanKey3x | column1 | -1729236392647 | value1 | + // | scanKey3x | column2 | -1729236392861 | value2 | + // | scanKey3x | column2 | -1729236392790 | value1 | + // | zScanKey1 | column1 | -1729236392931 | value1 | + // | zScanKey2 | column1 | -1729236393002 | value1 | + // +-----------+---------+----------------+--------+ scan = new Scan(); scan.addFamily(family.getBytes()); @@ -2328,6 +3579,189 @@ public void testScan() throws Exception { hTable.delete(deleteKey3Family); hTable.delete(deleteZKey1Family); hTable.delete(deleteZKey2Family); + + // test single cf setColumnFamilyTimeRange + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), timeStamp2, toBytes(value1)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timeStamp4, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), timeStamp5, toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), timeStamp7, toBytes(value2)); + + putKey3Column1Value1 = new Put(toBytes(key3)); + putKey3Column1Value1.add(toBytes(family), toBytes(column1), timeStamp8, toBytes(value1)); + + putKey3Column1Value2 = new Put(toBytes(key3)); + putKey3Column1Value2.add(toBytes(family), toBytes(column1), timeStamp9, toBytes(value2)); + + putKey3Column2Value1 = new Put(toBytes(key3)); + putKey3Column2Value1.add(toBytes(family), toBytes(column2), timeStamp10, toBytes(value1)); + + putKey3Column2Value2 = new Put(toBytes(key3)); + putKey3Column2Value2.add(toBytes(family), toBytes(column2), timeStamp11, toBytes(value2)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column1Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + tryPut(hTable, putKey3Column1Value1); + tryPut(hTable, putKey3Column1Value2); + tryPut(hTable, putKey3Column2Value1); + tryPut(hTable, putKey3Column2Value2); + + // scan key1 + key2 + scan = new Scan(toBytes(key1), toBytes(key3)); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, maxTimeStamp); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(8, res_count); + + // scan key1 + scan = new Scan(toBytes(key1), toBytes(key2)); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, maxTimeStamp); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(4, res_count); + + // scan key1 + scan = new Scan(toBytes(key1), toBytes(key2)); + scan.addFamily(toBytes(family)); + // set invalid timeRange + scan.setTimeRange(minTimeStamp, maxTimeStamp); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp1, timeStamp3); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(2, res_count); + + // scan all + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp2, timeStamp7); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(5, res_count); + + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + boolean foundKey3 = false; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (!foundKey3) { + if (key3.equals(Bytes.toString(kv.getRow()))) { + foundKey3 = true; + } + } + ++res_count; + } + } + Assert.assertTrue(foundKey3); + Assert.assertEquals(6, res_count); + + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); + scan.setMaxVersions(); + final Scan multiFamScan = scan; + Assert.assertThrows(IOException.class, () -> { + hTable.getScanner(multiFamScan); + }); + + scan = new Scan(); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); + scan.setMaxVersions(); + final Scan missFamScan = scan; + Assert.assertThrows(IOException.class, () -> { + hTable.getScanner(missFamScan); + }); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); } @Test @@ -2356,12 +3790,6 @@ public void testReversedScan() throws Exception { Delete deleteZKey2Family = new Delete(toBytes(zKey2)); deleteZKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteZKey1Family); - hTable.delete(deleteZKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -2451,7 +3879,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); // reverse scan with MaxVersion @@ -2469,7 +3897,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 3); + Assert.assertEquals(3, res_count); scanner.close(); // reverse scan with pageFilter @@ -2489,7 +3917,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); // reverse scan with not_exist_start_row @@ -2507,7 +3935,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); // reverse scan with abnormal range @@ -2525,7 +3953,7 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 0); + Assert.assertEquals(0, res_count); scanner.close(); // reverse scan with abnormal range @@ -2543,13 +3971,205 @@ public void testReversedScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 13); + Assert.assertEquals(13, res_count); scanner.close(); hTable.delete(deleteKey1Family); hTable.delete(deleteKey2Family); hTable.delete(deleteKey3Family); + // test single cf setColumnFamilyTimeRange + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp8 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp9 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp10 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp11 = System.currentTimeMillis(); + Thread.sleep(5); + long maxTimeStamp = System.currentTimeMillis(); + + putKey1Column1Value1 = new Put(toBytes(key1)); + putKey1Column1Value1.add(toBytes(family), toBytes(column1), minTimeStamp, toBytes(value1)); + + putKey1Column1Value2 = new Put(toBytes(key1)); + putKey1Column1Value2.add(toBytes(family), toBytes(column1), timeStamp1, toBytes(value2)); + + putKey1Column2Value1 = new Put(toBytes(key1)); + putKey1Column2Value1.add(toBytes(family), toBytes(column2), timeStamp2, toBytes(value1)); + + putKey1Column2Value2 = new Put(toBytes(key1)); + putKey1Column2Value2.add(toBytes(family), toBytes(column2), timeStamp3, toBytes(value2)); + + putKey2Column1Value1 = new Put(toBytes(key2)); + putKey2Column1Value1.add(toBytes(family), toBytes(column1), timeStamp4, toBytes(value1)); + + putKey2Column1Value2 = new Put(toBytes(key2)); + putKey2Column1Value2.add(toBytes(family), toBytes(column1), timeStamp5, toBytes(value2)); + + putKey2Column2Value1 = new Put(toBytes(key2)); + putKey2Column2Value1.add(toBytes(family), toBytes(column2), timeStamp6, toBytes(value1)); + + putKey2Column2Value2 = new Put(toBytes(key2)); + putKey2Column2Value2.add(toBytes(family), toBytes(column2), timeStamp7, toBytes(value2)); + + putKey3Column1Value1 = new Put(toBytes(key3)); + putKey3Column1Value1.add(toBytes(family), toBytes(column1), timeStamp8, toBytes(value1)); + + putKey3Column1Value2 = new Put(toBytes(key3)); + putKey3Column1Value2.add(toBytes(family), toBytes(column1), timeStamp9, toBytes(value2)); + + putKey3Column2Value1 = new Put(toBytes(key3)); + putKey3Column2Value1.add(toBytes(family), toBytes(column2), timeStamp10, toBytes(value1)); + + putKey3Column2Value2 = new Put(toBytes(key3)); + putKey3Column2Value2.add(toBytes(family), toBytes(column2), timeStamp11, toBytes(value2)); + + tryPut(hTable, putKey1Column1Value1); + tryPut(hTable, putKey1Column1Value2); + tryPut(hTable, putKey1Column2Value1); + tryPut(hTable, putKey1Column2Value2); + tryPut(hTable, putKey2Column1Value1); + tryPut(hTable, putKey2Column1Value2); + tryPut(hTable, putKey2Column2Value1); + tryPut(hTable, putKey2Column2Value2); + tryPut(hTable, putKey3Column1Value1); + tryPut(hTable, putKey3Column1Value2); + tryPut(hTable, putKey3Column2Value1); + tryPut(hTable, putKey3Column2Value2); + + // scan key1 + scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(family.getBytes()); + scan.setColumnFamilyTimeRange(toBytes(family), minTimeStamp, timeStamp3); + scan.setReversed(true); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(3, res_count); + + // scan key1 + key2 + scan = new Scan(); + scan.setStartRow(toBytes(key2)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp2, timeStamp7); + scan.setReversed(true); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(5, res_count); + + // scan key2 + scan = new Scan(); + scan.setStartRow(toBytes(key2)); + scan.setStopRow(toBytes(key1)); + scan.addFamily(toBytes(family)); + // set invalid timeRange + scan.setTimeRange(minTimeStamp, maxTimeStamp); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp4, timeStamp6); + scan.setReversed(true); + scan.setMaxVersions(10); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(2, res_count); + + // scan all + scan = new Scan(); + scan.setStartRow(toBytes(key3)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp2, timeStamp7); + scan.setReversed(true); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + Assert.assertTrue(key1.equals(Bytes.toString(kv.getRow())) + || key2.equals(Bytes.toString(kv.getRow()))); + ++res_count; + } + } + Assert.assertEquals(5, res_count); + + scan = new Scan(); + scan.setStartRow(toBytes(key3)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setReversed(true); + scan.setMaxVersions(); + scanner = hTable.getScanner(scan); + res_count = 0; + boolean foundKey3 = false; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (!foundKey3) { + if (key3.equals(Bytes.toString(kv.getRow()))) { + foundKey3 = true; + } + } + ++res_count; + } + } + Assert.assertTrue(foundKey3); + Assert.assertEquals(6, res_count); + + scan = new Scan(); + scan.setStartRow(toBytes(key3)); + scan.setStopRow("scanKey0x".getBytes()); + scan.addFamily(toBytes(family)); + scan.setColumnFamilyTimeRange(toBytes(family), timeStamp3, timeStamp9); + scan.setColumnFamilyTimeRange(toBytes("mockFamily"), timeStamp3, timeStamp9); + scan.setReversed(true); + scan.setMaxVersions(); + final Scan errorScan = scan; + Assert.assertThrows(IOException.class, () -> { + hTable.getScanner(errorScan); + }); + + hTable.delete(deleteKey1Family); + hTable.delete(deleteKey2Family); + hTable.delete(deleteKey3Family); + } @Test @@ -2563,7 +4183,6 @@ public void testPartitionScan() throws Exception { String column2 = "column2"; String value1 = "value1"; String value2 = "value2"; - String value3 = "value3"; String family = "partitionFamily1"; // delete previous data @@ -2578,12 +4197,6 @@ public void testPartitionScan() throws Exception { Delete deleteZKey2Family = new Delete(toBytes(zKey2)); deleteZKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteZKey1Family); - hTable.delete(deleteZKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -2700,7 +4313,7 @@ public void testPartitionScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 7); + Assert.assertEquals(7, res_count); scanner.close(); // scan with prefixFilter @@ -2720,7 +4333,7 @@ public void testPartitionScan() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 2); + Assert.assertEquals(2, res_count); scanner.close(); // scan with singleColumnValueFilter @@ -2851,9 +4464,6 @@ public void testCheckAndPut() throws IOException, InterruptedException { String column = "checkAndPut"; String value = "value"; String family = "family1"; - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); Get get = new Get(key.getBytes()); get.setMaxVersions(Integer.MAX_VALUE); get.addColumn(family.getBytes(), column.getBytes()); @@ -2903,15 +4513,12 @@ public void testCheckAndDelete() throws IOException { String column2 = "checkAndDeleteColumn2"; String value = "value"; String family = "family1"; - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); Put put = new Put(key.getBytes()); put.add(family.getBytes(), column.getBytes(), value.getBytes()); hTable.put(put); // check delete column - delete = new Delete(key.getBytes()); + Delete delete = new Delete(key.getBytes()); delete.deleteColumn(family.getBytes(), column.getBytes()); boolean ret = hTable.checkAndDelete(key.getBytes(), family.getBytes(), column.getBytes(), value.getBytes(), delete); @@ -3001,9 +4608,6 @@ public void testCheckAndMutate() throws IOException { String value1 = "value1"; String value2 = "value2"; String family = "family1"; - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); long t = System.currentTimeMillis(); // put @@ -3143,9 +4747,6 @@ public void testCheckAndMutate() throws IOException { public void testAppend() throws IOException { String column = "appendColumn"; String key = "appendKey"; - Delete delete = new Delete(key.getBytes()); - delete.deleteColumns("family1".getBytes(), column.getBytes()); - hTable.delete(delete); // append an absent column is not supported yet // Append append = new Append(key.getBytes()); @@ -3172,9 +4773,6 @@ public void testAppend() throws IOException { public void testIncrement() throws IOException { String column = "incrementColumn"; String key = "incrementKey"; - Delete delete = new Delete(key.getBytes()); - delete.deleteColumns("family1".getBytes(), column.getBytes()); - hTable.delete(delete); // increment an absent column is not supported yet // Increment increment = new Increment(key.getBytes()); @@ -3220,9 +4818,6 @@ public void testIncrement() throws IOException { public void testExist() throws IOException { String column = "existColumn"; String key = "existKey"; - Delete delete = new Delete(key.getBytes()); - delete.deleteColumns("family1".getBytes(), column.getBytes()); - hTable.delete(delete); Get get = new Get(key.getBytes()); get.addFamily("family1".getBytes()); @@ -3246,8 +4841,6 @@ public void testExist() throws IOException { get.setTimeStamp(timestamp + 1); Assert.assertFalse(hTable.exists(get)); - - hTable.delete(delete); } @Ignore @@ -3257,7 +4850,6 @@ public void testMutateRow() throws IOException { String column2 = "mutationRowColumn2"; String key = "mutationRowKey"; String family1 = "family1"; - String family2 = "family2"; String value = "value"; Delete deleteFamily = new Delete(key.getBytes()); @@ -3337,9 +4929,6 @@ public void testQualifyNull() throws Exception { String value = "value"; String value1 = "value1"; String family = "family1"; - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); Put put = new Put(key.getBytes()); put.add(family.getBytes(), null, value.getBytes()); hTable.put(put); @@ -3427,6 +5016,8 @@ public void testFamilyBlank() throws Exception { fail(); } catch (IllegalArgumentException e) { Assert.assertTrue(e.getMessage().contains("family is blank")); + } catch (NoSuchColumnFamilyException e) { + Assert.assertTrue(e.getMessage().contains("does not exist")); } Put put = new Put(key.getBytes()); put.add(null, null, value.getBytes()); @@ -3464,9 +5055,6 @@ public void testScannerMultiVersion() throws Exception { String column = "column"; String value1 = "value1"; String family = "family1"; - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); Put put = new Put(key.getBytes()); put.add(family.getBytes(), Bytes.toBytes(column), value.getBytes()); hTable.put(put); @@ -3737,9 +5325,6 @@ public static byte[] toByteArray(long value) { public void testIncrementConcurrency() throws Exception { String column = "incrementColumn"; String key = "incrementKey"; - Delete delete = new Delete(key.getBytes()); - delete.deleteColumns("family1".getBytes(), column.getBytes()); - hTable.delete(delete); for (int i = 0; i < 100; i++) { Increment increment = new Increment(key.getBytes()); @@ -3785,16 +5370,12 @@ public void testFilterSpecialValue() throws IOException { byte[] columnBytes = specialBytes; byte[] valueBytes = specialBytes; - Delete delete = new Delete(keyBytes); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); - Put put = new Put(keyBytes); put.add(family.getBytes(), columnBytes, valueBytes); hTable.put(put); // check delete column - delete = new Delete(keyBytes); + Delete delete = new Delete(keyBytes); delete.deleteColumn(family.getBytes(), columnBytes); boolean ret = hTable.checkAndDelete(keyBytes, family.getBytes(), columnBytes, valueBytes, delete); @@ -3830,6 +5411,5 @@ public void testFilterSpecialValue() throws IOException { result = hTable.get(get); Assert.assertEquals(1, result.raw().length); Assert.assertArrayEquals(keyBytes, result.getRow()); - } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/LoggerTest.java b/src/test/java/com/alipay/oceanbase/hbase/LoggerTest.java index d1909ea0..0ab0617b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/LoggerTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/LoggerTest.java @@ -17,12 +17,13 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Increment; import org.apache.hadoop.hbase.client.Scan; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.api.support.membermodification.MemberModifier; @@ -42,7 +43,7 @@ public class LoggerTest { HTableInterface hTableMock; - @Before + @BeforeClass public void setup() throws IOException { Configuration c = ObHTableTestUtil.newConfiguration(); c.set("rs.list.acquire.read.timeout", "10000"); diff --git a/src/test/java/com/alipay/oceanbase/hbase/NativeHBaseTest.java b/src/test/java/com/alipay/oceanbase/hbase/NativeHBaseTest.java new file mode 100644 index 00000000..545d95b9 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/NativeHBaseTest.java @@ -0,0 +1,46 @@ +package com.alipay.oceanbase.hbase; + +import com.alipay.oceanbase.hbase.util.NativeHBaseUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.*; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class NativeHBaseTest extends HTableTestBase { + + static Admin admin; + static TableName tableName1 = TableName.valueOf("test"); + static TableName tableName2 = TableName.valueOf("test_multi_cf"); + + static { + try { + admin = NativeHBaseUtil.getAdmin(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @BeforeClass + public static void setup() throws IOException { + hTable = NativeHBaseUtil.getTable(tableName1); + multiCfHTable = NativeHBaseUtil.getTable(tableName2); + } + + @Before + public void cleanData() throws IOException { + admin.disableTable(tableName1); + admin.disableTable(tableName2); + admin.truncateTable(tableName1, true); + admin.truncateTable(tableName2, true); + } + + @AfterClass + public static void finish() throws IOException { + hTable.close(); + multiCfHTable.close(); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index a4a78a61..1fa79204 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -17,25 +17,28 @@ package com.alipay.oceanbase.hbase; -import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; -import org.junit.Assert; -import org.junit.Test; +import org.junit.*; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.*; +import static com.alipay.oceanbase.hbase.constants.OHConstants.SOCKET_TIMEOUT; +import static org.apache.hadoop.hbase.ipc.RpcClient.SOCKET_TIMEOUT_CONNECT; import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.*; public class OHConnectionTest { - protected Table hTable; - protected Connection connection; + protected static Table hTable; + protected Connection connection; @Test public void testConnectionBySet() throws Exception { @@ -43,10 +46,15 @@ public void testConnectionBySet() throws Exception { c.set(ClusterConnection.HBASE_CLIENT_CONNECTION_IMPL, "com.alipay.oceanbase.hbase.util.OHConnectionImpl"); c.set("rs.list.acquire.read.timeout", "10000"); + // test set rpc connection timeout, the first one is the latest version + c.set(SOCKET_TIMEOUT_CONNECT, "15000"); + // the second one is the deprecated version + c.set(SOCKET_TIMEOUT, "12000"); connection = ConnectionFactory.createConnection(c); TableName tableName = TableName.valueOf("test"); hTable = connection.getTable(tableName); testBasic(); + hTable.close(); } @Test @@ -54,10 +62,45 @@ public void testConnectionByXml() throws Exception { Configuration c = ObHTableTestUtil.newConfiguration(); c.set("rs.list.acquire.read.timeout", "10000"); + // can set rpc connection timeout in xml connection = ConnectionFactory.createConnection(c); TableName tableName = TableName.valueOf("test"); hTable = connection.getTable(tableName); testBasic(); + hTable.close(); + } + + @Test + public void testRefreshTableEntry() throws Exception { + hTable = ObHTableTestUtil.newOHTableClient("n1:test"); + ((OHTableClient) hTable).init(); + ((OHTableClient) hTable).refreshTableEntry("family1", false); + ((OHTableClient) hTable).refreshTableEntry("family1", true); + } + + @Test + public void testNew() throws Exception { + OHTableClient hTable2 = ObHTableTestUtil.newOHTableClient("n1:test"); + hTable2.init(); + hTable2.getConfiguration().set("rs.list.acquire.read.timeout", "10000"); + + assertTrue(hTable2.isAutoFlush()); + hTable2.setAutoFlush(false); + assertFalse(hTable2.isAutoFlush()); + hTable2.setAutoFlush(true, true); + assertTrue(hTable2.isAutoFlush()); + hTable2.setWriteBufferSize(10000000L); + assertEquals(10000000L, hTable2.getWriteBufferSize()); + assertEquals("n1:test", hTable2.getTableNameString()); + assertEquals("n1:test", new String(hTable2.getTableName())); + hTable2.flushCommits(); + hTable2.close(); + assertTrue(true); + } + + @After + public void after() throws IOException { + hTable.close(); } private void testBasic() throws Exception { @@ -194,26 +237,31 @@ PRIMARY KEY (`K`, `Q`, `T`) public void testBufferedMutatorWithFlush() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); conf.set("rs.list.acquire.read.timeout", "10000"); - BufferedMutator putBufferMutator = null; - BufferedMutator delBufferedMutator = null; + conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); + BufferedMutator bufferMutator = null; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // use defualt params - putBufferMutator = connection.getBufferedMutator(tableName); - delBufferedMutator = connection.getBufferedMutator(tableName); + bufferMutator = connection.getBufferedMutator(tableName); + - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; long timestamp = System.currentTimeMillis(); // only support Put and Delete - // for other type of operations, BufferedMutator will not set its type for them Append append = new Append(Bytes.toBytes(key)); append.add("family_group".getBytes(), column1.getBytes(), toBytes("_suffix")); - final BufferedMutator apMut = putBufferMutator; + final BufferedMutator apMut = bufferMutator; Assert.assertThrows(IllegalArgumentException.class, () -> { apMut.mutate(append); }); @@ -221,37 +269,52 @@ public void testBufferedMutatorWithFlush() throws Exception { List mutations = new ArrayList<>(); // test Put Put put1 = new Put(Bytes.toBytes(key)); - put1.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); + put1.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); mutations.add(put1); Put put2 = new Put(Bytes.toBytes(key)); - put2.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); + put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); mutations.add(put2); // test add Mutations with List - putBufferMutator.mutate(mutations); - putBufferMutator.flush(); + bufferMutator.mutate(mutations); + bufferMutator.flush(); Get get = new Get(toBytes(key)); Result r = hTable.get(get); Assert.assertEquals(2, r.raw().length); - Delete del = new Delete(Bytes.toBytes(key)); - final BufferedMutator noCfMut = putBufferMutator; - // test mutation without setting family - Assert.assertThrows(FeatureNotSupportedException.class, () -> { - noCfMut.mutate(del); - }); - del.deleteFamily(Bytes.toBytes("family_group")); - // test reuse different type bufferedMutator - final BufferedMutator difTypeMut = putBufferMutator; + Put put3 = new Put(Bytes.toBytes(key)); + final BufferedMutator noCfMut = bufferMutator; + // test Put without setting family Assert.assertThrows(IllegalArgumentException.class, () -> { - difTypeMut.mutate(del); + noCfMut.mutate(put3); }); + put3.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "2"), timestamp, Bytes.toBytes(value)); // test add Mutation directly - delBufferedMutator.mutate(del); - delBufferedMutator.flush(); + bufferMutator.mutate(put3); + bufferMutator.flush(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + // test Delete + Delete del = new Delete(toBytes(key)); + del.deleteFamily(toBytes(family)); + bufferMutator.mutate(del); + bufferMutator.flush(); r = hTable.get(get); Assert.assertEquals(0, r.raw().length); + + // test hybrid mutations + mutations.clear(); + mutations.add(put1); + mutations.add(put2); + mutations.add(del); + mutations.add(put3); + bufferMutator.mutate(mutations); + bufferMutator.flush(); + + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); } catch (Exception ex) { if (ex instanceof RetriesExhaustedWithDetailsException) { ((RetriesExhaustedWithDetailsException) ex).getCauses().get(0).printStackTrace(); @@ -260,17 +323,14 @@ public void testBufferedMutatorWithFlush() throws Exception { } Assert.assertTrue(false); } finally { - if (putBufferMutator != null ) { - putBufferMutator.close(); + if (bufferMutator != null ) { + bufferMutator.close(); // test flush after closed - putBufferMutator.flush(); - } - if (delBufferedMutator != null) { - delBufferedMutator.close(); + bufferMutator.flush(); // test add mutations after closed - Delete delete = new Delete(Bytes.toBytes("putKey")); - delete.deleteFamily(Bytes.toBytes("family_group")); - final BufferedMutator closedMutator = delBufferedMutator; + Delete delete = new Delete(Bytes.toBytes(key)); + delete.deleteFamily(Bytes.toBytes(family)); + final BufferedMutator closedMutator = bufferMutator; Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); }); @@ -279,6 +339,7 @@ public void testBufferedMutatorWithFlush() throws Exception { } /* + USE n1; CREATE TABLEGROUP `n1:test` SHARDING = 'ADAPTIVE'; CREATE TABLE `n1:test$family_group` ( `K` varbinary(1024) NOT NULL, @@ -292,27 +353,32 @@ PRIMARY KEY (`K`, `Q`, `T`) public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); conf.set("rs.list.acquire.read.timeout", "10000"); - BufferedMutator putBufferMutator = null; - BufferedMutator delBufferedMutator = null; + conf.set(SOCKET_TIMEOUT_CONNECT, "15000"); + BufferedMutator bufferMutator = null; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + String family = "family_group"; try { // use n1 database - TableName tableName = TableName.valueOf("n1:test"); + TableName tableName = TableName.valueOf("n1","test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + Assert.assertEquals("n1:test", Bytes.toString(((OHTable) hTable).getTableName())); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // use defualt params - putBufferMutator = connection.getBufferedMutator(tableName); - delBufferedMutator = connection.getBufferedMutator(tableName); + bufferMutator = connection.getBufferedMutator(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; long timestamp = System.currentTimeMillis(); // only support Put and Delete - // for other type of operations, BufferedMutator will not set its type for them Append append = new Append(Bytes.toBytes(key)); append.add("family_group".getBytes(), column1.getBytes(), toBytes("_suffix")); - final BufferedMutator apMut = putBufferMutator; + final BufferedMutator apMut = bufferMutator; Assert.assertThrows(IllegalArgumentException.class, () -> { apMut.mutate(append); }); @@ -320,37 +386,52 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { List mutations = new ArrayList<>(); // test Put Put put1 = new Put(Bytes.toBytes(key)); - put1.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); + put1.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1), timestamp, Bytes.toBytes(value)); mutations.add(put1); Put put2 = new Put(Bytes.toBytes(key)); - put2.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); + put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "1"), timestamp, Bytes.toBytes(value + "4")); mutations.add(put2); // test add Mutations with List - putBufferMutator.mutate(mutations); - putBufferMutator.flush(); + bufferMutator.mutate(mutations); + bufferMutator.flush(); Get get = new Get(toBytes(key)); Result r = hTable.get(get); Assert.assertEquals(2, r.raw().length); - Delete del = new Delete(Bytes.toBytes(key)); - final BufferedMutator noCfMut = putBufferMutator; - // test mutation without setting family - Assert.assertThrows(FeatureNotSupportedException.class, () -> { - noCfMut.mutate(del); - }); - del.deleteFamily(Bytes.toBytes("family_group")); - final BufferedMutator difTypeMut = putBufferMutator; - // test reuse different type bufferedMutator + Put put3 = new Put(Bytes.toBytes(key)); + final BufferedMutator noCfMut = bufferMutator; + // test Put without setting family Assert.assertThrows(IllegalArgumentException.class, () -> { - difTypeMut.mutate(del); + noCfMut.mutate(put3); }); + put3.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "2"), timestamp, Bytes.toBytes(value)); // test add Mutation directly - delBufferedMutator.mutate(del); - delBufferedMutator.flush(); + bufferMutator.mutate(put3); + bufferMutator.flush(); + r = hTable.get(get); + Assert.assertEquals(3, r.raw().length); + + // test Delete + Delete del = new Delete(toBytes(key)); + del.deleteFamily(toBytes(family)); + bufferMutator.mutate(del); + bufferMutator.flush(); r = hTable.get(get); Assert.assertEquals(0, r.raw().length); + + // test hybrid mutations + mutations.clear(); + mutations.add(put1); + mutations.add(put2); + mutations.add(del); + mutations.add(put3); + bufferMutator.mutate(mutations); + bufferMutator.flush(); + + r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); } catch (Exception ex) { if (ex instanceof RetriesExhaustedWithDetailsException) { ((RetriesExhaustedWithDetailsException) ex).getCauses().get(0).printStackTrace(); @@ -359,17 +440,14 @@ public void testBufferedMutatorUseNameSpaceWithFlush() throws Exception { } Assert.assertTrue(false); } finally { - if (putBufferMutator != null ) { - putBufferMutator.close(); + if (bufferMutator != null ) { + bufferMutator.close(); // test flush after closed - putBufferMutator.flush(); - } - if (delBufferedMutator != null) { - delBufferedMutator.close(); + bufferMutator.flush(); // test add mutations after closed - Delete delete = new Delete(Bytes.toBytes("putKey")); - delete.deleteFamily(Bytes.toBytes("family_group")); - final BufferedMutator closedMutator = delBufferedMutator; + Delete delete = new Delete(Bytes.toBytes(key)); + delete.deleteFamily(Bytes.toBytes(family)); + final BufferedMutator closedMutator = bufferMutator; Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); }); @@ -391,35 +469,52 @@ PRIMARY KEY (`K`, `Q`, `T`) public void testBufferedMutatorWithAutoFlush() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); conf.set("rs.list.acquire.read.timeout", "10000"); - BufferedMutator putBufferMutator = null; + BufferedMutator bufferMutator = null; BufferedMutatorParams params = null; long bufferSize = 45000L; int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); - putBufferMutator = connection.getBufferedMutator(params); - - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); + bufferMutator = connection.getBufferedMutator(params); List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); for (int j = 0; j < 4; ++j) { Put put = new Put(Bytes.toBytes(key)); - put.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "_" + i + "_" + j), + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "_" + i + "_" + j), timestamp, Bytes.toBytes(value + "_" + i + "_" + j)); mutations.add(put); } - putBufferMutator.mutate(mutations); + if (i % 10 == 0) { // 0, 10, 20, 30, 40 + for(int j = 0; j < 4; ++j) { + Delete del = new Delete(Bytes.toBytes(key)); + del.addColumns(toBytes(family), toBytes(column1 + "_" + i + "_" + j)); + mutations.add(del); + } + } + bufferMutator.mutate(mutations); } + Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); + Result r = hTable.get(get); + count = r.raw().length; + Assert.assertTrue(count > 0); } catch (Exception ex) { if (ex instanceof RetriesExhaustedWithDetailsException) { ((RetriesExhaustedWithDetailsException) ex).getCauses().get(0).printStackTrace(); @@ -428,27 +523,26 @@ public void testBufferedMutatorWithAutoFlush() throws Exception { } Assert.assertTrue(false); } finally { - if (putBufferMutator != null) { - putBufferMutator.close(); - Get get = new Get(toBytes("putKey")); + if (bufferMutator != null) { + bufferMutator.close(); + Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); Result r = hTable.get(get); - for (KeyValue keyValue : r.raw()) { - ++count; - } - Assert.assertEquals(200, count); - Delete delete = new Delete(toBytes("putKey")); - delete.deleteFamily(toBytes("family_group")); + count = r.raw().length; + Assert.assertEquals(180, count); + Delete delete = new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); hTable.delete(delete); r = hTable.get(get); Assert.assertEquals(0, r.raw().length); // test add mutations after closed - final BufferedMutator closedMutator = putBufferMutator; + final BufferedMutator closedMutator = bufferMutator; Assert.assertThrows(IllegalStateException.class, () -> { closedMutator.mutate(delete); }); // test flush after closed - putBufferMutator.flush(); + bufferMutator.flush(); } if (params != null) { @@ -477,10 +571,20 @@ public void testBufferedMutatorWithUserPool() throws Exception { BufferedMutatorParams params = null; long bufferSize = 45000L; int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); @@ -493,22 +597,29 @@ public void testBufferedMutatorWithUserPool() throws Exception { ohBufferMutator = connection.getBufferedMutator(params); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - List mutations = new ArrayList<>(); for (int i = 0; i < 50; ++i) { mutations.clear(); for (int j = 0; j < 4; ++j) { Put put = new Put(Bytes.toBytes(key)); - put.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(column1 + "_" + i + "_" + j), + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column1 + "_" + i + "_" + j), timestamp, Bytes.toBytes(value + "_" + i + "_" + j)); mutations.add(put); } + if (i % 10 == 0) { // 0, 10, 20, 30, 40 + for(int j = 0; j < 4; ++j) { + Delete del = new Delete(Bytes.toBytes(key)); + del.addColumns(toBytes(family), toBytes(column1 + "_" + i + "_" + j)); + mutations.add(del); + } + } ohBufferMutator.mutate(mutations); } + Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); + Result r = hTable.get(get); + count = r.raw().length; + Assert.assertTrue(count > 0); } catch (Exception ex) { if (ex instanceof RetriesExhaustedWithDetailsException) { ((RetriesExhaustedWithDetailsException) ex).getCauses().get(0).printStackTrace(); @@ -519,14 +630,13 @@ public void testBufferedMutatorWithUserPool() throws Exception { } finally { if (ohBufferMutator != null) { ohBufferMutator.close(); - Get get = new Get(toBytes("putKey")); + Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); Result r = hTable.get(get); - for (KeyValue keyValue : r.raw()) { - ++count; - } - Assert.assertEquals(200, count); - Delete delete = new Delete(toBytes("putKey")); - delete.deleteFamily(toBytes("family_group")); + count = r.raw().length; + Assert.assertEquals(180, count); + Delete delete = new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); hTable.delete(delete); r = hTable.get(get); @@ -566,10 +676,20 @@ public void testBufferedMutatorConcurrent() throws Exception { ExecutorService executorService = Executors.newFixedThreadPool(10); long bufferSize = 45000L; int count = 0; + String key = "putKey"; + String column1 = "putColumn1"; + String value = "value333444"; + long timestamp = System.currentTimeMillis(); + String family = "family_group"; try { TableName tableName = TableName.valueOf("test"); connection = ConnectionFactory.createConnection(conf); hTable = connection.getTable(tableName); + + Delete delete= new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); + hTable.delete(delete); + // set params params = new BufferedMutatorParams(tableName); params.writeBufferSize(bufferSize); @@ -582,24 +702,19 @@ public void testBufferedMutatorConcurrent() throws Exception { ohBufferMutator = connection.getBufferedMutator(params); - String key = "putKey"; - String column1 = "putColumn1"; - String value = "value333444"; - long timestamp = System.currentTimeMillis(); - + // BufferedMutator is not concurrently safe for (int i = 0; i < 50; ++i) { final int taskId = i; final BufferedMutator thrBufferMutator = ohBufferMutator; executorService.submit(() -> { List mutations = new ArrayList<>(); for (int j = 0; j < 4; ++j) { - String thrKey = key; String thrColumn = column1 + "_" + taskId + "_" + j; String thrValue = value + "_" + taskId + "_" + j; long thrTimestamp = timestamp; - Put put = new Put(Bytes.toBytes(thrKey)); - put.addColumn(Bytes.toBytes("family_group"), Bytes.toBytes(thrColumn), + Put put = new Put(Bytes.toBytes(key)); + put.addColumn(Bytes.toBytes(family), Bytes.toBytes(thrColumn), thrTimestamp, Bytes.toBytes(thrValue)); mutations.add(put); } @@ -634,14 +749,13 @@ public void testBufferedMutatorConcurrent() throws Exception { } if (ohBufferMutator != null) { ohBufferMutator.close(); - Get get = new Get(toBytes("putKey")); + Get get = new Get(toBytes(key)); + get.addFamily(toBytes(family)); Result r = hTable.get(get); - for (KeyValue keyValue : r.raw()) { - ++count; - } + count = r.raw().length; Assert.assertEquals(200, count); - Delete delete = new Delete(toBytes("putKey")); - delete.deleteFamily(toBytes("family_group")); + Delete delete = new Delete(toBytes(key)); + delete.deleteFamily(toBytes(family)); hTable.delete(delete); r = hTable.get(get); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 380125f1..6d53616b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -17,6 +17,7 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.util.Pair; import org.junit.Assert; diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java index 044f45d4..01c4057f 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java @@ -17,24 +17,33 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.junit.*; -import java.io.IOException; +import java.util.LinkedList; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class OHTableClientTest extends HTableTestBase { - @Before - public void before() throws Exception { + @BeforeClass + public static void before() throws Exception { hTable = ObHTableTestUtil.newOHTableClient("test"); + // hTable = ObHTableTestUtil.newOHTableClient("n1:test"); ((OHTableClient) hTable).init(); + multiCfHTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); + ((OHTableClient) multiCfHTable).init(); + List tableGroups = new LinkedList<>(); + tableGroups.add("test"); + tableGroups.add("test_multi_cf"); + ObHTableTestUtil.prepareClean(tableGroups); } - @After - public void finish() throws IOException { - hTable.close(); + @Before + public void prepareCase() { + ObHTableTestUtil.cleanData(); } @Test @@ -46,6 +55,7 @@ public void testRefreshTableEntry() throws Exception { @Test public void testNew() throws Exception { OHTableClient hTable2 = ObHTableTestUtil.newOHTableClient("test"); + // OHTableClient hTable2 = ObHTableTestUtil.newOHTableClient("n1:test"); hTable2.init(); hTable2.getConfiguration().set("rs.list.acquire.read.timeout", "10000"); @@ -57,14 +67,18 @@ public void testNew() throws Exception { hTable2.setWriteBufferSize(10000000L); assertEquals(10000000L, hTable2.getWriteBufferSize()); assertEquals("test", hTable2.getTableNameString()); + // assertEquals("n1:test", hTable2.getTableNameString()); assertEquals("test", new String(hTable2.getTableName())); + // assertEquals("n1:test", new String(hTable2.getTableName())); hTable2.flushCommits(); hTable2.close(); assertTrue(true); } - @After - public void after() throws IOException { + @AfterClass + public static void finish() throws Exception { hTable.close(); + multiCfHTable.close(); + ObHTableTestUtil.closeConn(); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTestLoadTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTestLoadTest.java index 2c636436..ed9df2fa 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTestLoadTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTestLoadTest.java @@ -17,24 +17,36 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.rpc.exception.ObTableNotExistException; import org.apache.hadoop.hbase.client.Delete; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import java.io.IOException; +import java.util.LinkedList; +import java.util.List; import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_ENABLE; import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_SUFFIX; public class OHTableClientTestLoadTest extends HTableTestBase { - @Before - public void before() throws Exception { + @BeforeClass + public static void before() throws Exception { hTable = ObHTableTestUtil.newOHTableClient("test"); ((OHTableClient) hTable).init(); hTable.getConfiguration().set(HBASE_HTABLE_TEST_LOAD_ENABLE, "true"); + multiCfHTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); + ((OHTableClient) multiCfHTable).init(); + multiCfHTable.getConfiguration().set(HBASE_HTABLE_TEST_LOAD_ENABLE, "true"); + List tableGroups = new LinkedList<>(); + tableGroups.add("test"); + tableGroups.add("test_multi_cf"); + ObHTableTestUtil.prepareClean(tableGroups); + } + + @Before + public void prepareCase() { + ObHTableTestUtil.cleanData(); } @Test @@ -53,9 +65,11 @@ public void test_refresh_table_entry() throws Exception { } - @After - public void after() throws IOException { + @AfterClass + public static void after() throws Exception { hTable.close(); + multiCfHTable.close(); + ObHTableTestUtil.closeConn(); } @Test diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java deleted file mode 100644 index 1793bb7f..00000000 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java +++ /dev/null @@ -1,639 +0,0 @@ -/*- - * #%L - * com.oceanbase:obkv-hbase-client - * %% - * Copyright (C) 2022 - 2024 OceanBase Group - * %% - * OBKV HBase Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.hbase; - -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.filter.PrefixFilter; -import org.junit.*; -import org.junit.rules.ExpectedException; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -import static org.apache.hadoop.hbase.util.Bytes.toBytes; -import static org.junit.Assert.*; - -public class OHTableMultiColumnFamilyTest { - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - protected HTableInterface hTable; - - @Before - public void before() throws Exception { - hTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); - ((OHTableClient) hTable).init(); - } - - @After - public void finish() throws IOException { - hTable.close(); - } - - @Test - public void testMultiColumnFamilyPut() throws Exception { - byte[] family1 = "family_with_group1".getBytes(); - byte[] family2 = "family_with_group2".getBytes(); - byte[] family3 = "family_with_group3".getBytes(); - - byte[] family1_column1 = "family1_column1".getBytes(); - byte[] family1_column2 = "family1_column2".getBytes(); - byte[] family1_column3 = "family1_column3".getBytes(); - byte[] family2_column1 = "family2_column1".getBytes(); - byte[] family2_column2 = "family2_column2".getBytes(); - byte[] family3_column1 = "family3_column1".getBytes(); - byte[] family1_value = "VVV1".getBytes(); - byte[] family2_value = "VVV2".getBytes(); - byte[] family3_value = "VVV3".getBytes(); - - Map expectedValues = new HashMap<>(); - expectedValues.put(family1_column1, family1_value); - expectedValues.put(family1_column2, family1_value); - expectedValues.put(family1_column3, family1_value); - expectedValues.put(family2_column1, family2_value); - expectedValues.put(family2_column2, family2_value); - expectedValues.put(family3_column1, family3_value); - - int rows = 30; - - for (int i = 0; i < rows; ++i) { - Put put = new Put(toBytes("Key" + i)); - put.add(family1, family1_column1, family1_value); - put.add(family1, family1_column2, family1_value); - put.add(family1, family1_column3, family1_value); - put.add(family2, family2_column1, family2_value); - put.add(family2, family2_column2, family2_value); - put.add(family3, family3_column1, family3_value); - hTable.put(put); - } - hTable.flushCommits(); - - Scan scan = new Scan(); - scan.setStartRow(toBytes("Key")); - scan.setStopRow(toBytes("Kf")); - ResultScanner scanner = hTable.getScanner(scan); - int count = 0; - - for (Result result : scanner) { - KeyValue[] keyValues = result.raw(); - long timestamp = keyValues[0].getTimestamp(); - for (int i = 1; i < keyValues.length; ++i) { - assertEquals(timestamp, keyValues[i].getTimestamp()); - byte[] qualifier = keyValues[i].getQualifier(); - byte[] expectedValue = expectedValues.get(qualifier); - if (expectedValue != null) { - assertEquals(expectedValue, keyValues[i].getValue()); - } - } - count++; - } - assertEquals(count, rows); - } - - @Ignore - public void testMultiColumnFamilyAppend() throws Exception { - byte[] family1 = "family_with_group1".getBytes(); - byte[] family2 = "family_with_group2".getBytes(); - byte[] family3 = "family_with_group3".getBytes(); - - byte[] family1_column1 = "family1_column1".getBytes(); - byte[] family1_column2 = "family1_column2".getBytes(); - byte[] family1_column3 = "family1_column3".getBytes(); - byte[] family2_column1 = "family2_column1".getBytes(); - byte[] family2_column2 = "family2_column2".getBytes(); - byte[] family3_column1 = "family3_column1".getBytes(); - byte[] family1_value = "VVV1".getBytes(); - byte[] family2_value = "VVV2".getBytes(); - byte[] family3_value = "VVV3".getBytes(); - - Map expectedValues = new HashMap<>(); - expectedValues.put(family1_column1, family1_value); - expectedValues.put(family1_column2, family1_value); - expectedValues.put(family1_column3, family1_value); - expectedValues.put(family2_column1, family2_value); - expectedValues.put(family2_column2, family2_value); - expectedValues.put(family3_column1, family3_value); - - int rows = 30; - - for (int i = 0; i < rows; ++i) { - Append append = new Append(toBytes("Key" + i)); - append.add(family1, family1_column1, family1_value); - append.add(family1, family1_column2, family1_value); - append.add(family1, family1_column3, family1_value); - append.add(family2, family2_column1, family2_value); - append.add(family2, family2_column2, family2_value); - append.add(family3, family3_column1, family3_value); - hTable.append(append); - } - hTable.flushCommits(); - - Scan scan = new Scan(); - scan.setStartRow(toBytes("Key")); - scan.setStopRow(toBytes("Kf")); - ResultScanner scanner = hTable.getScanner(scan); - int count = 0; - - for (Result result : scanner) { - KeyValue[] keyValues = result.raw(); - long timestamp = keyValues[0].getTimestamp(); - for (int i = 1; i < keyValues.length; ++i) { - assertEquals(timestamp, keyValues[i].getTimestamp()); - byte[] qualifier = keyValues[i].getQualifier(); - byte[] expectedValue = expectedValues.get(qualifier); - if (expectedValue != null) { - assertEquals(expectedValue, keyValues[i].getValue()); - } - } - count++; - } - assertEquals(count, rows); - } - - @Test - public void testMultiColumnFamilyReverseScan() throws Exception { - byte[] family1 = "family_with_group1".getBytes(); - byte[] family2 = "family_with_group2".getBytes(); - byte[] family3 = "family_with_group3".getBytes(); - - byte[] family1_column1 = "family1_column1".getBytes(); - byte[] family1_column2 = "family1_column2".getBytes(); - byte[] family1_column3 = "family1_column3".getBytes(); - byte[] family2_column1 = "family2_column1".getBytes(); - byte[] family2_column2 = "family2_column2".getBytes(); - byte[] family3_column1 = "family3_column1".getBytes(); - byte[] family1_value = "VVV1".getBytes(); - byte[] family2_value = "VVV2".getBytes(); - byte[] family3_value = "VVV3".getBytes(); - - Map expectedValues = new HashMap<>(); - expectedValues.put(family1_column1, family1_value); - expectedValues.put(family1_column2, family1_value); - expectedValues.put(family1_column3, family1_value); - expectedValues.put(family2_column1, family2_value); - expectedValues.put(family2_column2, family2_value); - expectedValues.put(family3_column1, family3_value); - - int rows = 30; - - for (int i = 0; i < rows; ++i) { - Put put = new Put(toBytes("Key" + i)); - put.add(family1, family1_column1, family1_value); - put.add(family1, family1_column2, family1_value); - put.add(family1, family1_column3, family1_value); - put.add(family2, family2_column1, family2_value); - put.add(family2, family2_column2, family2_value); - put.add(family3, family3_column1, family3_value); - hTable.put(put); - } - - Scan scan = new Scan(); - scan.addFamily(family1); - scan.addFamily(family2); - scan.setReversed(true); - ResultScanner scanner2 = hTable.getScanner(scan); - - for (Result result : scanner2) { - KeyValue[] keyValues = result.raw(); - long timestamp = keyValues[0].getTimestamp(); - for (int i = 1; i < keyValues.length; ++i) { - assertEquals(timestamp, keyValues[i].getTimestamp()); - byte[] qualifier = keyValues[i].getQualifier(); - byte[] expectedValue = expectedValues.get(qualifier); - if (expectedValue != null) { - assertEquals(expectedValue, keyValues[i].getValue()); - } - } - } - } - - @Test - public void testMultiColumnFamilyScanWithColumns() throws Exception { - byte[] family1 = "family_with_group1".getBytes(); - byte[] family2 = "family_with_group2".getBytes(); - byte[] family3 = "family_with_group3".getBytes(); - - byte[] family1_column1 = "family1_column1".getBytes(); - byte[] family1_column2 = "family1_column2".getBytes(); - byte[] family1_column3 = "family1_column3".getBytes(); - byte[] family2_column1 = "family2_column1".getBytes(); - byte[] family2_column2 = "family2_column2".getBytes(); - byte[] family3_column1 = "family3_column1".getBytes(); - byte[] family1_value = "VVV1".getBytes(); - byte[] family2_value = "VVV2".getBytes(); - byte[] family3_value = "VVV3".getBytes(); - - Map expectedValues = new HashMap<>(); - expectedValues.put(family1_column1, family1_value); - expectedValues.put(family1_column2, family1_value); - expectedValues.put(family1_column3, family1_value); - expectedValues.put(family2_column1, family2_value); - expectedValues.put(family2_column2, family2_value); - expectedValues.put(family3_column1, family3_value); - - int rows = 30; - - for (int i = 0; i < rows; ++i) { - Put put = new Put(toBytes("Key" + i)); - put.add(family1, family1_column1, family1_value); - put.add(family1, family1_column2, family1_value); - put.add(family1, family1_column3, family1_value); - put.add(family2, family2_column1, family2_value); - put.add(family2, family2_column2, family2_value); - put.add(family3, family3_column1, family3_value); - hTable.put(put); - } - - Scan scan = new Scan(); - scan.setStartRow(toBytes("Key")); - scan.setStopRow(toBytes("Kf")); - scan.addColumn(family1, family1_column1); - scan.addColumn(family2, family2_column1); - ResultScanner scanner = hTable.getScanner(scan); - - for (Result result : scanner) { - KeyValue[] keyValues = result.raw(); - long timestamp = keyValues[0].getTimestamp(); - for (int i = 1; i < keyValues.length; ++i) { - assertEquals(timestamp, keyValues[i].getTimestamp()); - byte[] qualifier = keyValues[i].getQualifier(); - byte[] expectedValue = expectedValues.get(qualifier); - if (expectedValue != null) { - assertEquals(expectedValue, keyValues[i].getValue()); - } - } - assertEquals(2, keyValues.length); - } - scanner.close(); - scan = new Scan(); - scan.setStartRow(toBytes("Key")); - scan.setStopRow(toBytes("Kf")); - scan.addColumn(family1, family1_column1); - scan.addColumn(family1, family1_column2); - scan.addColumn(family1, family1_column3); - scan.addColumn(family2, family2_column1); - scan.addColumn(family2, family2_column2); - scanner = hTable.getScanner(scan); - - for (Result result : scanner) { - KeyValue[] keyValues = result.raw(); - long timestamp = keyValues[0].getTimestamp(); - for (int i = 1; i < keyValues.length; ++i) { - assertEquals(timestamp, keyValues[i].getTimestamp()); - byte[] qualifier = keyValues[i].getQualifier(); - byte[] expectedValue = expectedValues.get(qualifier); - if (expectedValue != null) { - assertEquals(expectedValue, keyValues[i].getValue()); - } - } - assertEquals(5, keyValues.length); - } - scanner.close(); - scan = new Scan(); - scan.setStartRow(toBytes("Key")); - scan.setStopRow(toBytes("Kf")); - scan.addFamily(family1); - scan.addFamily(family2); - - scanner = hTable.getScanner(scan); - - for (Result result : scanner) { - KeyValue[] keyValues = result.raw(); - long timestamp = keyValues[0].getTimestamp(); - for (int i = 1; i < keyValues.length; ++i) { - assertEquals(timestamp, keyValues[i].getTimestamp()); - byte[] qualifier = keyValues[i].getQualifier(); - byte[] expectedValue = expectedValues.get(qualifier); - if (expectedValue != null) { - assertEquals(expectedValue, keyValues[i].getValue()); - } - } - assertEquals(5, keyValues.length); - } - scanner.close(); - scan = new Scan(); - scan.setStartRow(toBytes("Key")); - scan.setStopRow(toBytes("Kf")); - scan.addFamily(family1); - scan.addFamily(family3); - - scanner = hTable.getScanner(scan); - - for (Result result : scanner) { - KeyValue[] keyValues = result.raw(); - long timestamp = keyValues[0].getTimestamp(); - for (int i = 1; i < keyValues.length; ++i) { - assertEquals(timestamp, keyValues[i].getTimestamp()); - byte[] qualifier = keyValues[i].getQualifier(); - byte[] expectedValue = expectedValues.get(qualifier); - if (expectedValue != null) { - assertEquals(expectedValue, keyValues[i].getValue()); - } - } - // f1c1 f1c2 f1c3 f3c1 - assertEquals(4, keyValues.length); - } - scanner.close(); - } - - @Test - public void testMultiColumnFamilyScanWithFilter() throws Exception { - byte[] family1 = "family_with_group1".getBytes(); - byte[] family2 = "family_with_group2".getBytes(); - byte[] family3 = "family_with_group3".getBytes(); - - byte[] family1_column1 = "family1_column1".getBytes(); - byte[] family1_column2 = "family1_column2".getBytes(); - byte[] family1_column3 = "family1_column3".getBytes(); - byte[] family2_column1 = "family2_column1".getBytes(); - byte[] family2_column2 = "family2_column2".getBytes(); - byte[] family3_column1 = "family3_column1".getBytes(); - byte[] family1_value = "VVV1".getBytes(); - byte[] family2_value = "VVV2".getBytes(); - byte[] family3_value = "VVV3".getBytes(); - - Map expectedValues = new HashMap<>(); - expectedValues.put(family1_column1, family1_value); - expectedValues.put(family1_column2, family1_value); - expectedValues.put(family1_column3, family1_value); - expectedValues.put(family2_column1, family2_value); - expectedValues.put(family2_column2, family2_value); - expectedValues.put(family3_column1, family3_value); - - int rows = 30; - - for (int i = 0; i < rows; ++i) { - Put put = new Put(toBytes("Key" + i)); - put.add(family1, family1_column1, family1_value); - put.add(family1, family1_column2, family1_value); - put.add(family1, family1_column3, family1_value); - put.add(family2, family2_column1, family2_value); - put.add(family2, family2_column2, family2_value); - put.add(family3, family3_column1, family3_value); - hTable.put(put); - } - - PrefixFilter filter = new PrefixFilter(toBytes("Key1")); - Scan scan = new Scan(); - scan.setStartRow(toBytes("Key")); - scan.setStopRow(toBytes("Kf")); - scan.setFilter(filter); - ResultScanner scanner = hTable.getScanner(scan); - - // Key1, Key10, Key11, Key12, Key13, Key14, Key15, Key16, Key17, Key18, Key19 - int count = 0; - for (Result result : scanner) { - KeyValue[] keyValues = result.raw(); - long timestamp = keyValues[0].getTimestamp(); - for (int i = 1; i < keyValues.length; ++i) { - assertEquals(timestamp, keyValues[i].getTimestamp()); - byte[] qualifier = keyValues[i].getQualifier(); - byte[] expectedValue = expectedValues.get(qualifier); - if (expectedValue != null) { - assertEquals(expectedValue, keyValues[i].getValue()); - } - } - assertEquals(6, keyValues.length); - count++; - } - assertEquals(11, count); - } - - @Test - public void testMultiColumnFamilyGet() throws Exception { - byte[] family1 = "family_with_group1".getBytes(); - byte[] family2 = "family_with_group2".getBytes(); - byte[] family3 = "family_with_group3".getBytes(); - - byte[] family1_column1 = "family1_column1".getBytes(); - byte[] family1_column2 = "family1_column2".getBytes(); - byte[] family1_column3 = "family1_column3".getBytes(); - byte[] family2_column1 = "family2_column1".getBytes(); - byte[] family2_column2 = "family2_column2".getBytes(); - byte[] family3_column1 = "family3_column1".getBytes(); - byte[] family1_value = "VVV1".getBytes(); - byte[] family2_value = "VVV2".getBytes(); - byte[] family3_value = "VVV3".getBytes(); - - Map expectedValues = new HashMap<>(); - expectedValues.put(family1_column1, family1_value); - expectedValues.put(family1_column2, family1_value); - expectedValues.put(family1_column3, family1_value); - expectedValues.put(family2_column1, family2_value); - expectedValues.put(family2_column2, family2_value); - expectedValues.put(family3_column1, family3_value); - - int rows = 3; - - for (int i = 0; i < rows; ++i) { - Put put = new Put(toBytes("Key" + i)); - put.add(family1, family1_column1, family1_value); - put.add(family1, family1_column2, family1_value); - put.add(family1, family1_column3, family1_value); - put.add(family2, family2_column1, family2_value); - put.add(family2, family2_column2, family2_value); - put.add(family3, family3_column1, family3_value); - hTable.put(put); - } - hTable.flushCommits(); - - // get with empty family - // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 - Get get = new Get(toBytes("Key1")); - Result result = hTable.get(get); - KeyValue[] keyValues = result.raw(); - long timestamp = keyValues[0].getTimestamp(); - for (int i = 1; i < keyValues.length; ++i) { - assertEquals(timestamp, keyValues[i].getTimestamp()); - byte[] qualifier = keyValues[i].getQualifier(); - byte[] expectedValue = expectedValues.get(qualifier); - if (expectedValue != null) { - assertEquals(expectedValue, keyValues[i].getValue()); - } - } - assertEquals(6, keyValues.length); - - // f1c1 f2c1 f2c2 - Get get2 = new Get(toBytes("Key1")); - get2.addColumn(family1, family1_column1); - get2.addColumn(family2, family2_column1); - get2.addColumn(family2, family2_column2); - Result result2 = hTable.get(get2); - keyValues = result2.raw(); - timestamp = keyValues[0].getTimestamp(); - for (int i = 1; i < keyValues.length; ++i) { - assertEquals(timestamp, keyValues[i].getTimestamp()); - byte[] qualifier = keyValues[i].getQualifier(); - byte[] expectedValue = expectedValues.get(qualifier); - if (expectedValue != null) { - assertEquals(expectedValue, keyValues[i].getValue()); - } - } - System.out.println(Arrays.toString(result2.raw())); - assertEquals(3, keyValues.length); - - //f2c1 f2c2 - Get get3 = new Get(toBytes("Key1")); - get3.addFamily(family1); - get3.addColumn(family2, family2_column1); - get3.addColumn(family2, family2_column2); - Result result3 = hTable.get(get3); - keyValues = result3.raw(); - timestamp = keyValues[0].getTimestamp(); - for (int i = 1; i < keyValues.length; ++i) { - assertEquals(timestamp, keyValues[i].getTimestamp()); - byte[] qualifier = keyValues[i].getQualifier(); - byte[] expectedValue = expectedValues.get(qualifier); - if (expectedValue != null) { - assertEquals(expectedValue, keyValues[i].getValue()); - } - } - assertEquals(5, keyValues.length); - } - - @Test - public void testMultiColumnFamilyDelete() throws Exception { - byte[] family1 = "family_with_group1".getBytes(); - byte[] family2 = "family_with_group2".getBytes(); - byte[] family3 = "family_with_group3".getBytes(); - - byte[] family1_column1 = "family1_column1".getBytes(); - byte[] family1_column2 = "family1_column2".getBytes(); - byte[] family1_column3 = "family1_column3".getBytes(); - byte[] family2_column1 = "family2_column1".getBytes(); - byte[] family2_column2 = "family2_column2".getBytes(); - byte[] family3_column1 = "family3_column1".getBytes(); - byte[] family1_value = "VVV1".getBytes(); - byte[] family2_value = "VVV2".getBytes(); - byte[] family3_value = "VVV3".getBytes(); - - int rows = 10; - - for (int i = 0; i < rows; ++i) { - Put put = new Put(toBytes("Key" + i)); - put.add(family1, family1_column1, family1_value); - put.add(family1, family1_column2, family1_value); - put.add(family1, family1_column3, family1_value); - put.add(family2, family2_column1, family2_value); - put.add(family2, family2_column2, family2_value); - put.add(family3, family3_column1, family3_value); - hTable.put(put); - } - - // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 - Delete delete = new Delete(toBytes("Key1")); - delete.deleteColumns(family1, family1_column1); - delete.deleteColumns(family2, family2_column1); - hTable.delete(delete); - // f1c2 f1c3 f2c2 f3c1 - Get get = new Get(toBytes("Key1")); - Result result = hTable.get(get); - KeyValue[] keyValues = result.raw(); - assertEquals(4, keyValues.length); - assertFalse(result.containsColumn(family1, family1_column1)); - assertFalse(result.containsColumn(family2, family2_column1)); - - assertTrue(result.containsColumn(family1, family1_column2)); - assertArrayEquals(result.getValue(family1, family1_column2), family1_value); - assertTrue(result.containsColumn(family1, family1_column3)); - assertArrayEquals(result.getValue(family1, family1_column3), family1_value); - assertTrue(result.containsColumn(family2, family2_column2)); - assertArrayEquals(result.getValue(family2, family2_column2), family2_value); - assertTrue(result.containsColumn(family3, family3_column1)); - assertArrayEquals(result.getValue(family3, family3_column1), family3_value); - - // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 - delete = new Delete(toBytes("Key2")); - delete.deleteFamily(family1); - delete.deleteFamily(family2); - // f3c1 - hTable.delete(delete); - get = new Get(toBytes("Key2")); - result = hTable.get(get); - keyValues = result.raw(); - assertEquals(1, keyValues.length); - - // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 - delete = new Delete(toBytes("Key3")); - delete.deleteFamily(family1); - delete.deleteColumns(family2, family2_column1); - hTable.delete(delete); - // f2c2 f3c1 - get = new Get(toBytes("Key3")); - result = hTable.get(get); - keyValues = result.raw(); - assertEquals(2, keyValues.length); - - // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 - delete = new Delete(toBytes("Key4")); - hTable.delete(delete); - // null - get = new Get(toBytes("Key4")); - result = hTable.get(get); - keyValues = result.raw(); - assertEquals(0, keyValues.length); - - // f1c1 f2c1 f2c2 - delete = new Delete(toBytes("Key5")); - delete.deleteColumns(family1, family1_column2); - delete.deleteColumns(family1, family1_column3); - delete.deleteColumns(family3, family3_column1); - hTable.delete(delete); - // null - get = new Get(toBytes("Key5")); - result = hTable.get(get); - keyValues = result.raw(); - assertEquals(3, keyValues.length); - - for (int i = 0; i < rows; ++i) { - Put put = new Put(toBytes("Key" + i)); - put.add(family1, family1_column1, family1_value); - put.add(family1, family1_column2, family1_value); - put.add(family1, family1_column3, family1_value); - put.add(family2, family2_column1, family2_value); - put.add(family2, family2_column2, family2_value); - put.add(family3, family3_column1, family3_value); - hTable.put(put); - } - - delete = new Delete(toBytes("Key6")); - delete.deleteColumn(family1, family1_column2); - delete.deleteColumn(family2, family2_column1); - hTable.delete(delete); - get = new Get(toBytes("Key6")); - result = hTable.get(get); - keyValues = result.raw(); - assertEquals(6, keyValues.length); - - long lastTimestamp = result.getColumnCells(family1, family1_column1).get(0).getTimestamp(); - assertEquals(lastTimestamp, result.getColumnCells(family1, family1_column3).get(0) - .getTimestamp()); - assertEquals(lastTimestamp, result.getColumnCells(family2, family2_column2).get(0) - .getTimestamp()); - assertEquals(lastTimestamp, result.getColumnCells(family3, family3_column1).get(0) - .getTimestamp()); - - long oldTimestamp = result.getColumnCells(family1, family1_column2).get(0).getTimestamp(); - assertEquals(oldTimestamp, result.getColumnCells(family2, family2_column1).get(0) - .getTimestamp()); - assertTrue(lastTimestamp > oldTimestamp); - } -} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolLoadTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolLoadTest.java index 4e66fc0b..d639d4a4 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolLoadTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolLoadTest.java @@ -17,15 +17,17 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.rpc.exception.ObTableNotExistException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.HTableInterface; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import java.io.IOException; +import java.sql.SQLException; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.Executors; import static com.alipay.oceanbase.hbase.constants.OHConstants.*; @@ -34,10 +36,10 @@ import static org.junit.Assert.assertTrue; public class OHTablePoolLoadTest extends HTableTestBase { - private OHTablePool ohTablePool; + private static OHTablePool ohTablePool; - @Before - public void setup() throws IOException { + @BeforeClass + public static void setup() throws Exception { Configuration c = new Configuration(); c.set(HBASE_HTABLE_TEST_LOAD_ENABLE, "true"); ohTablePool = new OHTablePool(c, 10); @@ -56,6 +58,16 @@ public void setup() throws IOException { } ohTablePool.setRuntimeBatchExecutor("test", Executors.newFixedThreadPool(3)); hTable = ohTablePool.getTable("test"); + multiCfHTable = ohTablePool.getTable("test_multi_cf"); + List tableGroups = new LinkedList<>(); + tableGroups.add("test"); + tableGroups.add("test_multi_cf"); + ObHTableTestUtil.prepareClean(tableGroups); + } + + @Before + public void prepareCase() { + ObHTableTestUtil.cleanData(); } @Test @@ -152,4 +164,11 @@ public void testNew() throws IOException { hTable2.close(); assertTrue(true); } + + @AfterClass + public static void finish() throws IOException, SQLException { + hTable.close(); + multiCfHTable.close(); + ObHTableTestUtil.closeConn(); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolTest.java index 7ed492d3..d915afdc 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolTest.java @@ -17,25 +17,26 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.remoting.util.ConcurrentHashSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.util.PoolMap; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import java.io.IOException; +import java.sql.SQLException; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import static com.alipay.oceanbase.hbase.util.ObTableClientManager.OB_TABLE_CLIENT_INSTANCE; public class OHTablePoolTest extends HTableTestBase { - protected OHTablePool ohTablePool; + protected static OHTablePool ohTablePool; - private OHTablePool newOHTablePool(final int maxSize, final PoolMap.PoolType poolType) { + private static OHTablePool newOHTablePool(final int maxSize, final PoolMap.PoolType poolType) { OHTablePool pool = new OHTablePool(new Configuration(), maxSize, poolType); pool.setFullUserName("test", ObHTableTestUtil.FULL_USER_NAME); pool.setPassword("test", ObHTableTestUtil.PASSWORD); @@ -52,17 +53,29 @@ private OHTablePool newOHTablePool(final int maxSize, final PoolMap.PoolType poo return pool; } - @Before - public void setup() throws IOException { + @BeforeClass + public static void setup() throws Exception { Configuration c = new Configuration(); ohTablePool = newOHTablePool(10, null); ohTablePool.setRuntimeBatchExecutor("test", Executors.newFixedThreadPool(3)); hTable = ohTablePool.getTable("test"); + multiCfHTable = ohTablePool.getTable("test_multi_cf"); + List tableGroups = new LinkedList<>(); + tableGroups.add("test"); + tableGroups.add("test_multi_cf"); + ObHTableTestUtil.prepareClean(tableGroups); + } + + @Before + public void prepareCase() { + ObHTableTestUtil.cleanData(); } - @After - public void finish() throws IOException { + @AfterClass + public static void finish() throws IOException, SQLException { hTable.close(); + multiCfHTable.close(); + ObHTableTestUtil.closeConn(); } public void test_current_get_close(final OHTablePool ohTablePool, int concurrency, int maxSize) { diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java index b14ca220..f36969ca 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java @@ -17,16 +17,18 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.exception.ObTableNotExistException; import com.alipay.sofa.common.thread.SofaThreadPoolExecutor; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTableInterface; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import java.io.IOException; +import java.sql.SQLException; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.SynchronousQueue; import static com.alipay.oceanbase.hbase.util.TableHBaseLoggerFactory.TABLE_HBASE_LOGGER_SPACE; @@ -35,12 +37,23 @@ import static org.junit.Assert.fail; public class OHTableTest extends HTableTestBase { - @Before - public void setup() throws IOException { + @BeforeClass + public static void setup() throws Exception { Configuration c = ObHTableTestUtil.newConfiguration(); c.set("rs.list.acquire.read.timeout", "10000"); + hTable = new OHTable(c, "test"); + multiCfHTable = new OHTable(c, "test_multi_cf"); + List tableGroups = new LinkedList<>(); + tableGroups.add("test"); +// tableGroups.add("test_multi_cf"); + ObHTableTestUtil.prepareClean(tableGroups); + } + + @Before + public void prepareCase() { + ObHTableTestUtil.cleanData(); } @Test @@ -93,4 +106,11 @@ public void testNew() throws Exception { } + @AfterClass + public static void finish() throws IOException, SQLException { + hTable.close(); + multiCfHTable.close(); + ObHTableTestUtil.closeConn(); + } + } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHVarPrefixPartitionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHVarPrefixPartitionTest.java index 3e22e9b0..2dc41621 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHVarPrefixPartitionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHVarPrefixPartitionTest.java @@ -1,5 +1,6 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.junit.After; import org.junit.Before; diff --git a/src/test/java/com/alipay/oceanbase/hbase/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/ObHTableTestUtil.java deleted file mode 100644 index ba9d25f2..00000000 --- a/src/test/java/com/alipay/oceanbase/hbase/ObHTableTestUtil.java +++ /dev/null @@ -1,59 +0,0 @@ -/*- - * #%L - * OBKV HBase Client Framework - * %% - * Copyright (C) 2022 OceanBase Group - * %% - * OBKV HBase Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.hbase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; - -import static com.alipay.oceanbase.hbase.constants.OHConstants.*; - -public class ObHTableTestUtil { - // please consult your dba for the following configuration. - public static String PARAM_URL = ""; - public static String FULL_USER_NAME = ""; - public static String PASSWORD = ""; - public static String SYS_USER_NAME = ""; - public static String SYS_PASSWORD = ""; - public static String ODP_ADDR = ""; - public static int ODP_PORT = 0; - public static boolean ODP_MODE = false; - public static String DATABASE = ""; - - public static Configuration newConfiguration() { - Configuration conf = HBaseConfiguration.create(); - conf.set(HBASE_OCEANBASE_FULL_USER_NAME, FULL_USER_NAME); - conf.set(HBASE_OCEANBASE_PASSWORD, PASSWORD); - if (ODP_MODE) { - // ODP mode - conf.set(HBASE_OCEANBASE_ODP_ADDR, ODP_ADDR); - conf.setInt(HBASE_OCEANBASE_ODP_PORT, ODP_PORT); - conf.setBoolean(HBASE_OCEANBASE_ODP_MODE, ODP_MODE); - conf.set(HBASE_OCEANBASE_DATABASE, DATABASE); - } else { - // OCP mode - conf.set(HBASE_OCEANBASE_PARAM_URL, PARAM_URL); - conf.set(HBASE_OCEANBASE_SYS_USER_NAME, SYS_USER_NAME); - conf.set(HBASE_OCEANBASE_SYS_PASSWORD, SYS_PASSWORD); - } - return conf; - } - - public static OHTableClient newOHTableClient(String tableName) { - return new OHTableClient(tableName, newConfiguration()); - } -} \ No newline at end of file diff --git a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java index b228f0a4..bc567fa3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/filter/HBaseFilterUtilsTest.java @@ -19,11 +19,15 @@ import org.apache.hadoop.hbase.filter.*; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeSet; public class HBaseFilterUtilsTest { private static final CompareFilter.CompareOp[] ops = { CompareFilter.CompareOp.LESS, @@ -134,6 +138,39 @@ public void testSingleColumnValueFilter() throws IOException { } } + @Test + public void testSingleColumnValueExcludeFilter() throws IOException { + for (int i = 0; i < ops.length; i++) { + String expect = String + .format( + "SingleColumnValueExcludeFilter('family','qualifier',%s,'binary:value',false,true)", + opFlags[i]); + SingleColumnValueExcludeFilter filter = new SingleColumnValueExcludeFilter( + "family".getBytes(), "qualifier".getBytes(), ops[i], "value".getBytes()); + Assert.assertArrayEquals(expect.getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + } + + @Test + public void testDependentColumnFilter() throws IOException { + DependentColumnFilter filter = new DependentColumnFilter("family".getBytes(), + "qualifier".getBytes()); + String expect = "DependentColumnFilter('family','qualifier',false)"; + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + filter = new DependentColumnFilter("family".getBytes(), "qualifier".getBytes(), true); + expect = "DependentColumnFilter('family','qualifier',true)"; + Assert.assertArrayEquals(expect.getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + for (int i = 0; i < ops.length; ++i) { + filter = new DependentColumnFilter("family".getBytes(), "qualifier".getBytes(), false, + ops[i], new BinaryComparator("value".getBytes())); + expect = String.format( + "DependentColumnFilter('family','qualifier',false,%s,'binary:value')", opFlags[i]); + Assert.assertArrayEquals(expect.getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + } + @Test public void testPageFilter() throws IOException { PageFilter filter = new PageFilter(128); @@ -159,6 +196,59 @@ public void testColumnPrefixFilter() throws IOException { HBaseFilterUtils.toParseableByteArray(filter)); } + @Test + public void testFuzzyRowFilter() throws IOException { + List> fuzzyKey = new ArrayList<>(); + fuzzyKey.add(new Pair(Bytes.toBytes("abc"), Bytes.toBytes("101"))); + fuzzyKey.add(new Pair(Bytes.toBytes("ddd"), Bytes.toBytes("010"))); + + FuzzyRowFilter filter = new FuzzyRowFilter(fuzzyKey); + System.out.println(Bytes.toString(HBaseFilterUtils.toParseableByteArray(filter))); + Assert.assertArrayEquals("FuzzyRowFilter('abc','101','ddd','010')".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + } + + @Test + public void testMultiRowRangeFilter() throws IOException { + List ranges = new ArrayList<>(); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("a"), true, Bytes.toBytes("b"), false)); + ranges.add(new MultiRowRangeFilter.RowRange(Bytes.toBytes("c"), true, Bytes.toBytes("d$%%"), false)); + + MultiRowRangeFilter filter = new MultiRowRangeFilter(ranges); + System.out.println(Bytes.toString(HBaseFilterUtils.toParseableByteArray(filter))); + Assert.assertArrayEquals("MultiRowRangeFilter('a',true,'b',false,'c',true,'d$%%',false)".getBytes(), HBaseFilterUtils.toParseableByteArray(filter)); + } + + @Test + public void testInclusiveStopFilter() throws IOException { + InclusiveStopFilter filter = new InclusiveStopFilter(Bytes.toBytes("aaa")); + Assert.assertArrayEquals("InclusiveStopFilter('aaa')".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + + @Test + public void testColumnRangeFilter() throws IOException { + ColumnRangeFilter filter = new ColumnRangeFilter(Bytes.toBytes("a"), true, + Bytes.toBytes("b"), false); + Assert.assertArrayEquals("ColumnRangeFilter('a',true,'b',false)".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + + @Test + public void testMultipleColumnPrefixFilter() throws IOException { + byte[][] prefix = { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("d"), }; + MultipleColumnPrefixFilter filter = new MultipleColumnPrefixFilter(prefix); + Assert.assertArrayEquals("MultipleColumnPrefixFilter('a','b','d')".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + + @Test + public void testFamilyFilter() throws IOException { + FamilyFilter filter = new FamilyFilter(CompareFilter.CompareOp.NOT_EQUAL, + new BinaryComparator(Bytes.toBytes("cf"))); + Assert.assertArrayEquals("FamilyFilter(!=,'binary:cf')".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + @Test public void testColumnCountGetFilter() throws IOException { ColumnCountGetFilter filter = new ColumnCountGetFilter(513); @@ -166,6 +256,16 @@ public void testColumnCountGetFilter() throws IOException { HBaseFilterUtils.toParseableByteArray(filter)); } + @Test + public void testFirstKeyValueMatchingQualifiersFilter() throws IOException { + TreeSet qualifiers = new TreeSet<>(Bytes.BYTES_COMPARATOR); + qualifiers.add(Bytes.toBytes("q1")); + qualifiers.add(Bytes.toBytes("q2")); + FirstKeyValueMatchingQualifiersFilter filter = new FirstKeyValueMatchingQualifiersFilter(qualifiers); + Assert.assertArrayEquals("FirstKeyValueMatchingQualifiersFilter('q1','q2')".getBytes(), + HBaseFilterUtils.toParseableByteArray(filter)); + } + @Test public void testPrefixFilter() throws IOException { PrefixFilter filter = new PrefixFilter("prefix".getBytes()); diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/NativeHBaseUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/NativeHBaseUtil.java new file mode 100644 index 00000000..7d6d82b5 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/NativeHBaseUtil.java @@ -0,0 +1,105 @@ +package com.alipay.oceanbase.hbase.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Table; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.LinkedHashMap; +import java.util.Map; + +public class NativeHBaseUtil { + public static String SQL_PATH = "src/test/java/unit_test_db.sql"; + public static String MASTER_IP_PORT = ""; + public static String ZK_QUORUM = ""; + public static String ZK_PORT = ""; + + @Test + public void createTable() throws IOException { + Configuration config = new Configuration(); + if (!MASTER_IP_PORT.isEmpty()) { + config.set("hbase.master", MASTER_IP_PORT); + } + config.set("hbase.zookeeper.quorum", ZK_QUORUM); + config.set("hbase.zookeeper.property.clientPort", ZK_PORT); + // 建立连接 + Connection connection = ConnectionFactory.createConnection(config); + Admin admin = connection.getAdmin(); + + // 读取建表语句 + String sql = new String(Files.readAllBytes(Paths.get(SQL_PATH))); + String[] sqlList = sql.split(";"); + Map tableMap = new LinkedHashMap<>(); + for (String singleSql : sqlList) { + String namespace = null; + String realTableName; + String family; + + if (singleSql.contains("CREATE TABLE ")) { + singleSql.trim(); + String[] splits = singleSql.split(" "); + realTableName = splits[2].substring(1, splits[2].length() - 1); + if (realTableName.contains(":")) { + String[] tmpStr = realTableName.split(":", 2); + namespace = tmpStr[0]; + realTableName = tmpStr[1]; + } + String[] tmpStr = realTableName.split("\\$", 2); + realTableName = tmpStr[0]; + family = tmpStr[1]; + HTableDescriptor hTableDescriptor = tableMap.get(realTableName); + if (hTableDescriptor == null) { + hTableDescriptor = new HTableDescriptor(TableName.valueOf(namespace, realTableName)); + tableMap.put(realTableName, hTableDescriptor); + } + if (family != null) { + try { + HColumnDescriptor hColumnDescriptor = new HColumnDescriptor(family); + hColumnDescriptor.setMaxVersions(1000); + hTableDescriptor.addFamily(hColumnDescriptor); + + } catch (Exception e) { + System.out.println("family has exist"); + } + } + // 不分区，结果不会有变化 + } + } + for (Map.Entry entry : tableMap.entrySet()) { + if (admin.tableExists(entry.getValue().getTableName())) { + admin.disableTable(entry.getValue().getTableName()); + admin.deleteTable(entry.getValue().getTableName()); + } + admin.createTable(entry.getValue()); + } + } + + static public Table getTable(TableName tableName) throws IOException { + Configuration config = new Configuration(); + config.set("hbase.master", MASTER_IP_PORT); + config.set("hbase.zookeeper.quorum", ZK_QUORUM); + config.set("hbase.zookeeper.property.clientPort", ZK_PORT); + // 建立连接 + Connection connection = ConnectionFactory.createConnection(config); + return connection.getTable(tableName); + } + + static public Admin getAdmin() throws IOException { + Configuration config = new Configuration(); + config.set("hbase.master", MASTER_IP_PORT); + config.set("hbase.zookeeper.quorum", ZK_QUORUM); + config.set("hbase.zookeeper.property.clientPort", ZK_PORT); + // 建立连接 + Connection connection = ConnectionFactory.createConnection(config); + return connection.getAdmin(); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/OHTableHotkeyThrottleUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/OHTableHotkeyThrottleUtil.java index 950ce5db..aaa3c5a9 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/OHTableHotkeyThrottleUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/OHTableHotkeyThrottleUtil.java @@ -18,7 +18,6 @@ package com.alipay.oceanbase.hbase.util; import com.alipay.oceanbase.hbase.OHTableClient; -import com.alipay.oceanbase.hbase.ObHTableTestUtil; import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; import org.apache.hadoop.hbase.client.*; import org.junit.Assert; diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java new file mode 100644 index 00000000..41e68bb2 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -0,0 +1,159 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import com.alipay.oceanbase.hbase.OHTableClient; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HTableDescriptor; + +import java.sql.Connection; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.*; + +import static com.alipay.oceanbase.hbase.constants.OHConstants.*; + +public class ObHTableTestUtil { + // please consult your dba for the following configuration. + public static String PARAM_URL = ""; + public static String FULL_USER_NAME = ""; + public static String PASSWORD = ""; + public static String SYS_USER_NAME = ""; + public static String SYS_PASSWORD = ""; + public static String ODP_ADDR = ""; + public static int ODP_PORT = 0; + public static boolean ODP_MODE = false; + public static String DATABASE = ""; + public static String JDBC_IP = ""; + public static String JDBC_PORT = ""; + public static String JDBC_DATABASE = ""; + public static String JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " + + JDBC_DATABASE + "?" + "useUnicode=TRUE&" + + "characterEncoding=utf-8&" + + "socketTimeout=3000000&" + "connectTimeout=60000"; + + public static String SQL_FORMAT = "truncate %s"; + public static List tableNameList; + public static Connection conn; + public static Statement stmt; + + static { + tableNameList = new LinkedList<>(); + conn = getConnection(); + try { + stmt = conn.createStatement(); + } catch (SQLException e) { + System.out.println("sql error " + e); + } + } + + public static void prepareClean(List tableGroupList) throws Exception { + for (String tableGroup : tableGroupList) { + tableNameList.addAll(getOTableNameList(tableGroup)); + } + } + + public static void cleanData() { + try { + for (String realTableName : tableNameList) { + try { + if (realTableName.contains("'")) { + realTableName = "`" + realTableName + "`"; + } + stmt.execute(String.format(SQL_FORMAT, realTableName)); + } catch (Exception e) { + System.out.println("clean table data error ." + realTableName + " exception:" + + e); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static void closeConn() throws SQLException { + stmt.close(); + conn.close(); + } + + public static Configuration newConfiguration() { + Configuration conf = HBaseConfiguration.create(); + conf.set(HBASE_OCEANBASE_FULL_USER_NAME, FULL_USER_NAME); + conf.set(HBASE_OCEANBASE_PASSWORD, PASSWORD); + if (ODP_MODE) { + // ODP mode + conf.set(HBASE_OCEANBASE_ODP_ADDR, ODP_ADDR); + conf.setInt(HBASE_OCEANBASE_ODP_PORT, ODP_PORT); + conf.setBoolean(HBASE_OCEANBASE_ODP_MODE, ODP_MODE); + conf.set(HBASE_OCEANBASE_DATABASE, DATABASE); + } else { + // OCP mode + conf.set(HBASE_OCEANBASE_PARAM_URL, PARAM_URL); + conf.set(HBASE_OCEANBASE_SYS_USER_NAME, SYS_USER_NAME); + conf.set(HBASE_OCEANBASE_SYS_PASSWORD, SYS_PASSWORD); + } + return conf; + } + + public static OHTableClient newOHTableClient(String tableName) { + return new OHTableClient(tableName, newConfiguration()); + } + + static public List getOTableNameList(String tableGroup) throws IOException { + // 读取建表语句 + List res = new LinkedList<>(); + String sql = new String(Files.readAllBytes(Paths.get(NativeHBaseUtil.SQL_PATH))); + String[] sqlList = sql.split(";"); + Map tableMap = new LinkedHashMap<>(); + for (String singleSql : sqlList) { + String realTableName; + if (singleSql.contains("CREATE TABLE ")) { + singleSql.trim(); + String[] splits = singleSql.split(" "); + String tableGroupName = splits[2].substring(1, splits[2].length() - 1); + if (tableGroupName.contains(":")) { + String[] tmpStr = tableGroupName.split(":", 2); + tableGroupName = tmpStr[1]; + } + realTableName = tableGroupName.split("\\$", 2)[0]; + if (realTableName.equals(tableGroup)) { + res.add(tableGroupName); + } + } + } + return res; + } + + static public Connection getConnection() { + try { + Class.forName("com.mysql.cj.jdbc.Driver"); + String[] userNames = FULL_USER_NAME.split("#"); + Connection conn = DriverManager.getConnection(JDBC_URL, userNames[0], PASSWORD); + + return conn; + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/src/test/java/unit_test_db.sql b/src/test/java/unit_test_db.sql index 6d4432dd..cbb172fe 100644 --- a/src/test/java/unit_test_db.sql +++ b/src/test/java/unit_test_db.sql @@ -212,3 +212,56 @@ CREATE TABLE `test_multi_cf$family_with_group3` ( `V` varbinary(1024) DEFAULT NULL, PRIMARY KEY (`K`, `Q`, `T`) ) TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3; + +CREATE DATABASE IF NOT EXISTS `n1`; +USE `n1`; +CREATE TABLE `n1:test$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +); + +CREATE TABLE `n1:test_t$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +); + +CREATE TABLEGROUP `n1:test` SHARDING = 'ADAPTIVE'; +CREATE TABLE `n1:test$family'1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = `n1:test`; + +CREATE TABLE `n1:test$family_with_local_index` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + key `idx1`(T) local, + PRIMARY KEY (`K`, `Q`, `T`) +); + +CREATE TABLE `n1:test$family_group` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = `n1:test`; + +CREATE TABLE `n1:test$partitionFamily1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) partition by key(`K`) partitions 17; + diff --git a/src/main/resources/hbase-site.xml b/src/test/resources/hbase-site.xml similarity index 100% rename from src/main/resources/hbase-site.xml rename to src/test/resources/hbase-site.xml