diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java index 2b47e01972a..597e0fc6cb0 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java @@ -58,6 +58,8 @@ import org.junit.runners.Parameterized.Parameters; import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Category(ParallelStatsDisabledTest.class) @@ -71,6 +73,8 @@ public class DerivedTableIT extends ParallelStatsDisabledIT { private String[] plans; private String tableName; + private static final Logger LOGGER = LoggerFactory.getLogger(DerivedTableIT.class); + public DerivedTableIT(String[] indexDDL, String[] plans) { this.indexDDL = indexDDL; @@ -114,7 +118,7 @@ public static synchronized Collection data() { { "CREATE INDEX "+dynamicTableName+"_DERIVED_IDX ON "+dynamicTableName+" (a_byte) INCLUDE (A_STRING, B_STRING)" }, { - "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX\n" + + "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX \n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]\n" + "CLIENT MERGE SORT\n" + "CLIENT SORTED BY [\"B_STRING\"]\n" + @@ -122,7 +126,7 @@ public static synchronized Collection data() { "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + "CLIENT SORTED BY [A DESC]", - "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX\n" + + "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX \n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]\n" + "CLIENT MERGE SORT\n" + "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + @@ -130,7 +134,7 @@ public static synchronized Collection data() { "CLIENT SORTED BY [A DESC]"}}); testCases.add(new String[][] { {}, { - "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+"\n" + + "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+" \n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]\n" + "CLIENT MERGE SORT\n" + "CLIENT SORTED BY [B_STRING]\n" + @@ -138,7 +142,7 @@ public static synchronized Collection data() { "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + "CLIENT SORTED BY [A DESC]", - "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+"\n" + + "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+" \n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]\n" + "CLIENT MERGE SORT\n" + "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + @@ -378,8 +382,12 @@ public void testDerivedTableWithGroupBy() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals(plans[0], QueryUtil.getExplainPlan(rs)); + rs = conn.createStatement().executeQuery("EXPLAIN WITH REGIONS " + query); + String explainPlanOutput = QueryUtil.getExplainPlan(rs); + LOGGER.info("Explain plan output: {}", explainPlanOutput); + String[] splitExplainPlan = explainPlanOutput.split("\\n \\(region locations = \\[region="); + String[] secondSplitExplainPlan = splitExplainPlan[1].split("]\\)"); + assertEquals(plans[0], splitExplainPlan[0] + secondSplitExplainPlan[1]); // distinct b (groupby a, b) groupby a orderby a query = "SELECT DISTINCT COLLECTDISTINCT(t.b) FROM (SELECT b_string b, a_string a FROM "+tableName+" GROUP BY a_string, b_string) AS t GROUP BY t.a ORDER BY t.a DESC"; @@ -400,8 +408,12 @@ public void testDerivedTableWithGroupBy() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals(plans[1], QueryUtil.getExplainPlan(rs)); + rs = conn.createStatement().executeQuery("EXPLAIN WITH REGIONS " + query); + explainPlanOutput = QueryUtil.getExplainPlan(rs); + LOGGER.info("Explain plan output: {}", explainPlanOutput); + splitExplainPlan = explainPlanOutput.split("\\n \\(region locations = \\[region="); + secondSplitExplainPlan = splitExplainPlan[1].split("]\\)"); + assertEquals(plans[1], splitExplainPlan[0] + secondSplitExplainPlan[1]); // (orderby) groupby query = "SELECT t.a_string, count(*) FROM (SELECT * FROM "+tableName+" order by a_integer) AS t where a_byte != 8 group by t.a_string"; diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java index 3c76cf3cefd..096a301ff79 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java @@ -50,12 +50,17 @@ import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; +import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FlappingLocalIndexIT extends BaseLocalIndexIT { + private static final Logger LOGGER = LoggerFactory.getLogger(FlappingLocalIndexIT.class); + public FlappingLocalIndexIT(boolean isNamespaceMapped) { super(isNamespaceMapped); } @@ -159,6 +164,13 @@ public void testLocalIndexScan() throws Exception { String query = "SELECT * FROM " + tableName +" where v1 like 'a%'"; + String explainPlanOutput = + QueryUtil.getExplainPlan(conn1.createStatement().executeQuery("EXPLAIN WITH REGIONS " + query)); + LOGGER.info("Explain plan output: {}", explainPlanOutput); + // MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN is set as 2 + assertTrue("Expected total " + numRegions + " regions", + explainPlanOutput.contains("...total size = " + numRegions)); + ExplainPlan plan = conn1.prepareStatement(query) .unwrap(PhoenixPreparedStatement.class).optimizeQuery() .getExplainPlan(); @@ -176,6 +188,7 @@ public void testLocalIndexScan() throws Exception { explainPlanAttributes.getServerWhereFilter()); assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertEquals(numRegions, explainPlanAttributes.getRegionLocations().size()); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java index ad8e969e807b..7f780305971 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java @@ -120,7 +120,7 @@ public void testDebugLogs() throws Exception { // sleep for sometime to let query log committed Thread.sleep(delay); - try (ResultSet explainRS = conn.createStatement().executeQuery("Explain " + query); + try (ResultSet explainRS = conn.createStatement().executeQuery("Explain with regions " + query); ResultSet rs = conn.createStatement().executeQuery(logQuery)) { boolean foundQueryLog = false; @@ -300,7 +300,7 @@ private void testPreparedStatement(LogLevel loglevel) throws Exception{ // sleep for sometime to let query log committed Thread.sleep(delay); - String explainQuery = "Explain " + "SELECT * FROM " + tableName + " where V = 'value5'"; + String explainQuery = "EXPLAIN WITH REGIONS " + "SELECT * FROM " + tableName + " where V = 'value5'"; try (ResultSet explainRS = conn.createStatement() .executeQuery(explainQuery); ResultSet rs = conn.createStatement().executeQuery(logQuery)) { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java index 4c8863fbec7..ec297c15959 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java @@ -61,6 +61,7 @@ public static synchronized void doSetup() throws Exception { // setting update frequency to a large value to test out that we are // generating stats for local indexes clientProps.put(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB, "120000"); + clientProps.put(QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, "2"); setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator())); } diff --git a/phoenix-core/src/main/antlr3/PhoenixSQL.g b/phoenix-core/src/main/antlr3/PhoenixSQL.g index 5d57c3669f4..a8cbb71085d 100644 --- a/phoenix-core/src/main/antlr3/PhoenixSQL.g +++ b/phoenix-core/src/main/antlr3/PhoenixSQL.g @@ -152,6 +152,7 @@ tokens REVOKE = 'revoke'; SHOW = 'show'; UNCOVERED = 'uncovered'; + REGIONS = 'regions'; } @@ -212,6 +213,7 @@ import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.parse.LikeParseNode.LikeType; import org.apache.phoenix.trace.util.Tracing; import org.apache.phoenix.parse.AddJarsStatement; +import org.apache.phoenix.parse.ExplainType; } @lexer::header { @@ -452,7 +454,14 @@ oneStatement returns [BindableStatement ret] finally{ contextStack.pop(); } explain_node returns [BindableStatement ret] - : EXPLAIN q=oneStatement {$ret=factory.explain(q);} + : EXPLAIN (w=WITH)? (r=REGIONS)? q=oneStatement + { + if ((w==null && r!=null) || (w!=null && r==null)) { + throw new RuntimeException("Valid usage: EXPLAIN {query} OR EXPLAIN WITH REGIONS {query}"); + } + ret = (w==null && r==null) ? factory.explain(q, ExplainType.DEFAULT) + : factory.explain(q, ExplainType.WITH_REGIONS); + } ; // Parse a create table statement. diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java index 258a7ee6b36..a41f776345c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java @@ -18,8 +18,10 @@ package org.apache.phoenix.compile; +import java.util.List; import java.util.Set; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.client.Consistency; import org.apache.phoenix.parse.HintNode; import org.apache.phoenix.parse.HintNode.Hint; @@ -73,6 +75,7 @@ public class ExplainPlanAttributes { // be null private final ExplainPlanAttributes rhsJoinQueryExplainPlan; private final Set serverMergeColumns; + private final List regionLocations; private static final ExplainPlanAttributes EXPLAIN_PLAN_INSTANCE = new ExplainPlanAttributes(); @@ -112,6 +115,7 @@ private ExplainPlanAttributes() { this.clientSortAlgo = null; this.rhsJoinQueryExplainPlan = null; this.serverMergeColumns = null; + this.regionLocations = null; } public ExplainPlanAttributes(String abstractExplainPlan, @@ -132,7 +136,7 @@ public ExplainPlanAttributes(String abstractExplainPlan, Integer clientSequenceCount, String clientCursorName, String clientSortAlgo, ExplainPlanAttributes rhsJoinQueryExplainPlan, - Set serverMergeColumns) { + Set serverMergeColumns, List regionLocations) { this.abstractExplainPlan = abstractExplainPlan; this.splitsChunk = splitsChunk; this.estimatedRows = estimatedRows; @@ -167,6 +171,7 @@ public ExplainPlanAttributes(String abstractExplainPlan, this.clientSortAlgo = clientSortAlgo; this.rhsJoinQueryExplainPlan = rhsJoinQueryExplainPlan; this.serverMergeColumns = serverMergeColumns; + this.regionLocations = regionLocations; } public String getAbstractExplainPlan() { @@ -305,6 +310,10 @@ public Set getServerMergeColumns() { return serverMergeColumns; } + public List getRegionLocations() { + return regionLocations; + } + public static ExplainPlanAttributes getDefaultExplainPlan() { return EXPLAIN_PLAN_INSTANCE; } @@ -344,6 +353,7 @@ public static class ExplainPlanAttributesBuilder { private String clientSortAlgo; private ExplainPlanAttributes rhsJoinQueryExplainPlan; private Set serverMergeColumns; + private List regionLocations; public ExplainPlanAttributesBuilder() { // default @@ -396,6 +406,7 @@ public ExplainPlanAttributesBuilder( this.rhsJoinQueryExplainPlan = explainPlanAttributes.getRhsJoinQueryExplainPlan(); this.serverMergeColumns = explainPlanAttributes.getServerMergeColumns(); + this.regionLocations = explainPlanAttributes.getRegionLocations(); } public ExplainPlanAttributesBuilder setAbstractExplainPlan( @@ -599,6 +610,12 @@ public ExplainPlanAttributesBuilder setServerMergeColumns( return this; } + public ExplainPlanAttributesBuilder setRegionLocations( + List regionLocations) { + this.regionLocations = regionLocations; + return this; + } + public ExplainPlanAttributes build() { return new ExplainPlanAttributes(abstractExplainPlan, splitsChunk, estimatedRows, estimatedSizeInBytes, iteratorTypeAndScanSize, @@ -611,7 +628,8 @@ public ExplainPlanAttributes build() { clientFilterBy, clientAggregate, clientSortedBy, clientAfterAggregate, clientDistinctFilter, clientOffset, clientRowLimit, clientSequenceCount, clientCursorName, - clientSortAlgo, rhsJoinQueryExplainPlan, serverMergeColumns); + clientSortAlgo, rhsJoinQueryExplainPlan, serverMergeColumns, + regionLocations); } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 08a946d0850..2c6885bbf5d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -1537,7 +1537,6 @@ private List recreateIterators(ConnectionQueryServices se @Override public void close() throws SQLException { - // Don't call cancel on already started work, as it causes the HConnection // to get into a funk. Instead, just cancel queued work. boolean cancelledWork = false; @@ -1724,7 +1723,7 @@ private void explainUtil(List planSteps, } } - explain(buf.toString(), planSteps, explainPlanAttributesBuilder); + explain(buf.toString(), planSteps, explainPlanAttributesBuilder, scans); } @Override diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java index c9ef46d00a7..23b7632cb59 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java @@ -17,15 +17,23 @@ */ package org.apache.phoenix.iterate; +import java.io.IOException; +import java.sql.SQLException; import java.text.Format; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import java.util.Set; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.client.Consistency; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.apache.hadoop.hbase.filter.PageFilter; @@ -46,6 +54,8 @@ import org.apache.phoenix.parse.HintNode.Hint; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.KeyRange.Bound; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.SortOrder; @@ -55,11 +65,17 @@ import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.ScanUtil; import org.apache.phoenix.util.StringUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class ExplainTable { + + private static final Logger LOGGER = LoggerFactory.getLogger(ExplainTable.class); private static final List EVERYTHING = Collections.singletonList(KeyRange.EVERYTHING_RANGE); public static final String POINT_LOOKUP_ON_STRING = "POINT LOOKUP ON "; + public static final String REGION_LOCATIONS = " (region locations = "; + protected final StatementContext context; protected final TableRef tableRef; protected final GroupBy groupBy; @@ -67,7 +83,7 @@ public abstract class ExplainTable { protected final HintNode hint; protected final Integer limit; protected final Integer offset; - + public ExplainTable(StatementContext context, TableRef table) { this(context, table, GroupBy.EMPTY_GROUP_BY, OrderBy.EMPTY_ORDER_BY, HintNode.EMPTY_HINT_NODE, null, null); } @@ -111,8 +127,65 @@ private String explainSkipScan() { return buf.toString(); } - protected void explain(String prefix, List planSteps, - ExplainPlanAttributesBuilder explainPlanAttributesBuilder) { + /** + * Get regions that represent the given range of start and end key for the given table, and + * all the regions to the regionLocations list. + * + * @param tableName the table name. + * @param startKey the start rowkey. + * @param endKey the end rowkey. + * @param includeEndKey true if end key needs to be included. + * @param reload true if reload from meta is necessary. + * @param regionBoundaries set of region boundaries to get the unique list of region locations. + * @param regionLocations the list of region locations as output. + * @throws IOException if something goes wrong while creating connection or querying region + * locations. + */ + private void getRegionsInRange(final byte[] tableName, + final byte[] startKey, + final byte[] endKey, + final boolean includeEndKey, + final boolean reload, + Set regionBoundaries, + List regionLocations) + throws IOException, SQLException { + final boolean endKeyIsEndOfTable = Bytes.equals(endKey, HConstants.EMPTY_END_ROW); + if ((Bytes.compareTo(startKey, endKey) > 0) && !endKeyIsEndOfTable) { + throw new IllegalArgumentException( + "Invalid range: " + Bytes.toStringBinary(startKey) + " > " + + Bytes.toStringBinary(endKey)); + } + byte[] currentKey = startKey; + try (Table table = context.getConnection().getQueryServices().getTable(tableName)) { + // include all regions that include key range from the given start key + // and end key + do { + HRegionLocation regionLocation = + table.getRegionLocator().getRegionLocation(currentKey, reload); + RegionBoundary regionBoundary = + new RegionBoundary(regionLocation.getRegion().getStartKey(), + regionLocation.getRegion().getEndKey()); + if (!regionBoundaries.contains(regionBoundary)) { + regionLocations.add(regionLocation); + regionBoundaries.add(regionBoundary); + } + currentKey = regionLocation.getRegion().getEndKey(); + // condition1 = currentKey != END_ROW_KEY + // condition2 = endKeyIsEndOfTable == true + // condition3 = currentKey < endKey + // condition4 = includeEndKey == true + // condition5 = currentKey == endKey + // while (condition1 && (condition2 || condition3 || (condition4 && condition5))) + } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW) + && (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0 + || (includeEndKey && Bytes.compareTo(currentKey, endKey) == 0))); + } + } + + protected void explain(String prefix, + List planSteps, + ExplainPlanAttributesBuilder explainPlanAttributesBuilder, + List> scansList) { StringBuilder buf = new StringBuilder(prefix); ScanRanges scanRanges = context.getScanRanges(); Scan scan = context.getScan(); @@ -275,6 +348,7 @@ protected void explain(String prefix, List planSteps, if (groupByLimitBytes != null) { groupByLimit = (Integer) PInteger.INSTANCE.toObject(groupByLimitBytes); } + getRegionLocations(planSteps, explainPlanAttributesBuilder, scansList); groupBy.explain(planSteps, groupByLimit, explainPlanAttributesBuilder); if (scan.getAttribute(BaseScannerRegionObserver.SPECIFIC_ARRAY_INDEX) != null) { planSteps.add(" SERVER ARRAY ELEMENT PROJECTION"); @@ -284,6 +358,108 @@ protected void explain(String prefix, List planSteps, } } + /** + * Retrieve region locations and set the values in the explain plan output. + * + * @param planSteps list of plan steps to add explain plan output to. + * @param explainPlanAttributesBuilder explain plan v2 attributes builder instance. + * @param scansList list of the list of scans, to be used for parallel scans. + */ + private void getRegionLocations(List planSteps, + ExplainPlanAttributesBuilder explainPlanAttributesBuilder, + List> scansList) { + String regionLocationPlan = getRegionLocationsForExplainPlan(explainPlanAttributesBuilder, + scansList); + if (regionLocationPlan.length() > 0) { + planSteps.add(regionLocationPlan); + } + } + + /** + * Retrieve region locations from hbase client and set the values for the explain plan output. + * If the list of region locations exceed max limit, print only list with the max limit and + * print num of total list size. + * + * @param explainPlanAttributesBuilder explain plan v2 attributes builder instance. + * @param scansList list of the list of scans, to be used for parallel scans. + * @return region locations to be added to the explain plan output. + */ + private String getRegionLocationsForExplainPlan( + ExplainPlanAttributesBuilder explainPlanAttributesBuilder, + List> scansList) { + try { + StringBuilder buf = new StringBuilder().append(REGION_LOCATIONS); + Set regionBoundaries = new HashSet<>(); + List regionLocations = new ArrayList<>(); + for (List scans : scansList) { + for (Scan eachScan : scans) { + getRegionsInRange(tableRef.getTable().getPhysicalName().getBytes(), + eachScan.getStartRow(), + eachScan.getStopRow(), + true, + false, + regionBoundaries, + regionLocations); + } + } + int maxLimitRegionLoc = context.getConnection().getQueryServices().getConfiguration() + .getInt(QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, + QueryServicesOptions.DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN); + if (explainPlanAttributesBuilder != null) { + explainPlanAttributesBuilder.setRegionLocations( + Collections.unmodifiableList(regionLocations)); + } + if (regionLocations.size() > maxLimitRegionLoc) { + int originalSize = regionLocations.size(); + List trimmedRegionLocations = + regionLocations.subList(0, maxLimitRegionLoc); + buf.append(trimmedRegionLocations); + buf.append("...total size = "); + buf.append(originalSize); + } else { + buf.append(regionLocations); + } + buf.append(") "); + return buf.toString(); + } catch (IOException | SQLException | UnsupportedOperationException e) { + LOGGER.error("Explain table unable to add region locations.", e); + return ""; + } + } + + /** + * Region boundary class with start and end key of the region. + */ + private static class RegionBoundary { + private final byte[] startKey; + private final byte[] endKey; + + RegionBoundary(byte[] startKey, byte[] endKey) { + this.startKey = startKey; + this.endKey = endKey; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RegionBoundary that = (RegionBoundary) o; + return Bytes.compareTo(startKey, that.startKey) == 0 + && Bytes.compareTo(endKey, that.endKey) == 0; + } + + @Override + public int hashCode() { + int result = Arrays.hashCode(startKey); + result = 31 * result + Arrays.hashCode(endKey); + return result; + } + } + private void appendPKColumnValue(StringBuilder buf, byte[] range, Boolean isNull, int slotIndex, boolean changeViewIndexId) { if (Boolean.TRUE.equals(isNull)) { buf.append("null"); @@ -414,4 +590,5 @@ private String appendKeyRanges() { buf.setCharAt(buf.length()-1, ']'); return buf.toString(); } + } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java index 44e2e98dcca..5b019d4cc4e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java @@ -109,6 +109,7 @@ import org.apache.phoenix.execute.visitor.QueryPlanVisitor; import org.apache.phoenix.expression.KeyValueColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; +import org.apache.phoenix.iterate.ExplainTable; import org.apache.phoenix.iterate.MaterializedResultIterator; import org.apache.phoenix.iterate.ParallelScanGrouper; import org.apache.phoenix.iterate.ResultIterator; @@ -140,6 +141,7 @@ import org.apache.phoenix.parse.DeclareCursorStatement; import org.apache.phoenix.parse.DeleteJarStatement; import org.apache.phoenix.parse.DeleteStatement; +import org.apache.phoenix.parse.ExplainType; import org.apache.phoenix.parse.ShowCreateTableStatement; import org.apache.phoenix.parse.ShowCreateTable; import org.apache.phoenix.parse.DropColumnStatement; @@ -789,8 +791,8 @@ EXPLAIN_PLAN_TABLE_NAME, new KeyValueColumnExpression( private static class ExecutableExplainStatement extends ExplainStatement implements CompilableStatement { - public ExecutableExplainStatement(BindableStatement statement) { - super(statement); + ExecutableExplainStatement(BindableStatement statement, ExplainType explainType) { + super(statement, explainType); } @Override @@ -816,6 +818,13 @@ public QueryPlan compilePlan(PhoenixStatement stmt, Sequence.ValueOp seqAction) } final StatementPlan plan = compilePlan; List planSteps = plan.getExplainPlan().getPlanSteps(); + ExplainType explainType = getExplainType(); + if (explainType == ExplainType.DEFAULT) { + List updatedExplainPlanSteps = new ArrayList<>(planSteps); + updatedExplainPlanSteps.removeIf(planStep -> planStep != null + && planStep.contains(ExplainTable.REGION_LOCATIONS)); + planSteps = Collections.unmodifiableList(updatedExplainPlanSteps); + } List tuples = Lists.newArrayListWithExpectedSize(planSteps.size()); Long estimatedBytesToScan = plan.getEstimatedBytesToScan(); Long estimatedRowsToScan = plan.getEstimatedRowsToScan(); @@ -1914,8 +1923,8 @@ public AlterSessionStatement alterSession(Map props) { } @Override - public ExplainStatement explain(BindableStatement statement) { - return new ExecutableExplainStatement(statement); + public ExplainStatement explain(BindableStatement statement, ExplainType explainType) { + return new ExecutableExplainStatement(statement, explainType); } @Override diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainStatement.java index 49ce5737a4b..3b28ca5c0d5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainStatement.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainStatement.java @@ -21,9 +21,11 @@ public class ExplainStatement implements BindableStatement { private final BindableStatement statement; - - public ExplainStatement(BindableStatement statement) { + private final ExplainType explainType; + + public ExplainStatement(BindableStatement statement, ExplainType explainType) { this.statement = statement; + this.explainType = explainType; } public BindableStatement getStatement() { @@ -39,4 +41,8 @@ public int getBindCount() { public Operation getOperation() { return Operation.QUERY; } + + public ExplainType getExplainType() { + return explainType; + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainType.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainType.java new file mode 100644 index 00000000000..fc35939d35f --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainType.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.phoenix.parse; + +/** + * Explain type attributes used to differentiate output of the explain plan. + */ +public enum ExplainType { + WITH_REGIONS, + DEFAULT +} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java index 77350cc0672..660598b4248 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java @@ -216,8 +216,8 @@ public static String createTempAlias() { return "$" + tempAliasCounter.incrementAndGet(); } - public ExplainStatement explain(BindableStatement statement) { - return new ExplainStatement(statement); + public ExplainStatement explain(BindableStatement statement, ExplainType explainType) { + return new ExplainStatement(statement, explainType); } public AliasedNode aliasedNode(String alias, ParseNode expression) { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java index da9da65ef17..e4885a79108 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -403,6 +403,14 @@ public interface QueryServices extends SQLCloseable { * Region server holding the SYSTEM.CATALOG table in batch oriented jobs. */ String SKIP_SYSTEM_TABLES_EXISTENCE_CHECK = "phoenix.skip.system.tables.existence.check"; + + /** + * Config key to represent max region locations to be displayed as part of the Explain plan + * output. + */ + String MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN = + "phoenix.max.region.locations.size.explain.plan"; + /** * Get executor service used for parallel scans */ diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index 697554605f9..adb3b55e2fa 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -57,6 +57,7 @@ import static org.apache.phoenix.query.QueryServices.MAX_CLIENT_METADATA_CACHE_SIZE_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_MEMORY_PERC_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_MUTATION_SIZE_ATTRIB; +import static org.apache.phoenix.query.QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN; import static org.apache.phoenix.query.QueryServices.MAX_SERVER_CACHE_SIZE_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_SERVER_CACHE_TIME_TO_LIVE_MS_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_SERVER_METADATA_CACHE_SIZE_ATTRIB; @@ -399,6 +400,7 @@ public class QueryServicesOptions { public static final int DEFAULT_SCAN_PAGE_SIZE = 32768; public static final boolean DEFAULT_APPLY_TIME_ZONE_DISPLACMENT = false; public static final boolean DEFAULT_PHOENIX_TABLE_TTL_ENABLED = true; + public static final int DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN = 5; private final Configuration config; @@ -489,7 +491,9 @@ public static QueryServicesOptions withDefaults() { .setIfUnset(INDEX_CREATE_DEFAULT_STATE, DEFAULT_CREATE_INDEX_STATE) .setIfUnset(SKIP_SYSTEM_TABLES_EXISTENCE_CHECK, DEFAULT_SKIP_SYSTEM_TABLES_EXISTENCE_CHECK) - .setIfUnset(MAX_IN_LIST_SKIP_SCAN_SIZE, DEFAULT_MAX_IN_LIST_SKIP_SCAN_SIZE); + .setIfUnset(MAX_IN_LIST_SKIP_SCAN_SIZE, DEFAULT_MAX_IN_LIST_SKIP_SCAN_SIZE) + .setIfUnset(MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, + DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN); // HBase sets this to 1, so we reset it to something more appropriate. // Hopefully HBase will change this, because we can't know if a user set