From cb5c064bfdd23281de3ca36d1fb8e2d3565b0653 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Tue, 2 May 2023 16:00:51 -0700 Subject: [PATCH 01/10] PHOENIX-6907 Explain Plan should output region locations with servers --- .../phoenix/end2end/DerivedTableIT.java | 24 ++- .../phoenix/end2end/FlappingLocalIndexIT.java | 13 ++ .../end2end/index/BaseLocalIndexIT.java | 1 + .../compile/ExplainPlanAttributes.java | 21 ++- .../phoenix/iterate/BaseResultIterators.java | 2 +- .../apache/phoenix/iterate/ExplainTable.java | 141 +++++++++++++++++- .../apache/phoenix/query/QueryServices.java | 6 + .../phoenix/query/QueryServicesOptions.java | 5 +- 8 files changed, 202 insertions(+), 11 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java index 2b47e01972a..92fba2f4513 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java @@ -58,6 +58,8 @@ import org.junit.runners.Parameterized.Parameters; import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Category(ParallelStatsDisabledTest.class) @@ -71,6 +73,8 @@ public class DerivedTableIT extends ParallelStatsDisabledIT { private String[] plans; private String tableName; + private static final Logger LOGGER = LoggerFactory.getLogger(DerivedTableIT.class); + public DerivedTableIT(String[] indexDDL, String[] plans) { this.indexDDL = indexDDL; @@ -114,7 +118,7 @@ public static synchronized Collection data() { { "CREATE INDEX "+dynamicTableName+"_DERIVED_IDX ON "+dynamicTableName+" (a_byte) INCLUDE (A_STRING, B_STRING)" }, { - "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX\n" + + "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX \n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]\n" + "CLIENT MERGE SORT\n" + "CLIENT SORTED BY [\"B_STRING\"]\n" + @@ -122,7 +126,7 @@ public static synchronized Collection data() { "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + "CLIENT SORTED BY [A DESC]", - "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX\n" + + "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX \n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]\n" + "CLIENT MERGE SORT\n" + "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + @@ -130,7 +134,7 @@ public static synchronized Collection data() { "CLIENT SORTED BY [A DESC]"}}); testCases.add(new String[][] { {}, { - "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+"\n" + + "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+" \n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]\n" + "CLIENT MERGE SORT\n" + "CLIENT SORTED BY [B_STRING]\n" + @@ -138,7 +142,7 @@ public static synchronized Collection data() { "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + "CLIENT SORTED BY [A DESC]", - "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+"\n" + + "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+" \n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]\n" + "CLIENT MERGE SORT\n" + "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + @@ -379,7 +383,11 @@ public void testDerivedTableWithGroupBy() throws Exception { assertFalse(rs.next()); rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals(plans[0], QueryUtil.getExplainPlan(rs)); + String explainPlanOutput = QueryUtil.getExplainPlan(rs); + LOGGER.info("Explain plan output: {}", explainPlanOutput); + String[] splitExplainPlan = explainPlanOutput.split("\\n \\(region locations = \\[region="); + String[] secondSplitExplainPlan = splitExplainPlan[1].split("]\\)"); + assertEquals(plans[0], splitExplainPlan[0] + secondSplitExplainPlan[1]); // distinct b (groupby a, b) groupby a orderby a query = "SELECT DISTINCT COLLECTDISTINCT(t.b) FROM (SELECT b_string b, a_string a FROM "+tableName+" GROUP BY a_string, b_string) AS t GROUP BY t.a ORDER BY t.a DESC"; @@ -401,7 +409,11 @@ public void testDerivedTableWithGroupBy() throws Exception { assertFalse(rs.next()); rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals(plans[1], QueryUtil.getExplainPlan(rs)); + explainPlanOutput = QueryUtil.getExplainPlan(rs); + LOGGER.info("Explain plan output: {}", explainPlanOutput); + splitExplainPlan = explainPlanOutput.split("\\n \\(region locations = \\[region="); + secondSplitExplainPlan = splitExplainPlan[1].split("]\\)"); + assertEquals(plans[1], splitExplainPlan[0] + secondSplitExplainPlan[1]); // (orderby) groupby query = "SELECT t.a_string, count(*) FROM (SELECT * FROM "+tableName+" order by a_integer) AS t where a_byte != 8 group by t.a_string"; diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java index 3c76cf3cefd..6ca16d04a2c 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java @@ -50,12 +50,17 @@ import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; +import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FlappingLocalIndexIT extends BaseLocalIndexIT { + private static final Logger LOGGER = LoggerFactory.getLogger(FlappingLocalIndexIT.class); + public FlappingLocalIndexIT(boolean isNamespaceMapped) { super(isNamespaceMapped); } @@ -159,6 +164,13 @@ public void testLocalIndexScan() throws Exception { String query = "SELECT * FROM " + tableName +" where v1 like 'a%'"; + String explainPlanOutput = + QueryUtil.getExplainPlan(conn1.createStatement().executeQuery("EXPLAIN " + query)); + LOGGER.info("Explain plan output: {}", explainPlanOutput); + // MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN is set as 2 + assertTrue("Expected total " + numRegions + " regions", + explainPlanOutput.contains("...total size = " + numRegions)); + ExplainPlan plan = conn1.prepareStatement(query) .unwrap(PhoenixPreparedStatement.class).optimizeQuery() .getExplainPlan(); @@ -176,6 +188,7 @@ public void testLocalIndexScan() throws Exception { explainPlanAttributes.getServerWhereFilter()); assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertEquals(numRegions, explainPlanAttributes.getRegionLocations().size()); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java index 4c8863fbec7..ec297c15959 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java @@ -61,6 +61,7 @@ public static synchronized void doSetup() throws Exception { // setting update frequency to a large value to test out that we are // generating stats for local indexes clientProps.put(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB, "120000"); + clientProps.put(QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, "2"); setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator())); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java index 258a7ee6b36..d5363da02f3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java @@ -18,8 +18,10 @@ package org.apache.phoenix.compile; +import java.util.List; import java.util.Set; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.client.Consistency; import org.apache.phoenix.parse.HintNode; import org.apache.phoenix.parse.HintNode.Hint; @@ -73,6 +75,7 @@ public class ExplainPlanAttributes { // be null private final ExplainPlanAttributes rhsJoinQueryExplainPlan; private final Set serverMergeColumns; + private final List regionLocations; private static final ExplainPlanAttributes EXPLAIN_PLAN_INSTANCE = new ExplainPlanAttributes(); @@ -112,6 +115,7 @@ private ExplainPlanAttributes() { this.clientSortAlgo = null; this.rhsJoinQueryExplainPlan = null; this.serverMergeColumns = null; + this.regionLocations = null; } public ExplainPlanAttributes(String abstractExplainPlan, @@ -132,7 +136,7 @@ public ExplainPlanAttributes(String abstractExplainPlan, Integer clientSequenceCount, String clientCursorName, String clientSortAlgo, ExplainPlanAttributes rhsJoinQueryExplainPlan, - Set serverMergeColumns) { + Set serverMergeColumns, List regionLocations) { this.abstractExplainPlan = abstractExplainPlan; this.splitsChunk = splitsChunk; this.estimatedRows = estimatedRows; @@ -167,6 +171,7 @@ public ExplainPlanAttributes(String abstractExplainPlan, this.clientSortAlgo = clientSortAlgo; this.rhsJoinQueryExplainPlan = rhsJoinQueryExplainPlan; this.serverMergeColumns = serverMergeColumns; + this.regionLocations = regionLocations; } public String getAbstractExplainPlan() { @@ -305,6 +310,10 @@ public Set getServerMergeColumns() { return serverMergeColumns; } + public List getRegionLocations() { + return regionLocations; + } + public static ExplainPlanAttributes getDefaultExplainPlan() { return EXPLAIN_PLAN_INSTANCE; } @@ -344,6 +353,7 @@ public static class ExplainPlanAttributesBuilder { private String clientSortAlgo; private ExplainPlanAttributes rhsJoinQueryExplainPlan; private Set serverMergeColumns; + private List regionLocations; public ExplainPlanAttributesBuilder() { // default @@ -396,6 +406,7 @@ public ExplainPlanAttributesBuilder( this.rhsJoinQueryExplainPlan = explainPlanAttributes.getRhsJoinQueryExplainPlan(); this.serverMergeColumns = explainPlanAttributes.getServerMergeColumns(); + this.regionLocations = explainPlanAttributes.getRegionLocations(); } public ExplainPlanAttributesBuilder setAbstractExplainPlan( @@ -599,6 +610,11 @@ public ExplainPlanAttributesBuilder setServerMergeColumns( return this; } + public ExplainPlanAttributesBuilder setRegionLocations(List regionLocations) { + this.regionLocations = regionLocations; + return this; + } + public ExplainPlanAttributes build() { return new ExplainPlanAttributes(abstractExplainPlan, splitsChunk, estimatedRows, estimatedSizeInBytes, iteratorTypeAndScanSize, @@ -611,7 +627,8 @@ public ExplainPlanAttributes build() { clientFilterBy, clientAggregate, clientSortedBy, clientAfterAggregate, clientDistinctFilter, clientOffset, clientRowLimit, clientSequenceCount, clientCursorName, - clientSortAlgo, rhsJoinQueryExplainPlan, serverMergeColumns); + clientSortAlgo, rhsJoinQueryExplainPlan, serverMergeColumns, + regionLocations); } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 08a946d0850..21b84bc8206 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -1724,7 +1724,7 @@ private void explainUtil(List planSteps, } } - explain(buf.toString(), planSteps, explainPlanAttributesBuilder); + explain(buf.toString(), planSteps, explainPlanAttributesBuilder, scans); } @Override diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java index c9ef46d00a7..4f53cd3e7be 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java @@ -17,15 +17,24 @@ */ package org.apache.phoenix.iterate; +import java.io.IOException; import java.text.Format; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import java.util.Set; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Consistency; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.apache.hadoop.hbase.filter.PageFilter; @@ -46,6 +55,8 @@ import org.apache.phoenix.parse.HintNode.Hint; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.KeyRange.Bound; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.SortOrder; @@ -111,8 +122,59 @@ private String explainSkipScan() { return buf.toString(); } + /** + * Get regions that represent the given range of start and end key for the given table, and all the regions + * to the regionLocations list. + * + * @param tableName the table name. + * @param startKey the start rowkey. + * @param endKey the end rowkey. + * @param includeEndKey true if end key needs to be included. + * @param reload true if reload from meta is necessary. + * @param regionBoundaries set of region boundaries to get the unique list of region locations. + * @param regionLocations the list of region locations as output. + * @throws IOException if something goes wrong while creating connection or querying region locations. + */ + private void getRegionsInRange(final String tableName, + final byte[] startKey, + final byte[] endKey, + final boolean includeEndKey, + final boolean reload, + Set regionBoundaries, + List regionLocations) + throws IOException { + final boolean endKeyIsEndOfTable = Bytes.equals(endKey, HConstants.EMPTY_END_ROW); + if ((Bytes.compareTo(startKey, endKey) > 0) && !endKeyIsEndOfTable) { + throw new IllegalArgumentException( + "Invalid range: " + Bytes.toStringBinary(startKey) + " > " + Bytes.toStringBinary(endKey)); + } + byte[] currentKey = startKey; + try (Connection connection = ConnectionFactory.createConnection( + context.getConnection().getQueryServices().getConfiguration()); + Table table = connection.getTable(TableName.valueOf(tableName))) { + do { + HRegionLocation regionLocation = table.getRegionLocator().getRegionLocation(currentKey, reload); + RegionBoundary regionBoundary = new RegionBoundary(regionLocation.getRegion().getStartKey(), + regionLocation.getRegion().getEndKey()); + if(!regionBoundaries.contains(regionBoundary)) { + regionLocations.add(regionLocation); + regionBoundaries.add(regionBoundary); + } + currentKey = regionLocation.getRegion().getEndKey(); + // condition1 = currentKey != END_ROW_KEY + // condition2 = endKeyIsEndOfTable == true + // condition3 = currentKey < endKey + // condition4 = includeEndKey == true + // condition5 = currentKey == endKey + // while (condition1 && (condition2 || condition3 || (condition4 && condition5))) + } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW) && + (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0 || + (includeEndKey && Bytes.compareTo(currentKey, endKey) == 0))); + } + } + protected void explain(String prefix, List planSteps, - ExplainPlanAttributesBuilder explainPlanAttributesBuilder) { + ExplainPlanAttributesBuilder explainPlanAttributesBuilder, List> scansList) { StringBuilder buf = new StringBuilder(prefix); ScanRanges scanRanges = context.getScanRanges(); Scan scan = context.getScan(); @@ -275,6 +337,7 @@ protected void explain(String prefix, List planSteps, if (groupByLimitBytes != null) { groupByLimit = (Integer) PInteger.INSTANCE.toObject(groupByLimitBytes); } + getRegionLocationsForExplainPlan(planSteps, explainPlanAttributesBuilder, scansList); groupBy.explain(planSteps, groupByLimit, explainPlanAttributesBuilder); if (scan.getAttribute(BaseScannerRegionObserver.SPECIFIC_ARRAY_INDEX) != null) { planSteps.add(" SERVER ARRAY ELEMENT PROJECTION"); @@ -284,6 +347,82 @@ protected void explain(String prefix, List planSteps, } } + /** + * Retrieve region locations from hbase client and set the values for the explain plan output. If the list + * of region locations exceed max limit, print only list with the max limit and print num of total list size. + * + * @param planSteps list of plan steps to add explain plan output to. + * @param explainPlanAttributesBuilder explain plan v2 attributes builder instance. + * @param scansList list of the list of scans, to be used for parallel scans. + */ + private void getRegionLocationsForExplainPlan(List planSteps, + ExplainPlanAttributesBuilder explainPlanAttributesBuilder, + List> scansList) { + StringBuilder buf; + try { + buf = new StringBuilder().append(" (region locations = "); + Set regionBoundaries = new HashSet<>(); + List regionLocations = new ArrayList<>(); + for (List scans : scansList) { + for (Scan eachScan : scans) { + getRegionsInRange(tableRef.getTable().getPhysicalName().getString(), + eachScan.getStartRow(), + eachScan.getStopRow(), + true, + false, + regionBoundaries, + regionLocations); + } + } + int maxLimitRegionLoc = context.getConnection().getQueryServices().getConfiguration() + .getInt(QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, + QueryServicesOptions.DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN); + if (explainPlanAttributesBuilder != null) { + explainPlanAttributesBuilder.setRegionLocations(Collections.unmodifiableList(regionLocations)); + } + if (regionLocations.size() > maxLimitRegionLoc) { + int originalSize = regionLocations.size(); + List trimmedRegionLocations = regionLocations.subList(0, maxLimitRegionLoc); + buf.append(trimmedRegionLocations); + buf.append("...total size = "); + buf.append(originalSize); + } else { + buf.append(regionLocations); + } + buf.append(") "); + planSteps.add(buf.toString()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + static class RegionBoundary { + private final byte[] startKey; + private final byte[] endKey; + + public RegionBoundary(byte[] startKey, byte[] endKey) { + this.startKey = startKey; + this.endKey = endKey; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RegionBoundary that = (RegionBoundary) o; + return Bytes.compareTo(startKey, that.startKey) == 0 && Bytes.compareTo(endKey, that.endKey) == 0; + } + + @Override + public int hashCode() { + return 0; + } + } + private void appendPKColumnValue(StringBuilder buf, byte[] range, Boolean isNull, int slotIndex, boolean changeViewIndexId) { if (Boolean.TRUE.equals(isNull)) { buf.append("null"); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java index da9da65ef17..9060bc4fae0 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -403,6 +403,12 @@ public interface QueryServices extends SQLCloseable { * Region server holding the SYSTEM.CATALOG table in batch oriented jobs. */ String SKIP_SYSTEM_TABLES_EXISTENCE_CHECK = "phoenix.skip.system.tables.existence.check"; + + /** + * Config key to represent max region locations to be displayed as part of the Explain plan output. + */ + String MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN = "phoenix.max.region.locations.size.explain.plan"; + /** * Get executor service used for parallel scans */ diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index 6fac191e68e..d60cfa7a0a2 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -57,6 +57,7 @@ import static org.apache.phoenix.query.QueryServices.MAX_CLIENT_METADATA_CACHE_SIZE_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_MEMORY_PERC_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_MUTATION_SIZE_ATTRIB; +import static org.apache.phoenix.query.QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN; import static org.apache.phoenix.query.QueryServices.MAX_SERVER_CACHE_SIZE_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_SERVER_CACHE_TIME_TO_LIVE_MS_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_SERVER_METADATA_CACHE_SIZE_ATTRIB; @@ -399,6 +400,7 @@ public class QueryServicesOptions { public static final int DEFAULT_SCAN_PAGE_SIZE = 32768; public static final boolean DEFAULT_APPLY_TIME_ZONE_DISPLACMENT = false; public static final boolean DEFAULT_PHOENIX_TABLE_TTL_ENABLED = true; + public static final int DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN = 10; private final Configuration config; @@ -489,7 +491,8 @@ public static QueryServicesOptions withDefaults() { .setIfUnset(INDEX_CREATE_DEFAULT_STATE, DEFAULT_CREATE_INDEX_STATE) .setIfUnset(SKIP_SYSTEM_TABLES_EXISTENCE_CHECK, DEFAULT_SKIP_SYSTEM_TABLES_EXISTENCE_CHECK) - .setIfUnset(MAX_IN_LIST_SKIP_SCAN_SIZE, DEFAULT_MAX_IN_LIST_SKIP_SCAN_SIZE); + .setIfUnset(MAX_IN_LIST_SKIP_SCAN_SIZE, DEFAULT_MAX_IN_LIST_SKIP_SCAN_SIZE) + .setIfUnset(MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN); // HBase sets this to 1, so we reset it to something more appropriate. // Hopefully HBase will change this, because we can't know if a user set From 1b7c1ad45014a9749d52cc04963d3c3e2eadf078 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Wed, 3 May 2023 18:38:10 -0700 Subject: [PATCH 02/10] Revert "PHOENIX-6907 Explain Plan should output region locations with servers" This reverts commit cb5c064bfdd23281de3ca36d1fb8e2d3565b0653. --- .../phoenix/end2end/DerivedTableIT.java | 24 +-- .../phoenix/end2end/FlappingLocalIndexIT.java | 13 -- .../end2end/index/BaseLocalIndexIT.java | 1 - .../compile/ExplainPlanAttributes.java | 21 +-- .../phoenix/iterate/BaseResultIterators.java | 2 +- .../apache/phoenix/iterate/ExplainTable.java | 141 +----------------- .../apache/phoenix/query/QueryServices.java | 6 - .../phoenix/query/QueryServicesOptions.java | 5 +- 8 files changed, 11 insertions(+), 202 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java index 92fba2f4513..2b47e01972a 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java @@ -58,8 +58,6 @@ import org.junit.runners.Parameterized.Parameters; import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @Category(ParallelStatsDisabledTest.class) @@ -73,8 +71,6 @@ public class DerivedTableIT extends ParallelStatsDisabledIT { private String[] plans; private String tableName; - private static final Logger LOGGER = LoggerFactory.getLogger(DerivedTableIT.class); - public DerivedTableIT(String[] indexDDL, String[] plans) { this.indexDDL = indexDDL; @@ -118,7 +114,7 @@ public static synchronized Collection data() { { "CREATE INDEX "+dynamicTableName+"_DERIVED_IDX ON "+dynamicTableName+" (a_byte) INCLUDE (A_STRING, B_STRING)" }, { - "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX \n" + + "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]\n" + "CLIENT MERGE SORT\n" + "CLIENT SORTED BY [\"B_STRING\"]\n" + @@ -126,7 +122,7 @@ public static synchronized Collection data() { "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + "CLIENT SORTED BY [A DESC]", - "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX \n" + + "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]\n" + "CLIENT MERGE SORT\n" + "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + @@ -134,7 +130,7 @@ public static synchronized Collection data() { "CLIENT SORTED BY [A DESC]"}}); testCases.add(new String[][] { {}, { - "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+" \n" + + "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+"\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]\n" + "CLIENT MERGE SORT\n" + "CLIENT SORTED BY [B_STRING]\n" + @@ -142,7 +138,7 @@ public static synchronized Collection data() { "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + "CLIENT SORTED BY [A DESC]", - "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+" \n" + + "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+"\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]\n" + "CLIENT MERGE SORT\n" + "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + @@ -383,11 +379,7 @@ public void testDerivedTableWithGroupBy() throws Exception { assertFalse(rs.next()); rs = conn.createStatement().executeQuery("EXPLAIN " + query); - String explainPlanOutput = QueryUtil.getExplainPlan(rs); - LOGGER.info("Explain plan output: {}", explainPlanOutput); - String[] splitExplainPlan = explainPlanOutput.split("\\n \\(region locations = \\[region="); - String[] secondSplitExplainPlan = splitExplainPlan[1].split("]\\)"); - assertEquals(plans[0], splitExplainPlan[0] + secondSplitExplainPlan[1]); + assertEquals(plans[0], QueryUtil.getExplainPlan(rs)); // distinct b (groupby a, b) groupby a orderby a query = "SELECT DISTINCT COLLECTDISTINCT(t.b) FROM (SELECT b_string b, a_string a FROM "+tableName+" GROUP BY a_string, b_string) AS t GROUP BY t.a ORDER BY t.a DESC"; @@ -409,11 +401,7 @@ public void testDerivedTableWithGroupBy() throws Exception { assertFalse(rs.next()); rs = conn.createStatement().executeQuery("EXPLAIN " + query); - explainPlanOutput = QueryUtil.getExplainPlan(rs); - LOGGER.info("Explain plan output: {}", explainPlanOutput); - splitExplainPlan = explainPlanOutput.split("\\n \\(region locations = \\[region="); - secondSplitExplainPlan = splitExplainPlan[1].split("]\\)"); - assertEquals(plans[1], splitExplainPlan[0] + secondSplitExplainPlan[1]); + assertEquals(plans[1], QueryUtil.getExplainPlan(rs)); // (orderby) groupby query = "SELECT t.a_string, count(*) FROM (SELECT * FROM "+tableName+" order by a_integer) AS t where a_byte != 8 group by t.a_string"; diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java index 6ca16d04a2c..3c76cf3cefd 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java @@ -50,17 +50,12 @@ import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class FlappingLocalIndexIT extends BaseLocalIndexIT { - private static final Logger LOGGER = LoggerFactory.getLogger(FlappingLocalIndexIT.class); - public FlappingLocalIndexIT(boolean isNamespaceMapped) { super(isNamespaceMapped); } @@ -164,13 +159,6 @@ public void testLocalIndexScan() throws Exception { String query = "SELECT * FROM " + tableName +" where v1 like 'a%'"; - String explainPlanOutput = - QueryUtil.getExplainPlan(conn1.createStatement().executeQuery("EXPLAIN " + query)); - LOGGER.info("Explain plan output: {}", explainPlanOutput); - // MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN is set as 2 - assertTrue("Expected total " + numRegions + " regions", - explainPlanOutput.contains("...total size = " + numRegions)); - ExplainPlan plan = conn1.prepareStatement(query) .unwrap(PhoenixPreparedStatement.class).optimizeQuery() .getExplainPlan(); @@ -188,7 +176,6 @@ public void testLocalIndexScan() throws Exception { explainPlanAttributes.getServerWhereFilter()); assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertEquals(numRegions, explainPlanAttributes.getRegionLocations().size()); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java index ec297c15959..4c8863fbec7 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java @@ -61,7 +61,6 @@ public static synchronized void doSetup() throws Exception { // setting update frequency to a large value to test out that we are // generating stats for local indexes clientProps.put(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB, "120000"); - clientProps.put(QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, "2"); setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator())); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java index d5363da02f3..258a7ee6b36 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java @@ -18,10 +18,8 @@ package org.apache.phoenix.compile; -import java.util.List; import java.util.Set; -import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.client.Consistency; import org.apache.phoenix.parse.HintNode; import org.apache.phoenix.parse.HintNode.Hint; @@ -75,7 +73,6 @@ public class ExplainPlanAttributes { // be null private final ExplainPlanAttributes rhsJoinQueryExplainPlan; private final Set serverMergeColumns; - private final List regionLocations; private static final ExplainPlanAttributes EXPLAIN_PLAN_INSTANCE = new ExplainPlanAttributes(); @@ -115,7 +112,6 @@ private ExplainPlanAttributes() { this.clientSortAlgo = null; this.rhsJoinQueryExplainPlan = null; this.serverMergeColumns = null; - this.regionLocations = null; } public ExplainPlanAttributes(String abstractExplainPlan, @@ -136,7 +132,7 @@ public ExplainPlanAttributes(String abstractExplainPlan, Integer clientSequenceCount, String clientCursorName, String clientSortAlgo, ExplainPlanAttributes rhsJoinQueryExplainPlan, - Set serverMergeColumns, List regionLocations) { + Set serverMergeColumns) { this.abstractExplainPlan = abstractExplainPlan; this.splitsChunk = splitsChunk; this.estimatedRows = estimatedRows; @@ -171,7 +167,6 @@ public ExplainPlanAttributes(String abstractExplainPlan, this.clientSortAlgo = clientSortAlgo; this.rhsJoinQueryExplainPlan = rhsJoinQueryExplainPlan; this.serverMergeColumns = serverMergeColumns; - this.regionLocations = regionLocations; } public String getAbstractExplainPlan() { @@ -310,10 +305,6 @@ public Set getServerMergeColumns() { return serverMergeColumns; } - public List getRegionLocations() { - return regionLocations; - } - public static ExplainPlanAttributes getDefaultExplainPlan() { return EXPLAIN_PLAN_INSTANCE; } @@ -353,7 +344,6 @@ public static class ExplainPlanAttributesBuilder { private String clientSortAlgo; private ExplainPlanAttributes rhsJoinQueryExplainPlan; private Set serverMergeColumns; - private List regionLocations; public ExplainPlanAttributesBuilder() { // default @@ -406,7 +396,6 @@ public ExplainPlanAttributesBuilder( this.rhsJoinQueryExplainPlan = explainPlanAttributes.getRhsJoinQueryExplainPlan(); this.serverMergeColumns = explainPlanAttributes.getServerMergeColumns(); - this.regionLocations = explainPlanAttributes.getRegionLocations(); } public ExplainPlanAttributesBuilder setAbstractExplainPlan( @@ -610,11 +599,6 @@ public ExplainPlanAttributesBuilder setServerMergeColumns( return this; } - public ExplainPlanAttributesBuilder setRegionLocations(List regionLocations) { - this.regionLocations = regionLocations; - return this; - } - public ExplainPlanAttributes build() { return new ExplainPlanAttributes(abstractExplainPlan, splitsChunk, estimatedRows, estimatedSizeInBytes, iteratorTypeAndScanSize, @@ -627,8 +611,7 @@ public ExplainPlanAttributes build() { clientFilterBy, clientAggregate, clientSortedBy, clientAfterAggregate, clientDistinctFilter, clientOffset, clientRowLimit, clientSequenceCount, clientCursorName, - clientSortAlgo, rhsJoinQueryExplainPlan, serverMergeColumns, - regionLocations); + clientSortAlgo, rhsJoinQueryExplainPlan, serverMergeColumns); } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 21b84bc8206..08a946d0850 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -1724,7 +1724,7 @@ private void explainUtil(List planSteps, } } - explain(buf.toString(), planSteps, explainPlanAttributesBuilder, scans); + explain(buf.toString(), planSteps, explainPlanAttributesBuilder); } @Override diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java index 4f53cd3e7be..c9ef46d00a7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java @@ -17,24 +17,15 @@ */ package org.apache.phoenix.iterate; -import java.io.IOException; import java.text.Format; -import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import java.util.Set; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Consistency; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.apache.hadoop.hbase.filter.PageFilter; @@ -55,8 +46,6 @@ import org.apache.phoenix.parse.HintNode.Hint; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.KeyRange.Bound; -import org.apache.phoenix.query.QueryServices; -import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.SortOrder; @@ -122,59 +111,8 @@ private String explainSkipScan() { return buf.toString(); } - /** - * Get regions that represent the given range of start and end key for the given table, and all the regions - * to the regionLocations list. - * - * @param tableName the table name. - * @param startKey the start rowkey. - * @param endKey the end rowkey. - * @param includeEndKey true if end key needs to be included. - * @param reload true if reload from meta is necessary. - * @param regionBoundaries set of region boundaries to get the unique list of region locations. - * @param regionLocations the list of region locations as output. - * @throws IOException if something goes wrong while creating connection or querying region locations. - */ - private void getRegionsInRange(final String tableName, - final byte[] startKey, - final byte[] endKey, - final boolean includeEndKey, - final boolean reload, - Set regionBoundaries, - List regionLocations) - throws IOException { - final boolean endKeyIsEndOfTable = Bytes.equals(endKey, HConstants.EMPTY_END_ROW); - if ((Bytes.compareTo(startKey, endKey) > 0) && !endKeyIsEndOfTable) { - throw new IllegalArgumentException( - "Invalid range: " + Bytes.toStringBinary(startKey) + " > " + Bytes.toStringBinary(endKey)); - } - byte[] currentKey = startKey; - try (Connection connection = ConnectionFactory.createConnection( - context.getConnection().getQueryServices().getConfiguration()); - Table table = connection.getTable(TableName.valueOf(tableName))) { - do { - HRegionLocation regionLocation = table.getRegionLocator().getRegionLocation(currentKey, reload); - RegionBoundary regionBoundary = new RegionBoundary(regionLocation.getRegion().getStartKey(), - regionLocation.getRegion().getEndKey()); - if(!regionBoundaries.contains(regionBoundary)) { - regionLocations.add(regionLocation); - regionBoundaries.add(regionBoundary); - } - currentKey = regionLocation.getRegion().getEndKey(); - // condition1 = currentKey != END_ROW_KEY - // condition2 = endKeyIsEndOfTable == true - // condition3 = currentKey < endKey - // condition4 = includeEndKey == true - // condition5 = currentKey == endKey - // while (condition1 && (condition2 || condition3 || (condition4 && condition5))) - } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW) && - (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0 || - (includeEndKey && Bytes.compareTo(currentKey, endKey) == 0))); - } - } - protected void explain(String prefix, List planSteps, - ExplainPlanAttributesBuilder explainPlanAttributesBuilder, List> scansList) { + ExplainPlanAttributesBuilder explainPlanAttributesBuilder) { StringBuilder buf = new StringBuilder(prefix); ScanRanges scanRanges = context.getScanRanges(); Scan scan = context.getScan(); @@ -337,7 +275,6 @@ protected void explain(String prefix, List planSteps, if (groupByLimitBytes != null) { groupByLimit = (Integer) PInteger.INSTANCE.toObject(groupByLimitBytes); } - getRegionLocationsForExplainPlan(planSteps, explainPlanAttributesBuilder, scansList); groupBy.explain(planSteps, groupByLimit, explainPlanAttributesBuilder); if (scan.getAttribute(BaseScannerRegionObserver.SPECIFIC_ARRAY_INDEX) != null) { planSteps.add(" SERVER ARRAY ELEMENT PROJECTION"); @@ -347,82 +284,6 @@ protected void explain(String prefix, List planSteps, } } - /** - * Retrieve region locations from hbase client and set the values for the explain plan output. If the list - * of region locations exceed max limit, print only list with the max limit and print num of total list size. - * - * @param planSteps list of plan steps to add explain plan output to. - * @param explainPlanAttributesBuilder explain plan v2 attributes builder instance. - * @param scansList list of the list of scans, to be used for parallel scans. - */ - private void getRegionLocationsForExplainPlan(List planSteps, - ExplainPlanAttributesBuilder explainPlanAttributesBuilder, - List> scansList) { - StringBuilder buf; - try { - buf = new StringBuilder().append(" (region locations = "); - Set regionBoundaries = new HashSet<>(); - List regionLocations = new ArrayList<>(); - for (List scans : scansList) { - for (Scan eachScan : scans) { - getRegionsInRange(tableRef.getTable().getPhysicalName().getString(), - eachScan.getStartRow(), - eachScan.getStopRow(), - true, - false, - regionBoundaries, - regionLocations); - } - } - int maxLimitRegionLoc = context.getConnection().getQueryServices().getConfiguration() - .getInt(QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, - QueryServicesOptions.DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN); - if (explainPlanAttributesBuilder != null) { - explainPlanAttributesBuilder.setRegionLocations(Collections.unmodifiableList(regionLocations)); - } - if (regionLocations.size() > maxLimitRegionLoc) { - int originalSize = regionLocations.size(); - List trimmedRegionLocations = regionLocations.subList(0, maxLimitRegionLoc); - buf.append(trimmedRegionLocations); - buf.append("...total size = "); - buf.append(originalSize); - } else { - buf.append(regionLocations); - } - buf.append(") "); - planSteps.add(buf.toString()); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - static class RegionBoundary { - private final byte[] startKey; - private final byte[] endKey; - - public RegionBoundary(byte[] startKey, byte[] endKey) { - this.startKey = startKey; - this.endKey = endKey; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - RegionBoundary that = (RegionBoundary) o; - return Bytes.compareTo(startKey, that.startKey) == 0 && Bytes.compareTo(endKey, that.endKey) == 0; - } - - @Override - public int hashCode() { - return 0; - } - } - private void appendPKColumnValue(StringBuilder buf, byte[] range, Boolean isNull, int slotIndex, boolean changeViewIndexId) { if (Boolean.TRUE.equals(isNull)) { buf.append("null"); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java index 9060bc4fae0..da9da65ef17 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -403,12 +403,6 @@ public interface QueryServices extends SQLCloseable { * Region server holding the SYSTEM.CATALOG table in batch oriented jobs. */ String SKIP_SYSTEM_TABLES_EXISTENCE_CHECK = "phoenix.skip.system.tables.existence.check"; - - /** - * Config key to represent max region locations to be displayed as part of the Explain plan output. - */ - String MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN = "phoenix.max.region.locations.size.explain.plan"; - /** * Get executor service used for parallel scans */ diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index d60cfa7a0a2..6fac191e68e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -57,7 +57,6 @@ import static org.apache.phoenix.query.QueryServices.MAX_CLIENT_METADATA_CACHE_SIZE_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_MEMORY_PERC_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_MUTATION_SIZE_ATTRIB; -import static org.apache.phoenix.query.QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN; import static org.apache.phoenix.query.QueryServices.MAX_SERVER_CACHE_SIZE_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_SERVER_CACHE_TIME_TO_LIVE_MS_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_SERVER_METADATA_CACHE_SIZE_ATTRIB; @@ -400,7 +399,6 @@ public class QueryServicesOptions { public static final int DEFAULT_SCAN_PAGE_SIZE = 32768; public static final boolean DEFAULT_APPLY_TIME_ZONE_DISPLACMENT = false; public static final boolean DEFAULT_PHOENIX_TABLE_TTL_ENABLED = true; - public static final int DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN = 10; private final Configuration config; @@ -491,8 +489,7 @@ public static QueryServicesOptions withDefaults() { .setIfUnset(INDEX_CREATE_DEFAULT_STATE, DEFAULT_CREATE_INDEX_STATE) .setIfUnset(SKIP_SYSTEM_TABLES_EXISTENCE_CHECK, DEFAULT_SKIP_SYSTEM_TABLES_EXISTENCE_CHECK) - .setIfUnset(MAX_IN_LIST_SKIP_SCAN_SIZE, DEFAULT_MAX_IN_LIST_SKIP_SCAN_SIZE) - .setIfUnset(MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN); + .setIfUnset(MAX_IN_LIST_SKIP_SCAN_SIZE, DEFAULT_MAX_IN_LIST_SKIP_SCAN_SIZE); // HBase sets this to 1, so we reset it to something more appropriate. // Hopefully HBase will change this, because we can't know if a user set From a20c6b6ef5791454f0f206c5ffc71f2a22e9f634 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Wed, 3 May 2023 18:41:17 -0700 Subject: [PATCH 03/10] addendum --- .../end2end/index/BaseLocalIndexIT.java | 1 + .../compile/ExplainPlanAttributes.java | 21 ++- .../phoenix/iterate/BaseResultIterators.java | 2 +- .../apache/phoenix/iterate/ExplainTable.java | 141 +++++++++++++++++- .../apache/phoenix/query/QueryServices.java | 6 + .../phoenix/query/QueryServicesOptions.java | 5 +- 6 files changed, 171 insertions(+), 5 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java index 4c8863fbec7..ec297c15959 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java @@ -61,6 +61,7 @@ public static synchronized void doSetup() throws Exception { // setting update frequency to a large value to test out that we are // generating stats for local indexes clientProps.put(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB, "120000"); + clientProps.put(QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, "2"); setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator())); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java index 258a7ee6b36..d5363da02f3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java @@ -18,8 +18,10 @@ package org.apache.phoenix.compile; +import java.util.List; import java.util.Set; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.client.Consistency; import org.apache.phoenix.parse.HintNode; import org.apache.phoenix.parse.HintNode.Hint; @@ -73,6 +75,7 @@ public class ExplainPlanAttributes { // be null private final ExplainPlanAttributes rhsJoinQueryExplainPlan; private final Set serverMergeColumns; + private final List regionLocations; private static final ExplainPlanAttributes EXPLAIN_PLAN_INSTANCE = new ExplainPlanAttributes(); @@ -112,6 +115,7 @@ private ExplainPlanAttributes() { this.clientSortAlgo = null; this.rhsJoinQueryExplainPlan = null; this.serverMergeColumns = null; + this.regionLocations = null; } public ExplainPlanAttributes(String abstractExplainPlan, @@ -132,7 +136,7 @@ public ExplainPlanAttributes(String abstractExplainPlan, Integer clientSequenceCount, String clientCursorName, String clientSortAlgo, ExplainPlanAttributes rhsJoinQueryExplainPlan, - Set serverMergeColumns) { + Set serverMergeColumns, List regionLocations) { this.abstractExplainPlan = abstractExplainPlan; this.splitsChunk = splitsChunk; this.estimatedRows = estimatedRows; @@ -167,6 +171,7 @@ public ExplainPlanAttributes(String abstractExplainPlan, this.clientSortAlgo = clientSortAlgo; this.rhsJoinQueryExplainPlan = rhsJoinQueryExplainPlan; this.serverMergeColumns = serverMergeColumns; + this.regionLocations = regionLocations; } public String getAbstractExplainPlan() { @@ -305,6 +310,10 @@ public Set getServerMergeColumns() { return serverMergeColumns; } + public List getRegionLocations() { + return regionLocations; + } + public static ExplainPlanAttributes getDefaultExplainPlan() { return EXPLAIN_PLAN_INSTANCE; } @@ -344,6 +353,7 @@ public static class ExplainPlanAttributesBuilder { private String clientSortAlgo; private ExplainPlanAttributes rhsJoinQueryExplainPlan; private Set serverMergeColumns; + private List regionLocations; public ExplainPlanAttributesBuilder() { // default @@ -396,6 +406,7 @@ public ExplainPlanAttributesBuilder( this.rhsJoinQueryExplainPlan = explainPlanAttributes.getRhsJoinQueryExplainPlan(); this.serverMergeColumns = explainPlanAttributes.getServerMergeColumns(); + this.regionLocations = explainPlanAttributes.getRegionLocations(); } public ExplainPlanAttributesBuilder setAbstractExplainPlan( @@ -599,6 +610,11 @@ public ExplainPlanAttributesBuilder setServerMergeColumns( return this; } + public ExplainPlanAttributesBuilder setRegionLocations(List regionLocations) { + this.regionLocations = regionLocations; + return this; + } + public ExplainPlanAttributes build() { return new ExplainPlanAttributes(abstractExplainPlan, splitsChunk, estimatedRows, estimatedSizeInBytes, iteratorTypeAndScanSize, @@ -611,7 +627,8 @@ public ExplainPlanAttributes build() { clientFilterBy, clientAggregate, clientSortedBy, clientAfterAggregate, clientDistinctFilter, clientOffset, clientRowLimit, clientSequenceCount, clientCursorName, - clientSortAlgo, rhsJoinQueryExplainPlan, serverMergeColumns); + clientSortAlgo, rhsJoinQueryExplainPlan, serverMergeColumns, + regionLocations); } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 08a946d0850..21b84bc8206 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -1724,7 +1724,7 @@ private void explainUtil(List planSteps, } } - explain(buf.toString(), planSteps, explainPlanAttributesBuilder); + explain(buf.toString(), planSteps, explainPlanAttributesBuilder, scans); } @Override diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java index c9ef46d00a7..4f53cd3e7be 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java @@ -17,15 +17,24 @@ */ package org.apache.phoenix.iterate; +import java.io.IOException; import java.text.Format; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import java.util.Set; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Consistency; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.apache.hadoop.hbase.filter.PageFilter; @@ -46,6 +55,8 @@ import org.apache.phoenix.parse.HintNode.Hint; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.KeyRange.Bound; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.SortOrder; @@ -111,8 +122,59 @@ private String explainSkipScan() { return buf.toString(); } + /** + * Get regions that represent the given range of start and end key for the given table, and all the regions + * to the regionLocations list. + * + * @param tableName the table name. + * @param startKey the start rowkey. + * @param endKey the end rowkey. + * @param includeEndKey true if end key needs to be included. + * @param reload true if reload from meta is necessary. + * @param regionBoundaries set of region boundaries to get the unique list of region locations. + * @param regionLocations the list of region locations as output. + * @throws IOException if something goes wrong while creating connection or querying region locations. + */ + private void getRegionsInRange(final String tableName, + final byte[] startKey, + final byte[] endKey, + final boolean includeEndKey, + final boolean reload, + Set regionBoundaries, + List regionLocations) + throws IOException { + final boolean endKeyIsEndOfTable = Bytes.equals(endKey, HConstants.EMPTY_END_ROW); + if ((Bytes.compareTo(startKey, endKey) > 0) && !endKeyIsEndOfTable) { + throw new IllegalArgumentException( + "Invalid range: " + Bytes.toStringBinary(startKey) + " > " + Bytes.toStringBinary(endKey)); + } + byte[] currentKey = startKey; + try (Connection connection = ConnectionFactory.createConnection( + context.getConnection().getQueryServices().getConfiguration()); + Table table = connection.getTable(TableName.valueOf(tableName))) { + do { + HRegionLocation regionLocation = table.getRegionLocator().getRegionLocation(currentKey, reload); + RegionBoundary regionBoundary = new RegionBoundary(regionLocation.getRegion().getStartKey(), + regionLocation.getRegion().getEndKey()); + if(!regionBoundaries.contains(regionBoundary)) { + regionLocations.add(regionLocation); + regionBoundaries.add(regionBoundary); + } + currentKey = regionLocation.getRegion().getEndKey(); + // condition1 = currentKey != END_ROW_KEY + // condition2 = endKeyIsEndOfTable == true + // condition3 = currentKey < endKey + // condition4 = includeEndKey == true + // condition5 = currentKey == endKey + // while (condition1 && (condition2 || condition3 || (condition4 && condition5))) + } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW) && + (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0 || + (includeEndKey && Bytes.compareTo(currentKey, endKey) == 0))); + } + } + protected void explain(String prefix, List planSteps, - ExplainPlanAttributesBuilder explainPlanAttributesBuilder) { + ExplainPlanAttributesBuilder explainPlanAttributesBuilder, List> scansList) { StringBuilder buf = new StringBuilder(prefix); ScanRanges scanRanges = context.getScanRanges(); Scan scan = context.getScan(); @@ -275,6 +337,7 @@ protected void explain(String prefix, List planSteps, if (groupByLimitBytes != null) { groupByLimit = (Integer) PInteger.INSTANCE.toObject(groupByLimitBytes); } + getRegionLocationsForExplainPlan(planSteps, explainPlanAttributesBuilder, scansList); groupBy.explain(planSteps, groupByLimit, explainPlanAttributesBuilder); if (scan.getAttribute(BaseScannerRegionObserver.SPECIFIC_ARRAY_INDEX) != null) { planSteps.add(" SERVER ARRAY ELEMENT PROJECTION"); @@ -284,6 +347,82 @@ protected void explain(String prefix, List planSteps, } } + /** + * Retrieve region locations from hbase client and set the values for the explain plan output. If the list + * of region locations exceed max limit, print only list with the max limit and print num of total list size. + * + * @param planSteps list of plan steps to add explain plan output to. + * @param explainPlanAttributesBuilder explain plan v2 attributes builder instance. + * @param scansList list of the list of scans, to be used for parallel scans. + */ + private void getRegionLocationsForExplainPlan(List planSteps, + ExplainPlanAttributesBuilder explainPlanAttributesBuilder, + List> scansList) { + StringBuilder buf; + try { + buf = new StringBuilder().append(" (region locations = "); + Set regionBoundaries = new HashSet<>(); + List regionLocations = new ArrayList<>(); + for (List scans : scansList) { + for (Scan eachScan : scans) { + getRegionsInRange(tableRef.getTable().getPhysicalName().getString(), + eachScan.getStartRow(), + eachScan.getStopRow(), + true, + false, + regionBoundaries, + regionLocations); + } + } + int maxLimitRegionLoc = context.getConnection().getQueryServices().getConfiguration() + .getInt(QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, + QueryServicesOptions.DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN); + if (explainPlanAttributesBuilder != null) { + explainPlanAttributesBuilder.setRegionLocations(Collections.unmodifiableList(regionLocations)); + } + if (regionLocations.size() > maxLimitRegionLoc) { + int originalSize = regionLocations.size(); + List trimmedRegionLocations = regionLocations.subList(0, maxLimitRegionLoc); + buf.append(trimmedRegionLocations); + buf.append("...total size = "); + buf.append(originalSize); + } else { + buf.append(regionLocations); + } + buf.append(") "); + planSteps.add(buf.toString()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + static class RegionBoundary { + private final byte[] startKey; + private final byte[] endKey; + + public RegionBoundary(byte[] startKey, byte[] endKey) { + this.startKey = startKey; + this.endKey = endKey; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RegionBoundary that = (RegionBoundary) o; + return Bytes.compareTo(startKey, that.startKey) == 0 && Bytes.compareTo(endKey, that.endKey) == 0; + } + + @Override + public int hashCode() { + return 0; + } + } + private void appendPKColumnValue(StringBuilder buf, byte[] range, Boolean isNull, int slotIndex, boolean changeViewIndexId) { if (Boolean.TRUE.equals(isNull)) { buf.append("null"); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java index da9da65ef17..9060bc4fae0 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -403,6 +403,12 @@ public interface QueryServices extends SQLCloseable { * Region server holding the SYSTEM.CATALOG table in batch oriented jobs. */ String SKIP_SYSTEM_TABLES_EXISTENCE_CHECK = "phoenix.skip.system.tables.existence.check"; + + /** + * Config key to represent max region locations to be displayed as part of the Explain plan output. + */ + String MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN = "phoenix.max.region.locations.size.explain.plan"; + /** * Get executor service used for parallel scans */ diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index 6fac191e68e..d60cfa7a0a2 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -57,6 +57,7 @@ import static org.apache.phoenix.query.QueryServices.MAX_CLIENT_METADATA_CACHE_SIZE_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_MEMORY_PERC_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_MUTATION_SIZE_ATTRIB; +import static org.apache.phoenix.query.QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN; import static org.apache.phoenix.query.QueryServices.MAX_SERVER_CACHE_SIZE_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_SERVER_CACHE_TIME_TO_LIVE_MS_ATTRIB; import static org.apache.phoenix.query.QueryServices.MAX_SERVER_METADATA_CACHE_SIZE_ATTRIB; @@ -399,6 +400,7 @@ public class QueryServicesOptions { public static final int DEFAULT_SCAN_PAGE_SIZE = 32768; public static final boolean DEFAULT_APPLY_TIME_ZONE_DISPLACMENT = false; public static final boolean DEFAULT_PHOENIX_TABLE_TTL_ENABLED = true; + public static final int DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN = 10; private final Configuration config; @@ -489,7 +491,8 @@ public static QueryServicesOptions withDefaults() { .setIfUnset(INDEX_CREATE_DEFAULT_STATE, DEFAULT_CREATE_INDEX_STATE) .setIfUnset(SKIP_SYSTEM_TABLES_EXISTENCE_CHECK, DEFAULT_SKIP_SYSTEM_TABLES_EXISTENCE_CHECK) - .setIfUnset(MAX_IN_LIST_SKIP_SCAN_SIZE, DEFAULT_MAX_IN_LIST_SKIP_SCAN_SIZE); + .setIfUnset(MAX_IN_LIST_SKIP_SCAN_SIZE, DEFAULT_MAX_IN_LIST_SKIP_SCAN_SIZE) + .setIfUnset(MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN); // HBase sets this to 1, so we reset it to something more appropriate. // Hopefully HBase will change this, because we can't know if a user set From 7c4d9478905f306d3c97932380d189d384502ff8 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Fri, 12 May 2023 15:38:40 -0700 Subject: [PATCH 04/10] addendum --- .../phoenix/end2end/DerivedTableIT.java | 28 +++++--- .../phoenix/end2end/FlappingLocalIndexIT.java | 13 ++++ phoenix-core/src/main/antlr3/PhoenixSQL.g | 11 ++- .../phoenix/iterate/BaseResultIterators.java | 2 +- .../apache/phoenix/iterate/ExplainTable.java | 72 ++++++++++++++++--- .../apache/phoenix/jdbc/PhoenixStatement.java | 14 ++-- .../phoenix/parse/ExplainStatement.java | 10 ++- .../org/apache/phoenix/parse/ExplainType.java | 24 +++++++ .../phoenix/parse/ParseNodeFactory.java | 4 +- .../phoenix/query/QueryServicesOptions.java | 2 +- 10 files changed, 151 insertions(+), 29 deletions(-) create mode 100644 phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainType.java diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java index 2b47e01972a..597e0fc6cb0 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java @@ -58,6 +58,8 @@ import org.junit.runners.Parameterized.Parameters; import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Category(ParallelStatsDisabledTest.class) @@ -71,6 +73,8 @@ public class DerivedTableIT extends ParallelStatsDisabledIT { private String[] plans; private String tableName; + private static final Logger LOGGER = LoggerFactory.getLogger(DerivedTableIT.class); + public DerivedTableIT(String[] indexDDL, String[] plans) { this.indexDDL = indexDDL; @@ -114,7 +118,7 @@ public static synchronized Collection data() { { "CREATE INDEX "+dynamicTableName+"_DERIVED_IDX ON "+dynamicTableName+" (a_byte) INCLUDE (A_STRING, B_STRING)" }, { - "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX\n" + + "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX \n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]\n" + "CLIENT MERGE SORT\n" + "CLIENT SORTED BY [\"B_STRING\"]\n" + @@ -122,7 +126,7 @@ public static synchronized Collection data() { "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + "CLIENT SORTED BY [A DESC]", - "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX\n" + + "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX \n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]\n" + "CLIENT MERGE SORT\n" + "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + @@ -130,7 +134,7 @@ public static synchronized Collection data() { "CLIENT SORTED BY [A DESC]"}}); testCases.add(new String[][] { {}, { - "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+"\n" + + "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+" \n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]\n" + "CLIENT MERGE SORT\n" + "CLIENT SORTED BY [B_STRING]\n" + @@ -138,7 +142,7 @@ public static synchronized Collection data() { "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + "CLIENT SORTED BY [A DESC]", - "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+"\n" + + "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+" \n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]\n" + "CLIENT MERGE SORT\n" + "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + @@ -378,8 +382,12 @@ public void testDerivedTableWithGroupBy() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals(plans[0], QueryUtil.getExplainPlan(rs)); + rs = conn.createStatement().executeQuery("EXPLAIN WITH REGIONS " + query); + String explainPlanOutput = QueryUtil.getExplainPlan(rs); + LOGGER.info("Explain plan output: {}", explainPlanOutput); + String[] splitExplainPlan = explainPlanOutput.split("\\n \\(region locations = \\[region="); + String[] secondSplitExplainPlan = splitExplainPlan[1].split("]\\)"); + assertEquals(plans[0], splitExplainPlan[0] + secondSplitExplainPlan[1]); // distinct b (groupby a, b) groupby a orderby a query = "SELECT DISTINCT COLLECTDISTINCT(t.b) FROM (SELECT b_string b, a_string a FROM "+tableName+" GROUP BY a_string, b_string) AS t GROUP BY t.a ORDER BY t.a DESC"; @@ -400,8 +408,12 @@ public void testDerivedTableWithGroupBy() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals(plans[1], QueryUtil.getExplainPlan(rs)); + rs = conn.createStatement().executeQuery("EXPLAIN WITH REGIONS " + query); + explainPlanOutput = QueryUtil.getExplainPlan(rs); + LOGGER.info("Explain plan output: {}", explainPlanOutput); + splitExplainPlan = explainPlanOutput.split("\\n \\(region locations = \\[region="); + secondSplitExplainPlan = splitExplainPlan[1].split("]\\)"); + assertEquals(plans[1], splitExplainPlan[0] + secondSplitExplainPlan[1]); // (orderby) groupby query = "SELECT t.a_string, count(*) FROM (SELECT * FROM "+tableName+" order by a_integer) AS t where a_byte != 8 group by t.a_string"; diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java index 3c76cf3cefd..096a301ff79 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java @@ -50,12 +50,17 @@ import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; +import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FlappingLocalIndexIT extends BaseLocalIndexIT { + private static final Logger LOGGER = LoggerFactory.getLogger(FlappingLocalIndexIT.class); + public FlappingLocalIndexIT(boolean isNamespaceMapped) { super(isNamespaceMapped); } @@ -159,6 +164,13 @@ public void testLocalIndexScan() throws Exception { String query = "SELECT * FROM " + tableName +" where v1 like 'a%'"; + String explainPlanOutput = + QueryUtil.getExplainPlan(conn1.createStatement().executeQuery("EXPLAIN WITH REGIONS " + query)); + LOGGER.info("Explain plan output: {}", explainPlanOutput); + // MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN is set as 2 + assertTrue("Expected total " + numRegions + " regions", + explainPlanOutput.contains("...total size = " + numRegions)); + ExplainPlan plan = conn1.prepareStatement(query) .unwrap(PhoenixPreparedStatement.class).optimizeQuery() .getExplainPlan(); @@ -176,6 +188,7 @@ public void testLocalIndexScan() throws Exception { explainPlanAttributes.getServerWhereFilter()); assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertEquals(numRegions, explainPlanAttributes.getRegionLocations().size()); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); diff --git a/phoenix-core/src/main/antlr3/PhoenixSQL.g b/phoenix-core/src/main/antlr3/PhoenixSQL.g index 5d57c3669f4..a8cbb71085d 100644 --- a/phoenix-core/src/main/antlr3/PhoenixSQL.g +++ b/phoenix-core/src/main/antlr3/PhoenixSQL.g @@ -152,6 +152,7 @@ tokens REVOKE = 'revoke'; SHOW = 'show'; UNCOVERED = 'uncovered'; + REGIONS = 'regions'; } @@ -212,6 +213,7 @@ import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.parse.LikeParseNode.LikeType; import org.apache.phoenix.trace.util.Tracing; import org.apache.phoenix.parse.AddJarsStatement; +import org.apache.phoenix.parse.ExplainType; } @lexer::header { @@ -452,7 +454,14 @@ oneStatement returns [BindableStatement ret] finally{ contextStack.pop(); } explain_node returns [BindableStatement ret] - : EXPLAIN q=oneStatement {$ret=factory.explain(q);} + : EXPLAIN (w=WITH)? (r=REGIONS)? q=oneStatement + { + if ((w==null && r!=null) || (w!=null && r==null)) { + throw new RuntimeException("Valid usage: EXPLAIN {query} OR EXPLAIN WITH REGIONS {query}"); + } + ret = (w==null && r==null) ? factory.explain(q, ExplainType.DEFAULT) + : factory.explain(q, ExplainType.WITH_REGIONS); + } ; // Parse a create table statement. diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 21b84bc8206..02135bc2d4a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -1537,7 +1537,7 @@ private List recreateIterators(ConnectionQueryServices se @Override public void close() throws SQLException { - + this.closeExecutorService(); // Don't call cancel on already started work, as it causes the HConnection // to get into a funk. Instead, just cancel queued work. boolean cancelledWork = false; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java index 4f53cd3e7be..729c635e1aa 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java @@ -26,6 +26,12 @@ import java.util.List; import java.util.NoSuchElementException; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; @@ -63,14 +69,21 @@ import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PInteger; +import org.apache.phoenix.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.ScanUtil; import org.apache.phoenix.util.StringUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class ExplainTable { + + private static final Logger LOGGER = LoggerFactory.getLogger(ExplainTable.class); private static final List EVERYTHING = Collections.singletonList(KeyRange.EVERYTHING_RANGE); public static final String POINT_LOOKUP_ON_STRING = "POINT LOOKUP ON "; + public static final String REGION_LOCATIONS = " (region locations = "; + protected final StatementContext context; protected final TableRef tableRef; protected final GroupBy groupBy; @@ -78,6 +91,7 @@ public abstract class ExplainTable { protected final HintNode hint; protected final Integer limit; protected final Integer offset; + private final ExecutorService executorService; public ExplainTable(StatementContext context, TableRef table) { this(context, table, GroupBy.EMPTY_GROUP_BY, OrderBy.EMPTY_ORDER_BY, HintNode.EMPTY_HINT_NODE, null, null); @@ -92,6 +106,10 @@ public ExplainTable(StatementContext context, TableRef table, GroupBy groupBy, O this.hint = hintNode; this.limit = limit; this.offset = offset; + this.executorService = Executors.newFixedThreadPool(5, new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat("explain-region-info-%d") + .build()); } private String explainSkipScan() { @@ -173,8 +191,10 @@ private void getRegionsInRange(final String tableName, } } - protected void explain(String prefix, List planSteps, - ExplainPlanAttributesBuilder explainPlanAttributesBuilder, List> scansList) { + protected void explain(String prefix, + List planSteps, + ExplainPlanAttributesBuilder explainPlanAttributesBuilder, + List> scansList) { StringBuilder buf = new StringBuilder(prefix); ScanRanges scanRanges = context.getScanRanges(); Scan scan = context.getScan(); @@ -337,7 +357,7 @@ protected void explain(String prefix, List planSteps, if (groupByLimitBytes != null) { groupByLimit = (Integer) PInteger.INSTANCE.toObject(groupByLimitBytes); } - getRegionLocationsForExplainPlan(planSteps, explainPlanAttributesBuilder, scansList); + getRegionLocations(planSteps, explainPlanAttributesBuilder, scansList); groupBy.explain(planSteps, groupByLimit, explainPlanAttributesBuilder); if (scan.getAttribute(BaseScannerRegionObserver.SPECIFIC_ARRAY_INDEX) != null) { planSteps.add(" SERVER ARRAY ELEMENT PROJECTION"); @@ -347,20 +367,39 @@ protected void explain(String prefix, List planSteps, } } + /** + * Retrieve region locations and set the values in the explain plan output. + * + * @param planSteps list of plan steps to add explain plan output to. + * @param explainPlanAttributesBuilder explain plan v2 attributes builder instance. + * @param scansList list of the list of scans, to be used for parallel scans. + */ + private void getRegionLocations(List planSteps, + ExplainPlanAttributesBuilder explainPlanAttributesBuilder, + List> scansList) { + Future task = executorService.submit( + () -> getRegionLocationsForExplainPlan(explainPlanAttributesBuilder, scansList)); + try { + String regionLocations = task.get(5, TimeUnit.SECONDS); + planSteps.add(regionLocations); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + LOGGER.error("Unable to retrieve region locations task result in 5s", e); + task.cancel(true); + } + } + /** * Retrieve region locations from hbase client and set the values for the explain plan output. If the list * of region locations exceed max limit, print only list with the max limit and print num of total list size. * - * @param planSteps list of plan steps to add explain plan output to. * @param explainPlanAttributesBuilder explain plan v2 attributes builder instance. * @param scansList list of the list of scans, to be used for parallel scans. + * @return region locations to be added to the explain plan output. */ - private void getRegionLocationsForExplainPlan(List planSteps, - ExplainPlanAttributesBuilder explainPlanAttributesBuilder, + private String getRegionLocationsForExplainPlan(ExplainPlanAttributesBuilder explainPlanAttributesBuilder, List> scansList) { - StringBuilder buf; try { - buf = new StringBuilder().append(" (region locations = "); + StringBuilder buf = new StringBuilder().append(REGION_LOCATIONS); Set regionBoundaries = new HashSet<>(); List regionLocations = new ArrayList<>(); for (List scans : scansList) { @@ -390,9 +429,10 @@ private void getRegionLocationsForExplainPlan(List planSteps, buf.append(regionLocations); } buf.append(") "); - planSteps.add(buf.toString()); + return buf.toString(); } catch (IOException e) { - throw new RuntimeException(e); + LOGGER.error("Explain table unable to add region locations.", e); + return ""; } } @@ -553,4 +593,16 @@ private String appendKeyRanges() { buf.setCharAt(buf.length()-1, ']'); return buf.toString(); } + + /** + * shutdown executor service, should be called by result iterator close. + */ + protected void closeExecutorService() { + this.executorService.shutdown(); + try { + this.executorService.awaitTermination(2, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOGGER.error("Interrupted while waiting for executor service shutdown.", e); + } + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java index 44e2e98dcca..ef96db5926f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java @@ -109,6 +109,7 @@ import org.apache.phoenix.execute.visitor.QueryPlanVisitor; import org.apache.phoenix.expression.KeyValueColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; +import org.apache.phoenix.iterate.ExplainTable; import org.apache.phoenix.iterate.MaterializedResultIterator; import org.apache.phoenix.iterate.ParallelScanGrouper; import org.apache.phoenix.iterate.ResultIterator; @@ -140,6 +141,7 @@ import org.apache.phoenix.parse.DeclareCursorStatement; import org.apache.phoenix.parse.DeleteJarStatement; import org.apache.phoenix.parse.DeleteStatement; +import org.apache.phoenix.parse.ExplainType; import org.apache.phoenix.parse.ShowCreateTableStatement; import org.apache.phoenix.parse.ShowCreateTable; import org.apache.phoenix.parse.DropColumnStatement; @@ -789,8 +791,8 @@ EXPLAIN_PLAN_TABLE_NAME, new KeyValueColumnExpression( private static class ExecutableExplainStatement extends ExplainStatement implements CompilableStatement { - public ExecutableExplainStatement(BindableStatement statement) { - super(statement); + public ExecutableExplainStatement(BindableStatement statement, ExplainType explainType) { + super(statement, explainType); } @Override @@ -816,6 +818,10 @@ public QueryPlan compilePlan(PhoenixStatement stmt, Sequence.ValueOp seqAction) } final StatementPlan plan = compilePlan; List planSteps = plan.getExplainPlan().getPlanSteps(); + ExplainType explainType = getExplainType(); + if (explainType == ExplainType.DEFAULT) { + planSteps.removeIf(planStep -> planStep != null && planStep.contains(ExplainTable.REGION_LOCATIONS)); + } List tuples = Lists.newArrayListWithExpectedSize(planSteps.size()); Long estimatedBytesToScan = plan.getEstimatedBytesToScan(); Long estimatedRowsToScan = plan.getEstimatedRowsToScan(); @@ -1914,8 +1920,8 @@ public AlterSessionStatement alterSession(Map props) { } @Override - public ExplainStatement explain(BindableStatement statement) { - return new ExecutableExplainStatement(statement); + public ExplainStatement explain(BindableStatement statement, ExplainType explainType) { + return new ExecutableExplainStatement(statement, explainType); } @Override diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainStatement.java index 49ce5737a4b..3b28ca5c0d5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainStatement.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainStatement.java @@ -21,9 +21,11 @@ public class ExplainStatement implements BindableStatement { private final BindableStatement statement; - - public ExplainStatement(BindableStatement statement) { + private final ExplainType explainType; + + public ExplainStatement(BindableStatement statement, ExplainType explainType) { this.statement = statement; + this.explainType = explainType; } public BindableStatement getStatement() { @@ -39,4 +41,8 @@ public int getBindCount() { public Operation getOperation() { return Operation.QUERY; } + + public ExplainType getExplainType() { + return explainType; + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainType.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainType.java new file mode 100644 index 00000000000..ef4655b1898 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainType.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.phoenix.parse; + +public enum ExplainType { + WITH_REGIONS, + DEFAULT +} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java index 77350cc0672..660598b4248 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java @@ -216,8 +216,8 @@ public static String createTempAlias() { return "$" + tempAliasCounter.incrementAndGet(); } - public ExplainStatement explain(BindableStatement statement) { - return new ExplainStatement(statement); + public ExplainStatement explain(BindableStatement statement, ExplainType explainType) { + return new ExplainStatement(statement, explainType); } public AliasedNode aliasedNode(String alias, ParseNode expression) { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index d60cfa7a0a2..62010416475 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -400,7 +400,7 @@ public class QueryServicesOptions { public static final int DEFAULT_SCAN_PAGE_SIZE = 32768; public static final boolean DEFAULT_APPLY_TIME_ZONE_DISPLACMENT = false; public static final boolean DEFAULT_PHOENIX_TABLE_TTL_ENABLED = true; - public static final int DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN = 10; + public static final int DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN = 5; private final Configuration config; From bf61eedca702afed74b32935a46a5d2699b0be34 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Fri, 12 May 2023 15:42:12 -0700 Subject: [PATCH 05/10] addendum --- .../src/main/java/org/apache/phoenix/iterate/ExplainTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java index 729c635e1aa..f73f4625a5c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java @@ -436,7 +436,7 @@ private String getRegionLocationsForExplainPlan(ExplainPlanAttributesBuilder exp } } - static class RegionBoundary { + private static class RegionBoundary { private final byte[] startKey; private final byte[] endKey; From 0a2b642d7cce6199e23c466d5903d278ed5c7383 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Sat, 13 May 2023 10:43:36 -0700 Subject: [PATCH 06/10] addendum --- .../main/java/org/apache/phoenix/jdbc/PhoenixStatement.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java index ef96db5926f..c51796d673f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java @@ -820,7 +820,10 @@ public QueryPlan compilePlan(PhoenixStatement stmt, Sequence.ValueOp seqAction) List planSteps = plan.getExplainPlan().getPlanSteps(); ExplainType explainType = getExplainType(); if (explainType == ExplainType.DEFAULT) { - planSteps.removeIf(planStep -> planStep != null && planStep.contains(ExplainTable.REGION_LOCATIONS)); + List updatedExplainPlanSteps = new ArrayList<>(planSteps); + updatedExplainPlanSteps.removeIf( + planStep -> planStep != null && planStep.contains(ExplainTable.REGION_LOCATIONS)); + planSteps = Collections.unmodifiableList(updatedExplainPlanSteps); } List tuples = Lists.newArrayListWithExpectedSize(planSteps.size()); Long estimatedBytesToScan = plan.getEstimatedBytesToScan(); From 7d9766999ca637c19afe2912119a901379fc7ab5 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Mon, 15 May 2023 11:06:54 -0700 Subject: [PATCH 07/10] addendum --- .../src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java index ad8e969e807b..7f780305971 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java @@ -120,7 +120,7 @@ public void testDebugLogs() throws Exception { // sleep for sometime to let query log committed Thread.sleep(delay); - try (ResultSet explainRS = conn.createStatement().executeQuery("Explain " + query); + try (ResultSet explainRS = conn.createStatement().executeQuery("Explain with regions " + query); ResultSet rs = conn.createStatement().executeQuery(logQuery)) { boolean foundQueryLog = false; @@ -300,7 +300,7 @@ private void testPreparedStatement(LogLevel loglevel) throws Exception{ // sleep for sometime to let query log committed Thread.sleep(delay); - String explainQuery = "Explain " + "SELECT * FROM " + tableName + " where V = 'value5'"; + String explainQuery = "EXPLAIN WITH REGIONS " + "SELECT * FROM " + tableName + " where V = 'value5'"; try (ResultSet explainRS = conn.createStatement() .executeQuery(explainQuery); ResultSet rs = conn.createStatement().executeQuery(logQuery)) { From 58a92afdea9242279f6d349226de8fcda1d46215 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Wed, 28 Jun 2023 15:52:00 -0700 Subject: [PATCH 08/10] addendum --- .../phoenix/iterate/BaseResultIterators.java | 1 - .../apache/phoenix/iterate/ExplainTable.java | 79 ++++++------------- 2 files changed, 26 insertions(+), 54 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 02135bc2d4a..2c6885bbf5d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -1537,7 +1537,6 @@ private List recreateIterators(ConnectionQueryServices se @Override public void close() throws SQLException { - this.closeExecutorService(); // Don't call cancel on already started work, as it causes the HConnection // to get into a funk. Instead, just cancel queued work. boolean cancelledWork = false; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java index f73f4625a5c..fb1a2509673 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java @@ -18,6 +18,7 @@ package org.apache.phoenix.iterate; import java.io.IOException; +import java.sql.SQLException; import java.text.Format; import java.util.ArrayList; import java.util.Collections; @@ -26,18 +27,9 @@ import java.util.List; import java.util.NoSuchElementException; import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Consistency; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; @@ -69,7 +61,6 @@ import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PInteger; -import org.apache.phoenix.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.ScanUtil; import org.apache.phoenix.util.StringUtil; @@ -91,8 +82,7 @@ public abstract class ExplainTable { protected final HintNode hint; protected final Integer limit; protected final Integer offset; - private final ExecutorService executorService; - + public ExplainTable(StatementContext context, TableRef table) { this(context, table, GroupBy.EMPTY_GROUP_BY, OrderBy.EMPTY_ORDER_BY, HintNode.EMPTY_HINT_NODE, null, null); } @@ -106,10 +96,6 @@ public ExplainTable(StatementContext context, TableRef table, GroupBy groupBy, O this.hint = hintNode; this.limit = limit; this.offset = offset; - this.executorService = Executors.newFixedThreadPool(5, new ThreadFactoryBuilder() - .setDaemon(true) - .setNameFormat("explain-region-info-%d") - .build()); } private String explainSkipScan() { @@ -153,27 +139,27 @@ private String explainSkipScan() { * @param regionLocations the list of region locations as output. * @throws IOException if something goes wrong while creating connection or querying region locations. */ - private void getRegionsInRange(final String tableName, + private void getRegionsInRange(final byte[] tableName, final byte[] startKey, final byte[] endKey, final boolean includeEndKey, final boolean reload, Set regionBoundaries, List regionLocations) - throws IOException { + throws IOException, SQLException { final boolean endKeyIsEndOfTable = Bytes.equals(endKey, HConstants.EMPTY_END_ROW); if ((Bytes.compareTo(startKey, endKey) > 0) && !endKeyIsEndOfTable) { throw new IllegalArgumentException( "Invalid range: " + Bytes.toStringBinary(startKey) + " > " + Bytes.toStringBinary(endKey)); } byte[] currentKey = startKey; - try (Connection connection = ConnectionFactory.createConnection( - context.getConnection().getQueryServices().getConfiguration()); - Table table = connection.getTable(TableName.valueOf(tableName))) { + try (Table table = context.getConnection().getQueryServices().getTable(tableName)) { do { - HRegionLocation regionLocation = table.getRegionLocator().getRegionLocation(currentKey, reload); - RegionBoundary regionBoundary = new RegionBoundary(regionLocation.getRegion().getStartKey(), - regionLocation.getRegion().getEndKey()); + HRegionLocation regionLocation = + table.getRegionLocator().getRegionLocation(currentKey, reload); + RegionBoundary regionBoundary = + new RegionBoundary(regionLocation.getRegion().getStartKey(), + regionLocation.getRegion().getEndKey()); if(!regionBoundaries.contains(regionBoundary)) { regionLocations.add(regionLocation); regionBoundaries.add(regionBoundary); @@ -377,34 +363,29 @@ protected void explain(String prefix, private void getRegionLocations(List planSteps, ExplainPlanAttributesBuilder explainPlanAttributesBuilder, List> scansList) { - Future task = executorService.submit( - () -> getRegionLocationsForExplainPlan(explainPlanAttributesBuilder, scansList)); - try { - String regionLocations = task.get(5, TimeUnit.SECONDS); - planSteps.add(regionLocations); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - LOGGER.error("Unable to retrieve region locations task result in 5s", e); - task.cancel(true); - } + planSteps.add( + getRegionLocationsForExplainPlan(explainPlanAttributesBuilder, scansList)); } /** - * Retrieve region locations from hbase client and set the values for the explain plan output. If the list - * of region locations exceed max limit, print only list with the max limit and print num of total list size. + * Retrieve region locations from hbase client and set the values for the explain plan output. + * If the list of region locations exceed max limit, print only list with the max limit and + * print num of total list size. * * @param explainPlanAttributesBuilder explain plan v2 attributes builder instance. * @param scansList list of the list of scans, to be used for parallel scans. * @return region locations to be added to the explain plan output. */ - private String getRegionLocationsForExplainPlan(ExplainPlanAttributesBuilder explainPlanAttributesBuilder, - List> scansList) { + private String getRegionLocationsForExplainPlan( + ExplainPlanAttributesBuilder explainPlanAttributesBuilder, + List> scansList) { try { StringBuilder buf = new StringBuilder().append(REGION_LOCATIONS); Set regionBoundaries = new HashSet<>(); List regionLocations = new ArrayList<>(); for (List scans : scansList) { for (Scan eachScan : scans) { - getRegionsInRange(tableRef.getTable().getPhysicalName().getString(), + getRegionsInRange(tableRef.getTable().getPhysicalName().getBytes(), eachScan.getStartRow(), eachScan.getStopRow(), true, @@ -417,11 +398,13 @@ private String getRegionLocationsForExplainPlan(ExplainPlanAttributesBuilder exp .getInt(QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, QueryServicesOptions.DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN); if (explainPlanAttributesBuilder != null) { - explainPlanAttributesBuilder.setRegionLocations(Collections.unmodifiableList(regionLocations)); + explainPlanAttributesBuilder.setRegionLocations( + Collections.unmodifiableList(regionLocations)); } if (regionLocations.size() > maxLimitRegionLoc) { int originalSize = regionLocations.size(); - List trimmedRegionLocations = regionLocations.subList(0, maxLimitRegionLoc); + List trimmedRegionLocations = + regionLocations.subList(0, maxLimitRegionLoc); buf.append(trimmedRegionLocations); buf.append("...total size = "); buf.append(originalSize); @@ -430,7 +413,7 @@ private String getRegionLocationsForExplainPlan(ExplainPlanAttributesBuilder exp } buf.append(") "); return buf.toString(); - } catch (IOException e) { + } catch (IOException | SQLException e) { LOGGER.error("Explain table unable to add region locations.", e); return ""; } @@ -454,7 +437,8 @@ public boolean equals(Object o) { return false; } RegionBoundary that = (RegionBoundary) o; - return Bytes.compareTo(startKey, that.startKey) == 0 && Bytes.compareTo(endKey, that.endKey) == 0; + return Bytes.compareTo(startKey, that.startKey) == 0 && + Bytes.compareTo(endKey, that.endKey) == 0; } @Override @@ -594,15 +578,4 @@ private String appendKeyRanges() { return buf.toString(); } - /** - * shutdown executor service, should be called by result iterator close. - */ - protected void closeExecutorService() { - this.executorService.shutdown(); - try { - this.executorService.awaitTermination(2, TimeUnit.SECONDS); - } catch (InterruptedException e) { - LOGGER.error("Interrupted while waiting for executor service shutdown.", e); - } - } } From e38c416f8a2ed1565131a92a5b9520048f65895e Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Thu, 29 Jun 2023 00:12:07 -0700 Subject: [PATCH 09/10] addendum --- .../compile/ExplainPlanAttributes.java | 3 +- .../apache/phoenix/iterate/ExplainTable.java | 36 +++++++++++-------- .../apache/phoenix/jdbc/PhoenixStatement.java | 6 ++-- .../org/apache/phoenix/parse/ExplainType.java | 16 +++++---- .../apache/phoenix/query/QueryServices.java | 6 ++-- .../phoenix/query/QueryServicesOptions.java | 3 +- 6 files changed, 42 insertions(+), 28 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java index d5363da02f3..a41f776345c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java @@ -610,7 +610,8 @@ public ExplainPlanAttributesBuilder setServerMergeColumns( return this; } - public ExplainPlanAttributesBuilder setRegionLocations(List regionLocations) { + public ExplainPlanAttributesBuilder setRegionLocations( + List regionLocations) { this.regionLocations = regionLocations; return this; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java index fb1a2509673..84c9a167a4e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java @@ -127,8 +127,8 @@ private String explainSkipScan() { } /** - * Get regions that represent the given range of start and end key for the given table, and all the regions - * to the regionLocations list. + * Get regions that represent the given range of start and end key for the given table, and + * all the regions to the regionLocations list. * * @param tableName the table name. * @param startKey the start rowkey. @@ -137,7 +137,8 @@ private String explainSkipScan() { * @param reload true if reload from meta is necessary. * @param regionBoundaries set of region boundaries to get the unique list of region locations. * @param regionLocations the list of region locations as output. - * @throws IOException if something goes wrong while creating connection or querying region locations. + * @throws IOException if something goes wrong while creating connection or querying region + * locations. */ private void getRegionsInRange(final byte[] tableName, final byte[] startKey, @@ -150,7 +151,8 @@ private void getRegionsInRange(final byte[] tableName, final boolean endKeyIsEndOfTable = Bytes.equals(endKey, HConstants.EMPTY_END_ROW); if ((Bytes.compareTo(startKey, endKey) > 0) && !endKeyIsEndOfTable) { throw new IllegalArgumentException( - "Invalid range: " + Bytes.toStringBinary(startKey) + " > " + Bytes.toStringBinary(endKey)); + "Invalid range: " + Bytes.toStringBinary(startKey) + " > " + + Bytes.toStringBinary(endKey)); } byte[] currentKey = startKey; try (Table table = context.getConnection().getQueryServices().getTable(tableName)) { @@ -160,7 +162,7 @@ private void getRegionsInRange(final byte[] tableName, RegionBoundary regionBoundary = new RegionBoundary(regionLocation.getRegion().getStartKey(), regionLocation.getRegion().getEndKey()); - if(!regionBoundaries.contains(regionBoundary)) { + if (!regionBoundaries.contains(regionBoundary)) { regionLocations.add(regionLocation); regionBoundaries.add(regionBoundary); } @@ -171,9 +173,9 @@ private void getRegionsInRange(final byte[] tableName, // condition4 = includeEndKey == true // condition5 = currentKey == endKey // while (condition1 && (condition2 || condition3 || (condition4 && condition5))) - } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW) && - (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0 || - (includeEndKey && Bytes.compareTo(currentKey, endKey) == 0))); + } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW) + && (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0 + || (includeEndKey && Bytes.compareTo(currentKey, endKey) == 0))); } } @@ -363,8 +365,11 @@ protected void explain(String prefix, private void getRegionLocations(List planSteps, ExplainPlanAttributesBuilder explainPlanAttributesBuilder, List> scansList) { - planSteps.add( - getRegionLocationsForExplainPlan(explainPlanAttributesBuilder, scansList)); + String regionLocationPlan = getRegionLocationsForExplainPlan(explainPlanAttributesBuilder, + scansList); + if (regionLocationPlan.length() > 0) { + planSteps.add(regionLocationPlan); + } } /** @@ -413,17 +418,20 @@ private String getRegionLocationsForExplainPlan( } buf.append(") "); return buf.toString(); - } catch (IOException | SQLException e) { + } catch (IOException | SQLException | UnsupportedOperationException e) { LOGGER.error("Explain table unable to add region locations.", e); return ""; } } + /** + * Region boundary class with start and end key of the region. + */ private static class RegionBoundary { private final byte[] startKey; private final byte[] endKey; - public RegionBoundary(byte[] startKey, byte[] endKey) { + RegionBoundary(byte[] startKey, byte[] endKey) { this.startKey = startKey; this.endKey = endKey; } @@ -437,8 +445,8 @@ public boolean equals(Object o) { return false; } RegionBoundary that = (RegionBoundary) o; - return Bytes.compareTo(startKey, that.startKey) == 0 && - Bytes.compareTo(endKey, that.endKey) == 0; + return Bytes.compareTo(startKey, that.startKey) == 0 + && Bytes.compareTo(endKey, that.endKey) == 0; } @Override diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java index c51796d673f..5b019d4cc4e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java @@ -791,7 +791,7 @@ EXPLAIN_PLAN_TABLE_NAME, new KeyValueColumnExpression( private static class ExecutableExplainStatement extends ExplainStatement implements CompilableStatement { - public ExecutableExplainStatement(BindableStatement statement, ExplainType explainType) { + ExecutableExplainStatement(BindableStatement statement, ExplainType explainType) { super(statement, explainType); } @@ -821,8 +821,8 @@ public QueryPlan compilePlan(PhoenixStatement stmt, Sequence.ValueOp seqAction) ExplainType explainType = getExplainType(); if (explainType == ExplainType.DEFAULT) { List updatedExplainPlanSteps = new ArrayList<>(planSteps); - updatedExplainPlanSteps.removeIf( - planStep -> planStep != null && planStep.contains(ExplainTable.REGION_LOCATIONS)); + updatedExplainPlanSteps.removeIf(planStep -> planStep != null + && planStep.contains(ExplainTable.REGION_LOCATIONS)); planSteps = Collections.unmodifiableList(updatedExplainPlanSteps); } List tuples = Lists.newArrayListWithExpectedSize(planSteps.size()); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainType.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainType.java index ef4655b1898..fc35939d35f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainType.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainType.java @@ -1,11 +1,10 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -18,6 +17,9 @@ package org.apache.phoenix.parse; +/** + * Explain type attributes used to differentiate output of the explain plan. + */ public enum ExplainType { WITH_REGIONS, DEFAULT diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java index 9060bc4fae0..e4885a79108 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -405,9 +405,11 @@ public interface QueryServices extends SQLCloseable { String SKIP_SYSTEM_TABLES_EXISTENCE_CHECK = "phoenix.skip.system.tables.existence.check"; /** - * Config key to represent max region locations to be displayed as part of the Explain plan output. + * Config key to represent max region locations to be displayed as part of the Explain plan + * output. */ - String MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN = "phoenix.max.region.locations.size.explain.plan"; + String MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN = + "phoenix.max.region.locations.size.explain.plan"; /** * Get executor service used for parallel scans diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index 62010416475..402c0ae88ed 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -492,7 +492,8 @@ public static QueryServicesOptions withDefaults() { .setIfUnset(SKIP_SYSTEM_TABLES_EXISTENCE_CHECK, DEFAULT_SKIP_SYSTEM_TABLES_EXISTENCE_CHECK) .setIfUnset(MAX_IN_LIST_SKIP_SCAN_SIZE, DEFAULT_MAX_IN_LIST_SKIP_SCAN_SIZE) - .setIfUnset(MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN); + .setIfUnset(MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, + DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN); // HBase sets this to 1, so we reset it to something more appropriate. // Hopefully HBase will change this, because we can't know if a user set From 81d6d0a0db500a161ffa84a361b10560c6207c5f Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Fri, 30 Jun 2023 19:34:33 -0700 Subject: [PATCH 10/10] addendum --- .../main/java/org/apache/phoenix/iterate/ExplainTable.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java index 84c9a167a4e..23b7632cb59 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java @@ -21,6 +21,7 @@ import java.sql.SQLException; import java.text.Format; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; @@ -156,6 +157,8 @@ private void getRegionsInRange(final byte[] tableName, } byte[] currentKey = startKey; try (Table table = context.getConnection().getQueryServices().getTable(tableName)) { + // include all regions that include key range from the given start key + // and end key do { HRegionLocation regionLocation = table.getRegionLocator().getRegionLocation(currentKey, reload); @@ -451,7 +454,9 @@ public boolean equals(Object o) { @Override public int hashCode() { - return 0; + int result = Arrays.hashCode(startKey); + result = 31 * result + Arrays.hashCode(endKey); + return result; } }