PHOENIX-6907 Explain Plan should output region locations with servers

phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java

@@ -58,6 +58,8 @@
 import org.junit.runners.Parameterized.Parameters;
 
 import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 @Category(ParallelStatsDisabledTest.class)
@@ -71,6 +73,8 @@ public class DerivedTableIT extends ParallelStatsDisabledIT {
     private String[] plans;
     private String tableName;
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(DerivedTableIT.class);
+
 
     public DerivedTableIT(String[] indexDDL, String[] plans) {
         this.indexDDL = indexDDL;
@@ -114,31 +118,31 @@ public static synchronized Collection<Object> data() {
                 {
                         "CREATE INDEX "+dynamicTableName+"_DERIVED_IDX ON "+dynamicTableName+" (a_byte) INCLUDE (A_STRING, B_STRING)"
                 }, {
-                "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX\n" +
+                "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX \n" +
                         "    SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]\n" +
                         "CLIENT MERGE SORT\n" +
                         "CLIENT SORTED BY [\"B_STRING\"]\n" +
                         "CLIENT SORTED BY [A]\n" +
                         "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" +
                         "CLIENT SORTED BY [A DESC]",
 
-                "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX\n" +
+                "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX \n" +
                         "    SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]\n" +
                         "CLIENT MERGE SORT\n" +
                         "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" +
                         "CLIENT DISTINCT ON [COLLECTDISTINCT(B)]\n" +
                         "CLIENT SORTED BY [A DESC]"}});
         testCases.add(new String[][] {
                 {}, {
-                "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+"\n" +
+                "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+" \n" +
                         "    SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]\n" +
                         "CLIENT MERGE SORT\n" +
                         "CLIENT SORTED BY [B_STRING]\n" +
                         "CLIENT SORTED BY [A]\n" +
                         "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" +
                         "CLIENT SORTED BY [A DESC]",
 
-                "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+"\n" +
+                "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+" \n" +
                         "    SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]\n" +
                         "CLIENT MERGE SORT\n" +
                         "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" +
@@ -378,8 +382,12 @@ public void testDerivedTableWithGroupBy() throws Exception {
 
             assertFalse(rs.next());
 
-            rs = conn.createStatement().executeQuery("EXPLAIN " + query);
-            assertEquals(plans[0], QueryUtil.getExplainPlan(rs));
+            rs = conn.createStatement().executeQuery("EXPLAIN WITH REGIONS " + query);
+            String explainPlanOutput = QueryUtil.getExplainPlan(rs);
+            LOGGER.info("Explain plan output: {}", explainPlanOutput);
+            String[] splitExplainPlan = explainPlanOutput.split("\\n \\(region locations = \\[region=");
+            String[] secondSplitExplainPlan = splitExplainPlan[1].split("]\\)");
+            assertEquals(plans[0], splitExplainPlan[0] + secondSplitExplainPlan[1]);
 
             // distinct b (groupby a, b) groupby a orderby a
             query = "SELECT DISTINCT COLLECTDISTINCT(t.b) FROM (SELECT b_string b, a_string a FROM "+tableName+" GROUP BY a_string, b_string) AS t GROUP BY t.a ORDER BY t.a DESC";
@@ -400,8 +408,12 @@ public void testDerivedTableWithGroupBy() throws Exception {
 
             assertFalse(rs.next());
 
-            rs = conn.createStatement().executeQuery("EXPLAIN " + query);
-            assertEquals(plans[1], QueryUtil.getExplainPlan(rs));
+            rs = conn.createStatement().executeQuery("EXPLAIN WITH REGIONS " + query);
+            explainPlanOutput = QueryUtil.getExplainPlan(rs);
+            LOGGER.info("Explain plan output: {}", explainPlanOutput);
+            splitExplainPlan = explainPlanOutput.split("\\n \\(region locations = \\[region=");
+            secondSplitExplainPlan = splitExplainPlan[1].split("]\\)");
+            assertEquals(plans[1], splitExplainPlan[0] + secondSplitExplainPlan[1]);
 
             // (orderby) groupby
             query = "SELECT t.a_string, count(*) FROM (SELECT * FROM "+tableName+" order by a_integer) AS t where a_byte != 8 group by t.a_string";

phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java

@@ -50,12 +50,17 @@
 import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.util.QueryUtil;
 import org.apache.phoenix.util.SchemaUtil;
 import org.apache.phoenix.util.TestUtil;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class FlappingLocalIndexIT extends BaseLocalIndexIT {
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(FlappingLocalIndexIT.class);
+
     public FlappingLocalIndexIT(boolean isNamespaceMapped) {
         super(isNamespaceMapped);
     }
@@ -159,6 +164,13 @@ public void testLocalIndexScan() throws Exception {
 
             String query = "SELECT * FROM " + tableName +" where v1 like 'a%'";
 
+            String explainPlanOutput =
+                    QueryUtil.getExplainPlan(conn1.createStatement().executeQuery("EXPLAIN WITH REGIONS " + query));
+            LOGGER.info("Explain plan output: {}", explainPlanOutput);
+            // MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN is set as 2
+            assertTrue("Expected total " + numRegions + " regions",
+                    explainPlanOutput.contains("...total size = " + numRegions));
+
             ExplainPlan plan = conn1.prepareStatement(query)
                 .unwrap(PhoenixPreparedStatement.class).optimizeQuery()
                 .getExplainPlan();
@@ -176,6 +188,7 @@ public void testLocalIndexScan() throws Exception {
                 explainPlanAttributes.getServerWhereFilter());
             assertEquals("CLIENT MERGE SORT",
                 explainPlanAttributes.getClientSortAlgo());
+            assertEquals(numRegions, explainPlanAttributes.getRegionLocations().size());
 
             rs = conn1.createStatement().executeQuery(query);
             assertTrue(rs.next());

phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java

@@ -120,7 +120,7 @@ public void testDebugLogs() throws Exception {
 
         // sleep for sometime to let query log committed
         Thread.sleep(delay);
-        try (ResultSet explainRS = conn.createStatement().executeQuery("Explain " + query);
+        try (ResultSet explainRS = conn.createStatement().executeQuery("Explain with regions " + query);
              ResultSet rs = conn.createStatement().executeQuery(logQuery)) {
             boolean foundQueryLog = false;
 
@@ -300,7 +300,7 @@ private void testPreparedStatement(LogLevel loglevel) throws Exception{
 
             // sleep for sometime to let query log committed
             Thread.sleep(delay);
-            String explainQuery = "Explain " + "SELECT * FROM " + tableName + " where V = 'value5'";
+            String explainQuery = "EXPLAIN WITH REGIONS " + "SELECT * FROM " + tableName + " where V = 'value5'";
             try (ResultSet explainRS = conn.createStatement()
                     .executeQuery(explainQuery);
                  ResultSet rs = conn.createStatement().executeQuery(logQuery)) {

phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java

@@ -61,6 +61,7 @@ public static synchronized void doSetup() throws Exception {
         // setting update frequency to a large value to test out that we are
         // generating stats for local indexes
         clientProps.put(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB, "120000");
+        clientProps.put(QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, "2");
         setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator()));
     }
 

phoenix-core/src/main/antlr3/PhoenixSQL.g

@@ -152,6 +152,7 @@ tokens
     REVOKE = 'revoke';
     SHOW = 'show';
     UNCOVERED = 'uncovered';
+    REGIONS = 'regions';
 }
 
 
@@ -212,6 +213,7 @@ import org.apache.phoenix.util.SchemaUtil;
 import org.apache.phoenix.parse.LikeParseNode.LikeType;
 import org.apache.phoenix.trace.util.Tracing;
 import org.apache.phoenix.parse.AddJarsStatement;
+import org.apache.phoenix.parse.ExplainType;
 }
 
 @lexer::header {
@@ -452,7 +454,14 @@ oneStatement returns [BindableStatement ret]
 finally{ contextStack.pop(); }
 
 explain_node returns [BindableStatement ret]
-    :   EXPLAIN q=oneStatement {$ret=factory.explain(q);}
+    :   EXPLAIN (w=WITH)? (r=REGIONS)? q=oneStatement
+     {
+        if ((w==null && r!=null) || (w!=null && r==null)) {
+            throw new RuntimeException("Valid usage: EXPLAIN {query} OR EXPLAIN WITH REGIONS {query}");
+        }
+        ret = (w==null && r==null) ? factory.explain(q, ExplainType.DEFAULT)
+         : factory.explain(q, ExplainType.WITH_REGIONS);
+     }
     ;
 
 // Parse a create table statement.

phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java

@@ -18,8 +18,10 @@
 
 package org.apache.phoenix.compile;
 
+import java.util.List;
 import java.util.Set;
 
+import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.client.Consistency;
 import org.apache.phoenix.parse.HintNode;
 import org.apache.phoenix.parse.HintNode.Hint;
@@ -73,6 +75,7 @@ public class ExplainPlanAttributes {
     // be null
     private final ExplainPlanAttributes rhsJoinQueryExplainPlan;
     private final Set<PColumn> serverMergeColumns;
+    private final List<HRegionLocation> regionLocations;
 
     private static final ExplainPlanAttributes EXPLAIN_PLAN_INSTANCE =
         new ExplainPlanAttributes();
@@ -112,6 +115,7 @@ private ExplainPlanAttributes() {
         this.clientSortAlgo = null;
         this.rhsJoinQueryExplainPlan = null;
         this.serverMergeColumns = null;
+        this.regionLocations = null;
     }
 
     public ExplainPlanAttributes(String abstractExplainPlan,
@@ -132,7 +136,7 @@ public ExplainPlanAttributes(String abstractExplainPlan,
             Integer clientSequenceCount, String clientCursorName,
             String clientSortAlgo,
             ExplainPlanAttributes rhsJoinQueryExplainPlan,
-            Set<PColumn> serverMergeColumns) {
+            Set<PColumn> serverMergeColumns, List<HRegionLocation> regionLocations) {
         this.abstractExplainPlan = abstractExplainPlan;
         this.splitsChunk = splitsChunk;
         this.estimatedRows = estimatedRows;
@@ -167,6 +171,7 @@ public ExplainPlanAttributes(String abstractExplainPlan,
         this.clientSortAlgo = clientSortAlgo;
         this.rhsJoinQueryExplainPlan = rhsJoinQueryExplainPlan;
         this.serverMergeColumns = serverMergeColumns;
+        this.regionLocations = regionLocations;
     }
 
     public String getAbstractExplainPlan() {
@@ -305,6 +310,10 @@ public Set<PColumn> getServerMergeColumns() {
         return serverMergeColumns;
     }
 
+    public List<HRegionLocation> getRegionLocations() {
+        return regionLocations;
+    }
+
     public static ExplainPlanAttributes getDefaultExplainPlan() {
         return EXPLAIN_PLAN_INSTANCE;
     }
@@ -344,6 +353,7 @@ public static class ExplainPlanAttributesBuilder {
         private String clientSortAlgo;
         private ExplainPlanAttributes rhsJoinQueryExplainPlan;
         private Set<PColumn> serverMergeColumns;
+        private List<HRegionLocation> regionLocations;
 
         public ExplainPlanAttributesBuilder() {
             // default
@@ -396,6 +406,7 @@ public ExplainPlanAttributesBuilder(
             this.rhsJoinQueryExplainPlan =
                 explainPlanAttributes.getRhsJoinQueryExplainPlan();
             this.serverMergeColumns = explainPlanAttributes.getServerMergeColumns();
+            this.regionLocations = explainPlanAttributes.getRegionLocations();
         }
 
         public ExplainPlanAttributesBuilder setAbstractExplainPlan(
@@ -599,6 +610,12 @@ public ExplainPlanAttributesBuilder setServerMergeColumns(
             return this;
         }
 
+        public ExplainPlanAttributesBuilder setRegionLocations(
+                List<HRegionLocation> regionLocations) {
+            this.regionLocations = regionLocations;
+            return this;
+        }
+
         public ExplainPlanAttributes build() {
             return new ExplainPlanAttributes(abstractExplainPlan, splitsChunk,
                 estimatedRows, estimatedSizeInBytes, iteratorTypeAndScanSize,
@@ -611,7 +628,8 @@ public ExplainPlanAttributes build() {
                 clientFilterBy, clientAggregate, clientSortedBy,
                 clientAfterAggregate, clientDistinctFilter, clientOffset,
                 clientRowLimit, clientSequenceCount, clientCursorName,
-                clientSortAlgo, rhsJoinQueryExplainPlan, serverMergeColumns);
+                clientSortAlgo, rhsJoinQueryExplainPlan, serverMergeColumns,
+                regionLocations);
         }
     }
 }

phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java

@@ -1537,7 +1537,6 @@ private List<PeekingResultIterator> recreateIterators(ConnectionQueryServices se
 
     @Override
     public void close() throws SQLException {
-
         // Don't call cancel on already started work, as it causes the HConnection
         // to get into a funk. Instead, just cancel queued work.
         boolean cancelledWork = false;
@@ -1724,7 +1723,7 @@ private void explainUtil(List<String> planSteps,
             }
         }
 
-        explain(buf.toString(), planSteps, explainPlanAttributesBuilder);
+        explain(buf.toString(), planSteps, explainPlanAttributesBuilder, scans);
     }
 
     @Override