Skip to content

HIVE-29612: LATERAL VIEW OUTER doesn't work with CBO enabled #6482

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
soumyakanti3578 wants to merge 3 commits into
base: master
Choose a base branch
from
Open

HIVE-29612: LATERAL VIEW OUTER doesn't work with CBO enabled #6482

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f2ec801
HIVE-29612: LATERAL VIEW OUTER doesn't work with CBO enabled
soumyakanti3578 May 13, 2026
da242cd
Refactor to address review comments, and update isCBOSupportedLateral…
soumyakanti3578 May 15, 2026
544b33f
Refactor again to capture missing calls to isASTNodeLateralViewOrOuter
soumyakanti3578 May 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 33 additions & 3 deletions .../java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableFunctionScan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelInput;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelWriter;
import org.apache.calcite.rel.core.TableFunctionScan;
import org.apache.calcite.rel.metadata.RelColumnMapping;
import org.apache.calcite.rel.type.RelDataType;
Expand All @@ -34,6 +35,9 @@

public class HiveTableFunctionScan extends TableFunctionScan implements HiveRelNode {

// Whether this is a LATERAL VIEW OUTER
private final boolean outer;

/**
* @param cluster
* cluster - Cluster that this relational expression belongs to
Expand All @@ -49,28 +53,54 @@ public class HiveTableFunctionScan extends TableFunctionScan implements HiveRelN
* rowType - Row type produced by function
* @param columnMappings
* columnMappings - Column mappings associated with this function
* @param outer
* outer - true if this is a LATERAL VIEW OUTER
*/
protected HiveTableFunctionScan(RelOptCluster cluster, RelTraitSet traitSet, List<RelNode> inputs,
RexNode rexCall, Type elementType, RelDataType rowType, Set<RelColumnMapping> columnMappings) {
RexNode rexCall, Type elementType, RelDataType rowType, Set<RelColumnMapping> columnMappings,
boolean outer) {
super(cluster, traitSet, inputs, rexCall, elementType, rowType, columnMappings);
this.outer = outer;
}

public HiveTableFunctionScan(RelInput input) {
super(input);
this.outer = input.getBoolean("outer", false);
}

public static HiveTableFunctionScan create(RelOptCluster cluster, RelTraitSet traitSet,
List<RelNode> inputs, RexNode rexCall, Type elementType, RelDataType rowType,
Set<RelColumnMapping> columnMappings) throws CalciteSemanticException {
return new HiveTableFunctionScan(cluster, traitSet, inputs, rexCall, elementType, rowType,
columnMappings);
columnMappings, false);
}

public static HiveTableFunctionScan create(RelOptCluster cluster, RelTraitSet traitSet,
List<RelNode> inputs, RexNode rexCall, Type elementType, RelDataType rowType,
Set<RelColumnMapping> columnMappings, boolean outer) throws CalciteSemanticException {
return new HiveTableFunctionScan(cluster, traitSet, inputs, rexCall, elementType, rowType,
columnMappings, outer);
}

/** Returns true if this represents a LATERAL VIEW OUTER. */
public boolean isOuter() {
return outer;
}

@Override
public RelWriter explainTerms(RelWriter pw) {
super.explainTerms(pw);
if (outer) {
pw.item("outer", true);
}
return pw;
}

@Override
public TableFunctionScan copy(RelTraitSet traitSet, List<RelNode> inputs, RexNode rexCall,
Type elementType, RelDataType rowType, Set<RelColumnMapping> columnMappings) {
return new HiveTableFunctionScan(getCluster(), traitSet, inputs, rexCall,
elementType, rowType, columnMappings);
elementType, rowType, columnMappings, outer);
}

/**
Expand Down
15 changes: 11 additions & 4 deletions ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -703,14 +703,21 @@ private static QueryBlockInfo createASTLateralView(TableFunctionScan tfs, QueryB
sel.add(selexpr.node());

// place the SELECT clause under the LATERAL VIEW clause
ASTBuilder lateralview = ASTBuilder.construct(HiveParser.TOK_LATERAL_VIEW, "TOK_LATERAL_VIEW");
lateralview.add(sel.node());
final boolean isOuterLateralView = tfs instanceof HiveTableFunctionScan htfs && htfs.isOuter();
final int lateralViewToken = isOuterLateralView
? HiveParser.TOK_LATERAL_VIEW_OUTER
: HiveParser.TOK_LATERAL_VIEW;
final String lateralViewText = isOuterLateralView
? "TOK_LATERAL_VIEW_OUTER"
: "TOK_LATERAL_VIEW";
ASTBuilder lateralView = ASTBuilder.construct(lateralViewToken, lateralViewText);
lateralView.add(sel.node());

// finally, add the LATERAL VIEW clause under the left side source which is the base table.
lateralview.add(tableFunctionSource.ast);
lateralView.add(tableFunctionSource.ast);

Schema outputSchema = new Schema(tableFunctionSource.schema, new Schema(alias, lvFields));
return new QueryBlockInfo(outputSchema, lateralview.node());
return new QueryBlockInfo(outputSchema, lateralView.node());
}

private boolean isLateralView(RelNode relNode) {
Expand Down
12 changes: 6 additions & 6 deletions ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1041,9 +1041,9 @@ boolean isCBOExecuted() {
}

@Override
boolean isCBOSupportedLateralView(ASTNode lateralView) {
// LATERAL VIEW OUTER not supported in CBO
return lateralView.getToken().getType() != HiveParser.TOK_LATERAL_VIEW_OUTER;
boolean isCBOSupportedLateralView() {
// Both LATERAL VIEW and LATERAL VIEW OUTER are supported in CBO.
return !this.conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP);
Copy link
Copy Markdown
Contributor

@thomasrebele thomasrebele May 21, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Previously the result didn't depend on the HIVE_CBO_RETPATH_HIVEOP conf option. Maybe it should still return true if lateralView.getToken().getType() == HiveParser.TOK_LATERAL_VIEW?

Could you please explain why using that conf option here? It's description is Flag to control calcite plan to hive operator conversion, but I don't understand its purpose.

Copy link
Copy Markdown
Contributor

@kasakrisz kasakrisz May 21, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@thomasrebele
CBO return path is a different implementation of transforming the logical plan to Hive operators. It skips the AST conversion and converts Calcite operators directly to Hive operators. Maybe I'm missing something but I haven't found any change regarding this conversion in this patch.

@soumyakanti3578
Does lateral views in general is supported in CBO return path?
If yes could you please implement the lateral view outer too. I assume it would affect only 1-2 files and it would fit into the scope of this patch.

Copy link
Copy Markdown
Contributor Author

@soumyakanti3578 soumyakanti3578 May 21, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Neither LATERAL VIEW nor LATERAL VIEW OUTER work with CBO return path. I'll create a separate ticket for them, as I don't think there's a ticket for them here: https://issues.apache.org/jira/browse/HIVE-9132

So the current condition in isCBOSupportedLateralView() is true for both types of LATERAL VIEW when CBO is enabled (If CBO is not enabled we take a different path), and it is false only when HIVE_CBO_RETPATH_HIVEOP is true (non default path for CBO). So I think the condition correctly reflects the current state.

}

@Override
Expand Down Expand Up @@ -2980,7 +2980,7 @@ private RelNode genJoinLogicalPlan(QB qb, ASTNode joinParseTree, Map<String, Rel
leftRel = aliasToRel.get(leftTableAlias);
} else if (SemanticAnalyzer.isJoinToken(left)) {
leftRel = genJoinLogicalPlan(qb, left, aliasToRel, outerNameToPosMap, outerRR);
} else if (left.getToken().getType() == HiveParser.TOK_LATERAL_VIEW) {
} else if (isASTNodeLateralViewOrOuter(left)) {
leftRel = genLateralViewPlans(qb, left, aliasToRel);
} else {
assert (false);
Expand All @@ -2994,7 +2994,7 @@ private RelNode genJoinLogicalPlan(QB qb, ASTNode joinParseTree, Map<String, Rel
|| (right.getToken().getType() == HiveParser.TOK_PTBLFUNCTION)) {
rightTableAlias = getTableAlias(right);
rightRel = aliasToRel.get(rightTableAlias);
} else if (right.getToken().getType() == HiveParser.TOK_LATERAL_VIEW) {
} else if (isASTNodeLateralViewOrOuter(right)) {
rightRel = genLateralViewPlans(qb, right, aliasToRel);
} else {
assert (false);
Expand Down Expand Up @@ -3374,7 +3374,7 @@ private RelNode genLateralViewPlans(QB qb, ASTNode lateralView, Map<String, RelN

// next token is either the table alias name or another lateral view (which we will call
// recursively)
RelNode inputRel = next.getToken().getType() == HiveParser.TOK_LATERAL_VIEW
RelNode inputRel = isASTNodeLateralViewOrOuter(next)
? genLateralViewPlans(qb, next, aliasToRel)
: aliasToRel.get(getTableAlias(next));

Expand Down
22 changes: 14 additions & 8 deletions ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1683,8 +1683,7 @@ private void processJoin(QB qb, ASTNode join) throws SemanticException {
"PTF invocation in a Join must have an alias"));
}

} else if (child.getToken().getType() == HiveParser.TOK_LATERAL_VIEW ||
child.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER) {
} else if (isASTNodeLateralViewOrOuter(child)) {
// SELECT * FROM src1 LATERAL VIEW udtf() AS myTable JOIN src2 ...
// is not supported. Instead, the lateral view must be in a subquery
// SELECT * FROM (SELECT * FROM src1 LATERAL VIEW udtf() AS myTable) a
Expand Down Expand Up @@ -1713,9 +1712,7 @@ private String processLateralView(QB qb, ASTNode lateralView)
int numChildren = lateralView.getChildCount();
assert (numChildren == 2);

if (!isCBOSupportedLateralView(lateralView)) {
queryProperties.setCBOSupportedLateralViews(false);
}
queryProperties.setCBOSupportedLateralViews(isCBOSupportedLateralView());

ASTNode next = (ASTNode) lateralView.getChild(1);
String alias = null;
Expand Down Expand Up @@ -1889,8 +1886,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx,
processTable(qb, frm);
} else if (frm.getToken().getType() == HiveParser.TOK_SUBQUERY) {
processSubQuery(qb, frm);
} else if (frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW ||
frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER) {
} else if (isASTNodeLateralViewOrOuter(frm)) {
queryProperties.setHasLateralViews(true);
processLateralView(qb, frm);
} else if (isJoinToken(frm)) {
Expand Down Expand Up @@ -11123,7 +11119,7 @@ boolean isCBOExecuted() {
return false;
}

boolean isCBOSupportedLateralView(ASTNode lateralView) {
boolean isCBOSupportedLateralView() {
return false;
}

Expand Down Expand Up @@ -15526,4 +15522,14 @@ public void startAnalysis() {
queryState.createHMSCache();
}
}

/**
* Utility method to determine if an AST node represents a lateral view or lateral view outer.
* @param node AST node
* @return true if node is of lateral view or lateral view outer; false otherwise.
*/
public static boolean isASTNodeLateralViewOrOuter(ASTNode node) {
Copy link
Copy Markdown
Contributor

@kasakrisz kasakrisz May 21, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit.: I think we can name this just simply isASTNodeLateralView because IIUC lateral view outer is just a variation of lateral views. WDYT?

Copy link
Copy Markdown
Contributor Author

@soumyakanti3578 soumyakanti3578 May 21, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I can rename this method in the next commit. 👍🏼

return node.getToken().getType() == HiveParser.TOK_LATERAL_VIEW
|| node.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER;
}
}
14 changes: 7 additions & 7 deletions ql/src/java/org/apache/hadoop/hive/ql/parse/relnodegen/LateralViewPlan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan;
Expand Down Expand Up @@ -91,14 +90,15 @@ public class LateralViewPlan {

public LateralViewPlan(ASTNode lateralView, RelOptCluster cluster, RelNode inputRel,
RowResolver inputRR, UnparseTranslator unparseTranslator,
HiveConf conf, FunctionHelper functionHelper
) throws SemanticException {
HiveConf conf, FunctionHelper functionHelper) throws SemanticException {
// initialize global variables containing helper information
this.cluster = cluster;
this.unparseTranslator = unparseTranslator;
this.conf = conf;
this.functionHelper = functionHelper;

boolean isOuter = lateralView.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER;

// AST should have form of LATERAL_VIEW -> SELECT -> SELEXPR -> FUNCTION -> function info tree
ASTNode selExprAST = (ASTNode) lateralView.getChild(0).getChild(0);
ASTNode functionAST = (ASTNode) selExprAST.getChild(0);
Expand All @@ -118,7 +118,7 @@ public LateralViewPlan(ASTNode lateralView, RelOptCluster cluster, RelNode input

this.lateralViewRel = HiveTableFunctionScan.create(cluster,
TraitsUtil.getDefaultTraitSet(cluster), ImmutableList.of(inputRel), udtfCall,
null, retType, createColumnMappings(inputRel));
null, retType, createColumnMappings(inputRel), isOuter);
}

public static void validateLateralView(ASTNode lateralView) throws SemanticException {
Expand All @@ -127,9 +127,9 @@ public static void validateLateralView(ASTNode lateralView) throws SemanticExcep
" children.");
}
ASTNode next = (ASTNode) lateralView.getChild(1);
if (!TABLE_ALIAS_TOKEN_TYPES.contains(next.getToken().getType()) &&
HiveParser.TOK_LATERAL_VIEW != next.getToken().getType()) {
throw new SemanticException(ASTErrorUtils.getMsg(
if (!TABLE_ALIAS_TOKEN_TYPES.contains(next.getToken().getType()) &&
!SemanticAnalyzer.isASTNodeLateralViewOrOuter(next)) {
throw new SemanticException(ASTErrorUtils.getMsg(
ErrorMsg.LATERAL_VIEW_INVALID_CHILD.getMsg(), lateralView));
}
}
Expand Down
16 changes: 16 additions & 0 deletions ql/src/test/queries/clientpositive/lateral_view_outer_cbo.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
CREATE TABLE test (id string, items array<string>);
INSERT INTO test VALUES ('A', array('a', 'b')), ('B', array('c')), ('D', array());

CREATE VIEW v AS
SELECT test.id AS id, item
FROM test
LATERAL VIEW OUTER explode(test.items) lv AS item;

-- CBO plan should contain `outer=[true]` in HiveTableFunctionScan node.
EXPLAIN CBO
SELECT id, item FROM v ORDER BY id, item;
-- Explain plan should contain `outer lateral view: true` in the UDTF Operator
EXPLAIN
SELECT id, item FROM v ORDER BY id, item;
-- One of the output row should be ('D', NULL) since it's an outer lateral view.
SELECT id, item FROM v ORDER BY id, item;
Loading
Loading