From c9bfdc3bff2279e575612e5561f50c4c8cdc501d Mon Sep 17 00:00:00 2001 From: Andreas Reichel Date: Thu, 20 May 2021 11:56:34 +0700 Subject: [PATCH 1/4] Allow Complex Parsing of Functions Fixes issues #1190 #1103 --- .../jsqlparser/parser/AbstractJSqlParser.java | 3 ++ .../jsqlparser/parser/CCJSqlParserUtil.java | 38 +++++++++++++++++-- .../sf/jsqlparser/parser/feature/Feature.java | 7 ++++ .../net/sf/jsqlparser/parser/JSqlParserCC.jjt | 4 +- .../parser/CCJSqlParserUtilTest.java | 37 ++++++++++++++++++ .../select/NestedBracketsPerformanceTest.java | 32 +++++++++++----- 6 files changed, 106 insertions(+), 15 deletions(-) diff --git a/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java b/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java index 205839335..22bd1eb04 100644 --- a/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java +++ b/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java @@ -25,6 +25,9 @@ public P withSquareBracketQuotation(boolean allowSquareBracketQuotation) { return withFeature(Feature.allowSquareBracketQuotation, allowSquareBracketQuotation); } + public P withAllowComplexParsing(boolean allowComplexParsing) { + return withFeature(Feature.allowComplexParsing, allowComplexParsing); + } public P withFeature(Feature f, boolean enabled) { getConfiguration().setValue(f, enabled); return me(); diff --git a/src/main/java/net/sf/jsqlparser/parser/CCJSqlParserUtil.java b/src/main/java/net/sf/jsqlparser/parser/CCJSqlParserUtil.java index 9d5acdc61..afadfc8cb 100644 --- a/src/main/java/net/sf/jsqlparser/parser/CCJSqlParserUtil.java +++ b/src/main/java/net/sf/jsqlparser/parser/CCJSqlParserUtil.java @@ -24,6 +24,7 @@ * @author toben */ public final class CCJSqlParserUtil { + public final static int ALLOWED_NESTING_DEPTH = 7; private CCJSqlParserUtil() { } @@ -53,7 +54,9 @@ public static Statement parse(String sql) throws JSQLParserException { * @throws JSQLParserException */ public static Statement parse(String sql, Consumer consumer) throws JSQLParserException { - CCJSqlParser parser = newParser(sql); + boolean allowComplexParsing = getNestingDepth(sql)<=ALLOWED_NESTING_DEPTH; + + CCJSqlParser parser = newParser(sql).withAllowComplexParsing(allowComplexParsing); if (consumer != null) { consumer.accept(parser); } @@ -110,7 +113,9 @@ public static Expression parseExpression(String expression, boolean allowPartial } public static Expression parseExpression(String expression, boolean allowPartialParse, Consumer consumer) throws JSQLParserException { - CCJSqlParser parser = newParser(expression); + boolean allowComplexParsing = getNestingDepth(expression)<=ALLOWED_NESTING_DEPTH; + + CCJSqlParser parser = newParser(expression).withAllowComplexParsing(allowComplexParsing); if (consumer != null) { consumer.accept(parser); } @@ -154,7 +159,9 @@ public static Expression parseCondExpression(String condExpr, boolean allowParti } public static Expression parseCondExpression(String condExpr, boolean allowPartialParse, Consumer consumer) throws JSQLParserException { - CCJSqlParser parser = newParser(condExpr); + boolean allowComplexParsing = getNestingDepth(condExpr)<=ALLOWED_NESTING_DEPTH; + + CCJSqlParser parser = newParser(condExpr).withAllowComplexParsing(allowComplexParsing); if (consumer != null) { consumer.accept(parser); } @@ -190,7 +197,9 @@ public static Statement parseStatement(CCJSqlParser parser) throws JSQLParserExc * @return the statements parsed */ public static Statements parseStatements(String sqls) throws JSQLParserException { - CCJSqlParser parser = newParser(sqls); + boolean allowComplexParsing = getNestingDepth(sqls)<=ALLOWED_NESTING_DEPTH; + + CCJSqlParser parser = newParser(sqls).withAllowComplexParsing(allowComplexParsing); return parseStatements(parser); } @@ -225,5 +234,26 @@ public static void streamStatements(StatementListener listener, InputStream is, throw new JSQLParserException(ex); } } + + public static int getNestingDepth(String sql) { + int maxlevel=0; + int level=0; + + char[] chars = sql.toCharArray(); + for (char c:chars) { + switch(c) { + case '(': + level++; + break; + case ')': + if (maxlevel parser.withSquareBracketQuotation(true)); assertEquals("[travel_data].[travel_id]", result.toString()); } + @Test + public void testNestingDepth() throws Exception { + assertEquals(2, + CCJSqlParserUtil.getNestingDepth("SELECT concat(concat('A','B'),'B') FROM mytbl")); + assertEquals(20, CCJSqlParserUtil.getNestingDepth( + "concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat('A','B'),'B'),'B'),'B'),'B'),'B'),'B'),'B'),'B'),'B'),'B'),'B'),'B'),'B'),'B'),'B'),'B'),'B'),'B'),'B') FROM mytbl")); + assertEquals(4, CCJSqlParserUtil.getNestingDepth("" + + "-- MERGE 1\n" + + "MERGE INTO cfe.impairment imp\n" + " USING ( WITH x AS (\n" + + " SELECT a.id_instrument\n" + + " , a.id_currency\n" + + " , a.id_instrument_type\n" + + " , b.id_portfolio\n" + + " , c.attribute_value product_code\n" + + " , t.valid_date\n" + " , t.ccf\n" + + " FROM cfe.instrument a\n" + + " INNER JOIN cfe.impairment b\n" + + " ON a.id_instrument = b.id_instrument\n" + + " LEFT JOIN cfe.instrument_attribute c\n" + + " ON a.id_instrument = c.id_instrument\n" + + " AND c.id_attribute = 'product'\n" + + " INNER JOIN cfe.ext_ccf t\n" + + " ON ( a.id_currency LIKE t.id_currency )\n" + + " AND ( a.id_instrument_type LIKE t.id_instrument_type )\n" + + " AND ( b.id_portfolio LIKE t.id_portfolio\n" + + " OR ( b.id_portfolio IS NULL\n" + + " AND t.id_portfolio = '%' ) )\n" + + " AND ( c.attribute_value LIKE t.product_code\n" + + " OR ( c.attribute_value IS NULL\n" + + " AND t.product_code = '%' ) ) )\n" + + "SELECT /*+ PARALLEL */ *\n" + " FROM x x1\n" + + " WHERE x1.valid_date = ( SELECT max\n" + + " FROM x\n" + + " WHERE id_instrument = x1.id_instrument ) ) s\n" + + " ON ( imp.id_instrument = s.id_instrument )\n" + "WHEN MATCHED THEN\n" + + " UPDATE SET imp.ccf = s.ccf\n" + ";")); + } } diff --git a/src/test/java/net/sf/jsqlparser/statement/select/NestedBracketsPerformanceTest.java b/src/test/java/net/sf/jsqlparser/statement/select/NestedBracketsPerformanceTest.java index 396197777..62750995d 100644 --- a/src/test/java/net/sf/jsqlparser/statement/select/NestedBracketsPerformanceTest.java +++ b/src/test/java/net/sf/jsqlparser/statement/select/NestedBracketsPerformanceTest.java @@ -23,17 +23,17 @@ */ public class NestedBracketsPerformanceTest { - @Test + @Test(timeout = 1000) public void testIssue766() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed("SELECT concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat('1','2'),'3'),'4'),'5'),'6'),'7'),'8'),'9'),'10'),'11'),'12'),'13'),'14'),'15'),'16'),'17'),'18'),'19'),'20'),'21'),col1 FROM tbl t1", true); } - @Test + @Test(timeout = 1000) public void testIssue766_2() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed("SELECT concat(concat(concat('1', '2'), '3'), '4'), col1 FROM tbl t1"); } - @Test + @Test(timeout = 1000) public void testIssue235() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed("SELECT CASE WHEN ( CASE WHEN ( CASE WHEN ( CASE WHEN ( 1 ) THEN 0 END ) THEN 0 END ) THEN 0 END ) THEN 0 END FROM a", true); } @@ -50,7 +50,7 @@ public void testIssue856() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed(sql); } - @Test + @Test(timeout = 1000) public void testRecursiveBracketExpressionIssue1019() { assertEquals("IF(1=1, 1, 2)", buildRecursiveBracketExpression("IF(1=1, $1, 2)", "1", 0)); assertEquals("IF(1=1, IF(1=1, 1, 2), 2)", buildRecursiveBracketExpression("IF(1=1, $1, 2)", "1", 1)); @@ -62,12 +62,12 @@ public void testRecursiveBracketExpressionIssue1019_2() throws JSQLParserExcepti doIncreaseOfParseTimeTesting("IF(1=1, $1, 2)", "1", 10); } - @Test + @Test(timeout = 1000) public void testIssue1013() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed("SELECT ((((((((((((((((tblA)))))))))))))))) FROM mytable"); } - @Test + @Test(timeout = 1000) public void testIssue1013_2() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed("SELECT * FROM ((((((((((((((((tblA))))))))))))))))"); } @@ -77,7 +77,7 @@ public void testIssue1013_3() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed("SELECT * FROM (((tblA)))"); } - @Test + @Test(timeout = 1000) public void testIssue1013_4() throws JSQLParserException { String s = "tblA"; for (int i = 1; i < 100; i++) { @@ -95,9 +95,9 @@ public void testIssue1013_4() throws JSQLParserException { * * @throws JSQLParserException */ - @Test + @Test(timeout = 1000) public void testIncreaseOfParseTime() throws JSQLParserException { - doIncreaseOfParseTimeTesting("concat($1,'B')", "'A'", 20); + doIncreaseOfParseTimeTesting("concat($1,'B')", "'A'", 50); } private void doIncreaseOfParseTimeTesting(String template, String finalExpression, int maxDepth) throws JSQLParserException { @@ -136,4 +136,16 @@ private String buildRecursiveBracketExpression(String template, String finalExpr } return template.replace("$1", buildRecursiveBracketExpression(template, finalExpression, depth - 1)); } -} + + @Test(timeout = 1000) + public void testIssue1103() throws JSQLParserException { + assertSqlCanBeParsedAndDeparsed( + "SELECT\n" + "ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(\n" + + "ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(\n" + + "ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(\n" + + "ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(0\n" + ",0),0),0),0),0),0),0),0)\n" + + ",0),0),0),0),0),0),0),0)\n" + ",0),0),0),0),0),0),0),0)\n" + + ",0),0),0),0),0),0),0),0)", + true); + } + } From a51a20492cfd81e2925202364bbc987d9a070423 Mon Sep 17 00:00:00 2001 From: Andreas Reichel Date: Thu, 20 May 2021 13:18:35 +0700 Subject: [PATCH 2/4] Apply Complex Parsing to PrimaryExpression() Fixes issue #1194 --- .../jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt | 8 ++++++++ .../net/sf/jsqlparser/statement/select/SelectTest.java | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index 07b9e4400..f7e96a33e 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -3370,6 +3370,14 @@ Expression PrimaryExpression() #PrimaryExpression: | LOOKAHEAD("(" retval=SubSelect() ")") "(" retval=SubSelect() ")" + | LOOKAHEAD({getAsBoolean(Feature.allowComplexParsing)}) "(" list = ComplexExpressionList() ")" + { + if (list.getExpressions().size() == 1) { + retval = new Parenthesis(list.getExpressions().get(0)); + } else { + retval = new RowConstructor().withExprList(list); + } + } | "(" list = SimpleExpressionList() ")" { if (list.getExpressions().size() == 1) { diff --git a/src/test/java/net/sf/jsqlparser/statement/select/SelectTest.java b/src/test/java/net/sf/jsqlparser/statement/select/SelectTest.java index 105182e9a..cae4607d6 100644 --- a/src/test/java/net/sf/jsqlparser/statement/select/SelectTest.java +++ b/src/test/java/net/sf/jsqlparser/statement/select/SelectTest.java @@ -4525,4 +4525,14 @@ public void testProblematicDeparsingIssue1183_2() throws JSQLParserException { public void testKeywordCostsIssue1185() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed("WITH costs AS (SELECT * FROM MY_TABLE1 AS ALIAS_TABLE1) SELECT * FROM TESTSTMT"); } + + @Test + public void testFunctionWithComplexParameters_Issue1190() throws JSQLParserException { + assertSqlCanBeParsedAndDeparsed("SELECT to_char(a = '3') FROM dual", true); + } + + @Test + public void testConditionsWithExtraBrackets_Issue1194() throws JSQLParserException { + assertSqlCanBeParsedAndDeparsed("SELECT (col IS NULL) FROM tbl", true); + } } From 42abed4fe7cbffaf91f5f9dd3c77f6e32c5f923c Mon Sep 17 00:00:00 2001 From: Andreas Reichel Date: Thu, 20 May 2021 13:27:27 +0700 Subject: [PATCH 3/4] Increase Test Timeout to 2 seconds for slow CI Servers. --- .../select/NestedBracketsPerformanceTest.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/test/java/net/sf/jsqlparser/statement/select/NestedBracketsPerformanceTest.java b/src/test/java/net/sf/jsqlparser/statement/select/NestedBracketsPerformanceTest.java index 62750995d..c7aa9ad8c 100644 --- a/src/test/java/net/sf/jsqlparser/statement/select/NestedBracketsPerformanceTest.java +++ b/src/test/java/net/sf/jsqlparser/statement/select/NestedBracketsPerformanceTest.java @@ -23,22 +23,22 @@ */ public class NestedBracketsPerformanceTest { - @Test(timeout = 1000) + @Test(timeout = 2000) public void testIssue766() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed("SELECT concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat(concat('1','2'),'3'),'4'),'5'),'6'),'7'),'8'),'9'),'10'),'11'),'12'),'13'),'14'),'15'),'16'),'17'),'18'),'19'),'20'),'21'),col1 FROM tbl t1", true); } - @Test(timeout = 1000) + @Test(timeout = 2000) public void testIssue766_2() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed("SELECT concat(concat(concat('1', '2'), '3'), '4'), col1 FROM tbl t1"); } - @Test(timeout = 1000) + @Test(timeout = 2000) public void testIssue235() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed("SELECT CASE WHEN ( CASE WHEN ( CASE WHEN ( CASE WHEN ( 1 ) THEN 0 END ) THEN 0 END ) THEN 0 END ) THEN 0 END FROM a", true); } - @Test(timeout = 100000) + @Test(timeout = 200000) @Ignore public void testIssue496() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed("select isNull(charLen(TEST_ID,0)+ isNull(charLen(TEST_DVC,0)+ isNull(charLen(TEST_NO,0)+ isNull(charLen(ATEST_ID,0)+ isNull(charLen(TESTNO,0)+ isNull(charLen(TEST_CTNT,0)+ isNull(charLen(TEST_MESG_CTNT,0)+ isNull(charLen(TEST_DTM,0)+ isNull(charLen(TEST_DTT,0)+ isNull(charLen(TEST_ADTT,0)+ isNull(charLen(TEST_TCD,0)+ isNull(charLen(TEST_PD,0)+ isNull(charLen(TEST_VAL,0)+ isNull(charLen(TEST_YN,0)+ isNull(charLen(TEST_DTACM,0)+ isNull(charLen(TEST_MST,0) from test_info_m"); @@ -50,7 +50,7 @@ public void testIssue856() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed(sql); } - @Test(timeout = 1000) + @Test(timeout = 2000) public void testRecursiveBracketExpressionIssue1019() { assertEquals("IF(1=1, 1, 2)", buildRecursiveBracketExpression("IF(1=1, $1, 2)", "1", 0)); assertEquals("IF(1=1, IF(1=1, 1, 2), 2)", buildRecursiveBracketExpression("IF(1=1, $1, 2)", "1", 1)); @@ -62,12 +62,12 @@ public void testRecursiveBracketExpressionIssue1019_2() throws JSQLParserExcepti doIncreaseOfParseTimeTesting("IF(1=1, $1, 2)", "1", 10); } - @Test(timeout = 1000) + @Test(timeout = 2000) public void testIssue1013() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed("SELECT ((((((((((((((((tblA)))))))))))))))) FROM mytable"); } - @Test(timeout = 1000) + @Test(timeout = 2000) public void testIssue1013_2() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed("SELECT * FROM ((((((((((((((((tblA))))))))))))))))"); } @@ -77,7 +77,7 @@ public void testIssue1013_3() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed("SELECT * FROM (((tblA)))"); } - @Test(timeout = 1000) + @Test(timeout = 2000) public void testIssue1013_4() throws JSQLParserException { String s = "tblA"; for (int i = 1; i < 100; i++) { @@ -95,13 +95,13 @@ public void testIssue1013_4() throws JSQLParserException { * * @throws JSQLParserException */ - @Test(timeout = 1000) + @Test(timeout = 2000) public void testIncreaseOfParseTime() throws JSQLParserException { doIncreaseOfParseTimeTesting("concat($1,'B')", "'A'", 50); } private void doIncreaseOfParseTimeTesting(String template, String finalExpression, int maxDepth) throws JSQLParserException { - long oldDurationTime = 1000; + long oldDurationTime = 2000; int countProblematic = 0; for (int i = 0; i < maxDepth; i++) { String sql = "SELECT " + buildRecursiveBracketExpression(template, finalExpression, i) + " FROM mytbl"; @@ -137,7 +137,7 @@ private String buildRecursiveBracketExpression(String template, String finalExpr return template.replace("$1", buildRecursiveBracketExpression(template, finalExpression, depth - 1)); } - @Test(timeout = 1000) + @Test(timeout = 2000) public void testIssue1103() throws JSQLParserException { assertSqlCanBeParsedAndDeparsed( "SELECT\n" + "ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(ROUND(\n" From cb1cb16575ebeef2163bf74eb192827d014984cb Mon Sep 17 00:00:00 2001 From: Andreas Reichel Date: Wed, 26 May 2021 13:57:55 +0700 Subject: [PATCH 4/4] Appease Codazy --- src/main/java/net/sf/jsqlparser/parser/CCJSqlParserUtil.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/net/sf/jsqlparser/parser/CCJSqlParserUtil.java b/src/main/java/net/sf/jsqlparser/parser/CCJSqlParserUtil.java index afadfc8cb..21824ca1d 100644 --- a/src/main/java/net/sf/jsqlparser/parser/CCJSqlParserUtil.java +++ b/src/main/java/net/sf/jsqlparser/parser/CCJSqlParserUtil.java @@ -251,6 +251,8 @@ public static int getNestingDepth(String sql) { } level--; break; + default: + // Codazy/PMD insists in a Default statement } } return maxlevel;