From d8c5bb0b93925cc1faf5960c79c5e9e2159a6324 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Tue, 8 Apr 2025 17:05:40 +0800 Subject: [PATCH 1/9] fix --- .../trees/plans/commands/info/BaseViewInfo.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java index a762b11c1f6a2e..d7ad12c1898be8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java @@ -63,6 +63,10 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalView; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.NullType; +import org.apache.doris.nereids.types.TinyIntType; +import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; @@ -162,8 +166,9 @@ protected String rewriteProjectsToUserDefineAlias(String resSql) { protected void createFinalCols(List outputs) throws org.apache.doris.common.AnalysisException { if (simpleColumnDefinitions.isEmpty()) { for (Slot output : outputs) { - Column column = new Column(output.getName(), output.getDataType().toCatalogDataType(), - output.nullable()); + DataType dataType = TypeCoercionUtils.replaceSpecifiedType(output.getDataType(), NullType.class, + TinyIntType.INSTANCE); + Column column = new Column(output.getName(), dataType.toCatalogDataType(), output.nullable()); finalCols.add(column); } } else { @@ -171,8 +176,10 @@ protected void createFinalCols(List outputs) throws org.apache.doris.commo ErrorReport.reportAnalysisException(ErrorCode.ERR_VIEW_WRONG_LIST); } for (int i = 0; i < simpleColumnDefinitions.size(); ++i) { + DataType dataType = TypeCoercionUtils.replaceSpecifiedType(outputs.get(i).getDataType(), NullType.class, + TinyIntType.INSTANCE); Column column = new Column(simpleColumnDefinitions.get(i).getName(), - outputs.get(i).getDataType().toCatalogDataType(), outputs.get(i).nullable()); + dataType.toCatalogDataType(), outputs.get(i).nullable()); column.setComment(simpleColumnDefinitions.get(i).getComment()); finalCols.add(column); } From c5eb968e91ac29a11110c816eb601f8d6dc7af21 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 10 Apr 2025 13:55:35 +0800 Subject: [PATCH 2/9] fix --- .../plans/commands/info/BaseViewInfo.java | 8 +++-- .../create_view_nereids_fix_test.groovy | 29 +++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java index d7ad12c1898be8..f4fc9ecf18910d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java @@ -168,7 +168,8 @@ protected void createFinalCols(List outputs) throws org.apache.doris.commo for (Slot output : outputs) { DataType dataType = TypeCoercionUtils.replaceSpecifiedType(output.getDataType(), NullType.class, TinyIntType.INSTANCE); - Column column = new Column(output.getName(), dataType.toCatalogDataType(), output.nullable()); + Column column = new Column(output.getName(), dataType.toCatalogDataType(), + !output.isColumnFromTable() || output.nullable()); finalCols.add(column); } } else { @@ -176,10 +177,11 @@ protected void createFinalCols(List outputs) throws org.apache.doris.commo ErrorReport.reportAnalysisException(ErrorCode.ERR_VIEW_WRONG_LIST); } for (int i = 0; i < simpleColumnDefinitions.size(); ++i) { - DataType dataType = TypeCoercionUtils.replaceSpecifiedType(outputs.get(i).getDataType(), NullType.class, + Slot output = outputs.get(i); + DataType dataType = TypeCoercionUtils.replaceSpecifiedType(output.getDataType(), NullType.class, TinyIntType.INSTANCE); Column column = new Column(simpleColumnDefinitions.get(i).getName(), - dataType.toCatalogDataType(), outputs.get(i).nullable()); + dataType.toCatalogDataType(), !output.isColumnFromTable() || output.nullable()); column.setComment(simpleColumnDefinitions.get(i).getComment()); finalCols.add(column); } diff --git a/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy b/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy new file mode 100644 index 00000000000000..0884063a18a603 --- /dev/null +++ b/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("create_view_nereids_fix_test") { + sql "drop view if exists test_null" + sql "CREATE VIEW test_null COMMENT '测试null类型' AS SELECT NULL AS `col1`; " + def res = sql "desc test_null" + mustContain(res[0][1], "tinyint") + + sql "drop view if exists test_null_array" + sql "CREATE VIEW test_null_array COMMENT '测试null类型' AS SELECT [NULL, NULL] AS `col1`; " + def res2 = sql "desc test_null_array" + mustContain(res2[0][1], "array") + mustContain(res2[0][2], "Yes") +} \ No newline at end of file From e97cf80d518a9cd6a0a799ac27e74d45c56f4b7f Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Tue, 15 Apr 2025 20:19:55 +0800 Subject: [PATCH 3/9] add jdbc exector code --- .../org/apache/doris/common/jni/vec/VectorColumn.java | 8 ++++---- .../create_view_nereids_fix_test.groovy | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java index 940a6ce4f16481..d2f2e42f866d73 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java @@ -1236,7 +1236,7 @@ public void appendArray(List[] batch, boolean isNullable) { } } } - childColumns[0].appendObjectColumn(nested, isNullable); + childColumns[0].appendObjectColumn(nested, true); } public ArrayList getArray(int rowId) { @@ -1300,8 +1300,8 @@ public void appendMap(Map[] batch, boolean isNullable) { } } } - childColumns[0].appendObjectColumn(keys, isNullable); - childColumns[1].appendObjectColumn(values, isNullable); + childColumns[0].appendObjectColumn(keys, true); + childColumns[1].appendObjectColumn(values, true); } public HashMap getMap(int rowId) { @@ -1363,7 +1363,7 @@ public void appendStruct(Map[] batch, boolean isNullable) { appendIndex++; } for (int j = 0; j < childColumns.length; ++j) { - childColumns[j].appendObjectColumn(columnData[j], isNullable); + childColumns[j].appendObjectColumn(columnData[j], true); } } diff --git a/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy b/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy index 0884063a18a603..3c4bd0fdd1bc54 100644 --- a/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy +++ b/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("create_view_nereids_fix_test") { +suite("create_view_nereids_fix_null") { sql "drop view if exists test_null" sql "CREATE VIEW test_null COMMENT '测试null类型' AS SELECT NULL AS `col1`; " def res = sql "desc test_null" @@ -25,5 +25,5 @@ suite("create_view_nereids_fix_test") { sql "CREATE VIEW test_null_array COMMENT '测试null类型' AS SELECT [NULL, NULL] AS `col1`; " def res2 = sql "desc test_null_array" mustContain(res2[0][1], "array") - mustContain(res2[0][2], "Yes") + mustContain(res2[0][2], "No") } \ No newline at end of file From 549b5f677fa08df41de6846537a6b4730130700e Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 17 Apr 2025 17:16:38 +0800 Subject: [PATCH 4/9] remove nullable change --- .../nereids/trees/plans/commands/info/BaseViewInfo.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java index f4fc9ecf18910d..af2f2378612bf9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java @@ -168,8 +168,7 @@ protected void createFinalCols(List outputs) throws org.apache.doris.commo for (Slot output : outputs) { DataType dataType = TypeCoercionUtils.replaceSpecifiedType(output.getDataType(), NullType.class, TinyIntType.INSTANCE); - Column column = new Column(output.getName(), dataType.toCatalogDataType(), - !output.isColumnFromTable() || output.nullable()); + Column column = new Column(output.getName(), dataType.toCatalogDataType(), output.nullable()); finalCols.add(column); } } else { @@ -181,7 +180,7 @@ protected void createFinalCols(List outputs) throws org.apache.doris.commo DataType dataType = TypeCoercionUtils.replaceSpecifiedType(output.getDataType(), NullType.class, TinyIntType.INSTANCE); Column column = new Column(simpleColumnDefinitions.get(i).getName(), - dataType.toCatalogDataType(), !output.isColumnFromTable() || output.nullable()); + dataType.toCatalogDataType(), output.nullable()); column.setComment(simpleColumnDefinitions.get(i).getComment()); finalCols.add(column); } From e8ba2bdfde70052dd23e8418a3398aadc7105efd Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 14 May 2025 18:32:48 +0800 Subject: [PATCH 5/9] add jdbc test --- .../create_view_nereids_fix_test.groovy | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy b/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy index 3c4bd0fdd1bc54..71f2175f5cd652 100644 --- a/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy +++ b/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy @@ -26,4 +26,27 @@ suite("create_view_nereids_fix_null") { def res2 = sql "desc test_null_array" mustContain(res2[0][1], "array") mustContain(res2[0][2], "No") + + String s3_endpoint = getS3Endpoint() + logger.info("s3_endpoint: " + s3_endpoint) + String bucket = getS3BucketName() + logger.info("bucket: " + bucket) + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-java-8.0.25.jar" + String dbname = context.config.getDbNameByFile(context.file) + String jdbcUrl = context.config.jdbcUrl + sql "drop catalog if exists jdbc_catalog" + sql """ + CREATE CATALOG jdbc_catalog PROPERTIES ( + "type"="jdbc", + "user"="root", + "password"="123456", + "jdbc_url"="${jdbcUrl}", + "driver_url"="${driver_url}", + "driver_class"="com.mysql.cj.jdbc.Driver" + ); + """ + sql "switch jdbc_catalog" + sql "use ${dbname}" + qt_test_null "select * from test_null" + qt_test_null_array "select * from test_null_array" } \ No newline at end of file From 3905759849ff876d0ede1b375a6b9f7ec0773f7b Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 14 May 2025 20:59:22 +0800 Subject: [PATCH 6/9] fix --- .../create_view_nereids_fix_test.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy b/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy index 71f2175f5cd652..0ec2c4bfe245eb 100644 --- a/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy +++ b/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy @@ -34,12 +34,16 @@ suite("create_view_nereids_fix_null") { String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-java-8.0.25.jar" String dbname = context.config.getDbNameByFile(context.file) String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + logger.info("jdbcUser: " + jdbcUser) + String jdbcPassword = context.config.jdbcPassword + logger.info("jdbcPassword: " + jdbcPassword) sql "drop catalog if exists jdbc_catalog" sql """ CREATE CATALOG jdbc_catalog PROPERTIES ( "type"="jdbc", - "user"="root", - "password"="123456", + "user"="${jdbcUser}", + "password"="${jdbcPassword}", "jdbc_url"="${jdbcUrl}", "driver_url"="${driver_url}", "driver_class"="com.mysql.cj.jdbc.Driver" From c40c691227f8e8967e6f1ea58dd245c7e44737d0 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 15 May 2025 10:05:20 +0800 Subject: [PATCH 7/9] add --- .../create_view_nereids/create_view_nereids_fix_test.out | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 regression-test/data/ddl_p0/create_view_nereids/create_view_nereids_fix_test.out diff --git a/regression-test/data/ddl_p0/create_view_nereids/create_view_nereids_fix_test.out b/regression-test/data/ddl_p0/create_view_nereids/create_view_nereids_fix_test.out new file mode 100644 index 00000000000000..62698117493550 --- /dev/null +++ b/regression-test/data/ddl_p0/create_view_nereids/create_view_nereids_fix_test.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test_null -- +\N + +-- !test_null_array -- +[null, null] + From 0f008706ffdce15fd8e5f0f88261a45c43256bf0 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 15 May 2025 12:04:49 +0800 Subject: [PATCH 8/9] fix --- ...view_nereids_fix_test.out => create_view_nereids_fix_null.out} | 0 ...ereids_fix_test.groovy => create_view_nereids_fix_null.groovy} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename regression-test/data/ddl_p0/create_view_nereids/{create_view_nereids_fix_test.out => create_view_nereids_fix_null.out} (100%) rename regression-test/suites/ddl_p0/create_view_nereids/{create_view_nereids_fix_test.groovy => create_view_nereids_fix_null.groovy} (100%) diff --git a/regression-test/data/ddl_p0/create_view_nereids/create_view_nereids_fix_test.out b/regression-test/data/ddl_p0/create_view_nereids/create_view_nereids_fix_null.out similarity index 100% rename from regression-test/data/ddl_p0/create_view_nereids/create_view_nereids_fix_test.out rename to regression-test/data/ddl_p0/create_view_nereids/create_view_nereids_fix_null.out diff --git a/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy b/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_null.groovy similarity index 100% rename from regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_test.groovy rename to regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_null.groovy From f406a93c51d99564a59aac9a54802e2d2b47998b Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 15 May 2025 19:41:53 +0800 Subject: [PATCH 9/9] fix --- .../create_view_nereids_fix_null.groovy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_null.groovy b/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_null.groovy index 0ec2c4bfe245eb..4ea07083be45cd 100644 --- a/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_null.groovy +++ b/regression-test/suites/ddl_p0/create_view_nereids/create_view_nereids_fix_null.groovy @@ -38,9 +38,9 @@ suite("create_view_nereids_fix_null") { logger.info("jdbcUser: " + jdbcUser) String jdbcPassword = context.config.jdbcPassword logger.info("jdbcPassword: " + jdbcPassword) - sql "drop catalog if exists jdbc_catalog" + sql "drop catalog if exists create_view_nereids_fix_null_catalog" sql """ - CREATE CATALOG jdbc_catalog PROPERTIES ( + CREATE CATALOG create_view_nereids_fix_null_catalog PROPERTIES ( "type"="jdbc", "user"="${jdbcUser}", "password"="${jdbcPassword}", @@ -49,8 +49,9 @@ suite("create_view_nereids_fix_null") { "driver_class"="com.mysql.cj.jdbc.Driver" ); """ - sql "switch jdbc_catalog" + sql "switch create_view_nereids_fix_null_catalog" sql "use ${dbname}" qt_test_null "select * from test_null" qt_test_null_array "select * from test_null_array" + sql "drop catalog create_view_nereids_fix_null_catalog;" } \ No newline at end of file