From 0fc5b57844534b839036771244805238566e2a21 Mon Sep 17 00:00:00 2001 From: linchen Date: Tue, 6 Sep 2022 17:48:36 +0800 Subject: [PATCH 1/4] [docs][1056]fix es7 support data type --- .../elasticsearch/es7-sink.md" | 8 ++++---- .../elasticsearch/es7-source.md" | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git "a/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/elasticsearch/es7-sink.md" "b/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/elasticsearch/es7-sink.md" index 0969c78da6..368fb4fc17 100644 --- "a/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/elasticsearch/es7-sink.md" +++ "b/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/elasticsearch/es7-sink.md" @@ -277,10 +277,10 @@ Elasticsearch 7.x # 五、数据类型 -|是否支持 | 类型名称 | -| --- | --- | -| 支持 |INTEGER,FLOAT,DOUBLE,LONG,DATE,TEXT,BYTE,BINARY,OBJECT,NESTED| -| 不支持 | IP,GEO_POINT,GEO_SHAPE| +|是否支持 | 类型名称 | +| --- |-----------------------------------------------------------------------------| +| 支持 | BOOLEAN,SHORT,INTEGER,FLOAT,DOUBLE,LONG,DATE,TEXT,BYTE,BINARY,OBJECT,NESTED | +| 不支持 | IP,GEO_POINT,GEO_SHAPE | # 六、脚本示例 见项目内`chunjun-examples`文件夹。 diff --git "a/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/elasticsearch/es7-source.md" "b/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/elasticsearch/es7-source.md" index af257bc1a8..337e885731 100644 --- "a/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/elasticsearch/es7-source.md" +++ "b/docs_zh/ChunJun\350\277\236\346\216\245\345\231\250/elasticsearch/es7-source.md" @@ -250,10 +250,10 @@ Elasticsearch 7.x ## 五、数据类型 -|是否支持 | 类型名称 | -| --- | --- | -| 支持 |INTEGER,FLOAT,DOUBLE,LONG,DATE,TEXT,BYTE,BINARY,OBJECT,NESTED| -| 不支持 | IP,GEO_POINT,GEO_SHAPE| +|是否支持 | 类型名称 | +| --- |-----------------------------------------------------------------------------| +| 支持 | BOOLEAN,SHORT,INTEGER,FLOAT,DOUBLE,LONG,DATE,TEXT,BYTE,BINARY,OBJECT,NESTED | +| 不支持 | IP,GEO_POINT,GEO_SHAPE | ## 六、脚本示例 见项目内`chunjun-examples`文件夹。 From 3876cec8b3b4221e91c232c45de3475060b72fa4 Mon Sep 17 00:00:00 2001 From: linchen Date: Thu, 27 Oct 2022 20:01:14 +0800 Subject: [PATCH 2/4] [hotfix-1360][es7]fix support keyword type --- .../connector/elasticsearch/ElasticsearchRawTypeMapper.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/chunjun-connectors/chunjun-connector-elasticsearch-base/src/main/java/com/dtstack/chunjun/connector/elasticsearch/ElasticsearchRawTypeMapper.java b/chunjun-connectors/chunjun-connector-elasticsearch-base/src/main/java/com/dtstack/chunjun/connector/elasticsearch/ElasticsearchRawTypeMapper.java index cd905a4b20..d78edd88f4 100644 --- a/chunjun-connectors/chunjun-connector-elasticsearch-base/src/main/java/com/dtstack/chunjun/connector/elasticsearch/ElasticsearchRawTypeMapper.java +++ b/chunjun-connectors/chunjun-connector-elasticsearch-base/src/main/java/com/dtstack/chunjun/connector/elasticsearch/ElasticsearchRawTypeMapper.java @@ -54,9 +54,10 @@ public static DataType apply(String type) { case "DOUBLE": return DataTypes.DOUBLE(); case "TEXT": - return DataTypes.STRING(); + case "STRING": case "BINARY": - return DataTypes.BYTES(); + case "KEYWORD": + return DataTypes.STRING(); case "DATE": return DataTypes.TIMESTAMP(); case "OBJECT": From 0219e699e55adb34812356105566428b85935a5f Mon Sep 17 00:00:00 2001 From: linchen Date: Sat, 5 Nov 2022 14:40:50 +0800 Subject: [PATCH 3/4] [bugfix][ftp]fix ftp connetor sql job error --- .../ftp/table/FtpDynamicTableFactory.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/table/FtpDynamicTableFactory.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/table/FtpDynamicTableFactory.java index dd165dd458..1de9bd4f63 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/table/FtpDynamicTableFactory.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/table/FtpDynamicTableFactory.java @@ -18,6 +18,7 @@ package com.dtstack.chunjun.connector.ftp.table; +import com.dtstack.chunjun.conf.FieldConf; import com.dtstack.chunjun.connector.ftp.conf.FtpConfig; import com.dtstack.chunjun.connector.ftp.options.FtpOptions; import com.dtstack.chunjun.connector.ftp.sink.FtpDynamicTableSink; @@ -38,9 +39,12 @@ import org.apache.flink.table.factories.DynamicTableSourceFactory; import org.apache.flink.table.factories.FactoryUtil; import org.apache.flink.table.factories.SerializationFormatFactory; +import org.apache.flink.table.types.logical.RowType; import org.apache.flink.table.utils.TableSchemaUtils; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Set; /** @@ -94,7 +98,19 @@ public DynamicTableSource createDynamicTableSource(Context context) { helper.discoverDecodingFormat( DeserializationFormatFactory.class, FtpOptions.FORMAT); + RowType rowType = (RowType) physicalSchema.toRowDataType().getLogicalType(); FtpConfig ftpConfig = getFtpConfByOptions(config); + String[] fieldNames = physicalSchema.getFieldNames(); + List columnList = new ArrayList<>(fieldNames.length); + for (int i = 0; i < fieldNames.length; i++) { + FieldConf field = new FieldConf(); + field.setName(fieldNames[i]); + field.setType(rowType.getTypeAt(i).asSummaryString()); + field.setIndex(i); + columnList.add(field); + } + ftpConfig.setColumn(columnList); + return new FtpDynamicTableSource(physicalSchema, ftpConfig, decodingFormat); } From b1ad958c4c41d709a0b64f04579e68b3594beaad Mon Sep 17 00:00:00 2001 From: linchen Date: Sat, 5 Nov 2022 14:50:26 +0800 Subject: [PATCH 4/4] [bugfix][ftp]fix ftp sql job error adn mvn spotless --- .../chunjun/connector/ftp/table/FtpDynamicTableFactory.java | 1 - 1 file changed, 1 deletion(-) diff --git a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/table/FtpDynamicTableFactory.java b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/table/FtpDynamicTableFactory.java index 1de9bd4f63..5f42828d42 100644 --- a/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/table/FtpDynamicTableFactory.java +++ b/chunjun-connectors/chunjun-connector-ftp/src/main/java/com/dtstack/chunjun/connector/ftp/table/FtpDynamicTableFactory.java @@ -111,7 +111,6 @@ public DynamicTableSource createDynamicTableSource(Context context) { } ftpConfig.setColumn(columnList); - return new FtpDynamicTableSource(physicalSchema, ftpConfig, decodingFormat); }