From e4656c97f31e252a3f7fee9976ce08091dc4d4c6 Mon Sep 17 00:00:00 2001 From: liuliu Date: Mon, 23 May 2022 15:42:37 +0800 Subject: [PATCH] [feat-861][protobuf]add protobuf format --- .../chunjun-connector-hbase-1.4/pom.xml | 4 + .../chunjun-connector-hive/pom.xml | 6 + .../chunjun-connector-inceptor/pom.xml | 4 + .../chunjun-connector-kafka/pom.xml | 158 +- .../org.apache.flink.table.factories.Factory | 2 - .../chunjun/constants/ConstantValue.java | 2 + .../util/ConnectorNameConvertUtil.java | 1 + .../flink/table/factories/FactoryUtil.java | 9 + chunjun-examples/sql/kafka/proto_proto.sql | 133 + chunjun-formats/chunjun-protobuf/pom.xml | 91 + .../format/protobuf/PbFormatFactory.java | 111 + .../format/protobuf/PbFormatOptions.java | 35 + .../chunjun/format/protobuf/PbFormatType.java | 113 + .../format/protobuf/PbMessageAdaptor.java | 96 + .../protobuf/deserialize/PbMessageGetter.java | 131 + .../format/protobuf/deserialize/PbParser.java | 53 + .../PbRowDataDeserializationSchema.java | 95 + .../deserialize/PbToRowDataPbConverter.java | 230 ++ .../protobuf/serialize/PbMessageSetter.java | 102 + .../PbRowDataSerializationSchema.java | 82 + .../serialize/RowDataToPbConverter.java | 328 ++ .../format/protobuf/util/FormatCheckUtil.java | 217 ++ .../format/protobuf/util/PbReflectUtil.java | 131 + .../protobuf/test/MessageTestOuterClass.java | 2950 +++++++++++++++++ .../protobuf/test/PbFormatFactoryTest.java | 143 + chunjun-formats/pom.xml | 4 + .../dtstack/chunjun/local/test/LocalTest.java | 16 +- 27 files changed, 5164 insertions(+), 83 deletions(-) create mode 100644 chunjun-examples/sql/kafka/proto_proto.sql create mode 100644 chunjun-formats/chunjun-protobuf/pom.xml create mode 100644 chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbFormatFactory.java create mode 100644 chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbFormatOptions.java create mode 100644 chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbFormatType.java create mode 100644 chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbMessageAdaptor.java create mode 100644 chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbMessageGetter.java create mode 100644 chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbParser.java create mode 100644 chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbRowDataDeserializationSchema.java create mode 100644 chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbToRowDataPbConverter.java create mode 100644 chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/serialize/PbMessageSetter.java create mode 100644 chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/serialize/PbRowDataSerializationSchema.java create mode 100644 chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/serialize/RowDataToPbConverter.java create mode 100644 chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/util/FormatCheckUtil.java create mode 100644 chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/util/PbReflectUtil.java create mode 100644 chunjun-formats/chunjun-protobuf/src/test/java/com/dtstack/chunjun/format/protobuf/test/MessageTestOuterClass.java create mode 100644 chunjun-formats/chunjun-protobuf/src/test/java/com/dtstack/chunjun/format/protobuf/test/PbFormatFactoryTest.java diff --git a/chunjun-connectors/chunjun-connector-hbase-1.4/pom.xml b/chunjun-connectors/chunjun-connector-hbase-1.4/pom.xml index 1e77e74b03..96c9b539c0 100644 --- a/chunjun-connectors/chunjun-connector-hbase-1.4/pom.xml +++ b/chunjun-connectors/chunjun-connector-hbase-1.4/pom.xml @@ -125,6 +125,10 @@ org.apache.http com.dtstack.chunjun.connector.hbase14.org.apache.http + + com.google.protobuf + com.dtstack.chunjun.connector.hbase14.com.google.protobuf + diff --git a/chunjun-connectors/chunjun-connector-hive/pom.xml b/chunjun-connectors/chunjun-connector-hive/pom.xml index 71e641fb64..5d609cd6ec 100644 --- a/chunjun-connectors/chunjun-connector-hive/pom.xml +++ b/chunjun-connectors/chunjun-connector-hive/pom.xml @@ -282,6 +282,12 @@ + + + com.google.protobuf + com.dtstack.chunjun.connector.hive.com.google.protobuf + + diff --git a/chunjun-connectors/chunjun-connector-inceptor/pom.xml b/chunjun-connectors/chunjun-connector-inceptor/pom.xml index fd841a2549..02d0f7316f 100644 --- a/chunjun-connectors/chunjun-connector-inceptor/pom.xml +++ b/chunjun-connectors/chunjun-connector-inceptor/pom.xml @@ -82,6 +82,10 @@ com.esotericsoftware.kryo shade.inceptor.com.esotericsoftware.kryo + + com.google.protobuf + shade.inceptor.com.google.protobuf + - - chunjun-connectors - com.dtstack.chunjun - 1.12-SNAPSHOT - - 4.0.0 + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + + chunjun-connectors + com.dtstack.chunjun + 1.12-SNAPSHOT + + 4.0.0 - chunjun-connector-kafka + chunjun-connector-kafka ChunJun : Connectors : Kafka - - - org.apache.flink - flink-connector-kafka_${scala.binary.version} - ${flink.version} - - - flink-core - org.apache.flink - - - + + + org.apache.flink + flink-connector-kafka_${scala.binary.version} + ${flink.version} + + + flink-core + org.apache.flink + + + com.pingcap.ticdc.cdc ticdc-decoder 5.2.0-SNAPSHOT - - - com.alibaba - fastjson - - + + + com.alibaba + fastjson + + - - com.alibaba - fastjson - 1.2.79 - + + com.alibaba + fastjson + 1.2.79 + org.apache.flink @@ -54,13 +54,13 @@ avro 1.10.0 - + - - - - org.apache.maven.plugins - maven-shade-plugin + + + + org.apache.maven.plugins + maven-shade-plugin 3.2.4 @@ -70,11 +70,11 @@ false - - - com.alibaba.fastjson - com.shade.alibaba.fastjson - + + + com.alibaba.fastjson + com.shade.alibaba.fastjson + org.apache.avro @@ -88,7 +88,11 @@ org.apache.commons.compress org.apache.flink.avro.shaded.org.apache.commons.compress - + + com.google.protobuf + com.dtstack.chunjun.connector.kafka.shaded.com.google.protobuf + + org.slf4j @@ -116,28 +120,50 @@ META-INF/*.RSA + + + org.apache.flink:flink-avro + + META-INF/services/* + + + + org.apache.flink:flink-connector-kafka + + META-INF/services/* + + + + org.glassfish.jersey.core:jersey-common + + META-INF/services/* + + + + + - - maven-antrun-plugin - - - copy-resources - - package - - run - - - - + maven-antrun-plugin + + + copy-resources + + package + + run + + + + - + - - - - - - - + + + + + + + diff --git a/chunjun-connectors/chunjun-connector-kafka/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/chunjun-connectors/chunjun-connector-kafka/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory index 63bf1d6180..d840aac69f 100644 --- a/chunjun-connectors/chunjun-connector-kafka/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory +++ b/chunjun-connectors/chunjun-connector-kafka/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -15,5 +15,3 @@ com.dtstack.chunjun.connector.kafka.table.KafkaDynamicTableFactory com.dtstack.chunjun.connector.upsertkafka.table.UpsertKafkaDynamicTableFactory -org.apache.flink.formats.avro.registry.confluent.RegistryAvroFormatFactory -org.apache.flink.formats.avro.registry.confluent.debezium.DebeziumAvroFormatFactory diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/constants/ConstantValue.java b/chunjun-core/src/main/java/com/dtstack/chunjun/constants/ConstantValue.java index b32bd16b33..e7b5eb51d4 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/constants/ConstantValue.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/constants/ConstantValue.java @@ -72,6 +72,8 @@ public class ConstantValue { public static final String CONNECTOR_DIR_NAME = "connector"; + public static final String FORMAT_DIR_NAME = "formats"; + public static final String DIRTY_DATA_DIR_NAME = "dirty-data-collector"; public static final String RESTORE_DIR_NAME = "restore-plugins"; diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/util/ConnectorNameConvertUtil.java b/chunjun-core/src/main/java/com/dtstack/chunjun/util/ConnectorNameConvertUtil.java index 1a0a0eb31a..ca3677e494 100644 --- a/chunjun-core/src/main/java/com/dtstack/chunjun/util/ConnectorNameConvertUtil.java +++ b/chunjun-core/src/main/java/com/dtstack/chunjun/util/ConnectorNameConvertUtil.java @@ -40,6 +40,7 @@ public class ConnectorNameConvertUtil { connectorNameMap.put("adbpostgresql", new Tuple2<>("postgresql", "postgresql")); connectorNameMap.put("dorisbatch", new Tuple2<>("doris", "doris")); connectorNameMap.put("gbase", new Tuple2<>("gBase", "gBase")); + connectorNameMap.put("protobuf", new Tuple2<>("pbformat", "pbformat")); } public static String convertClassPrefix(String originName) { diff --git a/chunjun-core/src/main/java/org/apache/flink/table/factories/FactoryUtil.java b/chunjun-core/src/main/java/org/apache/flink/table/factories/FactoryUtil.java index dceeef0132..748471fe08 100644 --- a/chunjun-core/src/main/java/org/apache/flink/table/factories/FactoryUtil.java +++ b/chunjun-core/src/main/java/org/apache/flink/table/factories/FactoryUtil.java @@ -49,6 +49,7 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Locale; import java.util.Optional; import java.util.ServiceConfigurationError; import java.util.ServiceLoader; @@ -654,6 +655,14 @@ private Optional discoverOptionalFormatFactory( if (identifier == null) { return Optional.empty(); } + if (identifier.toLowerCase(Locale.ROOT).endsWith("-x")) { + String s = identifier.substring(0, identifier.length() - 2); + FactoryHelper factoryHelper = factoryHelperThreadLocal.get(); + if (factoryHelper != null) { + factoryHelper.registerCachedFile( + s, context.getClassLoader(), ConstantValue.FORMAT_DIR_NAME); + } + } final F factory = discoverFactory(context.getClassLoader(), formatFactoryClass, identifier); String formatPrefix = formatPrefix(factory, formatOption); diff --git a/chunjun-examples/sql/kafka/proto_proto.sql b/chunjun-examples/sql/kafka/proto_proto.sql new file mode 100644 index 0000000000..dac088b590 --- /dev/null +++ b/chunjun-examples/sql/kafka/proto_proto.sql @@ -0,0 +1,133 @@ +--complex with flink original kafka-connector +CREATE TABLE reader ( + GroupInfo MAP + ,Messages ARRAY< + ROW< + TAGNAME VARCHAR , + TagValue ROW< + `Value` ROW< + ValueCase INTEGER, + ValueBool BOOlEAN, + ArrayBool ROW<`Values` ARRAY>, + ValueInt32 INTEGER, + ArrayInt32 ROW<`Values` ARRAY>, + ValueUint32 INTEGER, + ArrayUint32 ROW<`Values` ARRAY>, + ValueInt64 BIGINT, + ArrayInt64 ROW<`Values` ARRAY>, + ValueUint64 BIGINT, + ArrayUint64 ROW<`Values` ARRAY>, + ValueFloat FLOAT, + ArrayFloat ROW<`Values` ARRAY>, + ValueDouble DOUBLE, + ArrayDouble ROW<`Values` ARRAY>, + ValueString STRING, + ArrayString ROW<`Values` ARRAY>, + ValueBytes BINARY, + ValueTimestamp BIGINT + >, + boolx BOOLEAN, + Value2 ROW< + ValueCase INTEGER , + Value2Value BOOLEAN, + ArrayBool2 ROW<`Values` ARRAY>>, + booly BOOLEAN + >, + UaDataType INTEGER, + Quality BOOLEAN, + `Timestamp` BIGINT, + TagInfos MAP, + ExValues MAP + > + > + + + +-- , `topic` STRING METADATA VIRTUAL -- from Kafka connector +-- , `leader-epoch` int METADATA VIRTUAL -- from Kafka connector +-- , `offset` BIGINT METADATA VIRTUAL -- from Kafka connector +-- , ts TIMESTAMP(3) METADATA FROM 'timestamp' -- from Kafka connector +-- , `timestamp-type` STRING METADATA VIRTUAL -- from Kafka connector +-- , partition_id BIGINT METADATA FROM 'partition' VIRTUAL -- from Kafka connector + +) WITH ( + 'connector' = 'kafka-x' + ,'topic' = 'liuliu_proto_source' + ,'properties.bootstrap.servers' = 'flink01:9092' + ,'properties.group.id' = 'luna_g' + ,'scan.startup.mode' = 'earliest-offset' + ,'format' = 'protobuf-x' +-- ,'protobuf-x.class.name'='ZPMC.Message.MessageGroupOuterClass' +-- ,'protobuf-x.message.name'='MessageGroup' + ,'protobuf-x.message-class-name' = 'ZPMC.Message.MessageGroupOuterClass$MessageGroup' + ,'scan.parallelism' = '1' + ); + +CREATE TABLE writer ( + GroupInfo MAP + ,Messages ARRAY< + ROW< + TAGNAME VARCHAR , + TagValue ROW< + `Value` ROW< + ValueCase INTEGER, + ValueBool BOOlEAN, + ArrayBool ROW<`Values` ARRAY>, + ValueInt32 INTEGER, + ArrayInt32 ROW<`Values` ARRAY>, + ValueUint32 INTEGER, + ArrayUint32 ROW<`Values` ARRAY>, + ValueInt64 BIGINT, + ArrayInt64 ROW<`Values` ARRAY>, + ValueUint64 BIGINT, + ArrayUint64 ROW<`Values` ARRAY>, + ValueFloat FLOAT, + ArrayFloat ROW<`Values` ARRAY>, + ValueDouble DOUBLE, + ArrayDouble ROW<`Values` ARRAY>, + ValueString STRING, + ArrayString ROW<`Values` ARRAY>, + ValueBytes BINARY, + ValueTimestamp BIGINT + >, + boolx BOOLEAN, + Value2 ROW< + ValueCase INTEGER , + Value2Value BOOLEAN, + ArrayBool2 ROW<`Values` ARRAY>>, + booly BOOLEAN + >, + UaDataType INTEGER, + Quality BOOLEAN, + `Timestamp` BIGINT, + TagInfos MAP, + ExValues MAP + > + > + + +-- +-- , `topic` STRING METADATA VIRTUAL -- from Kafka connector +-- , `leader-epoch` int METADATA VIRTUAL -- from Kafka connector +-- , `offset` BIGINT METADATA VIRTUAL -- from Kafka connector +-- , ts TIMESTAMP(3) METADATA FROM 'timestamp' -- from Kafka connector +-- , `timestamp-type` STRING METADATA VIRTUAL -- from Kafka connector +-- , partition_id BIGINT METADATA FROM 'partition' VIRTUAL -- from Kafka connector + +) WITH ( + 'connector' = 'kafka-x' + ,'topic' = 'liuliu_proto_sink' + ,'properties.bootstrap.servers' = 'flink01:9092' + ,'properties.group.id' = 'luna_g' + ,'scan.startup.mode' = 'earliest-offset' + ,'format' = 'protobuf-x' +-- ,'protobuf-x.class.name'='ZPMC.Message.MessageGroupOuterClass' +-- ,'protobuf-x.message.name'='MessageGroup' + ,'protobuf-x.message-class-name' = 'ZPMC.Message.MessageGroupOuterClass$MessageGroup' + ,'scan.parallelism' = '1' + ); + +INSERT INTO writer +SELECT * +from reader + diff --git a/chunjun-formats/chunjun-protobuf/pom.xml b/chunjun-formats/chunjun-protobuf/pom.xml new file mode 100644 index 0000000000..d1cf68cc89 --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/pom.xml @@ -0,0 +1,91 @@ + + + + chunjun-formats + com.dtstack.chunjun + 1.12-SNAPSHOT + + 4.0.0 + + flinkx-protobuf + + + 8 + 8 + + + + + com.google.protobuf + protobuf-java + 3.20.1-rc-1 + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.0 + + + package + + shade + + + + + org.slf4j:slf4j-api + log4j:log4j + ch.qos.logback:* + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + maven-antrun-plugin + + + copy-resources + + package + + run + + + + + + + + + + + + + + + + + diff --git a/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbFormatFactory.java b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbFormatFactory.java new file mode 100644 index 0000000000..4d3cecc17b --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbFormatFactory.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.format.protobuf; + +import com.dtstack.chunjun.format.protobuf.deserialize.PbRowDataDeserializationSchema; +import com.dtstack.chunjun.format.protobuf.serialize.PbRowDataSerializationSchema; + +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.connector.format.DecodingFormat; +import org.apache.flink.table.connector.format.EncodingFormat; +import org.apache.flink.table.connector.sink.DynamicTableSink; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.factories.DeserializationFormatFactory; +import org.apache.flink.table.factories.DynamicTableFactory; +import org.apache.flink.table.factories.FactoryUtil; +import org.apache.flink.table.factories.SerializationFormatFactory; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.RowType; + +import java.util.HashSet; +import java.util.Set; + +/** @author liuliu 2022/4/8 */ +public class PbFormatFactory implements DeserializationFormatFactory, SerializationFormatFactory { + + public static final String IDENTIFIER = "protobuf-x"; + + @Override + public DecodingFormat> createDecodingFormat( + DynamicTableFactory.Context context, ReadableConfig formatOptions) { + FactoryUtil.validateFactoryOptions(this, formatOptions); + + String messageClassName = formatOptions.get(PbFormatOptions.MESSAGE_CLASS_NAME); + return new DecodingFormat>() { + @Override + public DeserializationSchema createRuntimeDecoder( + DynamicTableSource.Context context, DataType physicalDataType) { + final TypeInformation rowDataTypeInfo = + context.createTypeInformation(physicalDataType); + final RowType rowType = (RowType) physicalDataType.getLogicalType(); + return new PbRowDataDeserializationSchema( + rowType, rowDataTypeInfo, messageClassName); + } + + @Override + public ChangelogMode getChangelogMode() { + return ChangelogMode.insertOnly(); + } + }; + } + + @Override + public EncodingFormat> createEncodingFormat( + DynamicTableFactory.Context context, ReadableConfig formatOptions) { + FactoryUtil.validateFactoryOptions(this, formatOptions); + + String messageClassName = formatOptions.get(PbFormatOptions.MESSAGE_CLASS_NAME); + return new EncodingFormat>() { + @Override + public SerializationSchema createRuntimeEncoder( + DynamicTableSink.Context context, DataType physicalDataType) { + final RowType rowType = (RowType) physicalDataType.getLogicalType(); + return new PbRowDataSerializationSchema(rowType, messageClassName); + } + + @Override + public ChangelogMode getChangelogMode() { + return ChangelogMode.insertOnly(); + } + }; + } + + @Override + public String factoryIdentifier() { + return IDENTIFIER; + } + + @Override + public Set> requiredOptions() { + Set> options = new HashSet<>(); + options.add(PbFormatOptions.MESSAGE_CLASS_NAME); + return options; + } + + @Override + public Set> optionalOptions() { + return new HashSet<>(); + } +} diff --git a/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbFormatOptions.java b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbFormatOptions.java new file mode 100644 index 0000000000..5c9f93c994 --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbFormatOptions.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.format.protobuf; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; + +/** @author liuliu 2022/4/18 */ +public class PbFormatOptions { + private PbFormatOptions() {} + + public static final ConfigOption MESSAGE_CLASS_NAME = + ConfigOptions.key("message-class-name") + .stringType() + .noDefaultValue() + .withDescription( + "Required option to specify the full name of protobuf message class. The protobuf class " + + "must be located in the classpath both in client and task side"); +} diff --git a/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbFormatType.java b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbFormatType.java new file mode 100644 index 0000000000..e3e12e2658 --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbFormatType.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.format.protobuf; + +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; +import com.dtstack.chunjun.throwable.UnsupportedTypeException; + +import com.google.protobuf.AbstractMessage; +import com.google.protobuf.Descriptors; + +/** @author liuliu 2022/4/11 */ +public enum PbFormatType { + MAP, + ENUM, + ARRAY, + MESSAGE, + BOOLEAN, + INT, + LONG, + FLOAT, + DOUBLE, + STRING, + BYTE_STRING; + + PbFormatType() {} + + public static PbFormatType getTypeByFieldDescriptor( + Descriptors.FieldDescriptor fieldDescriptor) { + if (fieldDescriptor.isMapField()) { + return MAP; + } else if (fieldDescriptor.isRepeated()) { + return ARRAY; + } else { + return getTypeByTypeName(fieldDescriptor.getJavaType().name()); + } + } + + public static PbFormatType getArrayInnerTypeByFieldDescriptor( + Descriptors.FieldDescriptor fieldDescriptor) { + return getTypeByTypeName(fieldDescriptor.getJavaType().name()); + } + + public static PbFormatType getProtoTypeForMapKey(Class clazz) { + switch (clazz.getName()) { + case "java.lang.String": + return STRING; + case "java.lang.Integer": + case "int": + return INT; + case "java.lang.Long": + case "long": + return LONG; + default: + throw new ChunJunRuntimeException( + String.format( + "Map key expect any scalar type except for floating point types and bytes,but it is %s", + clazz.getName())); + } + } + + public static PbFormatType getProtoTypeForMapValue(Class clazz) { + if (com.google.protobuf.ProtocolMessageEnum.class.isAssignableFrom(clazz)) { + return ENUM; + } else if (AbstractMessage.class.isAssignableFrom(clazz)) { + return MESSAGE; + } + switch (clazz.getName()) { + case "java.lang.String": + return STRING; + case "java.lang.Integer": + case "int": + return INT; + case "java.lang.Double": + case "double": + return DOUBLE; + case "java.lang.Long": + case "long": + return LONG; + case "java.lang.Float": + case "float": + return FLOAT; + case "com.google.protobuf.ByteString": + return BYTE_STRING; + default: + throw new UnsupportedOperationException(clazz.getName()); + } + } + + public static PbFormatType getTypeByTypeName(String typeName) { + for (PbFormatType protoType : PbFormatType.values()) { + if (typeName.equalsIgnoreCase(protoType.name())) { + return protoType; + } + } + throw new UnsupportedTypeException(typeName); + } +} diff --git a/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbMessageAdaptor.java b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbMessageAdaptor.java new file mode 100644 index 0000000000..ce328124d2 --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/PbMessageAdaptor.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.format.protobuf; + +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; + +import com.google.protobuf.AbstractMessage; +import com.google.protobuf.Descriptors; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.List; + +/** @author liuliu 2022/4/27 */ +public class PbMessageAdaptor { + protected final int[] normalToOneof; + protected final int[] oneOfLastIndex; + protected int size; + + public PbMessageAdaptor( + List fieldDescriptorList, + List oneofDescriptorList) { + size = fieldDescriptorList.size(); + oneOfLastIndex = new int[oneofDescriptorList.size()]; + normalToOneof = new int[fieldDescriptorList.size()]; + Arrays.fill(normalToOneof, -1); + for (int i = 0; i < oneofDescriptorList.size(); i++) { + Descriptors.OneofDescriptor oneofDescriptor = oneofDescriptorList.get(i); + for (Descriptors.FieldDescriptor fieldDescriptor : oneofDescriptor.getFields()) { + normalToOneof[fieldDescriptor.getIndex()] = i; + oneOfLastIndex[i] = fieldDescriptor.getIndex(); + size--; + } + size++; + } + } + + /** Check whether the protobuf field corresponding to index belongs to a oneof field */ + public boolean isOneOf(int index) { + return normalToOneof[index] != -1; + } + + /** + * If the protobuf field corresponding to index belongs to a oneof field, then return the index + * of the last field of the oneof + */ + public Integer getLastOneOfIndex(int index) { + int oneofIndex = normalToOneof[index]; + if (oneofIndex != -1) { + return oneOfLastIndex[oneofIndex]; + } + throw new ChunJunRuntimeException(""); + } + + public Method obtainMethod(String methodName, Class clazz) { + try { + Method method = + Arrays.stream(clazz.getDeclaredMethods()) + .filter(m -> m.getName().equalsIgnoreCase(methodName)) + .filter( + m -> + m.getParameterTypes().length == 0 + || !AbstractMessage.Builder.class + .isAssignableFrom( + m.getParameterTypes()[0])) + .toArray(Method[]::new)[0]; + method.setAccessible(true); + return method; + } catch (Exception e) { + throw new ChunJunRuntimeException( + String.format( + "failed to obtain getMethod[%s] from class[%s]", + methodName, clazz.getName())); + } + } + + public int getSize() { + return size; + } +} diff --git a/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbMessageGetter.java b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbMessageGetter.java new file mode 100644 index 0000000000..d5a5f61cfe --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbMessageGetter.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.format.protobuf.deserialize; + +import com.dtstack.chunjun.format.protobuf.PbMessageAdaptor; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; + +import com.google.protobuf.AbstractMessage; +import com.google.protobuf.Descriptors; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.List; + +/** + * Reflect all GET methods in the order of the fields in the protobuf message, then we can get the + * values of all the fields in the Message object by index. + * + * @author liuliu 2022/4/13 + */ +public class PbMessageGetter extends PbMessageAdaptor { + + private Method[] normalMethods; + private Method[] oneofCaseMethods; + private Method[] oneOfNumberMethods; + + public PbMessageGetter( + List fieldDescriptorList, + List oneofDescriptorList, + Class clazz) { + super(fieldDescriptorList, oneofDescriptorList); + initMethods(fieldDescriptorList, oneofDescriptorList, clazz); + } + + /** + * Get the value in the protocolMessage object by fieldDescriptor index + * + * @param object protocolMessage + * @param index index of fieldDescriptor + */ + public Object getByIndex(Object object, int index) { + try { + return normalMethods[index].invoke(object); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new ChunJunRuntimeException( + String.format( + "failed to get filed from flink type[%s],index[%s]", object, index), + e); + } + } + + /** + * If the index is a member of oneof,returns the corresponding index of the valid value in oneof + * + * @param object protocolMessage + * @param index index of fieldDescriptor + */ + public Integer getOneofCase(Object object, int index) + throws InvocationTargetException, IllegalAccessException { + int oneofIndex = normalToOneof[index]; + return (Integer) + oneOfNumberMethods[oneofIndex].invoke(oneofCaseMethods[oneofIndex].invoke(object)); + } + + public void initMethods( + List fieldDescriptorList, + List oneofDescriptorList, + Class clazz) { + this.normalMethods = + fieldDescriptorList.stream() + .map(fieldDescriptor -> obtainNormalGetMethod(fieldDescriptor, clazz)) + .toArray(Method[]::new); + this.oneofCaseMethods = + oneofDescriptorList.stream() + .map(oneofDescriptor -> obtainOneofCaseMethod(oneofDescriptor, clazz)) + .toArray(Method[]::new); + this.oneOfNumberMethods = + oneofDescriptorList.stream() + .map(oneofDescriptor -> obtainOneofNumberMethod(oneofDescriptor, clazz)) + .toArray(Method[]::new); + } + + public Method obtainOneofNumberMethod( + Descriptors.OneofDescriptor oneofDescriptor, Class clazz) { + Class caseClass; + try { + caseClass = Class.forName(clazz.getName() + "$" + oneofDescriptor.getName() + "Case"); + } catch (ClassNotFoundException e) { + throw new ChunJunRuntimeException( + String.format( + "failed to get OneOfCase Class,caseName=%s", + oneofDescriptor.getName())); + } + return obtainMethod("getNumber", caseClass); + } + + public Method obtainOneofCaseMethod( + Descriptors.OneofDescriptor oneofDescriptor, Class clazz) { + return obtainMethod("get" + oneofDescriptor.getName() + "Case", clazz); + } + + /** + * obtain get Method . methodName = get+descriptorName. if repeated,methodName = + * get+descriptorName+List + */ + public Method obtainNormalGetMethod( + Descriptors.FieldDescriptor fieldDescriptor, Class clazz) { + StringBuilder stringBuilder = new StringBuilder("get"); + stringBuilder.append(fieldDescriptor.getName()); + if (!fieldDescriptor.isMapField() && fieldDescriptor.isRepeated()) { + stringBuilder.append("List"); + } + return obtainMethod(stringBuilder.toString(), clazz); + } +} diff --git a/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbParser.java b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbParser.java new file mode 100644 index 0000000000..1a4f3c594a --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbParser.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.format.protobuf.deserialize; + +import com.dtstack.chunjun.format.protobuf.util.PbReflectUtil; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; + +import java.lang.reflect.Method; + +/** + * trans byte[] to protobuf object by protobuf parseFrom method + * + * @author liuliu 2022/4/8 + */ +public class PbParser { + + private Class messageClass; + private Method parseMethod; + + public PbParser(String messageClassName) throws NoSuchMethodException { + messageClass = PbReflectUtil.getClassByClassName(messageClassName); + parseMethod = messageClass.getMethod("parseFrom", byte[].class); + parseMethod.setAccessible(true); + } + + public Object parse(byte[] bytes) { + try { + return parseMethod.invoke(messageClass, bytes); + } catch (Exception e) { + throw new ChunJunRuntimeException( + String.format( + "Failed to deserialize protocol data from byte[] to message object,messageClass=%s", + messageClass), + e); + } + } +} diff --git a/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbRowDataDeserializationSchema.java b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbRowDataDeserializationSchema.java new file mode 100644 index 0000000000..aea96a9b6d --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbRowDataDeserializationSchema.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.format.protobuf.deserialize; + +import com.dtstack.chunjun.format.protobuf.util.FormatCheckUtil; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; + +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.RowType; + +import java.util.Objects; + +/** @author liuliu 2022/4/8 */ +public class PbRowDataDeserializationSchema implements DeserializationSchema { + + private static final long serialVersionUID = 1L; + + private PbParser protoMessageTransformer; + private PbToRowDataPbConverter.ProtoToRowDataConverter runtimeConverter; + private TypeInformation typeInformation; + private String messageClassName; + private RowType rowType; + + public PbRowDataDeserializationSchema( + RowType rowType, TypeInformation typeInformation, String protoOutClassName) { + this.rowType = rowType; + this.typeInformation = typeInformation; + this.messageClassName = protoOutClassName; + new FormatCheckUtil(rowType, messageClassName).isValid(); + } + + @Override + public void open(InitializationContext context) throws Exception { + protoMessageTransformer = new PbParser(messageClassName); + runtimeConverter = PbToRowDataPbConverter.initMessageDataConverter(messageClassName); + } + + @Override + public RowData deserialize(byte[] message) { + if (message == null) { + return null; + } + Object protoMessage = protoMessageTransformer.parse(message); + try { + return (RowData) runtimeConverter.convert(protoMessage); + } catch (Exception e) { + throw new ChunJunRuntimeException("Failed to convert protobuf record to RowData", e); + } + } + + @Override + public boolean isEndOfStream(RowData nextElement) { + return false; + } + + @Override + public TypeInformation getProducedType() { + return typeInformation; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PbRowDataDeserializationSchema that = (PbRowDataDeserializationSchema) o; + return messageClassName.equals(that.messageClassName); + } + + @Override + public int hashCode() { + return Objects.hash(messageClassName); + } +} diff --git a/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbToRowDataPbConverter.java b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbToRowDataPbConverter.java new file mode 100644 index 0000000000..6076551c76 --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/deserialize/PbToRowDataPbConverter.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.format.protobuf.deserialize; + +import com.dtstack.chunjun.format.protobuf.PbFormatType; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.tuple.Tuple4; +import org.apache.flink.table.data.GenericArrayData; +import org.apache.flink.table.data.GenericMapData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.StringData; + +import com.google.protobuf.AbstractMessage; +import com.google.protobuf.ByteString; +import com.google.protobuf.Descriptors; + +import java.io.Serializable; +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.dtstack.chunjun.format.protobuf.util.PbReflectUtil.getClassByFieldDescriptor; +import static com.dtstack.chunjun.format.protobuf.util.PbReflectUtil.getDescriptorByMessageClassName; +import static com.dtstack.chunjun.format.protobuf.util.PbReflectUtil.getMapTypeTuple; + +/** @author liuliu 2022/4/11 Converte from proto to RowData */ +public class PbToRowDataPbConverter { + + /** proto class full name example as com.dtstack.messageOutClass */ + protected static String PROTO_CLASS_NAME; + /** proto class package name example as com.dtstack */ + protected static String PROTO_PACKAGE_NAME; + + public interface ProtoToRowDataConverter extends Serializable { + Object convert(Object object) throws InvocationTargetException, IllegalAccessException; + } + + /** + * create a runtime converter for row by specific protoMessageClassName and init {@link + * PbToRowDataPbConverter#PROTO_PACKAGE_NAME} and {@link + * PbToRowDataPbConverter#PROTO_CLASS_NAME} + */ + public static ProtoToRowDataConverter initMessageDataConverter(String messageClassName) { + try { + PROTO_CLASS_NAME = messageClassName.substring(0, messageClassName.lastIndexOf("$")); + PROTO_PACKAGE_NAME = PROTO_CLASS_NAME.substring(0, PROTO_CLASS_NAME.lastIndexOf(".")); + return createMessageDataConverter(messageClassName); + } catch (IndexOutOfBoundsException e) { + throw new ChunJunRuntimeException( + String.format("Incorrect proto message class name:%s", messageClassName), e); + } + } + + /** + * create a runtime converter for row by specific messageClassName example as + * com.dtstack.MessageOutClass$Message + */ + public static ProtoToRowDataConverter createMessageDataConverter(String protoMessageClassName) { + Tuple2> + descriptorByMessageClassName = + getDescriptorByMessageClassName(protoMessageClassName); + + return createMessageDataConverter( + descriptorByMessageClassName.f0.getFields(), + descriptorByMessageClassName.f0.getOneofs(), + descriptorByMessageClassName.f1); + } + + /** create a runtime converter for row by specific fieldDescriptor */ + public static ProtoToRowDataConverter createMessageDataConverter( + Descriptors.FieldDescriptor fieldDescriptor) { + + Descriptors.Descriptor messageType = fieldDescriptor.getMessageType(); + return createMessageDataConverter( + messageType.getFields(), + messageType.getOneofs(), + getClassByFieldDescriptor( + PROTO_CLASS_NAME, PROTO_PACKAGE_NAME, fieldDescriptor, null)); + } + + public static ProtoToRowDataConverter createMessageDataConverter( + List fieldDescriptorList, + List oneofDescriptorList, + Class clazz) { + PbMessageGetter pbMessageGetter = + new PbMessageGetter(fieldDescriptorList, oneofDescriptorList, clazz); + ProtoToRowDataConverter[] protoToRowDataConverters = + fieldDescriptorList.stream() + .map(PbToRowDataPbConverter::createNullableConverter) + .toArray(ProtoToRowDataConverter[]::new); + return object -> { + int size = pbMessageGetter.getSize(); + GenericRowData genericRowData = new GenericRowData(size); + for (int i = 0, index = 0; i < protoToRowDataConverters.length && index < size; ) { + if (pbMessageGetter.isOneOf(i)) { + int lastIndex = pbMessageGetter.getLastOneOfIndex(i); + GenericRowData oneofGenericRowData = new GenericRowData(lastIndex - i + 2); + oneofGenericRowData.setField(0, pbMessageGetter.getOneofCase(object, i)); + int currentCase = 1; + while (currentCase < oneofGenericRowData.getArity()) { + Object o = pbMessageGetter.getByIndex(object, i); + oneofGenericRowData.setField( + currentCase++, protoToRowDataConverters[i++].convert(o)); + } + genericRowData.setField(index++, oneofGenericRowData); + } else { + Object o = pbMessageGetter.getByIndex(object, i); + genericRowData.setField(index++, protoToRowDataConverters[i++].convert(o)); + } + } + return genericRowData; + }; + } + + /** Creates a runtime converter which is null safe. */ + private static ProtoToRowDataConverter createNullableConverter( + Descriptors.FieldDescriptor descriptor) { + return createNullableConverter( + descriptor, PbFormatType.getTypeByFieldDescriptor(descriptor)); + } + + private static ProtoToRowDataConverter createNullableConverter( + Descriptors.FieldDescriptor descriptor, PbFormatType protoType) { + final ProtoToRowDataConverter converter = createConverter(descriptor, protoType); + return protoObject -> { + if (protoObject == null) { + return null; + } + return converter.convert(protoObject); + }; + } + + /** + * In standard ProtocolBuffers, mapKey can be any integral or string type (so, any scalar type + * except for floating point types and bytes), mapValue not can be mapType and arrayType + * + * @return a runtime map converter + */ + public static ProtoToRowDataConverter createMapConverter( + Descriptors.FieldDescriptor fieldDescriptor) { + // get builder method + Tuple4 mapTypeTuple = + getMapTypeTuple(PROTO_CLASS_NAME, PROTO_PACKAGE_NAME, fieldDescriptor); + // create keyConverter and valueConverter + final ProtoToRowDataConverter keyConverter = createNullableConverter(null, mapTypeTuple.f0); + final ProtoToRowDataConverter valueConverter; + if (mapTypeTuple.f1 == PbFormatType.MESSAGE) { + valueConverter = createMessageDataConverter(mapTypeTuple.f3.getName()); + } else { + // scalar type/enum + valueConverter = createConverter(null, mapTypeTuple.f1); + } + // create runtime map converter + return object -> { + final Map map = (Map) object; + Map result = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + Object key = keyConverter.convert(entry.getKey()); + Object value = valueConverter.convert(entry.getValue()); + result.put(key, value); + } + return new GenericMapData(result); + }; + } + + /** + * In standard ProtocolBuffers, element can only be map、message、scalar + * + * @return a runtime array converter + */ + public static ProtoToRowDataConverter createArrayConverter( + Descriptors.FieldDescriptor fieldDescriptor) { + PbFormatType protoType = PbFormatType.getArrayInnerTypeByFieldDescriptor(fieldDescriptor); + final ProtoToRowDataConverter elementConverter = + createConverter(fieldDescriptor, protoType); + return object -> { + final List list = (List) object; + final int length = list.size(); + final Object[] array = new Object[length]; + for (int i = 0; i < length; ++i) { + array[i] = elementConverter.convert(list.get(i)); + } + return new GenericArrayData(array); + }; + } + + public static ProtoToRowDataConverter createConverter( + Descriptors.FieldDescriptor fieldDescriptor, PbFormatType protoType) { + switch (protoType) { + case ENUM: + case STRING: + return object -> StringData.fromString(object.toString()); + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case BOOLEAN: + return object -> object; + case BYTE_STRING: + return object -> ((ByteString) object).toByteArray(); + case MAP: + return createMapConverter(fieldDescriptor); + case MESSAGE: + return createMessageDataConverter(fieldDescriptor); + case ARRAY: + return createArrayConverter(fieldDescriptor); + default: + throw new UnsupportedOperationException(protoType.name()); + } + } +} diff --git a/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/serialize/PbMessageSetter.java b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/serialize/PbMessageSetter.java new file mode 100644 index 0000000000..0d67ddbd7c --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/serialize/PbMessageSetter.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.format.protobuf.serialize; + +import com.dtstack.chunjun.format.protobuf.PbMessageAdaptor; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; + +import com.google.protobuf.AbstractMessage; +import com.google.protobuf.Descriptors; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.List; + +/** + * Reflect all Builder#Set methods in the order of the fields in the Protobuf Message, then we can + * set the values by index. + * + * @author liuliu 2022/4/26 + */ +public class PbMessageSetter extends PbMessageAdaptor { + + private AbstractMessage.Builder builder; + private Method[] normalMethods; + + public PbMessageSetter( + List fieldDescriptorList, + List oneofDescriptorList, + Class clazz) { + super(fieldDescriptorList, oneofDescriptorList); + initBuilder(clazz); + initMethods(fieldDescriptorList, builder.getClass()); + } + + private void initBuilder(Class builderClass) { + try { + Method newBuilder = builderClass.getMethod("newBuilder"); + this.builder = (AbstractMessage.Builder) newBuilder.invoke(builderClass); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public void setByIndex(Object o, int index) throws ChunJunRuntimeException { + try { + + normalMethods[index].invoke(builder, o); + } catch (InvocationTargetException + | IllegalAccessException + | IllegalArgumentException exception) { + System.out.println(normalMethods[index].getName()); + Arrays.stream(normalMethods[index].getParameterTypes()) + .forEach(p -> System.out.println(p.getName())); + System.out.println(o); + throw new ChunJunRuntimeException( + String.format( + "failed to set field by protobuf builder,builder class[%s],index[%s],filed[%s]", + builder.getClass().getName(), index, o), + exception); + } + } + + public Object build() { + Object result = builder.build(); + builder.clear(); + return result; + } + + private void initMethods(List fieldDescriptorList, Class clazz) { + normalMethods = + fieldDescriptorList.stream() + .map( + f -> { + String action = "set"; + if (f.isMapField()) { + action = "putAll"; + } else if (f.isRepeated()) { + action = "addAll"; + } + return action + f.getName(); + }) + .map(methodName -> obtainMethod(methodName, clazz)) + .toArray(Method[]::new); + } +} diff --git a/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/serialize/PbRowDataSerializationSchema.java b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/serialize/PbRowDataSerializationSchema.java new file mode 100644 index 0000000000..22fb685bda --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/serialize/PbRowDataSerializationSchema.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.format.protobuf.serialize; + +import com.dtstack.chunjun.format.protobuf.util.FormatCheckUtil; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; + +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.RowType; + +import com.google.protobuf.AbstractMessage; + +import java.util.Objects; + +/** @author liuliu 2022/4/26 */ +public class PbRowDataSerializationSchema implements SerializationSchema { + private static final long serialVersionUID = 1L; + + private final RowType rowType; + private final String messageClassName; + private RowDataToPbConverter.RowDataToProtoConverter runtimeConverter; + + public PbRowDataSerializationSchema(RowType rowType, String messageClassName) { + this.rowType = rowType; + this.messageClassName = messageClassName; + new FormatCheckUtil(rowType, messageClassName).isValid(); + } + + @Override + public void open(InitializationContext context) { + runtimeConverter = RowDataToPbConverter.initMessageDataConverter(rowType, messageClassName); + } + + @Override + public byte[] serialize(RowData element) { + if (element == null) { + return null; + } + try { + return ((AbstractMessage) runtimeConverter.convert(element)).toByteArray(); + } catch (Exception e) { + throw new ChunJunRuntimeException( + String.format( + "Failed to convert RowData to protobuf record,RowData:%s", element), + e); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PbRowDataSerializationSchema that = (PbRowDataSerializationSchema) o; + return messageClassName.equals(that.messageClassName); + } + + @Override + public int hashCode() { + return Objects.hash(messageClassName); + } +} diff --git a/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/serialize/RowDataToPbConverter.java b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/serialize/RowDataToPbConverter.java new file mode 100644 index 0000000000..625ffe6223 --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/serialize/RowDataToPbConverter.java @@ -0,0 +1,328 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.format.protobuf.serialize; + +import com.dtstack.chunjun.format.protobuf.PbFormatType; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.tuple.Tuple4; +import org.apache.flink.table.data.GenericArrayData; +import org.apache.flink.table.data.GenericMapData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.binary.BinaryArrayData; +import org.apache.flink.table.data.binary.BinaryMapData; +import org.apache.flink.table.types.logical.ArrayType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.RowType; + +import com.google.protobuf.AbstractMessage; +import com.google.protobuf.ByteString; +import com.google.protobuf.Descriptors; +import com.google.protobuf.ProtocolMessageEnum; + +import java.io.Serializable; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.dtstack.chunjun.format.protobuf.util.PbReflectUtil.getClassByClassName; +import static com.dtstack.chunjun.format.protobuf.util.PbReflectUtil.getClassByFieldDescriptor; +import static com.dtstack.chunjun.format.protobuf.util.PbReflectUtil.getClassNameByFullName; +import static com.dtstack.chunjun.format.protobuf.util.PbReflectUtil.getDescriptorByMessageClassName; +import static com.dtstack.chunjun.format.protobuf.util.PbReflectUtil.getMapTypeTuple; + +/** @author liuliu 2022/4/26 */ +public class RowDataToPbConverter { + + /** proto class full name example as com.dtstack.messageOutClass */ + protected static String PROTO_CLASS_NAME; + /** proto class package name example as com.dtstack */ + protected static String PROTO_PACKAGE_NAME; + + public interface RowDataToProtoConverter extends Serializable { + Object convert(Object object) throws InvocationTargetException, IllegalAccessException; + } + + /** + * create a runtime converter for row by specific protoMessageClassName and init {@link + * RowDataToPbConverter#PROTO_PACKAGE_NAME} and {@link RowDataToPbConverter#PROTO_CLASS_NAME} + */ + public static RowDataToProtoConverter initMessageDataConverter( + LogicalType logicalType, String messageClassName) { + try { + PROTO_CLASS_NAME = messageClassName.substring(0, messageClassName.lastIndexOf("$")); + PROTO_PACKAGE_NAME = PROTO_CLASS_NAME.substring(0, PROTO_CLASS_NAME.lastIndexOf(".")); + return createMessageDataConverter(logicalType, messageClassName); + } catch (IndexOutOfBoundsException e) { + throw new ChunJunRuntimeException( + String.format("Incorrect proto message class name:%s", messageClassName), e); + } + } + + /** + * create a runtime converter for row by specific messageClassName example as + * com.dtstack.MessageOutClass$Message + */ + private static RowDataToProtoConverter createMessageDataConverter( + LogicalType logicalType, String protoMessageClassName) { + Tuple2> + descriptorByMessageClassName = + getDescriptorByMessageClassName(protoMessageClassName); + return createMessageDataConverter( + logicalType, + descriptorByMessageClassName.f0.getFields(), + descriptorByMessageClassName.f0.getOneofs(), + descriptorByMessageClassName.f1); + } + + /** create a runtime converter for row by specific fieldDescriptor */ + public static RowDataToProtoConverter createMessageDataConverter( + LogicalType logicalType, Descriptors.FieldDescriptor fieldDescriptor) { + + Descriptors.Descriptor messageType = fieldDescriptor.getMessageType(); + return createMessageDataConverter( + logicalType, + messageType.getFields(), + messageType.getOneofs(), + getClassByFieldDescriptor( + PROTO_CLASS_NAME, PROTO_PACKAGE_NAME, fieldDescriptor, null)); + } + + private static RowDataToProtoConverter createMessageDataConverter( + LogicalType logicalType, + List fieldDescriptorList, + List oneofDescriptorList, + Class clazz) { + RowDataToProtoConverter[] rowDataToProtoConverters = + new RowDataToProtoConverter[fieldDescriptorList.size()]; + PbMessageSetter pbMessageSetter = + new PbMessageSetter(fieldDescriptorList, oneofDescriptorList, clazz); + int currentIndex = 0; + for (LogicalType childLogicalType : logicalType.getChildren()) { + if (childLogicalType instanceof RowType && pbMessageSetter.isOneOf(currentIndex)) { + for (int i = 1; i < ((RowType) childLogicalType).getFieldCount(); i++) { + rowDataToProtoConverters[currentIndex] = + createNullableConverter( + ((RowType) childLogicalType).getTypeAt(i), + fieldDescriptorList.get(currentIndex++)); + } + } else { + rowDataToProtoConverters[currentIndex] = + createNullableConverter( + childLogicalType, fieldDescriptorList.get(currentIndex++)); + } + } + return object -> { + GenericRowData genericRowData = (GenericRowData) object; + for (int i = 0, index = 0; + i < fieldDescriptorList.size() && index < genericRowData.getArity(); + i++) { + if (pbMessageSetter.isOneOf(i)) { + GenericRowData oneofGenericRowData = + (GenericRowData) genericRowData.getField(index++); + int oneofCase = oneofGenericRowData.getInt(0); + int caseInRow = oneofCase - i; + pbMessageSetter.setByIndex( + rowDataToProtoConverters[oneofCase - 1].convert( + oneofGenericRowData.getField(caseInRow)), + oneofCase - 1); + i = pbMessageSetter.getLastOneOfIndex(i); + } else { + pbMessageSetter.setByIndex( + rowDataToProtoConverters[i].convert(genericRowData.getField(index++)), + i); + } + } + return pbMessageSetter.build(); + }; + } + + /** Creates a runtime converter which is null safe. */ + private static RowDataToProtoConverter createNullableConverter( + LogicalType logicalType, Descriptors.FieldDescriptor descriptor) { + return createNullableConverter( + logicalType, descriptor, PbFormatType.getTypeByFieldDescriptor(descriptor)); + } + + private static RowDataToProtoConverter createNullableConverter( + LogicalType logicalType, + Descriptors.FieldDescriptor descriptor, + PbFormatType protoType) { + final RowDataToProtoConverter converter = + createConverter(logicalType, descriptor, protoType); + return protoObject -> { + if (protoObject == null) { + return null; + } + return converter.convert(protoObject); + }; + } + + public static RowDataToProtoConverter createConverter( + LogicalType logicalType, + Descriptors.FieldDescriptor fieldDescriptor, + PbFormatType protoType) { + switch (protoType) { + case ENUM: + return createEnumConverter(fieldDescriptor); + case STRING: + return Object::toString; + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case BOOLEAN: + return object -> object; + case BYTE_STRING: + return object -> ByteString.copyFrom((byte[]) object); + case MAP: + return createMapConverter(logicalType, fieldDescriptor); + case MESSAGE: + return createMessageDataConverter(logicalType, fieldDescriptor); + case ARRAY: + return createArrayConverter(logicalType, fieldDescriptor); + default: + throw new UnsupportedOperationException(protoType.name()); + } + } + + /** + * In standard ProtocolBuffers, mapKey can be any integral or string type (so, any scalar type + * except for floating point types and bytes), mapValue not can be mapType and arrayType + * + * @return a runtime map converter + */ + private static RowDataToProtoConverter createMapConverter( + LogicalType logicalType, Descriptors.FieldDescriptor fieldDescriptor) { + Tuple4 mapTypeTuple = + getMapTypeTuple(PROTO_CLASS_NAME, PROTO_PACKAGE_NAME, fieldDescriptor); + LogicalType keyType = ((MapType) logicalType).getKeyType(); + LogicalType valueType = ((MapType) logicalType).getValueType(); + // create keyConverter and valueConverter + final RowDataToProtoConverter keyConverter = + createNullableConverter(keyType, null, mapTypeTuple.f0); + final RowDataToProtoConverter valueConverter; + if (mapTypeTuple.f1 == PbFormatType.MESSAGE) { + valueConverter = createMessageDataConverter(valueType, mapTypeTuple.f3.getName()); + } else { + // scalar type/enum + valueConverter = createConverter(valueType, null, mapTypeTuple.f1); + } + + return object -> { + Map result = new HashMap<>(); + Object[] keyArray; + Object[] valueArray; + if (object instanceof BinaryMapData) { + keyArray = ((BinaryMapData) object).keyArray().toObjectArray(keyType); + valueArray = ((BinaryMapData) object).valueArray().toObjectArray(valueType); + } else if (object instanceof GenericMapData) { + keyArray = + ((GenericArrayData) ((GenericMapData) object).keyArray()).toObjectArray(); + valueArray = + ((GenericArrayData) ((GenericMapData) object).valueArray()).toObjectArray(); + } else { + throw new UnsupportedOperationException( + "protobuf-x format Map serializer only support BinaryMapData and GenericMapData"); + } + for (int i = 0; i < keyArray.length; i++) { + result.put( + keyConverter.convert(keyArray[i]), valueConverter.convert(valueArray[i])); + } + return result; + }; + } + + /** + * In standard ProtocolBuffers, element can only be map、message、scalar + * + * @return a runtime array converter + */ + private static RowDataToProtoConverter createArrayConverter( + LogicalType logicalType, Descriptors.FieldDescriptor fieldDescriptor) { + PbFormatType protoType = PbFormatType.getArrayInnerTypeByFieldDescriptor(fieldDescriptor); + LogicalType elementType = ((ArrayType) logicalType).getElementType(); + final RowDataToProtoConverter elementConverter = + createConverter(elementType, fieldDescriptor, protoType); + return object -> { + List list = new ArrayList<>(); + Object[] elements; + if (object instanceof BinaryArrayData) { + elements = ((BinaryArrayData) object).toObjectArray(elementType); + } else if (object instanceof GenericArrayData) { + elements = ((GenericArrayData) object).toObjectArray(); + } else { + throw new UnsupportedOperationException( + "protobuf-x format Repeated serializer only support BinaryArrayData and GenericArrayData"); + } + for (Object o : elements) { + list.add(elementConverter.convert(o)); + } + return list; + }; + } + + private static RowDataToProtoConverter createEnumConverter( + Descriptors.FieldDescriptor fieldDescriptor) { + EnumMessageAssemblers assemblers = new EnumMessageAssemblers(fieldDescriptor); + return object -> assemblers.valueOf(object.toString()); + } + + private static class EnumMessageAssemblers { + + private final Method valueOfMethod; + private final String enumClassName; + private final Class enumClass; + + public EnumMessageAssemblers(Descriptors.FieldDescriptor fieldDescriptor) { + this.enumClassName = + getClassNameByFullName( + PROTO_CLASS_NAME, + PROTO_PACKAGE_NAME, + fieldDescriptor.getEnumType().getFullName()); + enumClass = getClassByClassName(enumClassName); + try { + this.valueOfMethod = enumClass.getDeclaredMethod("valueOf", String.class); + valueOfMethod.setAccessible(true); + } catch (NoSuchMethodException e) { + throw new ChunJunRuntimeException( + String.format( + "Failed to get valueOf method from enum class [%s]", + enumClassName)); + } + } + + public Object valueOf(String enumString) { + try { + return valueOfMethod.invoke(enumClass, enumString); + } catch (Exception e) { + throw new ChunJunRuntimeException( + String.format( + "Failed to generate an enum object of class [%s] from the string [%s]", + enumClassName, enumString), + e); + } + } + } +} diff --git a/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/util/FormatCheckUtil.java b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/util/FormatCheckUtil.java new file mode 100644 index 0000000000..c63aee12ec --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/util/FormatCheckUtil.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.format.protobuf.util; + +import com.dtstack.chunjun.format.protobuf.PbFormatType; +import com.dtstack.chunjun.format.protobuf.PbMessageAdaptor; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; + +import org.apache.flink.api.java.tuple.Tuple4; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.types.logical.ArrayType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.RowType; + +import com.google.protobuf.Descriptors; + +import java.util.ArrayList; +import java.util.List; + +import static com.dtstack.chunjun.format.protobuf.util.PbReflectUtil.getMapTypeTuple; + +/** @author liuliu 2022/5/11 */ +public class FormatCheckUtil { + + /** proto class full name example as com.dtstack.messageOutClass */ + protected String PROTO_CLASS_NAME; + /** proto class package name example as com.dtstack */ + protected String PROTO_PACKAGE_NAME; + + protected RowType rowType; + protected String messageClassName; + + public FormatCheckUtil(RowType rowType, String messageClassName) { + this.PROTO_CLASS_NAME = messageClassName.substring(0, messageClassName.lastIndexOf("$")); + this.PROTO_PACKAGE_NAME = PROTO_CLASS_NAME.substring(0, PROTO_CLASS_NAME.lastIndexOf(".")); + this.rowType = rowType; + this.messageClassName = messageClassName; + } + + /** Checks whether the DDL statement matches the Protobuf message */ + public void isValid() { + RowType messageLogicalType = createMessageLogicalType(messageClassName); + if (!rowTypeEqual(messageLogicalType, rowType)) { + throw new ChunJunRuntimeException( + String.format( + "protobuf class %s except rowType like\n" + " %s\n" + "but\n %s", + messageClassName, messageLogicalType, rowType)); + } + } + + private boolean logicalTypeEqual(LogicalType logicalType, LogicalType target) { + boolean flag; + if (logicalType instanceof RowType && target instanceof RowType) { + flag = rowTypeEqual((RowType) logicalType, (RowType) target); + } else if (logicalType instanceof MapType && target instanceof MapType) { + flag = mapTypeEqual((MapType) logicalType, (MapType) target); + } else if (logicalType instanceof ArrayType && target instanceof ArrayType) { + flag = arrayTypeEqual((ArrayType) logicalType, (ArrayType) target); + } else { + flag = logicalType.getClass() == target.getClass(); + } + return flag; + } + + private boolean arrayTypeEqual(ArrayType arrayType, ArrayType target) { + return logicalTypeEqual(arrayType.getElementType(), target.getElementType()); + } + + private boolean mapTypeEqual(MapType mapType, MapType target) { + return logicalTypeEqual(mapType.getKeyType(), target.getKeyType()) + && logicalTypeEqual(mapType.getValueType(), target.getValueType()); + } + + public boolean rowTypeEqual(RowType rowType, RowType target) { + List fields = rowType.getFields(); + List targetFields = target.getFields(); + int size = fields.size(); + if (size == targetFields.size()) { + for (int i = 0; i < size; i++) { + RowType.RowField field = fields.get(i); + RowType.RowField targetField = targetFields.get(i); + if (!logicalTypeEqual(field.getType(), targetField.getType())) { + return false; + } + } + return true; + } else { + return false; + } + } + + /** create rowType by protoMessageClass */ + public RowType createMessageLogicalType(String protoMessageClass) { + Descriptors.Descriptor descriptor = + PbReflectUtil.getDescriptorByMessageClassName(protoMessageClass).f0; + return createMessageLogicalType(descriptor); + } + + private RowType createMessageLogicalType(Descriptors.Descriptor descriptor) { + return createMessageLogicalType(descriptor.getFields(), descriptor.getOneofs()); + } + + private RowType createMessageLogicalType(Descriptors.FieldDescriptor descriptor) { + return createMessageLogicalType( + descriptor.getMessageType().getFields(), descriptor.getMessageType().getOneofs()); + } + + private RowType createMessageLogicalType( + List descriptorList, + List oneofDescriptorList) { + PbMessageAdaptor pbMessageAdaptor = + new PbMessageAdaptor(descriptorList, oneofDescriptorList); + + List rowFieldList = new ArrayList<>(); + int indexOfOneof = 0; + for (int i = 0; i < descriptorList.size(); ) { + if (pbMessageAdaptor.isOneOf(i)) { + + List oneofRowFieldList = new ArrayList<>(); + oneofRowFieldList.add( + new RowType.RowField("case", DataTypes.INT().getLogicalType())); + Integer lastOneOfIndex = pbMessageAdaptor.getLastOneOfIndex(i); + while (i <= lastOneOfIndex) { + Descriptors.FieldDescriptor descriptor = descriptorList.get(i++); + oneofRowFieldList.add( + new RowType.RowField( + descriptor.getName(), createLogicalType(descriptor))); + } + rowFieldList.add( + new RowType.RowField( + oneofDescriptorList.get(indexOfOneof++).getName(), + new RowType(oneofRowFieldList))); + + } else { + Descriptors.FieldDescriptor descriptor = descriptorList.get(i++); + rowFieldList.add( + new RowType.RowField(descriptor.getName(), createLogicalType(descriptor))); + } + } + return new RowType(rowFieldList); + } + + private LogicalType createLogicalType(Descriptors.FieldDescriptor fieldDescriptor) { + return createLogicalType( + fieldDescriptor, PbFormatType.getTypeByFieldDescriptor(fieldDescriptor)); + } + + private LogicalType createLogicalType(PbFormatType protoType) { + return createLogicalType(null, protoType); + } + + private LogicalType createLogicalType( + Descriptors.FieldDescriptor fieldDescriptor, PbFormatType protoType) { + switch (protoType) { + case ENUM: + case STRING: + return DataTypes.STRING().getLogicalType(); + case INT: + return DataTypes.INT().getLogicalType(); + case LONG: + return DataTypes.BIGINT().getLogicalType(); + case FLOAT: + return DataTypes.FLOAT().getLogicalType(); + case DOUBLE: + return DataTypes.DOUBLE().getLogicalType(); + case BOOLEAN: + return DataTypes.BOOLEAN().getLogicalType(); + case BYTE_STRING: + return DataTypes.BYTES().getLogicalType(); + case MAP: + return createMapLogicalType(fieldDescriptor); + case MESSAGE: + return createMessageLogicalType(fieldDescriptor); + case ARRAY: + return createArrayLogicalType(fieldDescriptor); + default: + throw new UnsupportedOperationException(protoType.name()); + } + } + + private LogicalType createArrayLogicalType(Descriptors.FieldDescriptor fieldDescriptor) { + PbFormatType protoType = PbFormatType.getArrayInnerTypeByFieldDescriptor(fieldDescriptor); + return new ArrayType(createLogicalType(fieldDescriptor, protoType)); + } + + private LogicalType createMapLogicalType(Descriptors.FieldDescriptor fieldDescriptor) { + // get builder method + Tuple4 mapTypeTuple = + getMapTypeTuple(PROTO_CLASS_NAME, PROTO_PACKAGE_NAME, fieldDescriptor); + LogicalType keyLogicalType = createLogicalType(mapTypeTuple.f0); + LogicalType valueLogicalType; + if (mapTypeTuple.f1 == PbFormatType.MESSAGE) { + valueLogicalType = createMessageLogicalType(mapTypeTuple.f3.getName()); + } else { + // scalar type/enum + valueLogicalType = createLogicalType(mapTypeTuple.f1); + } + return new MapType(keyLogicalType, valueLogicalType); + } +} diff --git a/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/util/PbReflectUtil.java b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/util/PbReflectUtil.java new file mode 100644 index 0000000000..222fdd990e --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/src/main/java/com/dtstack/chunjun/format/protobuf/util/PbReflectUtil.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.format.protobuf.util; + +import com.dtstack.chunjun.format.protobuf.PbFormatType; +import com.dtstack.chunjun.throwable.ChunJunRuntimeException; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.tuple.Tuple4; + +import com.google.protobuf.AbstractMessage; +import com.google.protobuf.Descriptors; + +import java.lang.reflect.Method; +import java.util.Arrays; + +/** @author liuliu 2022/4/26 */ +public abstract class PbReflectUtil { + + public static Tuple2> + getDescriptorByMessageClassName(String protoMessageClassName) { + Class clazz; + try { + clazz = getClassByClassName(protoMessageClassName); + return Tuple2.of(getDescriptorByClass(clazz), clazz); + } catch (Exception e) { + throw new ChunJunRuntimeException("failed to get proto Descriptor", e); + } + } + + public static Class getClassByClassName(String protoMessageClassName) { + try { + return Class.forName(protoMessageClassName); + } catch (Exception e) { + throw new ChunJunRuntimeException( + String.format( + "failed to get proto class by className %s", protoMessageClassName), + e); + } + } + + public static Descriptors.Descriptor getDescriptorByClass( + Class clazz) { + try { + Method getDescriptor = clazz.getMethod("getDescriptor"); + getDescriptor.setAccessible(true); + return (Descriptors.Descriptor) getDescriptor.invoke(clazz); + } catch (Exception e) { + throw new ChunJunRuntimeException( + String.format( + "failed to get proto Descriptor,final class name=%s", clazz.getName()), + e); + } + } + + public static String getClassNameByFullName( + String protoClassName, String protoPackageName, String fullName) { + return protoClassName + + fullName.substring(protoPackageName.length()).replaceAll("\\.", "\\$"); + } + + /** @param suffix example as $Builder */ + public static Class getClassByFieldDescriptor( + String protoClassName, + String protoPackageName, + Descriptors.FieldDescriptor fieldDescriptor, + String suffix) { + String className = null; + try { + className = + getClassNameByFullName( + protoClassName, + protoPackageName, + fieldDescriptor.getMessageType().getFullName()); + if (suffix != null) { + int index = + className.lastIndexOf("$") == -1 + ? className.lastIndexOf(".") + : className.lastIndexOf("$"); + className = className.substring(0, index) + suffix; + } + return Class.forName(className); + } catch (Exception e) { + throw new ChunJunRuntimeException( + String.format( + "cannot found class by fieldDescriptor,messageFullName=%s,final className=%s}", + fieldDescriptor.getMessageType().getFullName(), className)); + } + } + + public static Tuple4 getMapTypeTuple( + String protoClassName, + String protoPackageName, + Descriptors.FieldDescriptor fieldDescriptor) { + // get builder method + Class builderClass = + getClassByFieldDescriptor( + protoClassName, protoPackageName, fieldDescriptor, "$Builder"); + Method[] methods = + Arrays.stream(builderClass.getMethods()) + .filter( + method -> + method.getName() + .equalsIgnoreCase( + "put" + fieldDescriptor.getName())) + .toArray(Method[]::new); + // get map key and value type by builder method + assert methods.length == 1; + Class keyClass = methods[0].getParameterTypes()[0]; + Class valueClass = methods[0].getParameterTypes()[1]; + PbFormatType keyProtoType = PbFormatType.getProtoTypeForMapKey(keyClass); + PbFormatType valueProtoType = PbFormatType.getProtoTypeForMapValue(valueClass); + return Tuple4.of(keyProtoType, valueProtoType, keyClass, valueClass); + } +} diff --git a/chunjun-formats/chunjun-protobuf/src/test/java/com/dtstack/chunjun/format/protobuf/test/MessageTestOuterClass.java b/chunjun-formats/chunjun-protobuf/src/test/java/com/dtstack/chunjun/format/protobuf/test/MessageTestOuterClass.java new file mode 100644 index 0000000000..2cdd3470e3 --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/src/test/java/com/dtstack/chunjun/format/protobuf/test/MessageTestOuterClass.java @@ -0,0 +1,2950 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: MessageTest.proto + +package com.dtstack.chunjun.format.protobuf.test; + +public final class MessageTestOuterClass { + private MessageTestOuterClass() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistryLite registry) { + } + + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + registerAllExtensions( + (com.google.protobuf.ExtensionRegistryLite) registry); + } + public interface VariantOrBuilder extends + // @@protoc_insertion_point(interface_extends:com.dtstack.chunjun.format.protobuf.test.Variant) + com.google.protobuf.MessageOrBuilder { + + /** + * bool boolx = 1; + */ + boolean getBoolx(); + + /** + * bool ValueBool2 = 2; + */ + boolean getValueBool2(); + + /** + * int32 ValueInt32 = 3; + */ + int getValueInt32(); + + /** + * bool booly = 4; + */ + boolean getBooly(); + + public Variant.Value2Case getValue2Case(); + } + /** + * Protobuf type {@code com.dtstack.chunjun.format.protobuf.test.Variant} + */ + public static final class Variant extends + com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:com.dtstack.chunjun.format.protobuf.test.Variant) + VariantOrBuilder { + // Use Variant.newBuilder() to construct. + private Variant(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + private Variant() { + boolx_ = false; + booly_ = false; + } + + @Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return com.google.protobuf.UnknownFieldSet.getDefaultInstance(); + } + private Variant( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + int mutable_bitField0_ = 0; + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!input.skipField(tag)) { + done = true; + } + break; + } + case 8: { + + boolx_ = input.readBool(); + break; + } + case 16: { + value2Case_ = 2; + value2_ = input.readBool(); + break; + } + case 24: { + value2Case_ = 3; + value2_ = input.readInt32(); + break; + } + case 32: { + + booly_ = input.readBool(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e).setUnfinishedMessage(this); + } finally { + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_Variant_descriptor; + } + + protected FieldAccessorTable + internalGetFieldAccessorTable() { + return MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_Variant_fieldAccessorTable + .ensureFieldAccessorsInitialized( + Variant.class, Builder.class); + } + + private int value2Case_ = 0; + private Object value2_; + public enum Value2Case + implements com.google.protobuf.Internal.EnumLite { + VALUEBOOL2(2), + VALUEINT32(3), + VALUE2_NOT_SET(0); + private final int value; + private Value2Case(int value) { + this.value = value; + } + /** + * @deprecated Use {@link #forNumber(int)} instead. + */ + @Deprecated + public static Value2Case valueOf(int value) { + return forNumber(value); + } + + public static Value2Case forNumber(int value) { + switch (value) { + case 2: return VALUEBOOL2; + case 3: return VALUEINT32; + case 0: return VALUE2_NOT_SET; + default: return null; + } + } + public int getNumber() { + return this.value; + } + }; + + public Value2Case + getValue2Case() { + return Value2Case.forNumber( + value2Case_); + } + + public static final int BOOLX_FIELD_NUMBER = 1; + private boolean boolx_; + /** + * bool boolx = 1; + */ + public boolean getBoolx() { + return boolx_; + } + + public static final int VALUEBOOL2_FIELD_NUMBER = 2; + /** + * bool ValueBool2 = 2; + */ + public boolean getValueBool2() { + if (value2Case_ == 2) { + return (Boolean) value2_; + } + return false; + } + + public static final int VALUEINT32_FIELD_NUMBER = 3; + /** + * int32 ValueInt32 = 3; + */ + public int getValueInt32() { + if (value2Case_ == 3) { + return (Integer) value2_; + } + return 0; + } + + public static final int BOOLY_FIELD_NUMBER = 4; + private boolean booly_; + /** + * bool booly = 4; + */ + public boolean getBooly() { + return booly_; + } + + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) return true; + if (isInitialized == 0) return false; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + if (boolx_ != false) { + output.writeBool(1, boolx_); + } + if (value2Case_ == 2) { + output.writeBool( + 2, (boolean)((Boolean) value2_)); + } + if (value2Case_ == 3) { + output.writeInt32( + 3, (int)((Integer) value2_)); + } + if (booly_ != false) { + output.writeBool(4, booly_); + } + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) return size; + + size = 0; + if (boolx_ != false) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(1, boolx_); + } + if (value2Case_ == 2) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize( + 2, (boolean)((Boolean) value2_)); + } + if (value2Case_ == 3) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size( + 3, (int)((Integer) value2_)); + } + if (booly_ != false) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(4, booly_); + } + memoizedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof Variant)) { + return super.equals(obj); + } + Variant other = (Variant) obj; + + boolean result = true; + result = result && (getBoolx() + == other.getBoolx()); + result = result && (getBooly() + == other.getBooly()); + result = result && getValue2Case().equals( + other.getValue2Case()); + if (!result) return false; + switch (value2Case_) { + case 2: + result = result && (getValueBool2() + == other.getValueBool2()); + break; + case 3: + result = result && (getValueInt32() + == other.getValueInt32()); + break; + case 0: + default: + } + return result; + } + + @Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + hash = (37 * hash) + BOOLX_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean( + getBoolx()); + hash = (37 * hash) + BOOLY_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean( + getBooly()); + switch (value2Case_) { + case 2: + hash = (37 * hash) + VALUEBOOL2_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean( + getValueBool2()); + break; + case 3: + hash = (37 * hash) + VALUEINT32_FIELD_NUMBER; + hash = (53 * hash) + getValueInt32(); + break; + case 0: + default: + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static Variant parseFrom( + java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static Variant parseFrom( + java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static Variant parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static Variant parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static Variant parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static Variant parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static Variant parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static Variant parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + public static Variant parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input); + } + public static Variant parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input, extensionRegistry); + } + public static Variant parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static Variant parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + public static Builder newBuilder(Variant prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { + return this == DEFAULT_INSTANCE + ? new Builder() : new Builder().mergeFrom(this); + } + + @Override + protected Builder newBuilderForType( + BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code com.dtstack.chunjun.format.protobuf.test.Variant} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessageV3.Builder implements + // @@protoc_insertion_point(builder_implements:com.dtstack.chunjun.format.protobuf.test.Variant) + VariantOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_Variant_descriptor; + } + + protected FieldAccessorTable + internalGetFieldAccessorTable() { + return MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_Variant_fieldAccessorTable + .ensureFieldAccessorsInitialized( + Variant.class, Builder.class); + } + + // Construct using com.dtstack.chunjun.format.protobuf.test.MessageTestOuterClass.Variant.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3 + .alwaysUseFieldBuilders) { + } + } + public Builder clear() { + super.clear(); + boolx_ = false; + + booly_ = false; + + value2Case_ = 0; + value2_ = null; + return this; + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_Variant_descriptor; + } + + public Variant getDefaultInstanceForType() { + return Variant.getDefaultInstance(); + } + + public Variant build() { + Variant result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public Variant buildPartial() { + Variant result = new Variant(this); + result.boolx_ = boolx_; + if (value2Case_ == 2) { + result.value2_ = value2_; + } + if (value2Case_ == 3) { + result.value2_ = value2_; + } + result.booly_ = booly_; + result.value2Case_ = value2Case_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + public Builder setField( + com.google.protobuf.Descriptors.FieldDescriptor field, + Object value) { + return (Builder) super.setField(field, value); + } + public Builder clearField( + com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + public Builder clearOneof( + com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + public Builder setRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + int index, Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + public Builder addRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + Object value) { + return (Builder) super.addRepeatedField(field, value); + } + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof Variant) { + return mergeFrom((Variant)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(Variant other) { + if (other == Variant.getDefaultInstance()) return this; + if (other.getBoolx() != false) { + setBoolx(other.getBoolx()); + } + if (other.getBooly() != false) { + setBooly(other.getBooly()); + } + switch (other.getValue2Case()) { + case VALUEBOOL2: { + setValueBool2(other.getValueBool2()); + break; + } + case VALUEINT32: { + setValueInt32(other.getValueInt32()); + break; + } + case VALUE2_NOT_SET: { + break; + } + } + onChanged(); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Variant parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (Variant) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int value2Case_ = 0; + private Object value2_; + public Value2Case + getValue2Case() { + return Value2Case.forNumber( + value2Case_); + } + + public Builder clearValue2() { + value2Case_ = 0; + value2_ = null; + onChanged(); + return this; + } + + + private boolean boolx_ ; + /** + * bool boolx = 1; + */ + public boolean getBoolx() { + return boolx_; + } + /** + * bool boolx = 1; + */ + public Builder setBoolx(boolean value) { + + boolx_ = value; + onChanged(); + return this; + } + /** + * bool boolx = 1; + */ + public Builder clearBoolx() { + + boolx_ = false; + onChanged(); + return this; + } + + /** + * bool ValueBool2 = 2; + */ + public boolean getValueBool2() { + if (value2Case_ == 2) { + return (Boolean) value2_; + } + return false; + } + /** + * bool ValueBool2 = 2; + */ + public Builder setValueBool2(boolean value) { + value2Case_ = 2; + value2_ = value; + onChanged(); + return this; + } + /** + * bool ValueBool2 = 2; + */ + public Builder clearValueBool2() { + if (value2Case_ == 2) { + value2Case_ = 0; + value2_ = null; + onChanged(); + } + return this; + } + + /** + * int32 ValueInt32 = 3; + */ + public int getValueInt32() { + if (value2Case_ == 3) { + return (Integer) value2_; + } + return 0; + } + /** + * int32 ValueInt32 = 3; + */ + public Builder setValueInt32(int value) { + value2Case_ = 3; + value2_ = value; + onChanged(); + return this; + } + /** + * int32 ValueInt32 = 3; + */ + public Builder clearValueInt32() { + if (value2Case_ == 3) { + value2Case_ = 0; + value2_ = null; + onChanged(); + } + return this; + } + + private boolean booly_ ; + /** + * bool booly = 4; + */ + public boolean getBooly() { + return booly_; + } + /** + * bool booly = 4; + */ + public Builder setBooly(boolean value) { + + booly_ = value; + onChanged(); + return this; + } + /** + * bool booly = 4; + */ + public Builder clearBooly() { + + booly_ = false; + onChanged(); + return this; + } + public final Builder setUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return this; + } + + public final Builder mergeUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return this; + } + + + // @@protoc_insertion_point(builder_scope:com.dtstack.chunjun.format.protobuf.test.Variant) + } + + // @@protoc_insertion_point(class_scope:com.dtstack.chunjun.format.protobuf.test.Variant) + private static final Variant DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new Variant(); + } + + public static Variant getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + private static final com.google.protobuf.Parser + PARSER = new com.google.protobuf.AbstractParser() { + public Variant parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new Variant(input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public Variant getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface MessageItemOrBuilder extends + // @@protoc_insertion_point(interface_extends:com.dtstack.chunjun.format.protobuf.test.MessageItem) + com.google.protobuf.MessageOrBuilder { + + /** + * 
+     *默认optional
+     *
+ * + * string TagName = 1; + */ + String getTagName(); + /** + * 
+     *默认optional
+     *
+ * + * string TagName = 1; + */ + com.google.protobuf.ByteString + getTagNameBytes(); + + /** + * .com.dtstack.chunjun.format.protobuf.test.Variant TagValue = 2; + */ + boolean hasTagValue(); + /** + * .com.dtstack.chunjun.format.protobuf.test.Variant TagValue = 2; + */ + Variant getTagValue(); + /** + * .com.dtstack.chunjun.format.protobuf.test.Variant TagValue = 2; + */ + VariantOrBuilder getTagValueOrBuilder(); + + /** + * map<string, string> ExValues = 3; + */ + int getExValuesCount(); + /** + * map<string, string> ExValues = 3; + */ + boolean containsExValues( + String key); + /** + * Use {@link #getExValuesMap()} instead. + */ + @Deprecated + java.util.Map + getExValues(); + /** + * map<string, string> ExValues = 3; + */ + java.util.Map + getExValuesMap(); + /** + * map<string, string> ExValues = 3; + */ + + String getExValuesOrDefault( + String key, + String defaultValue); + /** + * map<string, string> ExValues = 3; + */ + + String getExValuesOrThrow( + String key); + } + /** + * Protobuf type {@code com.dtstack.chunjun.format.protobuf.test.MessageItem} + */ + public static final class MessageItem extends + com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:com.dtstack.chunjun.format.protobuf.test.MessageItem) + MessageItemOrBuilder { + // Use MessageItem.newBuilder() to construct. + private MessageItem(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + private MessageItem() { + tagName_ = ""; + } + + @Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return com.google.protobuf.UnknownFieldSet.getDefaultInstance(); + } + private MessageItem( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + int mutable_bitField0_ = 0; + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!input.skipField(tag)) { + done = true; + } + break; + } + case 10: { + String s = input.readStringRequireUtf8(); + + tagName_ = s; + break; + } + case 18: { + Variant.Builder subBuilder = null; + if (tagValue_ != null) { + subBuilder = tagValue_.toBuilder(); + } + tagValue_ = input.readMessage(Variant.parser(), extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(tagValue_); + tagValue_ = subBuilder.buildPartial(); + } + + break; + } + case 26: { + if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + exValues_ = com.google.protobuf.MapField.newMapField( + ExValuesDefaultEntryHolder.defaultEntry); + mutable_bitField0_ |= 0x00000004; + } + com.google.protobuf.MapEntry + exValues__ = input.readMessage( + ExValuesDefaultEntryHolder.defaultEntry.getParserForType(), extensionRegistry); + exValues_.getMutableMap().put( + exValues__.getKey(), exValues__.getValue()); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e).setUnfinishedMessage(this); + } finally { + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_descriptor; + } + + @SuppressWarnings({"rawtypes"}) + protected com.google.protobuf.MapField internalGetMapField( + int number) { + switch (number) { + case 3: + return internalGetExValues(); + default: + throw new RuntimeException( + "Invalid map field number: " + number); + } + } + protected FieldAccessorTable + internalGetFieldAccessorTable() { + return MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_fieldAccessorTable + .ensureFieldAccessorsInitialized( + MessageItem.class, Builder.class); + } + + private int bitField0_; + public static final int TAGNAME_FIELD_NUMBER = 1; + private volatile Object tagName_; + /** + * 
+     *默认optional
+     *
+ * + * string TagName = 1; + */ + public String getTagName() { + Object ref = tagName_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + tagName_ = s; + return s; + } + } + /** + * 
+     *默认optional
+     *
+ * + * string TagName = 1; + */ + public com.google.protobuf.ByteString + getTagNameBytes() { + Object ref = tagName_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (String) ref); + tagName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int TAGVALUE_FIELD_NUMBER = 2; + private Variant tagValue_; + /** + * .com.dtstack.chunjun.format.protobuf.test.Variant TagValue = 2; + */ + public boolean hasTagValue() { + return tagValue_ != null; + } + /** + * .com.dtstack.chunjun.format.protobuf.test.Variant TagValue = 2; + */ + public Variant getTagValue() { + return tagValue_ == null ? Variant.getDefaultInstance() : tagValue_; + } + /** + * .com.dtstack.chunjun.format.protobuf.test.Variant TagValue = 2; + */ + public VariantOrBuilder getTagValueOrBuilder() { + return getTagValue(); + } + + public static final int EXVALUES_FIELD_NUMBER = 3; + private static final class ExValuesDefaultEntryHolder { + static final com.google.protobuf.MapEntry< + String, String> defaultEntry = + com.google.protobuf.MapEntry + .newDefaultInstance( + MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_ExValuesEntry_descriptor, + com.google.protobuf.WireFormat.FieldType.STRING, + "", + com.google.protobuf.WireFormat.FieldType.STRING, + ""); + } + private com.google.protobuf.MapField< + String, String> exValues_; + private com.google.protobuf.MapField + internalGetExValues() { + if (exValues_ == null) { + return com.google.protobuf.MapField.emptyMapField( + ExValuesDefaultEntryHolder.defaultEntry); + } + return exValues_; + } + + public int getExValuesCount() { + return internalGetExValues().getMap().size(); + } + /** + * map<string, string> ExValues = 3; + */ + + public boolean containsExValues( + String key) { + if (key == null) { throw new NullPointerException(); } + return internalGetExValues().getMap().containsKey(key); + } + /** + * Use {@link #getExValuesMap()} instead. + */ + @Deprecated + public java.util.Map getExValues() { + return getExValuesMap(); + } + /** + * map<string, string> ExValues = 3; + */ + + public java.util.Map getExValuesMap() { + return internalGetExValues().getMap(); + } + /** + * map<string, string> ExValues = 3; + */ + + public String getExValuesOrDefault( + String key, + String defaultValue) { + if (key == null) { throw new NullPointerException(); } + java.util.Map map = + internalGetExValues().getMap(); + return map.containsKey(key) ? map.get(key) : defaultValue; + } + /** + * map<string, string> ExValues = 3; + */ + + public String getExValuesOrThrow( + String key) { + if (key == null) { throw new NullPointerException(); } + java.util.Map map = + internalGetExValues().getMap(); + if (!map.containsKey(key)) { + throw new IllegalArgumentException(); + } + return map.get(key); + } + + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) return true; + if (isInitialized == 0) return false; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + if (!getTagNameBytes().isEmpty()) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, tagName_); + } + if (tagValue_ != null) { + output.writeMessage(2, getTagValue()); + } + com.google.protobuf.GeneratedMessageV3 + .serializeStringMapTo( + output, + internalGetExValues(), + ExValuesDefaultEntryHolder.defaultEntry, + 3); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) return size; + + size = 0; + if (!getTagNameBytes().isEmpty()) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, tagName_); + } + if (tagValue_ != null) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(2, getTagValue()); + } + for (java.util.Map.Entry entry + : internalGetExValues().getMap().entrySet()) { + com.google.protobuf.MapEntry + exValues__ = ExValuesDefaultEntryHolder.defaultEntry.newBuilderForType() + .setKey(entry.getKey()) + .setValue(entry.getValue()) + .build(); + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(3, exValues__); + } + memoizedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof MessageItem)) { + return super.equals(obj); + } + MessageItem other = (MessageItem) obj; + + boolean result = true; + result = result && getTagName() + .equals(other.getTagName()); + result = result && (hasTagValue() == other.hasTagValue()); + if (hasTagValue()) { + result = result && getTagValue() + .equals(other.getTagValue()); + } + result = result && internalGetExValues().equals( + other.internalGetExValues()); + return result; + } + + @Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + hash = (37 * hash) + TAGNAME_FIELD_NUMBER; + hash = (53 * hash) + getTagName().hashCode(); + if (hasTagValue()) { + hash = (37 * hash) + TAGVALUE_FIELD_NUMBER; + hash = (53 * hash) + getTagValue().hashCode(); + } + if (!internalGetExValues().getMap().isEmpty()) { + hash = (37 * hash) + EXVALUES_FIELD_NUMBER; + hash = (53 * hash) + internalGetExValues().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static MessageItem parseFrom( + java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static MessageItem parseFrom( + java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static MessageItem parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static MessageItem parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static MessageItem parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static MessageItem parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static MessageItem parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static MessageItem parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + public static MessageItem parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input); + } + public static MessageItem parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input, extensionRegistry); + } + public static MessageItem parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static MessageItem parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + public static Builder newBuilder(MessageItem prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { + return this == DEFAULT_INSTANCE + ? new Builder() : new Builder().mergeFrom(this); + } + + @Override + protected Builder newBuilderForType( + BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code com.dtstack.chunjun.format.protobuf.test.MessageItem} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessageV3.Builder implements + // @@protoc_insertion_point(builder_implements:com.dtstack.chunjun.format.protobuf.test.MessageItem) + MessageItemOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_descriptor; + } + + @SuppressWarnings({"rawtypes"}) + protected com.google.protobuf.MapField internalGetMapField( + int number) { + switch (number) { + case 3: + return internalGetExValues(); + default: + throw new RuntimeException( + "Invalid map field number: " + number); + } + } + @SuppressWarnings({"rawtypes"}) + protected com.google.protobuf.MapField internalGetMutableMapField( + int number) { + switch (number) { + case 3: + return internalGetMutableExValues(); + default: + throw new RuntimeException( + "Invalid map field number: " + number); + } + } + protected FieldAccessorTable + internalGetFieldAccessorTable() { + return MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_fieldAccessorTable + .ensureFieldAccessorsInitialized( + MessageItem.class, Builder.class); + } + + // Construct using com.dtstack.chunjun.format.protobuf.test.MessageTestOuterClass.MessageItem.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3 + .alwaysUseFieldBuilders) { + } + } + public Builder clear() { + super.clear(); + tagName_ = ""; + + if (tagValueBuilder_ == null) { + tagValue_ = null; + } else { + tagValue_ = null; + tagValueBuilder_ = null; + } + internalGetMutableExValues().clear(); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_descriptor; + } + + public MessageItem getDefaultInstanceForType() { + return MessageItem.getDefaultInstance(); + } + + public MessageItem build() { + MessageItem result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public MessageItem buildPartial() { + MessageItem result = new MessageItem(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + result.tagName_ = tagName_; + if (tagValueBuilder_ == null) { + result.tagValue_ = tagValue_; + } else { + result.tagValue_ = tagValueBuilder_.build(); + } + result.exValues_ = internalGetExValues(); + result.exValues_.makeImmutable(); + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + public Builder setField( + com.google.protobuf.Descriptors.FieldDescriptor field, + Object value) { + return (Builder) super.setField(field, value); + } + public Builder clearField( + com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + public Builder clearOneof( + com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + public Builder setRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + int index, Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + public Builder addRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + Object value) { + return (Builder) super.addRepeatedField(field, value); + } + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof MessageItem) { + return mergeFrom((MessageItem)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(MessageItem other) { + if (other == MessageItem.getDefaultInstance()) return this; + if (!other.getTagName().isEmpty()) { + tagName_ = other.tagName_; + onChanged(); + } + if (other.hasTagValue()) { + mergeTagValue(other.getTagValue()); + } + internalGetMutableExValues().mergeFrom( + other.internalGetExValues()); + onChanged(); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + MessageItem parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (MessageItem) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + private Object tagName_ = ""; + /** + * 
+       *默认optional
+       *
+ * + * string TagName = 1; + */ + public String getTagName() { + Object ref = tagName_; + if (!(ref instanceof String)) { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + tagName_ = s; + return s; + } else { + return (String) ref; + } + } + /** + * 
+       *默认optional
+       *
+ * + * string TagName = 1; + */ + public com.google.protobuf.ByteString + getTagNameBytes() { + Object ref = tagName_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (String) ref); + tagName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * 
+       *默认optional
+       *
+ * + * string TagName = 1; + */ + public Builder setTagName( + String value) { + if (value == null) { + throw new NullPointerException(); + } + + tagName_ = value; + onChanged(); + return this; + } + /** + * 
+       *默认optional
+       *
+ * + * string TagName = 1; + */ + public Builder clearTagName() { + + tagName_ = getDefaultInstance().getTagName(); + onChanged(); + return this; + } + /** + * 
+       *默认optional
+       *
+ * + * string TagName = 1; + */ + public Builder setTagNameBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + checkByteStringIsUtf8(value); + + tagName_ = value; + onChanged(); + return this; + } + + private Variant tagValue_ = null; + private com.google.protobuf.SingleFieldBuilderV3< + Variant, Variant.Builder, VariantOrBuilder> tagValueBuilder_; + /** + * .com.dtstack.chunjun.format.protobuf.test.Variant TagValue = 2; + */ + public boolean hasTagValue() { + return tagValueBuilder_ != null || tagValue_ != null; + } + /** + * .com.dtstack.chunjun.format.protobuf.test.Variant TagValue = 2; + */ + public Variant getTagValue() { + if (tagValueBuilder_ == null) { + return tagValue_ == null ? Variant.getDefaultInstance() : tagValue_; + } else { + return tagValueBuilder_.getMessage(); + } + } + /** + * .com.dtstack.chunjun.format.protobuf.test.Variant TagValue = 2; + */ + public Builder setTagValue(Variant value) { + if (tagValueBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + tagValue_ = value; + onChanged(); + } else { + tagValueBuilder_.setMessage(value); + } + + return this; + } + /** + * .com.dtstack.chunjun.format.protobuf.test.Variant TagValue = 2; + */ + public Builder setTagValue( + Variant.Builder builderForValue) { + if (tagValueBuilder_ == null) { + tagValue_ = builderForValue.build(); + onChanged(); + } else { + tagValueBuilder_.setMessage(builderForValue.build()); + } + + return this; + } + /** + * .com.dtstack.chunjun.format.protobuf.test.Variant TagValue = 2; + */ + public Builder mergeTagValue(Variant value) { + if (tagValueBuilder_ == null) { + if (tagValue_ != null) { + tagValue_ = + Variant.newBuilder(tagValue_).mergeFrom(value).buildPartial(); + } else { + tagValue_ = value; + } + onChanged(); + } else { + tagValueBuilder_.mergeFrom(value); + } + + return this; + } + /** + * .com.dtstack.chunjun.format.protobuf.test.Variant TagValue = 2; + */ + public Builder clearTagValue() { + if (tagValueBuilder_ == null) { + tagValue_ = null; + onChanged(); + } else { + tagValue_ = null; + tagValueBuilder_ = null; + } + + return this; + } + /** + * .com.dtstack.chunjun.format.protobuf.test.Variant TagValue = 2; + */ + public Variant.Builder getTagValueBuilder() { + + onChanged(); + return getTagValueFieldBuilder().getBuilder(); + } + /** + * .com.dtstack.chunjun.format.protobuf.test.Variant TagValue = 2; + */ + public VariantOrBuilder getTagValueOrBuilder() { + if (tagValueBuilder_ != null) { + return tagValueBuilder_.getMessageOrBuilder(); + } else { + return tagValue_ == null ? + Variant.getDefaultInstance() : tagValue_; + } + } + /** + * .com.dtstack.chunjun.format.protobuf.test.Variant TagValue = 2; + */ + private com.google.protobuf.SingleFieldBuilderV3< + Variant, Variant.Builder, VariantOrBuilder> + getTagValueFieldBuilder() { + if (tagValueBuilder_ == null) { + tagValueBuilder_ = new com.google.protobuf.SingleFieldBuilderV3< + Variant, Variant.Builder, VariantOrBuilder>( + getTagValue(), + getParentForChildren(), + isClean()); + tagValue_ = null; + } + return tagValueBuilder_; + } + + private com.google.protobuf.MapField< + String, String> exValues_; + private com.google.protobuf.MapField + internalGetExValues() { + if (exValues_ == null) { + return com.google.protobuf.MapField.emptyMapField( + ExValuesDefaultEntryHolder.defaultEntry); + } + return exValues_; + } + private com.google.protobuf.MapField + internalGetMutableExValues() { + onChanged();; + if (exValues_ == null) { + exValues_ = com.google.protobuf.MapField.newMapField( + ExValuesDefaultEntryHolder.defaultEntry); + } + if (!exValues_.isMutable()) { + exValues_ = exValues_.copy(); + } + return exValues_; + } + + public int getExValuesCount() { + return internalGetExValues().getMap().size(); + } + /** + * map<string, string> ExValues = 3; + */ + + public boolean containsExValues( + String key) { + if (key == null) { throw new NullPointerException(); } + return internalGetExValues().getMap().containsKey(key); + } + /** + * Use {@link #getExValuesMap()} instead. + */ + @Deprecated + public java.util.Map getExValues() { + return getExValuesMap(); + } + /** + * map<string, string> ExValues = 3; + */ + + public java.util.Map getExValuesMap() { + return internalGetExValues().getMap(); + } + /** + * map<string, string> ExValues = 3; + */ + + public String getExValuesOrDefault( + String key, + String defaultValue) { + if (key == null) { throw new NullPointerException(); } + java.util.Map map = + internalGetExValues().getMap(); + return map.containsKey(key) ? map.get(key) : defaultValue; + } + /** + * map<string, string> ExValues = 3; + */ + + public String getExValuesOrThrow( + String key) { + if (key == null) { throw new NullPointerException(); } + java.util.Map map = + internalGetExValues().getMap(); + if (!map.containsKey(key)) { + throw new IllegalArgumentException(); + } + return map.get(key); + } + + public Builder clearExValues() { + internalGetMutableExValues().getMutableMap() + .clear(); + return this; + } + /** + * map<string, string> ExValues = 3; + */ + + public Builder removeExValues( + String key) { + if (key == null) { throw new NullPointerException(); } + internalGetMutableExValues().getMutableMap() + .remove(key); + return this; + } + /** + * Use alternate mutation accessors instead. + */ + @Deprecated + public java.util.Map + getMutableExValues() { + return internalGetMutableExValues().getMutableMap(); + } + /** + * map<string, string> ExValues = 3; + */ + public Builder putExValues( + String key, + String value) { + if (key == null) { throw new NullPointerException(); } + if (value == null) { throw new NullPointerException(); } + internalGetMutableExValues().getMutableMap() + .put(key, value); + return this; + } + /** + * map<string, string> ExValues = 3; + */ + + public Builder putAllExValues( + java.util.Map values) { + internalGetMutableExValues().getMutableMap() + .putAll(values); + return this; + } + public final Builder setUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return this; + } + + public final Builder mergeUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return this; + } + + + // @@protoc_insertion_point(builder_scope:com.dtstack.chunjun.format.protobuf.test.MessageItem) + } + + // @@protoc_insertion_point(class_scope:com.dtstack.chunjun.format.protobuf.test.MessageItem) + private static final MessageItem DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new MessageItem(); + } + + public static MessageItem getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + private static final com.google.protobuf.Parser + PARSER = new com.google.protobuf.AbstractParser() { + public MessageItem parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new MessageItem(input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public MessageItem getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface MessageTestOrBuilder extends + // @@protoc_insertion_point(interface_extends:com.dtstack.chunjun.format.protobuf.test.MessageTest) + com.google.protobuf.MessageOrBuilder { + + /** + * map<string, string> GroupInfo = 1; + */ + int getGroupInfoCount(); + /** + * map<string, string> GroupInfo = 1; + */ + boolean containsGroupInfo( + String key); + /** + * Use {@link #getGroupInfoMap()} instead. + */ + @Deprecated + java.util.Map + getGroupInfo(); + /** + * map<string, string> GroupInfo = 1; + */ + java.util.Map + getGroupInfoMap(); + /** + * map<string, string> GroupInfo = 1; + */ + + String getGroupInfoOrDefault( + String key, + String defaultValue); + /** + * map<string, string> GroupInfo = 1; + */ + + String getGroupInfoOrThrow( + String key); + + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + java.util.List + getMessagesList(); + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + MessageItem getMessages(int index); + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + int getMessagesCount(); + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + java.util.List + getMessagesOrBuilderList(); + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + MessageItemOrBuilder getMessagesOrBuilder( + int index); + } + /** + * Protobuf type {@code com.dtstack.chunjun.format.protobuf.test.MessageTest} + */ + public static final class MessageTest extends + com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:com.dtstack.chunjun.format.protobuf.test.MessageTest) + MessageTestOrBuilder { + // Use MessageTest.newBuilder() to construct. + private MessageTest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + private MessageTest() { + messages_ = java.util.Collections.emptyList(); + } + + @Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return com.google.protobuf.UnknownFieldSet.getDefaultInstance(); + } + private MessageTest( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + int mutable_bitField0_ = 0; + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!input.skipField(tag)) { + done = true; + } + break; + } + case 10: { + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + groupInfo_ = com.google.protobuf.MapField.newMapField( + GroupInfoDefaultEntryHolder.defaultEntry); + mutable_bitField0_ |= 0x00000001; + } + com.google.protobuf.MapEntry + groupInfo__ = input.readMessage( + GroupInfoDefaultEntryHolder.defaultEntry.getParserForType(), extensionRegistry); + groupInfo_.getMutableMap().put( + groupInfo__.getKey(), groupInfo__.getValue()); + break; + } + case 18: { + if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + messages_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000002; + } + messages_.add( + input.readMessage(MessageItem.parser(), extensionRegistry)); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + messages_ = java.util.Collections.unmodifiableList(messages_); + } + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_descriptor; + } + + @SuppressWarnings({"rawtypes"}) + protected com.google.protobuf.MapField internalGetMapField( + int number) { + switch (number) { + case 1: + return internalGetGroupInfo(); + default: + throw new RuntimeException( + "Invalid map field number: " + number); + } + } + protected FieldAccessorTable + internalGetFieldAccessorTable() { + return MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_fieldAccessorTable + .ensureFieldAccessorsInitialized( + MessageTest.class, Builder.class); + } + + public static final int GROUPINFO_FIELD_NUMBER = 1; + private static final class GroupInfoDefaultEntryHolder { + static final com.google.protobuf.MapEntry< + String, String> defaultEntry = + com.google.protobuf.MapEntry + .newDefaultInstance( + MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_GroupInfoEntry_descriptor, + com.google.protobuf.WireFormat.FieldType.STRING, + "", + com.google.protobuf.WireFormat.FieldType.STRING, + ""); + } + private com.google.protobuf.MapField< + String, String> groupInfo_; + private com.google.protobuf.MapField + internalGetGroupInfo() { + if (groupInfo_ == null) { + return com.google.protobuf.MapField.emptyMapField( + GroupInfoDefaultEntryHolder.defaultEntry); + } + return groupInfo_; + } + + public int getGroupInfoCount() { + return internalGetGroupInfo().getMap().size(); + } + /** + * map<string, string> GroupInfo = 1; + */ + + public boolean containsGroupInfo( + String key) { + if (key == null) { throw new NullPointerException(); } + return internalGetGroupInfo().getMap().containsKey(key); + } + /** + * Use {@link #getGroupInfoMap()} instead. + */ + @Deprecated + public java.util.Map getGroupInfo() { + return getGroupInfoMap(); + } + /** + * map<string, string> GroupInfo = 1; + */ + + public java.util.Map getGroupInfoMap() { + return internalGetGroupInfo().getMap(); + } + /** + * map<string, string> GroupInfo = 1; + */ + + public String getGroupInfoOrDefault( + String key, + String defaultValue) { + if (key == null) { throw new NullPointerException(); } + java.util.Map map = + internalGetGroupInfo().getMap(); + return map.containsKey(key) ? map.get(key) : defaultValue; + } + /** + * map<string, string> GroupInfo = 1; + */ + + public String getGroupInfoOrThrow( + String key) { + if (key == null) { throw new NullPointerException(); } + java.util.Map map = + internalGetGroupInfo().getMap(); + if (!map.containsKey(key)) { + throw new IllegalArgumentException(); + } + return map.get(key); + } + + public static final int MESSAGES_FIELD_NUMBER = 2; + private java.util.List messages_; + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public java.util.List getMessagesList() { + return messages_; + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public java.util.List + getMessagesOrBuilderList() { + return messages_; + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public int getMessagesCount() { + return messages_.size(); + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public MessageItem getMessages(int index) { + return messages_.get(index); + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public MessageItemOrBuilder getMessagesOrBuilder( + int index) { + return messages_.get(index); + } + + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) return true; + if (isInitialized == 0) return false; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + com.google.protobuf.GeneratedMessageV3 + .serializeStringMapTo( + output, + internalGetGroupInfo(), + GroupInfoDefaultEntryHolder.defaultEntry, + 1); + for (int i = 0; i < messages_.size(); i++) { + output.writeMessage(2, messages_.get(i)); + } + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) return size; + + size = 0; + for (java.util.Map.Entry entry + : internalGetGroupInfo().getMap().entrySet()) { + com.google.protobuf.MapEntry + groupInfo__ = GroupInfoDefaultEntryHolder.defaultEntry.newBuilderForType() + .setKey(entry.getKey()) + .setValue(entry.getValue()) + .build(); + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, groupInfo__); + } + for (int i = 0; i < messages_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(2, messages_.get(i)); + } + memoizedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof MessageTest)) { + return super.equals(obj); + } + MessageTest other = (MessageTest) obj; + + boolean result = true; + result = result && internalGetGroupInfo().equals( + other.internalGetGroupInfo()); + result = result && getMessagesList() + .equals(other.getMessagesList()); + return result; + } + + @Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (!internalGetGroupInfo().getMap().isEmpty()) { + hash = (37 * hash) + GROUPINFO_FIELD_NUMBER; + hash = (53 * hash) + internalGetGroupInfo().hashCode(); + } + if (getMessagesCount() > 0) { + hash = (37 * hash) + MESSAGES_FIELD_NUMBER; + hash = (53 * hash) + getMessagesList().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static MessageTest parseFrom( + java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static MessageTest parseFrom( + java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static MessageTest parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static MessageTest parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static MessageTest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static MessageTest parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static MessageTest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static MessageTest parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + public static MessageTest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input); + } + public static MessageTest parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input, extensionRegistry); + } + public static MessageTest parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static MessageTest parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + public static Builder newBuilder(MessageTest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { + return this == DEFAULT_INSTANCE + ? new Builder() : new Builder().mergeFrom(this); + } + + @Override + protected Builder newBuilderForType( + BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code com.dtstack.chunjun.format.protobuf.test.MessageTest} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessageV3.Builder implements + // @@protoc_insertion_point(builder_implements:com.dtstack.chunjun.format.protobuf.test.MessageTest) + MessageTestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_descriptor; + } + + @SuppressWarnings({"rawtypes"}) + protected com.google.protobuf.MapField internalGetMapField( + int number) { + switch (number) { + case 1: + return internalGetGroupInfo(); + default: + throw new RuntimeException( + "Invalid map field number: " + number); + } + } + @SuppressWarnings({"rawtypes"}) + protected com.google.protobuf.MapField internalGetMutableMapField( + int number) { + switch (number) { + case 1: + return internalGetMutableGroupInfo(); + default: + throw new RuntimeException( + "Invalid map field number: " + number); + } + } + protected FieldAccessorTable + internalGetFieldAccessorTable() { + return MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_fieldAccessorTable + .ensureFieldAccessorsInitialized( + MessageTest.class, Builder.class); + } + + // Construct using com.dtstack.chunjun.format.protobuf.test.MessageTestOuterClass.MessageTest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3 + .alwaysUseFieldBuilders) { + getMessagesFieldBuilder(); + } + } + public Builder clear() { + super.clear(); + internalGetMutableGroupInfo().clear(); + if (messagesBuilder_ == null) { + messages_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + } else { + messagesBuilder_.clear(); + } + return this; + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return MessageTestOuterClass.internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_descriptor; + } + + public MessageTest getDefaultInstanceForType() { + return MessageTest.getDefaultInstance(); + } + + public MessageTest build() { + MessageTest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public MessageTest buildPartial() { + MessageTest result = new MessageTest(this); + int from_bitField0_ = bitField0_; + result.groupInfo_ = internalGetGroupInfo(); + result.groupInfo_.makeImmutable(); + if (messagesBuilder_ == null) { + if (((bitField0_ & 0x00000002) == 0x00000002)) { + messages_ = java.util.Collections.unmodifiableList(messages_); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.messages_ = messages_; + } else { + result.messages_ = messagesBuilder_.build(); + } + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + public Builder setField( + com.google.protobuf.Descriptors.FieldDescriptor field, + Object value) { + return (Builder) super.setField(field, value); + } + public Builder clearField( + com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + public Builder clearOneof( + com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + public Builder setRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + int index, Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + public Builder addRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + Object value) { + return (Builder) super.addRepeatedField(field, value); + } + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof MessageTest) { + return mergeFrom((MessageTest)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(MessageTest other) { + if (other == MessageTest.getDefaultInstance()) return this; + internalGetMutableGroupInfo().mergeFrom( + other.internalGetGroupInfo()); + if (messagesBuilder_ == null) { + if (!other.messages_.isEmpty()) { + if (messages_.isEmpty()) { + messages_ = other.messages_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureMessagesIsMutable(); + messages_.addAll(other.messages_); + } + onChanged(); + } + } else { + if (!other.messages_.isEmpty()) { + if (messagesBuilder_.isEmpty()) { + messagesBuilder_.dispose(); + messagesBuilder_ = null; + messages_ = other.messages_; + bitField0_ = (bitField0_ & ~0x00000002); + messagesBuilder_ = + com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ? + getMessagesFieldBuilder() : null; + } else { + messagesBuilder_.addAllMessages(other.messages_); + } + } + } + onChanged(); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + MessageTest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (MessageTest) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + private com.google.protobuf.MapField< + String, String> groupInfo_; + private com.google.protobuf.MapField + internalGetGroupInfo() { + if (groupInfo_ == null) { + return com.google.protobuf.MapField.emptyMapField( + GroupInfoDefaultEntryHolder.defaultEntry); + } + return groupInfo_; + } + private com.google.protobuf.MapField + internalGetMutableGroupInfo() { + onChanged();; + if (groupInfo_ == null) { + groupInfo_ = com.google.protobuf.MapField.newMapField( + GroupInfoDefaultEntryHolder.defaultEntry); + } + if (!groupInfo_.isMutable()) { + groupInfo_ = groupInfo_.copy(); + } + return groupInfo_; + } + + public int getGroupInfoCount() { + return internalGetGroupInfo().getMap().size(); + } + /** + * map<string, string> GroupInfo = 1; + */ + + public boolean containsGroupInfo( + String key) { + if (key == null) { throw new NullPointerException(); } + return internalGetGroupInfo().getMap().containsKey(key); + } + /** + * Use {@link #getGroupInfoMap()} instead. + */ + @Deprecated + public java.util.Map getGroupInfo() { + return getGroupInfoMap(); + } + /** + * map<string, string> GroupInfo = 1; + */ + + public java.util.Map getGroupInfoMap() { + return internalGetGroupInfo().getMap(); + } + /** + * map<string, string> GroupInfo = 1; + */ + + public String getGroupInfoOrDefault( + String key, + String defaultValue) { + if (key == null) { throw new NullPointerException(); } + java.util.Map map = + internalGetGroupInfo().getMap(); + return map.containsKey(key) ? map.get(key) : defaultValue; + } + /** + * map<string, string> GroupInfo = 1; + */ + + public String getGroupInfoOrThrow( + String key) { + if (key == null) { throw new NullPointerException(); } + java.util.Map map = + internalGetGroupInfo().getMap(); + if (!map.containsKey(key)) { + throw new IllegalArgumentException(); + } + return map.get(key); + } + + public Builder clearGroupInfo() { + internalGetMutableGroupInfo().getMutableMap() + .clear(); + return this; + } + /** + * map<string, string> GroupInfo = 1; + */ + + public Builder removeGroupInfo( + String key) { + if (key == null) { throw new NullPointerException(); } + internalGetMutableGroupInfo().getMutableMap() + .remove(key); + return this; + } + /** + * Use alternate mutation accessors instead. + */ + @Deprecated + public java.util.Map + getMutableGroupInfo() { + return internalGetMutableGroupInfo().getMutableMap(); + } + /** + * map<string, string> GroupInfo = 1; + */ + public Builder putGroupInfo( + String key, + String value) { + if (key == null) { throw new NullPointerException(); } + if (value == null) { throw new NullPointerException(); } + internalGetMutableGroupInfo().getMutableMap() + .put(key, value); + return this; + } + /** + * map<string, string> GroupInfo = 1; + */ + + public Builder putAllGroupInfo( + java.util.Map values) { + internalGetMutableGroupInfo().getMutableMap() + .putAll(values); + return this; + } + + private java.util.List messages_ = + java.util.Collections.emptyList(); + private void ensureMessagesIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + messages_ = new java.util.ArrayList(messages_); + bitField0_ |= 0x00000002; + } + } + + private com.google.protobuf.RepeatedFieldBuilderV3< + MessageItem, MessageItem.Builder, MessageItemOrBuilder> messagesBuilder_; + + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public java.util.List getMessagesList() { + if (messagesBuilder_ == null) { + return java.util.Collections.unmodifiableList(messages_); + } else { + return messagesBuilder_.getMessageList(); + } + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public int getMessagesCount() { + if (messagesBuilder_ == null) { + return messages_.size(); + } else { + return messagesBuilder_.getCount(); + } + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public MessageItem getMessages(int index) { + if (messagesBuilder_ == null) { + return messages_.get(index); + } else { + return messagesBuilder_.getMessage(index); + } + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public Builder setMessages( + int index, MessageItem value) { + if (messagesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureMessagesIsMutable(); + messages_.set(index, value); + onChanged(); + } else { + messagesBuilder_.setMessage(index, value); + } + return this; + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public Builder setMessages( + int index, MessageItem.Builder builderForValue) { + if (messagesBuilder_ == null) { + ensureMessagesIsMutable(); + messages_.set(index, builderForValue.build()); + onChanged(); + } else { + messagesBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public Builder addMessages(MessageItem value) { + if (messagesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureMessagesIsMutable(); + messages_.add(value); + onChanged(); + } else { + messagesBuilder_.addMessage(value); + } + return this; + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public Builder addMessages( + int index, MessageItem value) { + if (messagesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureMessagesIsMutable(); + messages_.add(index, value); + onChanged(); + } else { + messagesBuilder_.addMessage(index, value); + } + return this; + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public Builder addMessages( + MessageItem.Builder builderForValue) { + if (messagesBuilder_ == null) { + ensureMessagesIsMutable(); + messages_.add(builderForValue.build()); + onChanged(); + } else { + messagesBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public Builder addMessages( + int index, MessageItem.Builder builderForValue) { + if (messagesBuilder_ == null) { + ensureMessagesIsMutable(); + messages_.add(index, builderForValue.build()); + onChanged(); + } else { + messagesBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public Builder addAllMessages( + Iterable values) { + if (messagesBuilder_ == null) { + ensureMessagesIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll( + values, messages_); + onChanged(); + } else { + messagesBuilder_.addAllMessages(values); + } + return this; + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public Builder clearMessages() { + if (messagesBuilder_ == null) { + messages_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + } else { + messagesBuilder_.clear(); + } + return this; + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public Builder removeMessages(int index) { + if (messagesBuilder_ == null) { + ensureMessagesIsMutable(); + messages_.remove(index); + onChanged(); + } else { + messagesBuilder_.remove(index); + } + return this; + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public MessageItem.Builder getMessagesBuilder( + int index) { + return getMessagesFieldBuilder().getBuilder(index); + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public MessageItemOrBuilder getMessagesOrBuilder( + int index) { + if (messagesBuilder_ == null) { + return messages_.get(index); } else { + return messagesBuilder_.getMessageOrBuilder(index); + } + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public java.util.List + getMessagesOrBuilderList() { + if (messagesBuilder_ != null) { + return messagesBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(messages_); + } + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public MessageItem.Builder addMessagesBuilder() { + return getMessagesFieldBuilder().addBuilder( + MessageItem.getDefaultInstance()); + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public MessageItem.Builder addMessagesBuilder( + int index) { + return getMessagesFieldBuilder().addBuilder( + index, MessageItem.getDefaultInstance()); + } + /** + * repeated .com.dtstack.chunjun.format.protobuf.test.MessageItem Messages = 2; + */ + public java.util.List + getMessagesBuilderList() { + return getMessagesFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilderV3< + MessageItem, MessageItem.Builder, MessageItemOrBuilder> + getMessagesFieldBuilder() { + if (messagesBuilder_ == null) { + messagesBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3< + MessageItem, MessageItem.Builder, MessageItemOrBuilder>( + messages_, + ((bitField0_ & 0x00000002) == 0x00000002), + getParentForChildren(), + isClean()); + messages_ = null; + } + return messagesBuilder_; + } + public final Builder setUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return this; + } + + public final Builder mergeUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return this; + } + + + // @@protoc_insertion_point(builder_scope:com.dtstack.chunjun.format.protobuf.test.MessageTest) + } + + // @@protoc_insertion_point(class_scope:com.dtstack.chunjun.format.protobuf.test.MessageTest) + private static final MessageTest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new MessageTest(); + } + + public static MessageTest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + private static final com.google.protobuf.Parser + PARSER = new com.google.protobuf.AbstractParser() { + public MessageTest parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new MessageTest(input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public MessageTest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + private static final com.google.protobuf.Descriptors.Descriptor + internal_static_com_dtstack_chunjun_format_protobuf_test_Variant_descriptor; + private static final + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internal_static_com_dtstack_chunjun_format_protobuf_test_Variant_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_descriptor; + private static final + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_ExValuesEntry_descriptor; + private static final + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_ExValuesEntry_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_descriptor; + private static final + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_GroupInfoEntry_descriptor; + private static final + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_GroupInfoEntry_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + String[] descriptorData = { + "\n\021MessageTest.proto\022(com.dtstack.chunjun" + + ".format.protobuf.test\"]\n\007Variant\022\r\n\005bool" + + "x\030\001 \001(\010\022\024\n\nValueBool2\030\002 \001(\010H\000\022\024\n\nValueIn" + + "t32\030\003 \001(\005H\000\022\r\n\005booly\030\004 \001(\010B\010\n\006Value2\"\353\001\n" + + "\013MessageItem\022\017\n\007TagName\030\001 \001(\t\022C\n\010TagValu" + + "e\030\002 \001(\01321.com.dtstack.chunjun.format.pro" + + "tobuf.test.Variant\022U\n\010ExValues\030\003 \003(\0132C.c" + + "om.dtstack.chunjun.format.protobuf.test." + + "MessageItem.ExValuesEntry\032/\n\rExValuesEnt" + + "ry\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001\"\341\001\n\013M", + "essageTest\022W\n\tGroupInfo\030\001 \003(\0132D.com.dtst" + + "ack.chunjun.format.protobuf.test.Message" + + "Test.GroupInfoEntry\022G\n\010Messages\030\002 \003(\01325." + + "com.dtstack.chunjun.format.protobuf.test" + + ".MessageItem\0320\n\016GroupInfoEntry\022\013\n\003key\030\001 " + + "\001(\t\022\r\n\005value\030\002 \001(\t:\0028\001b\006proto3" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor. InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + }, assigner); + internal_static_com_dtstack_chunjun_format_protobuf_test_Variant_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_com_dtstack_chunjun_format_protobuf_test_Variant_fieldAccessorTable = new + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_com_dtstack_chunjun_format_protobuf_test_Variant_descriptor, + new String[] { "Boolx", "ValueBool2", "ValueInt32", "Booly", "Value2", }); + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_descriptor = + getDescriptor().getMessageTypes().get(1); + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_fieldAccessorTable = new + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_descriptor, + new String[] { "TagName", "TagValue", "ExValues", }); + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_ExValuesEntry_descriptor = + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_descriptor.getNestedTypes().get(0); + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_ExValuesEntry_fieldAccessorTable = new + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageItem_ExValuesEntry_descriptor, + new String[] { "Key", "Value", }); + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_descriptor = + getDescriptor().getMessageTypes().get(2); + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_fieldAccessorTable = new + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_descriptor, + new String[] { "GroupInfo", "Messages", }); + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_GroupInfoEntry_descriptor = + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_descriptor.getNestedTypes().get(0); + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_GroupInfoEntry_fieldAccessorTable = new + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_com_dtstack_chunjun_format_protobuf_test_MessageTest_GroupInfoEntry_descriptor, + new String[] { "Key", "Value", }); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/chunjun-formats/chunjun-protobuf/src/test/java/com/dtstack/chunjun/format/protobuf/test/PbFormatFactoryTest.java b/chunjun-formats/chunjun-protobuf/src/test/java/com/dtstack/chunjun/format/protobuf/test/PbFormatFactoryTest.java new file mode 100644 index 0000000000..7f8d7bacfe --- /dev/null +++ b/chunjun-formats/chunjun-protobuf/src/test/java/com/dtstack/chunjun/format/protobuf/test/PbFormatFactoryTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.chunjun.format.protobuf.test; + +import com.dtstack.chunjun.format.protobuf.PbFormatFactory; +import com.dtstack.chunjun.format.protobuf.util.FormatCheckUtil; + +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.table.connector.format.DecodingFormat; +import org.apache.flink.table.connector.format.EncodingFormat; +import org.apache.flink.table.connector.sink.DynamicTableSink; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.factories.DynamicTableFactory; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.RowType; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import static com.dtstack.chunjun.format.protobuf.PbFormatOptions.MESSAGE_CLASS_NAME; + +/** @author liuliu 2022/5/4 */ +@RunWith(PowerMockRunner.class) +@PrepareForTest({PbFormatFactory.class}) +public class PbFormatFactoryTest { + + DecodingFormat> decodingFormat; + EncodingFormat> encodingFormat; + + DeserializationSchema runtimeDecoder; + SerializationSchema runtimeEncoder; + + @Before + public void init() throws Exception { + DynamicTableFactory.Context context = PowerMockito.mock(DynamicTableFactory.Context.class); + ReadableConfig readableConfig = PowerMockito.mock(ReadableConfig.class); + PowerMockito.when(readableConfig.get(MESSAGE_CLASS_NAME)) + .thenReturn( + "com.dtstack.chunjun.format.protobuf.test.MessageTestOuterClass$MessageTest"); + + PbFormatFactory pbFormatFactory = PowerMockito.mock(PbFormatFactory.class); + PowerMockito.when(pbFormatFactory, "createDecodingFormat", context, readableConfig) + .thenCallRealMethod(); + PowerMockito.when(pbFormatFactory, "createEncodingFormat", context, readableConfig) + .thenCallRealMethod(); + decodingFormat = pbFormatFactory.createDecodingFormat(context, readableConfig); + encodingFormat = pbFormatFactory.createEncodingFormat(context, readableConfig); + + DynamicTableSource.Context sourceContext = + PowerMockito.mock(DynamicTableSource.Context.class); + DynamicTableSink.Context sinkContext = PowerMockito.mock(DynamicTableSink.Context.class); + + FormatCheckUtil formatCheckUtil = + new FormatCheckUtil( + null, + "com.dtstack.chunjun.format.protobuf.test.MessageTestOuterClass$MessageTest"); + RowType messageLogicalType = + formatCheckUtil.createMessageLogicalType( + "com.dtstack.chunjun.format.protobuf.test.MessageTestOuterClass$MessageTest"); + DataType physicalDataType = PowerMockito.mock(DataType.class); + PowerMockito.when(physicalDataType.getLogicalType()).thenReturn(messageLogicalType); + PowerMockito.when(sourceContext.createTypeInformation(physicalDataType)).thenReturn(null); + PowerMockito.when(sinkContext.createTypeInformation(physicalDataType)).thenReturn(null); + + runtimeDecoder = decodingFormat.createRuntimeDecoder(sourceContext, physicalDataType); + + runtimeEncoder = encodingFormat.createRuntimeEncoder(sinkContext, physicalDataType); + } + + @Test + public void serializeTest() throws Exception { + + MessageTestOuterClass.MessageTest messageGroup = getMessageGroup(); + // serialize + runtimeDecoder.open(null); + RowData deserialize = runtimeDecoder.deserialize(messageGroup.toByteArray()); + assert deserialize.getMap(0).keyArray().getString(0).toString().equals("group"); + assert deserialize.getArray(1).getRow(0, 0).getRow(1, 0).getRow(1, 0).getInt(0) == 3; + } + + @Test + public void deserializeTest() throws Exception { + + MessageTestOuterClass.MessageTest messageGroup = getMessageGroup(); + runtimeDecoder.open(null); + RowData rowData = runtimeDecoder.deserialize(messageGroup.toByteArray()); + + runtimeEncoder.open(null); + assert messageGroup.equals( + MessageTestOuterClass.MessageTest.parseFrom(runtimeEncoder.serialize(rowData))); + } + + public static MessageTestOuterClass.MessageTest getMessageGroup() { + MessageTestOuterClass.MessageTest.Builder builder = + MessageTestOuterClass.MessageTest.newBuilder(); + builder.putGroupInfo("group", "test"); + builder.addMessages(0, getMessageItem()); + return builder.build(); + } + + public static MessageTestOuterClass.MessageItem getMessageItem() { + MessageTestOuterClass.MessageItem.Builder builder = + MessageTestOuterClass.MessageItem.newBuilder(); + + builder.setTagName("tag"); + builder.setTagValue(getVariant()); + builder.putExValues("2", "3"); + + return builder.build(); + } + + public static MessageTestOuterClass.Variant getVariant() { + MessageTestOuterClass.Variant.Builder builder = MessageTestOuterClass.Variant.newBuilder(); + builder.setBoolx(true); + builder.setValueInt32(1); + builder.setBooly(false); + + return builder.build(); + } +} diff --git a/chunjun-formats/pom.xml b/chunjun-formats/pom.xml index 802ce8a162..ad9548c858 100644 --- a/chunjun-formats/pom.xml +++ b/chunjun-formats/pom.xml @@ -13,6 +13,10 @@ ChunJun : Formats pom + + chunjun-protobuf + + com.dtstack.chunjun diff --git a/chunjun-local-test/src/main/java/com/dtstack/chunjun/local/test/LocalTest.java b/chunjun-local-test/src/main/java/com/dtstack/chunjun/local/test/LocalTest.java index a9337bb888..14279f26cd 100644 --- a/chunjun-local-test/src/main/java/com/dtstack/chunjun/local/test/LocalTest.java +++ b/chunjun-local-test/src/main/java/com/dtstack/chunjun/local/test/LocalTest.java @@ -175,21 +175,7 @@ public static void main(String[] args) throws Exception { // argsList.add(configJsonString); /* ---------------------------------------- pyFlink 测试 end --------------------------------------- */ } - // 防止加载chunjun-connector-kafka/target/classes/META-INF/services/下的spi文件 - URLClassLoader contextClassLoader = - (URLClassLoader) Thread.currentThread().getContextClassLoader(); - URL[] urls = - Arrays.stream(contextClassLoader.getURLs()) - .filter(URL -> !URL.getPath().contains("start-chunjun-connector-kafka")) - .toArray(URL[]::new); - URLClassLoader urlClassLoader = new URLClassLoader(urls, contextClassLoader.getParent()); - Thread.currentThread().setContextClassLoader(urlClassLoader); - Class
mainClass = - (Class
) Class.forName(Main.class.getName(), false, urlClassLoader); - Constructor constructor = mainClass.getConstructor(); - Object mainObject = constructor.newInstance(); - Method mainMethod = mainClass.getMethod("main", String[].class); - mainMethod.invoke(mainObject, (Object) argsList.toArray(new String[0])); + Main.main(argsList.toArray(new String[0])); } private static String readFile(String sqlPath) {