From 86fb725301e014bd7b3f8a9068a975e7ebcacb6d Mon Sep 17 00:00:00 2001 From: wangguowei Date: Mon, 2 Nov 2020 13:40:39 +0800 Subject: [PATCH 01/19] [SQL] migrate SchemaHandle to decoder --- pulsar-sql/presto-pulsar/pom.xml | 6 + .../pulsar/sql/presto/AvroSchemaHandler.java | 104 ----- .../pulsar/sql/presto/JSONSchemaHandler.java | 108 ----- .../sql/presto/KeyValueSchemaHandler.java | 105 ----- .../pulsar/sql/presto/PulsarColumnHandle.java | 78 ++-- .../sql/presto/PulsarColumnMetadata.java | 118 ++++-- .../sql/presto/PulsarConnectorModule.java | 16 +- .../PulsarDispatchingRowDecoderFactory.java | 77 ++++ .../sql/presto/PulsarFieldValueProviders.java | 61 +++ .../sql/presto/PulsarInternalColumn.java | 179 ++------ .../pulsar/sql/presto/PulsarMetadata.java | 316 ++------------ .../presto/PulsarPrimitiveSchemaHandler.java | 63 --- .../pulsar/sql/presto/PulsarRecordCursor.java | 307 +++++++------- .../pulsar/sql/presto/PulsarRecordSet.java | 13 +- .../sql/presto/PulsarRecordSetProvider.java | 21 +- .../sql/presto/PulsarRowDecoderFactory.java | 52 +++ .../sql/presto/PulsarSchemaHandlers.java | 69 ---- .../pulsar/sql/presto/PulsarSplitManager.java | 41 +- .../presto/PulsarSqlSchemaInfoProvider.java | 25 +- .../pulsar/sql/presto/SchemaHandler.java | 45 -- .../decoder/avro/PulsarAvroColumnDecoder.java | 347 ++++++++++++++++ .../decoder/avro/PulsarAvroRowDecoder.java | 75 ++++ .../avro/PulsarAvroRowDecoderFactory.java | 161 ++++++++ .../decoder/json/PulsarJsonFieldDecoder.java | 387 ++++++++++++++++++ .../decoder/json/PulsarJsonRowDecoder.java | 81 ++++ .../json/PulsarJsonRowDecoderFactory.java | 146 +++++++ .../primitive/PulsarPrimitiveRowDecoder.java | 62 +++ .../PulsarPrimitiveRowDecoderFactory.java | 93 +++++ 28 files changed, 1974 insertions(+), 1182 deletions(-) delete mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/AvroSchemaHandler.java delete mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/JSONSchemaHandler.java delete mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/KeyValueSchemaHandler.java create mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java create mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarFieldValueProviders.java delete mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarPrimitiveSchemaHandler.java create mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java delete mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSchemaHandlers.java delete mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/SchemaHandler.java create mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java create mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java create mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java create mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java create mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java create mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java create mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java create mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index c2caef58fc715..82883aa678247 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -109,6 +109,12 @@ ${joda.version} + + io.prestosql + presto-record-decoder + ${presto.version} + + diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/AvroSchemaHandler.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/AvroSchemaHandler.java deleted file mode 100644 index 966dc07b5c171..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/AvroSchemaHandler.java +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import com.google.common.annotations.VisibleForTesting; - -import io.airlift.log.Logger; -import io.netty.buffer.ByteBuf; - -import java.util.List; - -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord; -import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.SchemaInfo; - - -/** - * Schema handler for payload in the Avro format. - */ -public class AvroSchemaHandler implements SchemaHandler { - - private final List columnHandles; - - private final GenericAvroSchema genericAvroSchema; - - private final SchemaInfo schemaInfo; - - private static final Logger log = Logger.get(AvroSchemaHandler.class); - - AvroSchemaHandler(TopicName topicName, - PulsarConnectorConfig pulsarConnectorConfig, - SchemaInfo schemaInfo, - List columnHandles) throws PulsarClientException { - this(new PulsarSqlSchemaInfoProvider(topicName, - pulsarConnectorConfig.getPulsarAdmin()), schemaInfo, columnHandles); - } - - AvroSchemaHandler(PulsarSqlSchemaInfoProvider pulsarSqlSchemaInfoProvider, - SchemaInfo schemaInfo, List columnHandles) { - this.schemaInfo = schemaInfo; - this.genericAvroSchema = new GenericAvroSchema(schemaInfo); - this.genericAvroSchema.setSchemaInfoProvider(pulsarSqlSchemaInfoProvider); - this.columnHandles = columnHandles; - } - - @Override - public Object deserialize(ByteBuf payload) { - return genericAvroSchema.decode(payload); - } - - @Override - public Object deserialize(ByteBuf payload, byte[] schemaVersion) { - return genericAvroSchema.decode(payload, schemaVersion); - } - - @Override - public Object extractField(int index, Object currentRecord) { - try { - GenericAvroRecord record = (GenericAvroRecord) currentRecord; - PulsarColumnHandle pulsarColumnHandle = this.columnHandles.get(index); - String[] names = pulsarColumnHandle.getFieldNames(); - - if (names.length == 1) { - return record.getField(pulsarColumnHandle.getFieldNames()[0]); - } else { - for (int i = 0; i < names.length - 1; i++) { - record = (GenericAvroRecord) record.getField(names[i]); - } - return record.getField(names[names.length - 1]); - } - } catch (Exception ex) { - log.debug(ex, "%s", ex); - } - return null; - } - - @VisibleForTesting - GenericAvroSchema getSchema() { - return this.genericAvroSchema; - } - - @VisibleForTesting - SchemaInfo getSchemaInfo() { - return schemaInfo; - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/JSONSchemaHandler.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/JSONSchemaHandler.java deleted file mode 100644 index a31aa417f6ee7..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/JSONSchemaHandler.java +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import com.dslplatform.json.DslJson; -import io.airlift.log.Logger; -import io.netty.buffer.ByteBuf; -import io.netty.util.concurrent.FastThreadLocal; -import io.prestosql.spi.type.Type; -import java.io.IOException; -import java.math.BigDecimal; -import java.util.List; -import java.util.Map; - -/** - * Schema handler for payload in the JSON format. - */ -public class JSONSchemaHandler implements SchemaHandler { - - private static final Logger log = Logger.get(JSONSchemaHandler.class); - - private List columnHandles; - - private final DslJson dslJson = new DslJson<>(); - - private static final FastThreadLocal tmpBuffer = new FastThreadLocal() { - @Override - protected byte[] initialValue() { - return new byte[1024]; - } - }; - - public JSONSchemaHandler(List columnHandles) { - this.columnHandles = columnHandles; - } - - @Override - public Object deserialize(ByteBuf payload) { - // Since JSON deserializer only works on a byte[] we need to convert a direct mem buffer into - // a byte[]. - int size = payload.readableBytes(); - byte[] buffer = tmpBuffer.get(); - if (buffer.length < size) { - // If the thread-local buffer is not big enough, replace it with - // a bigger one - buffer = new byte[size * 2]; - tmpBuffer.set(buffer); - } - - payload.readBytes(buffer, 0, size); - - try { - return dslJson.deserialize(Map.class, buffer, size); - } catch (IOException e) { - log.error("Failed to deserialize Json object", e); - return null; - } - } - - @Override - public Object extractField(int index, Object currentRecord) { - try { - Map jsonObject = (Map) currentRecord; - PulsarColumnHandle pulsarColumnHandle = columnHandles.get(index); - - String[] fieldNames = pulsarColumnHandle.getFieldNames(); - Object field = jsonObject.get(fieldNames[0]); - if (field == null) { - return null; - } - for (int i = 1; i < fieldNames.length; i++) { - field = ((Map) field).get(fieldNames[i]); - if (field == null) { - return null; - } - } - - Type type = pulsarColumnHandle.getType(); - - Class javaType = type.getJavaType(); - - if (javaType == double.class) { - return ((BigDecimal) field).doubleValue(); - } - - return field; - } catch (Exception ex) { - log.debug(ex, "%s", ex); - } - return null; - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/KeyValueSchemaHandler.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/KeyValueSchemaHandler.java deleted file mode 100644 index 434dd44a4d80f..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/KeyValueSchemaHandler.java +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import com.google.common.annotations.VisibleForTesting; -import io.airlift.log.Logger; -import io.netty.buffer.ByteBuf; -import java.util.List; -import java.util.Objects; -import org.apache.pulsar.client.impl.schema.KeyValueSchemaInfo; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.KeyValue; -import org.apache.pulsar.common.schema.KeyValueEncodingType; -import org.apache.pulsar.common.schema.SchemaInfo; - - -/** - * Schema handler for payload in the KeyValue format. - */ -public class KeyValueSchemaHandler implements SchemaHandler { - - private static final Logger log = Logger.get(KeyValueSchemaHandler.class); - - private final List columnHandles; - - private final SchemaHandler keySchemaHandler; - - private final SchemaHandler valueSchemaHandler; - - private KeyValueEncodingType keyValueEncodingType; - - public KeyValueSchemaHandler(TopicName topicName, - PulsarConnectorConfig pulsarConnectorConfig, - SchemaInfo schemaInfo, - List columnHandles) { - this.columnHandles = columnHandles; - KeyValue kvSchemaInfo = KeyValueSchemaInfo.decodeKeyValueSchemaInfo(schemaInfo); - keySchemaHandler = PulsarSchemaHandlers.newPulsarSchemaHandler(topicName, pulsarConnectorConfig, - kvSchemaInfo.getKey(), columnHandles); - valueSchemaHandler = PulsarSchemaHandlers.newPulsarSchemaHandler(topicName, pulsarConnectorConfig, - kvSchemaInfo.getValue(), columnHandles); - keyValueEncodingType = KeyValueSchemaInfo.decodeKeyValueEncodingType(schemaInfo); - } - - @VisibleForTesting - KeyValueSchemaHandler(SchemaHandler keySchemaHandler, - SchemaHandler valueSchemaHandler, - List columnHandles) { - this.keySchemaHandler = keySchemaHandler; - this.valueSchemaHandler = valueSchemaHandler; - this.columnHandles = columnHandles; - } - - @Override - public Object deserialize(ByteBuf keyPayload, ByteBuf dataPayload, byte[] schemaVersion) { - Object keyObj; - Object valueObj; - ByteBuf keyByteBuf; - ByteBuf valueByteBuf; - if (Objects.equals(keyValueEncodingType, KeyValueEncodingType.INLINE)) { - dataPayload.resetReaderIndex(); - int keyLength = dataPayload.readInt(); - keyByteBuf = dataPayload.readSlice(keyLength); - - int valueLength = dataPayload.readInt(); - valueByteBuf = dataPayload.readSlice(valueLength); - } else { - keyByteBuf = keyPayload; - valueByteBuf = dataPayload; - } - - keyObj = keySchemaHandler.deserialize(keyByteBuf, schemaVersion); - valueObj = valueSchemaHandler.deserialize(valueByteBuf, schemaVersion); - return new KeyValue<>(keyObj, valueObj); - } - - @Override - public Object extractField(int index, Object currentRecord) { - PulsarColumnHandle pulsarColumnHandle = this.columnHandles.get(index); - KeyValue keyValue = (KeyValue) currentRecord; - if (pulsarColumnHandle.isKey()) { - return keySchemaHandler.extractField(index, keyValue.getKey()); - } else if (pulsarColumnHandle.isValue()) { - return valueSchemaHandler.extractField(index, keyValue.getValue()); - } - return null; - } - -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnHandle.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnHandle.java index 2cd85cff66a0d..e751d7a878b3e 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnHandle.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnHandle.java @@ -18,21 +18,21 @@ */ package org.apache.pulsar.sql.presto; -import static java.util.Objects.requireNonNull; - import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.connector.ColumnMetadata; import io.prestosql.spi.type.Type; -import java.util.Arrays; + import java.util.Objects; +import static java.util.Objects.requireNonNull; + /** * This class represents the basic information about a presto column. */ -public class PulsarColumnHandle implements ColumnHandle { +public class PulsarColumnHandle implements DecoderColumnHandle { private final String connectorId; @@ -56,17 +56,29 @@ public class PulsarColumnHandle implements ColumnHandle { */ private final boolean internal; - private final String[] fieldNames; - - private final Integer[] positionIndices; private HandleKeyValueType handleKeyValueType; + /** + * {@link org.apache.pulsar.sql.presto.PulsarColumnMetadata.DecoderExtraInfo#mapping} + */ + private String mapping; + /** + * {@link org.apache.pulsar.sql.presto.PulsarColumnMetadata.DecoderExtraInfo#dataFormat} + */ + private String dataFormat; + + /** + * {@link org.apache.pulsar.sql.presto.PulsarColumnMetadata.DecoderExtraInfo#formatHint} + */ + private String formatHint; + /** * Column Handle keyValue type, used for keyValue schema. */ public enum HandleKeyValueType { /** + * * The handle not for keyValue schema. */ NONE, @@ -87,16 +99,18 @@ public PulsarColumnHandle( @JsonProperty("type") Type type, @JsonProperty("hidden") boolean hidden, @JsonProperty("internal") boolean internal, - @JsonProperty("fieldNames") String[] fieldNames, - @JsonProperty("positionIndices") Integer[] positionIndices, + @JsonProperty("mapping") String mapping, + @JsonProperty("dataFormat") String dataFormat, + @JsonProperty("formatHint") String formatHint, @JsonProperty("handleKeyValueType") HandleKeyValueType handleKeyValueType) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.name = requireNonNull(name, "name is null"); this.type = requireNonNull(type, "type is null"); this.hidden = hidden; this.internal = internal; - this.fieldNames = fieldNames; - this.positionIndices = positionIndices; + this.mapping=mapping; + this.dataFormat=dataFormat; + this.formatHint=formatHint; if (handleKeyValueType == null) { this.handleKeyValueType = HandleKeyValueType.NONE; } else { @@ -114,6 +128,16 @@ public String getName() { return name; } + @JsonProperty + public String getMapping() { + return mapping; + } + + @JsonProperty + public String getDataFormat() { + return dataFormat; + } + @JsonProperty public Type getType() { return type; @@ -130,13 +154,8 @@ public boolean isInternal() { } @JsonProperty - public String[] getFieldNames() { - return fieldNames; - } - - @JsonProperty - public Integer[] getPositionIndices() { - return positionIndices; + public String getFormatHint() { + return formatHint; } @JsonProperty @@ -155,7 +174,7 @@ public boolean isValue() { } ColumnMetadata getColumnMetadata() { - return new ColumnMetadata(name, type, null, hidden); + return new PulsarColumnMetadata(name, type, null, null, hidden, internal, handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(mapping, dataFormat, formatHint)); } @Override @@ -184,12 +203,17 @@ public boolean equals(Object o) { if (type != null ? !type.equals(that.type) : that.type != null) { return false; } - if (!Arrays.deepEquals(fieldNames, that.fieldNames)) { + if (mapping != null ? !mapping.equals(that.mapping) : that.mapping != null) { + return false; + } + if (dataFormat != null ? !dataFormat.equals(that.dataFormat) : that.dataFormat != null) { return false; } - if (!Arrays.deepEquals(positionIndices, that.positionIndices)) { + + if (formatHint != null ? !formatHint.equals(that.formatHint) : that.formatHint != null) { return false; } + return Objects.equals(handleKeyValueType, that.handleKeyValueType); } @@ -200,8 +224,9 @@ public int hashCode() { result = 31 * result + (type != null ? type.hashCode() : 0); result = 31 * result + (hidden ? 1 : 0); result = 31 * result + (internal ? 1 : 0); - result = 31 * result + Arrays.hashCode(fieldNames); - result = 31 * result + Arrays.hashCode(positionIndices); + result = 31 * result + (mapping != null ? mapping.hashCode() : 0); + result = 31 * result + (dataFormat != null ? dataFormat.hashCode() : 0); + result = 31 * result + (formatHint != null ? formatHint.hashCode() : 0); result = 31 * result + (handleKeyValueType != null ? handleKeyValueType.hashCode() : 0); return result; } @@ -214,8 +239,9 @@ public String toString() { + ", type=" + type + ", hidden=" + hidden + ", internal=" + internal - + ", fieldNames=" + Arrays.toString(fieldNames) - + ", positionIndices=" + Arrays.toString(positionIndices) + + ", mapping=" + mapping + + ", dataFormat=" + dataFormat + + ", formatHint=" + formatHint + ", handleKeyValueType=" + handleKeyValueType + '}'; } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java index a24367dc23860..c126c98ebf05a 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java @@ -20,7 +20,7 @@ import io.prestosql.spi.connector.ColumnMetadata; import io.prestosql.spi.type.Type; -import java.util.Arrays; + import java.util.Objects; /** @@ -31,23 +31,27 @@ public class PulsarColumnMetadata extends ColumnMetadata { private boolean isInternal; // need this because presto ColumnMetadata saves name in lowercase private String nameWithCase; - private String[] fieldNames; - private Integer[] positionIndices; private PulsarColumnHandle.HandleKeyValueType handleKeyValueType; public final static String KEY_SCHEMA_COLUMN_PREFIX = "__key."; + + private DecoderExtraInfo decoderExtraInfo ; + public PulsarColumnMetadata(String name, Type type, String comment, String extraInfo, boolean hidden, boolean isInternal, - String[] fieldNames, Integer[] positionIndices, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + PulsarColumnHandle.HandleKeyValueType handleKeyValueType, DecoderExtraInfo decoderExtraInfo) { super(name, type, comment, extraInfo, hidden); this.nameWithCase = name; this.isInternal = isInternal; - this.fieldNames = fieldNames; - this.positionIndices = positionIndices; this.handleKeyValueType = handleKeyValueType; + this.decoderExtraInfo = decoderExtraInfo; } + public DecoderExtraInfo getDecoderExtraInfo() { + return decoderExtraInfo; + } + + public String getNameWithCase() { return nameWithCase; } @@ -56,13 +60,6 @@ public boolean isInternal() { return isInternal; } - public String[] getFieldNames() { - return fieldNames; - } - - public Integer[] getPositionIndices() { - return positionIndices; - } public PulsarColumnHandle.HandleKeyValueType getHandleKeyValueType() { return handleKeyValueType; @@ -88,8 +85,6 @@ public String toString() { return "PulsarColumnMetadata{" + "isInternal=" + isInternal + ", nameWithCase='" + nameWithCase + '\'' - + ", fieldNames=" + Arrays.toString(fieldNames) - + ", positionIndices=" + Arrays.toString(positionIndices) + ", handleKeyValueType=" + handleKeyValueType + '}'; } @@ -114,12 +109,6 @@ public boolean equals(Object o) { if (nameWithCase != null ? !nameWithCase.equals(that.nameWithCase) : that.nameWithCase != null) { return false; } - if (!Arrays.deepEquals(fieldNames, that.fieldNames)) { - return false; - } - if (!Arrays.deepEquals(positionIndices, that.positionIndices)) { - return false; - } return Objects.equals(handleKeyValueType, that.handleKeyValueType); } @@ -128,9 +117,90 @@ public int hashCode() { int result = super.hashCode(); result = 31 * result + (isInternal ? 1 : 0); result = 31 * result + (nameWithCase != null ? nameWithCase.hashCode() : 0); - result = 31 * result + Arrays.hashCode(fieldNames); - result = 31 * result + Arrays.hashCode(positionIndices); result = 31 * result + (handleKeyValueType != null ? handleKeyValueType.hashCode() : 0); return result; } + + + /** + * Decoder ExtraInfo for {@link org.apache.pulsar.sql.presto.PulsarColumnHandle} used by {@link io.prestosql.decoder.RowDecoder} + */ + public static class DecoderExtraInfo { + + public DecoderExtraInfo(String mapping, String dataFormat, String formatHint) { + this.mapping = mapping; + this.dataFormat = dataFormat; + this.formatHint = formatHint; + } + + public DecoderExtraInfo() {} + + //equals ColumnName in general, may used as alias or embedded field future + private String mapping; + //reserved dataFormat for used by RowDecoder + private String dataFormat; + //reserved formatHint for used by RowDecoder + private String formatHint; + + public String getMapping() { + return mapping; + } + + public void setMapping(String mapping) { + this.mapping = mapping; + } + + public String getDataFormat() { + return dataFormat; + } + + public void setDataFormat(String dataFormat) { + this.dataFormat = dataFormat; + } + + public String getFormatHint() { + return formatHint; + } + + public void setFormatHint(String formatHint) { + this.formatHint = formatHint; + } + + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + + DecoderExtraInfo that = (DecoderExtraInfo) o; + + if (mapping != that.mapping) { + return false; + } + if (dataFormat != null ? !dataFormat.equals(that.dataFormat) : that.dataFormat != null) { + return false; + } + return Objects.equals(formatHint, that.formatHint); + } + + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (mapping != null ? mapping.hashCode() : 0); + result = 31 * result + (dataFormat != null ? dataFormat.hashCode() : 0); + result = 31 * result + (formatHint != null ? formatHint.hashCode() : 0); + return result; + } + + } + + } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorModule.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorModule.java index 1388bdaf4ce81..ff2a638fd3b22 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorModule.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorModule.java @@ -18,20 +18,22 @@ */ package org.apache.pulsar.sql.presto; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonBinder.jsonBinder; -import static java.util.Objects.requireNonNull; - import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Scopes; +import io.prestosql.decoder.DecoderModule; import io.prestosql.spi.type.Type; import io.prestosql.spi.type.TypeId; import io.prestosql.spi.type.TypeManager; + import javax.inject.Inject; +import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.airlift.json.JsonBinder.jsonBinder; +import static java.util.Objects.requireNonNull; + /** * This class defines binding of classes in the Presto connector. */ @@ -55,9 +57,15 @@ public void configure(Binder binder) { binder.bind(PulsarMetadata.class).in(Scopes.SINGLETON); binder.bind(PulsarSplitManager.class).in(Scopes.SINGLETON); binder.bind(PulsarRecordSetProvider.class).in(Scopes.SINGLETON); + + binder.bind(PulsarDispatchingRowDecoderFactory.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(PulsarConnectorConfig.class); jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class); + + binder.install(new DecoderModule()); + } /** diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java new file mode 100644 index 0000000000000..62a71d031e10c --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto; + +import com.google.inject.Inject; +import io.airlift.log.Logger; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.RowDecoder; +import io.prestosql.spi.connector.ColumnMetadata; +import io.prestosql.spi.type.TypeManager; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; +import org.apache.pulsar.sql.presto.decoder.avro.PulsarAvroRowDecoderFactory; +import org.apache.pulsar.sql.presto.decoder.json.PulsarJsonRowDecoderFactory; +import org.apache.pulsar.sql.presto.decoder.primitive.PulsarPrimitiveRowDecoderFactory; + +import java.util.List; +import java.util.Set; + + +/** + * dispatcher RowDecoderFactory for {@link org.apache.pulsar.common.schema.SchemaType}. + */ +public class PulsarDispatchingRowDecoderFactory { + + private static final Logger log = Logger.get(PulsarDispatchingRowDecoderFactory.class); + + private TypeManager typeManager; + + @Inject + public PulsarDispatchingRowDecoderFactory(TypeManager typeManager) { + this.typeManager = typeManager; + } + + public RowDecoder createRowDecoder(SchemaInfo schemaInfo, Set columns) { + PulsarRowDecoderFactory rowDecoderFactory = createDecoderFactory(schemaInfo); + return rowDecoderFactory.createRowDecoder(schemaInfo, columns); + } + + + public List extractColumnMetadata(SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + PulsarRowDecoderFactory rowDecoderFactory = createDecoderFactory(schemaInfo); + return rowDecoderFactory.extractColumnMetadata(schemaInfo, handleKeyValueType); + } + + private PulsarRowDecoderFactory createDecoderFactory(SchemaInfo schemaInfo) { + if (SchemaType.AVRO.equals(schemaInfo.getType())) { + return new PulsarAvroRowDecoderFactory(typeManager); + } else if (SchemaType.JSON.equals(schemaInfo.getType())) { + return new PulsarJsonRowDecoderFactory(typeManager); + } else if (schemaInfo.getType().isPrimitive()) { + return new PulsarPrimitiveRowDecoderFactory(); + } else { + throw new RuntimeException("unsupported typd ..."); + } + } + + + + +} \ No newline at end of file diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarFieldValueProviders.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarFieldValueProviders.java new file mode 100644 index 0000000000000..18e0e080ad6ab --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarFieldValueProviders.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto; + +import io.prestosql.decoder.FieldValueProvider; + +public class PulsarFieldValueProviders { + + public static FieldValueProvider doubleValueProvider(double value) { + return new FieldValueProvider() { + @Override + public double getDouble() { + return value; + } + + @Override + public boolean isNull() { + return false; + } + }; + } + + /** + * FieldValueProvider for Time (Data,Timstamp etc.) with indicate Null instead of longValueProvider + * + * @param value + * @param isNull + * @return + */ + public static FieldValueProvider timeValueProvider(long value, boolean isNull) { + return new FieldValueProvider() { + @Override + public long getLong() { + return value; + } + + @Override + public boolean isNull() { + return isNull; + } + }; + } + + +} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java index 04e00c58c10da..a8ebb7b602bb7 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java @@ -18,179 +18,52 @@ */ package org.apache.pulsar.sql.presto; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Strings.isNullOrEmpty; -import static java.util.Objects.requireNonNull; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.prestosql.spi.type.BigintType; -import io.prestosql.spi.type.IntegerType; -import io.prestosql.spi.type.TimestampType; -import io.prestosql.spi.type.Type; -import io.prestosql.spi.type.VarcharType; +import io.prestosql.spi.type.*; + import java.util.Map; import java.util.Set; import java.util.function.Consumer; -import org.apache.pulsar.common.api.raw.RawMessage; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; +import static java.util.Objects.requireNonNull; /** * This abstract class represents internal columns. */ -public abstract class PulsarInternalColumn { - - /** - * Internal column representing the partition. - */ - public static class PartitionColumn extends PulsarInternalColumn { - - PartitionColumn(String name, Type type, String comment) { - super(name, type, comment); - } - - @Override - public Object getData(RawMessage message) { - return null; - } - } - - /** - * Internal column representing the event time. - */ - public static class EventTimeColumn extends PulsarInternalColumn { - - EventTimeColumn(String name, Type type, String comment) { - super(name, type, comment); - } - - @Override - public Object getData(RawMessage message) { - return message.getEventTime() == 0 ? null : message.getEventTime(); - } - } - - /** - * Internal column representing the publish time. - */ - public static class PublishTimeColumn extends PulsarInternalColumn { - - PublishTimeColumn(String name, Type type, String comment) { - super(name, type, comment); - } - - @Override - public Object getData(RawMessage message) { - return message.getPublishTime(); - } - } - - /** - * Internal column representing the message id. - */ - public static class MessageIdColumn extends PulsarInternalColumn { - - MessageIdColumn(String name, Type type, String comment) { - super(name, type, comment); - } +public class PulsarInternalColumn { - @Override - public Object getData(RawMessage message) { - return message.getMessageId().toString(); - } - } - - /** - * Internal column representing the sequence id. - */ - public static class SequenceIdColumn extends PulsarInternalColumn { - - SequenceIdColumn(String name, Type type, String comment) { - super(name, type, comment); - } - - @Override - public Object getData(RawMessage message) { - return message.getSequenceId(); - } - } - - /** - * Internal column representing the producer name. - */ - public static class ProducerNameColumn extends PulsarInternalColumn { - ProducerNameColumn(String name, Type type, String comment) { - super(name, type, comment); - } - - @Override - public Object getData(RawMessage message) { - return message.getProducerName(); - } - } - - /** - * Internal column representing the key. - */ - public static class KeyColumn extends PulsarInternalColumn { - - KeyColumn(String name, Type type, String comment) { - super(name, type, comment); - } - - @Override - public Object getData(RawMessage message) { - return message.getKey().orElse(null); - } - } + public static final PulsarInternalColumn PARTITION = new PulsarInternalColumn("__partition__", IntegerType.INTEGER, + "The partition number which the message belongs to"); - /** - * Internal column representing the message properties. - */ - public static class PropertiesColumn extends PulsarInternalColumn { - - private static final ObjectMapper mapper = new ObjectMapper(); - - PropertiesColumn(String name, Type type, String comment) { - super(name, type, comment); - } - - @Override - public Object getData(RawMessage message) { - try { - return mapper.writeValueAsString(message.getProperties()); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - } - - public static final PartitionColumn PARTITION = new PartitionColumn("__partition__", IntegerType.INTEGER, - "The partition number which the message belongs to"); - - public static final PulsarInternalColumn EVENT_TIME = new EventTimeColumn("__event_time__", TimestampType + public static final PulsarInternalColumn EVENT_TIME = new PulsarInternalColumn("__event_time__", TimestampType .TIMESTAMP, "Application defined timestamp in milliseconds of when the event occurred"); - public static final PulsarInternalColumn PUBLISH_TIME = new PublishTimeColumn("__publish_time__", + public static final PulsarInternalColumn PUBLISH_TIME = new PulsarInternalColumn("__publish_time__", TimestampType.TIMESTAMP, "The timestamp in milliseconds of when event as published"); - public static final PulsarInternalColumn MESSAGE_ID = new MessageIdColumn("__message_id__", VarcharType.VARCHAR, + public static final PulsarInternalColumn MESSAGE_ID = new PulsarInternalColumn("__message_id__", VarcharType.VARCHAR, "The message ID of the message used to generate this row"); - public static final PulsarInternalColumn SEQUENCE_ID = new SequenceIdColumn("__sequence_id__", BigintType.BIGINT, + public static final PulsarInternalColumn SEQUENCE_ID = new PulsarInternalColumn("__sequence_id__", BigintType.BIGINT, "The sequence ID of the message used to generate this row"); - public static final PulsarInternalColumn PRODUCER_NAME = new ProducerNameColumn("__producer_name__", VarcharType + public static final PulsarInternalColumn PRODUCER_NAME = new PulsarInternalColumn("__producer_name__", VarcharType .VARCHAR, "The name of the producer that publish the message used to generate this row"); - public static final PulsarInternalColumn KEY = new KeyColumn("__key__", VarcharType.VARCHAR, "The partition key " - + "for the topic"); + public static final PulsarInternalColumn KEY = new PulsarInternalColumn("__key__", VarcharType.VARCHAR, "The partition key " + + "for the topic"); - public static final PulsarInternalColumn PROPERTIES = new PropertiesColumn("__properties__", VarcharType.VARCHAR, + public static final PulsarInternalColumn PROPERTIES = new PulsarInternalColumn("__properties__", VarcharType.VARCHAR, "User defined properties"); + private static Set internalFields = ImmutableSet.of(PARTITION, EVENT_TIME, PUBLISH_TIME, MESSAGE_ID, SEQUENCE_ID, PRODUCER_NAME, KEY, + PROPERTIES); + + private final String name; private final Type type; private final String comment; @@ -218,17 +91,16 @@ PulsarColumnHandle getColumnHandle(String connectorId, boolean hidden) { getName(), getType(), hidden, - true, null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + true, getName(), null, null, PulsarColumnHandle.HandleKeyValueType.NONE); } PulsarColumnMetadata getColumnMetadata(boolean hidden) { - return new PulsarColumnMetadata(name, type, comment, null, hidden, true, null, null, - PulsarColumnHandle.HandleKeyValueType.NONE); + return new PulsarColumnMetadata(name, type, comment, null, hidden, true, + PulsarColumnHandle.HandleKeyValueType.NONE, null); } public static Set getInternalFields() { - return ImmutableSet.of(PARTITION, EVENT_TIME, PUBLISH_TIME, MESSAGE_ID, SEQUENCE_ID, PRODUCER_NAME, KEY, - PROPERTIES); + return internalFields; } public static Map getInternalFieldsMap() { @@ -242,5 +114,4 @@ public void accept(PulsarInternalColumn pulsarInternalColumn) { return builder.build(); } - public abstract Object getData(RawMessage message); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java index 01ef01a6c289f..45b073be1f221 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java @@ -18,64 +18,11 @@ */ package org.apache.pulsar.sql.presto; -import static io.prestosql.spi.StandardErrorCode.NOT_FOUND; -import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.prestosql.spi.StandardErrorCode.QUERY_REJECTED; -import static io.prestosql.spi.type.DateType.DATE; -import static io.prestosql.spi.type.TimeType.TIME; -import static io.prestosql.spi.type.TimestampType.TIMESTAMP; -import static java.util.Objects.requireNonNull; -import static org.apache.pulsar.sql.presto.PulsarConnectorUtils.restoreNamespaceDelimiterIfNeeded; -import static org.apache.pulsar.sql.presto.PulsarConnectorUtils.rewriteNamespaceDelimiterIfNeeded; -import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertColumnHandle; -import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertTableHandle; - -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airlift.log.Logger; import io.prestosql.spi.PrestoException; -import io.prestosql.spi.connector.ColumnHandle; -import io.prestosql.spi.connector.ColumnMetadata; -import io.prestosql.spi.connector.ConnectorMetadata; -import io.prestosql.spi.connector.ConnectorSession; -import io.prestosql.spi.connector.ConnectorTableHandle; -import io.prestosql.spi.connector.ConnectorTableLayout; -import io.prestosql.spi.connector.ConnectorTableLayoutHandle; -import io.prestosql.spi.connector.ConnectorTableLayoutResult; -import io.prestosql.spi.connector.ConnectorTableMetadata; -import io.prestosql.spi.connector.Constraint; -import io.prestosql.spi.connector.SchemaTableName; -import io.prestosql.spi.connector.SchemaTablePrefix; -import io.prestosql.spi.connector.TableNotFoundException; -import io.prestosql.spi.type.BigintType; -import io.prestosql.spi.type.BooleanType; -import io.prestosql.spi.type.DateType; -import io.prestosql.spi.type.DoubleType; -import io.prestosql.spi.type.IntegerType; -import io.prestosql.spi.type.RealType; -import io.prestosql.spi.type.SmallintType; -import io.prestosql.spi.type.TimeType; -import io.prestosql.spi.type.TimestampType; -import io.prestosql.spi.type.TinyintType; -import io.prestosql.spi.type.Type; -import io.prestosql.spi.type.VarbinaryType; -import io.prestosql.spi.type.VarcharType; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.Stack; -import java.util.stream.Collectors; -import javax.inject.Inject; -import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; -import org.apache.avro.SchemaParseException; -import org.apache.commons.lang3.StringUtils; +import io.prestosql.spi.connector.*; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -86,6 +33,18 @@ import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +import static io.prestosql.spi.StandardErrorCode.NOT_FOUND; +import static io.prestosql.spi.StandardErrorCode.QUERY_REJECTED; +import static java.util.Objects.requireNonNull; +import static org.apache.pulsar.sql.presto.PulsarConnectorUtils.restoreNamespaceDelimiterIfNeeded; +import static org.apache.pulsar.sql.presto.PulsarConnectorUtils.rewriteNamespaceDelimiterIfNeeded; +import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertColumnHandle; +import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertTableHandle; + /** * This connector helps to work with metadata. */ @@ -95,12 +54,16 @@ public class PulsarMetadata implements ConnectorMetadata { private final PulsarAdmin pulsarAdmin; private final PulsarConnectorConfig pulsarConnectorConfig; + private final PulsarDispatchingRowDecoderFactory decoderFactory; + private static final String INFORMATION_SCHEMA = "information_schema"; + private static final Logger log = Logger.get(PulsarMetadata.class); @Inject - public PulsarMetadata(PulsarConnectorId connectorId, PulsarConnectorConfig pulsarConnectorConfig) { + public PulsarMetadata(PulsarConnectorId connectorId, PulsarConnectorConfig pulsarConnectorConfig, PulsarDispatchingRowDecoderFactory decoderFactory) { + this.decoderFactory = decoderFactory; this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); this.pulsarConnectorConfig = pulsarConnectorConfig; try { @@ -224,8 +187,8 @@ public Map getColumnHandles(ConnectorSession session, Conn pulsarColumnMetadata.getType(), pulsarColumnMetadata.isHidden(), pulsarColumnMetadata.isInternal(), - pulsarColumnMetadata.getFieldNames(), - pulsarColumnMetadata.getPositionIndices(), + pulsarColumnMetadata.getDecoderExtraInfo().getMapping(), + pulsarColumnMetadata.getDecoderExtraInfo().getDataFormat(),pulsarColumnMetadata.getDecoderExtraInfo().getFormatHint(), pulsarColumnMetadata.getHandleKeyValueType()); columnHandles.put( @@ -316,7 +279,8 @@ private ConnectorTableMetadata getTableMetadata(SchemaTableName schemaTableName, } catch (PulsarAdminException e) { if (e.getStatusCode() == 404) { // use default schema because there is no schema - schemaInfo = PulsarSchemaHandlers.defaultSchema(); + schemaInfo = PulsarSqlSchemaInfoProvider.defaultSchema(); + } else if (e.getStatusCode() == 401) { throw new PrestoException(QUERY_REJECTED, String.format("Failed to get pulsar topic schema information for topic %s/%s: Unauthorized", @@ -338,15 +302,13 @@ private ConnectorTableMetadata getTableMetadata(SchemaTableName schemaTableName, /** * Convert pulsar schema into presto table metadata. */ - static List getPulsarColumns(TopicName topicName, - SchemaInfo schemaInfo, - boolean withInternalColumns, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + List getPulsarColumns(TopicName topicName, + SchemaInfo schemaInfo, + boolean withInternalColumns, + PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { SchemaType schemaType = schemaInfo.getType(); - if (schemaType.isStruct()) { - return getPulsarColumnsFromStructSchema(topicName, schemaInfo, withInternalColumns, handleKeyValueType); - } else if (schemaType.isPrimitive()) { - return getPulsarColumnsFromPrimitiveSchema(topicName, schemaInfo, withInternalColumns, handleKeyValueType); + if (schemaType.isStruct()|| schemaType.isPrimitive()) { + return getPulsarColumnsFromSchema(topicName, schemaInfo, withInternalColumns, handleKeyValueType); } else if (schemaType.equals(SchemaType.KEY_VALUE)) { return getPulsarColumnsFromKeyValueSchema(topicName, schemaInfo, withInternalColumns); } else { @@ -354,50 +316,16 @@ static List getPulsarColumns(TopicName topicName, } } - static List getPulsarColumnsFromPrimitiveSchema(TopicName topicName, - SchemaInfo schemaInfo, - boolean withInternalColumns, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { - ImmutableList.Builder builder = ImmutableList.builder(); - - ColumnMetadata valueColumn = new PulsarColumnMetadata( - PulsarColumnMetadata.getColumnName(handleKeyValueType, "__value__"), - convertPulsarType(schemaInfo.getType()), - "The value of the message with primitive type schema", null, false, false, - new String[0], - new Integer[0], handleKeyValueType); - - builder.add(valueColumn); - - if (withInternalColumns) { - PulsarInternalColumn.getInternalFields() - .stream() - .forEach(pulsarInternalColumn -> builder.add(pulsarInternalColumn.getColumnMetadata(false))); - } - return builder.build(); - } - static List getPulsarColumnsFromStructSchema(TopicName topicName, - SchemaInfo schemaInfo, - boolean withInternalColumns, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { - String schemaJson = new String(schemaInfo.getSchema()); - if (StringUtils.isBlank(schemaJson)) { - throw new PrestoException(NOT_SUPPORTED, "Topic " + topicName.toString() - + " does not have a valid schema"); - } - Schema schema; - try { - schema = PulsarConnectorUtils.parseSchema(schemaJson); - } catch (SchemaParseException ex) { - throw new PrestoException(NOT_SUPPORTED, "Topic " + topicName.toString() - + " does not have a valid schema"); - } + List getPulsarColumnsFromSchema(TopicName topicName, + SchemaInfo schemaInfo, + boolean withInternalColumns, + PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { ImmutableList.Builder builder = ImmutableList.builder(); - builder.addAll(getColumns(null, schema, new HashSet<>(), new Stack<>(), new Stack<>(), handleKeyValueType)); + builder.addAll(decoderFactory.extractColumnMetadata(schemaInfo,handleKeyValueType)); if (withInternalColumns) { PulsarInternalColumn.getInternalFields() @@ -407,9 +335,9 @@ static List getPulsarColumnsFromStructSchema(TopicName topicName return builder.build(); } - static List getPulsarColumnsFromKeyValueSchema(TopicName topicName, - SchemaInfo schemaInfo, - boolean withInternalColumns) { + List getPulsarColumnsFromKeyValueSchema(TopicName topicName, + SchemaInfo schemaInfo, + boolean withInternalColumns) { ImmutableList.Builder builder = ImmutableList.builder(); KeyValue kvSchemaInfo = KeyValueSchemaInfo.decodeKeyValueSchemaInfo(schemaInfo); SchemaInfo keySchemaInfo = kvSchemaInfo.getKey(); @@ -429,179 +357,7 @@ static List getPulsarColumnsFromKeyValueSchema(TopicName topicNa return builder.build(); } - @VisibleForTesting - static Type convertPulsarType(SchemaType pulsarType) { - switch (pulsarType) { - case BOOLEAN: - return BooleanType.BOOLEAN; - case INT8: - return TinyintType.TINYINT; - case INT16: - return SmallintType.SMALLINT; - case INT32: - return IntegerType.INTEGER; - case INT64: - return BigintType.BIGINT; - case FLOAT: - return RealType.REAL; - case DOUBLE: - return DoubleType.DOUBLE; - case NONE: - case BYTES: - return VarbinaryType.VARBINARY; - case STRING: - return VarcharType.VARCHAR; - case DATE: - return DateType.DATE; - case TIME: - return TimeType.TIME; - case TIMESTAMP: - return TimestampType.TIMESTAMP; - default: - log.error("Cannot convert type: %s", pulsarType); - return null; - } - } - @VisibleForTesting - static List getColumns(String fieldName, Schema fieldSchema, - Set fieldTypes, - Stack fieldNames, - Stack positionIndices, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { - - List columnMetadataList = new LinkedList<>(); - - if (isPrimitiveType(fieldSchema.getType())) { - columnMetadataList.add(new PulsarColumnMetadata( - PulsarColumnMetadata.getColumnName(handleKeyValueType, fieldName), - convertType(fieldSchema.getType(), fieldSchema.getLogicalType()), - null, null, false, false, - fieldNames.toArray(new String[fieldNames.size()]), - positionIndices.toArray(new Integer[positionIndices.size()]), handleKeyValueType)); - } else if (fieldSchema.getType() == Schema.Type.UNION) { - boolean canBeNull = false; - for (Schema type : fieldSchema.getTypes()) { - if (isPrimitiveType(type.getType())) { - PulsarColumnMetadata columnMetadata; - if (type.getType() != Schema.Type.NULL) { - if (!canBeNull) { - columnMetadata = new PulsarColumnMetadata( - PulsarColumnMetadata.getColumnName(handleKeyValueType, fieldName), - convertType(type.getType(), type.getLogicalType()), - null, null, false, false, - fieldNames.toArray(new String[fieldNames.size()]), - positionIndices.toArray(new Integer[positionIndices.size()]), handleKeyValueType); - } else { - columnMetadata = new PulsarColumnMetadata( - PulsarColumnMetadata.getColumnName(handleKeyValueType, fieldName), - convertType(type.getType(), type.getLogicalType()), - "field can be null", null, false, false, - fieldNames.toArray(new String[fieldNames.size()]), - positionIndices.toArray(new Integer[positionIndices.size()]), handleKeyValueType); - } - columnMetadataList.add(columnMetadata); - } else { - canBeNull = true; - } - } else { - List columns = getColumns(fieldName, type, fieldTypes, fieldNames, - positionIndices, handleKeyValueType); - columnMetadataList.addAll(columns); - } - } - } else if (fieldSchema.getType() == Schema.Type.RECORD) { - // check if we have seen this type before to prevent cyclic class definitions. - if (!fieldTypes.contains(fieldSchema.getFullName())) { - // add to types seen so far in traversal - fieldTypes.add(fieldSchema.getFullName()); - List fields = fieldSchema.getFields(); - for (int i = 0; i < fields.size(); i++) { - Schema.Field field = fields.get(i); - fieldNames.push(field.name()); - positionIndices.push(i); - List columns; - if (fieldName == null) { - columns = getColumns(field.name(), field.schema(), fieldTypes, fieldNames, positionIndices, - handleKeyValueType); - } else { - columns = getColumns(String.format("%s.%s", fieldName, field.name()), field.schema(), - fieldTypes, fieldNames, positionIndices, handleKeyValueType); - } - positionIndices.pop(); - fieldNames.pop(); - columnMetadataList.addAll(columns); - } - fieldTypes.remove(fieldSchema.getFullName()); - } else { - log.debug("Already seen type: %s", fieldSchema.getFullName()); - } - } else if (fieldSchema.getType() == Schema.Type.ARRAY) { - - } else if (fieldSchema.getType() == Schema.Type.MAP) { - - } else if (fieldSchema.getType() == Schema.Type.ENUM) { - PulsarColumnMetadata columnMetadata = new PulsarColumnMetadata( - PulsarColumnMetadata.getColumnName(handleKeyValueType, fieldName), - convertType(fieldSchema.getType(), fieldSchema.getLogicalType()), - null, null, false, false, - fieldNames.toArray(new String[fieldNames.size()]), - positionIndices.toArray(new Integer[positionIndices.size()]), handleKeyValueType); - columnMetadataList.add(columnMetadata); - - } else if (fieldSchema.getType() == Schema.Type.FIXED) { - - } else { - log.error("Unknown column type: {}", fieldSchema); - } - return columnMetadataList; - } - - @VisibleForTesting - static Type convertType(Schema.Type avroType, LogicalType logicalType) { - switch (avroType) { - case BOOLEAN: - return BooleanType.BOOLEAN; - case INT: - if (logicalType == LogicalTypes.timeMillis()) { - return TIME; - } else if (logicalType == LogicalTypes.date()) { - return DATE; - } - return IntegerType.INTEGER; - case LONG: - if (logicalType == LogicalTypes.timestampMillis()) { - return TIMESTAMP; - } - return BigintType.BIGINT; - case FLOAT: - return RealType.REAL; - case DOUBLE: - return DoubleType.DOUBLE; - case BYTES: - return VarbinaryType.VARBINARY; - case STRING: - return VarcharType.VARCHAR; - case ENUM: - return VarcharType.VARCHAR; - default: - log.error("Cannot convert type: %s", avroType); - return null; - } - } - - @VisibleForTesting - static boolean isPrimitiveType(Schema.Type type) { - return Schema.Type.NULL == type - || Schema.Type.BOOLEAN == type - || Schema.Type.INT == type - || Schema.Type.LONG == type - || Schema.Type.FLOAT == type - || Schema.Type.DOUBLE == type - || Schema.Type.BYTES == type - || Schema.Type.STRING == type; - - } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarPrimitiveSchemaHandler.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarPrimitiveSchemaHandler.java deleted file mode 100644 index 354550faa4778..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarPrimitiveSchemaHandler.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import io.netty.buffer.ByteBuf; - -import java.sql.Time; -import java.sql.Timestamp; -import java.util.Date; - -import org.apache.pulsar.client.impl.schema.AbstractSchema; -import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; -import org.apache.pulsar.common.schema.SchemaInfo; - -/** - * A presto schema handler that interprets data using pulsar schema. - */ -public class PulsarPrimitiveSchemaHandler implements SchemaHandler { - - private final SchemaInfo schemaInfo; - private final AbstractSchema schema; - - PulsarPrimitiveSchemaHandler(SchemaInfo schemaInfo) { - this.schemaInfo = schemaInfo; - this.schema = (AbstractSchema) AutoConsumeSchema.getSchema(schemaInfo); - } - - @Override - public Object deserialize(ByteBuf payload) { - Object currentRecord = schema.decode(payload); - switch (schemaInfo.getType()) { - case DATE: - return ((Date) currentRecord).getTime(); - case TIME: - return ((Time) currentRecord).getTime(); - case TIMESTAMP: - return ((Timestamp) currentRecord).getTime(); - default: - return currentRecord; - } - } - - @Override - public Object extractField(int index, Object currentRecord) { - return currentRecord; - } -} \ No newline at end of file diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index 53819524d6e66..a073390d7dac6 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -18,52 +18,49 @@ */ package org.apache.pulsar.sql.presto; -import static com.google.common.base.Preconditions.checkArgument; -import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.prestosql.spi.type.BigintType.BIGINT; -import static io.prestosql.spi.type.DateTimeEncoding.packDateTimeWithZone; -import static io.prestosql.spi.type.DateType.DATE; -import static io.prestosql.spi.type.IntegerType.INTEGER; -import static io.prestosql.spi.type.RealType.REAL; -import static io.prestosql.spi.type.SmallintType.SMALLINT; -import static io.prestosql.spi.type.TimeType.TIME; -import static io.prestosql.spi.type.TimestampType.TIMESTAMP; -import static io.prestosql.spi.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; -import static io.prestosql.spi.type.TinyintType.TINYINT; - +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import io.airlift.log.Logger; import io.airlift.slice.Slice; -import io.airlift.slice.Slices; import io.netty.buffer.ByteBuf; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; +import io.prestosql.decoder.RowDecoder; import io.prestosql.spi.PrestoException; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.connector.ColumnHandle; import io.prestosql.spi.connector.RecordCursor; import io.prestosql.spi.type.Type; -import io.prestosql.spi.type.VarbinaryType; -import io.prestosql.spi.type.VarcharType; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; -import org.apache.bookkeeper.mledger.AsyncCallbacks; -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.ManagedLedgerConfig; -import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.bookkeeper.mledger.ManagedLedgerFactory; -import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.ReadOnlyCursor; +import org.apache.bookkeeper.mledger.*; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.mledger.impl.ReadOnlyCursorImpl; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.api.raw.MessageParser; import org.apache.pulsar.common.api.raw.RawMessage; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.OffloadPolicies; -import org.apache.pulsar.sql.presto.PulsarInternalColumn.PartitionColumn; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; import org.jctools.queues.MessagePassingQueue; import org.jctools.queues.SpscArrayQueue; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicLong; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static io.prestosql.decoder.FieldValueProviders.bytesValueProvider; +import static io.prestosql.decoder.FieldValueProviders.longValueProvider; +import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; + /** * Implementation of a cursor to read records. */ @@ -75,10 +72,7 @@ public class PulsarRecordCursor implements RecordCursor { private ReadOnlyCursor cursor; private SpscArrayQueue messageQueue; private SpscArrayQueue entryQueue; - private Object currentRecord; private RawMessage currentMessage; - private Map internalColumnMap = PulsarInternalColumn.getInternalFieldsMap(); - private SchemaHandler schemaHandler; private int maxBatchSize; private long completedBytes = 0; private ReadEntries readEntries; @@ -98,10 +92,17 @@ public class PulsarRecordCursor implements RecordCursor { private int partition = -1; + private PulsarSqlSchemaInfoProvider schemaInfoProvider; + + private FieldValueProvider[] currentRowValues = null; + + + PulsarDispatchingRowDecoderFactory decoderFactory; + private static final Logger log = Logger.get(PulsarRecordCursor.class); public PulsarRecordCursor(List columnHandles, PulsarSplit pulsarSplit, - PulsarConnectorConfig pulsarConnectorConfig) { + PulsarConnectorConfig pulsarConnectorConfig, PulsarDispatchingRowDecoderFactory decoderFactory) { this.splitSize = pulsarSplit.getSplitSize(); // Set start time for split this.startTime = System.nanoTime(); @@ -127,12 +128,13 @@ public PulsarRecordCursor(List columnHandles, PulsarSplit pu pulsarSplit.getTableName()).getNamespaceObject(), offloadPolicies, pulsarConnectorConfig), new PulsarConnectorMetricsTracker(pulsarConnectorCache.getStatsProvider())); + this.decoderFactory = decoderFactory; } // Exposed for testing purposes PulsarRecordCursor(List columnHandles, PulsarSplit pulsarSplit, PulsarConnectorConfig pulsarConnectorConfig, ManagedLedgerFactory managedLedgerFactory, ManagedLedgerConfig managedLedgerConfig, - PulsarConnectorMetricsTracker pulsarConnectorMetricsTracker) { + PulsarConnectorMetricsTracker pulsarConnectorMetricsTracker) { this.splitSize = pulsarSplit.getSplitSize(); initialize(columnHandles, pulsarSplit, pulsarConnectorConfig, managedLedgerFactory, managedLedgerConfig, pulsarConnectorMetricsTracker); @@ -140,8 +142,9 @@ public PulsarRecordCursor(List columnHandles, PulsarSplit pu private void initialize(List columnHandles, PulsarSplit pulsarSplit, PulsarConnectorConfig pulsarConnectorConfig, ManagedLedgerFactory managedLedgerFactory, ManagedLedgerConfig managedLedgerConfig, - PulsarConnectorMetricsTracker pulsarConnectorMetricsTracker) { + PulsarConnectorMetricsTracker pulsarConnectorMetricsTracker) { this.columnHandles = columnHandles; + this.currentRowValues = new FieldValueProvider[columnHandles.size()]; this.pulsarSplit = pulsarSplit; this.partition = TopicName.getPartitionIndex(pulsarSplit.getTableName()); this.pulsarConnectorConfig = pulsarConnectorConfig; @@ -155,10 +158,13 @@ private void initialize(List columnHandles, PulsarSplit puls this.readOffloaded = pulsarConnectorConfig.getManagedLedgerOffloadDriver() != null; this.pulsarConnectorConfig = pulsarConnectorConfig; - this.schemaHandler = PulsarSchemaHandlers - .newPulsarSchemaHandler(this.topicName, - this.pulsarConnectorConfig, pulsarSplit.getSchemaInfo(), columnHandles); + try { + this.schemaInfoProvider = new PulsarSqlSchemaInfoProvider( this.topicName, pulsarConnectorConfig.getPulsarAdmin()); + } catch (PulsarClientException e) { + log.error(e, "Failed to init Pulsar SchemaInfo Provider"); + throw new RuntimeException(e); + } log.info("Initializing split with parameters: %s", pulsarSplit); try { @@ -420,148 +426,130 @@ public boolean advanceNextPosition() { //start time for deseralizing record metricsTracker.start_RECORD_DESERIALIZE_TIME(); - if (this.schemaHandler instanceof KeyValueSchemaHandler) { - ByteBuf keyByteBuf = null; - if (this.currentMessage.getKeyBytes().isPresent()) { - keyByteBuf = this.currentMessage.getKeyBytes().get(); - } - currentRecord = this.schemaHandler.deserialize(keyByteBuf, - this.currentMessage.getData(), this.currentMessage.getSchemaVersion()); - } else { - currentRecord = this.schemaHandler.deserialize(this.currentMessage.getData(), - this.currentMessage.getSchemaVersion()); + SchemaInfo schemaInfo; + try { + schemaInfo = schemaInfoProvider.getSchemaByVersion(this.currentMessage.getSchemaVersion()).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); } - metricsTracker.incr_NUM_RECORD_DESERIALIZED(); - // stats for time spend deserializing - metricsTracker.end_RECORD_DESERIALIZE_TIME(); + Map currentRowValuesMap = new HashMap<>(); - return true; - } + if(schemaInfo.getType().equals(SchemaType.KEY_VALUE) ){ + RowDecoder keyDecoder = decoderFactory.createRowDecoder( + schemaInfo, + columnHandles.stream() + .filter(col -> !col.isInternal()) + .filter(col -> PulsarColumnHandle.HandleKeyValueType.KEY.equals(col.getHandleKeyValueType())) + .collect(toImmutableSet())); - @VisibleForTesting - Object getRecord(int fieldIndex) { - if (this.currentRecord == null) { - return null; - } + RowDecoder messageDecoder = decoderFactory.createRowDecoder( + schemaInfo, + columnHandles.stream() + .filter(col -> !col.isInternal()) + .filter(col -> PulsarColumnHandle.HandleKeyValueType.VALUE.equals(col.getHandleKeyValueType())) + .collect(toImmutableSet())); - Object data; - PulsarColumnHandle pulsarColumnHandle = this.columnHandles.get(fieldIndex); + Optional> decodedKey; + if (this.currentMessage.getKeyBytes().isPresent()) { + ByteBuf keyByteBuf = this.currentMessage.getKeyBytes().get(); + decodedKey = keyDecoder.decodeRow(keyByteBuf.array(),null); + decodedKey.ifPresent(currentRowValuesMap::putAll); + } - if (pulsarColumnHandle.isInternal()) { - String fieldName = this.columnHandles.get(fieldIndex).getName(); - PulsarInternalColumn pulsarInternalColumn = this.internalColumnMap.get(fieldName); - if (pulsarInternalColumn instanceof PartitionColumn) { - data = this.partition; - } else { - data = pulsarInternalColumn.getData(this.currentMessage); + Optional> decodedValue = messageDecoder.decodeRow(toBytes(this.currentMessage.getData()),null); + decodedValue.ifPresent(currentRowValuesMap::putAll); + }else { + RowDecoder messageDecoder = decoderFactory.createRowDecoder( + schemaInfo, + columnHandles.stream() + .filter(col -> !col.isInternal()) + .filter(col -> PulsarColumnHandle.HandleKeyValueType.NONE.equals(col.getHandleKeyValueType())) + .collect(toImmutableSet())); + byte[] data = toBytes(this.currentMessage.getData()); + Optional> decodedValue = messageDecoder.decodeRow(data, null); + decodedValue.ifPresent(currentRowValuesMap::putAll); + } + + for (DecoderColumnHandle columnHandle : columnHandles) { + if (columnHandle.isInternal()) { + if(PulsarInternalColumn.PARTITION.getName().equals(columnHandle.getName())){ + currentRowValuesMap.put(columnHandle, longValueProvider(this.partition)); + }else if(PulsarInternalColumn.EVENT_TIME.getName().equals(columnHandle.getName())){ + currentRowValuesMap.put(columnHandle, PulsarFieldValueProviders.timeValueProvider(this.currentMessage.getEventTime(), this.currentMessage.getPublishTime() == 0)); + }else if(PulsarInternalColumn.PUBLISH_TIME.getName().equals(columnHandle.getName())){ + currentRowValuesMap.put(columnHandle, PulsarFieldValueProviders.timeValueProvider(this.currentMessage.getPublishTime(), this.currentMessage.getPublishTime() == 0)); + }else if(PulsarInternalColumn.MESSAGE_ID.getName().equals(columnHandle.getName())){ + currentRowValuesMap.put(columnHandle, bytesValueProvider(this.currentMessage.getMessageId().toString().getBytes())); + }else if(PulsarInternalColumn.SEQUENCE_ID.getName().equals(columnHandle.getName())){ + currentRowValuesMap.put(columnHandle, longValueProvider(this.currentMessage.getSequenceId())); + }else if(PulsarInternalColumn.PRODUCER_NAME.getName().equals(columnHandle.getName())){ + currentRowValuesMap.put(columnHandle, bytesValueProvider(this.currentMessage.getProducerName().getBytes())); + }else if(PulsarInternalColumn.KEY.getName().equals(columnHandle.getName())){ + String key = this.currentMessage.getKey().orElse(null); + currentRowValuesMap.put(columnHandle, bytesValueProvider(key ==null? null:key.getBytes())); + }else if(PulsarInternalColumn.PROPERTIES.getName().equals(columnHandle.getName())){ + try { + currentRowValuesMap.put(columnHandle, bytesValueProvider(new ObjectMapper().writeValueAsBytes(this.currentMessage.getProperties()))); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }else { + throw new IllegalArgumentException("unknown internal field " + columnHandle.getName()); + } } - } else { - data = this.schemaHandler.extractField(fieldIndex, this.currentRecord); } + for (int i = 0; i < columnHandles.size(); i++) { + ColumnHandle columnHandle = columnHandles.get(i); + currentRowValues[i] = currentRowValuesMap.get(columnHandle); + } + + metricsTracker.incr_NUM_RECORD_DESERIALIZED(); + + // stats for time spend deserializing + metricsTracker.end_RECORD_DESERIALIZE_TIME(); - return data; + return true; } + @Override public boolean getBoolean(int field) { - checkFieldType(field, boolean.class); - return (boolean) getRecord(field); + return getFieldValueProvider(field, boolean.class).getBoolean(); } @Override public long getLong(int field) { - checkFieldType(field, long.class); - - Object record = getRecord(field); - Type type = getType(field); - - if (type.equals(BIGINT)) { - return ((Number) record).longValue(); - } else if (type.equals(DATE)) { - return ((Number) record).longValue(); - } else if (type.equals(INTEGER)) { - return ((Number) record).intValue(); - } else if (type.equals(REAL)) { - return Float.floatToIntBits(((Number) record).floatValue()); - } else if (type.equals(SMALLINT)) { - return ((Number) record).shortValue(); - } else if (type.equals(TIME)) { - return ((Number) record).longValue(); - } else if (type.equals(TIMESTAMP)) { - if (record instanceof String) { - return Long.parseLong((String) record); - } else { - return ((Number) record).longValue(); - } - } else if (type.equals(TIMESTAMP_WITH_TIME_ZONE)) { - return packDateTimeWithZone(((Number) record).longValue(), 0); - } else if (type.equals(TINYINT)) { - return Byte.parseByte(record.toString()); - } else { - throw new PrestoException(NOT_SUPPORTED, "Unsupported type " + getType(field)); - } + return getFieldValueProvider(field, long.class).getLong(); } @Override public double getDouble(int field) { - checkFieldType(field, double.class); - Object record = getRecord(field); - return (double) record; + return getFieldValueProvider(field, double.class).getDouble(); } @Override public Slice getSlice(int field) { - checkFieldType(field, Slice.class); - - Object record = getRecord(field); - Type type = getType(field); - if (type == VarcharType.VARCHAR) { - return Slices.utf8Slice(record.toString()); - } else if (type == VarbinaryType.VARBINARY) { - return Slices.wrappedBuffer(toBytes(record)); - } else { - throw new PrestoException(NOT_SUPPORTED, "Unsupported type " + type); - } + return getFieldValueProvider(field, Slice.class).getSlice(); } - private byte[] toBytes(Object record) { - if (record instanceof ByteBuffer) { - ByteBuffer byteBuffer = (ByteBuffer) record; - if (byteBuffer.hasArray()) { - return byteBuffer.array(); - } - byte[] bytes = new byte[byteBuffer.position()]; - byteBuffer.flip(); - byteBuffer.get(bytes); - return bytes; - } else if (record instanceof ByteBuf) { - ByteBuf byteBuf = (ByteBuf) record; - if (byteBuf.hasArray()) { - return byteBuf.array(); - } - byte[] bytes = new byte[byteBuf.readableBytes()]; - byteBuf.readBytes(bytes); - return bytes; - } else { - try { - return (byte[]) record; - } catch (Exception e) { - throw new PrestoException(NOT_SUPPORTED, "Unsupported type " + record.getClass().getName()); - } - } + private FieldValueProvider getFieldValueProvider(int fieldIndex, Class expectedType) + { + checkArgument(fieldIndex < columnHandles.size(), "Invalid field index"); + checkFieldType(fieldIndex, expectedType); + return currentRowValues[fieldIndex]; } @Override public Object getObject(int field) { - throw new UnsupportedOperationException(); + return getFieldValueProvider(field, Block.class).getBlock(); } @Override public boolean isNull(int field) { - Object record = getRecord(field); - return record == null; + FieldValueProvider provider= currentRowValues[field]; + return provider == null || provider.isNull(); } @Override @@ -600,13 +588,36 @@ public void close() { } + private byte[] toBytes(Object record) { + if (record instanceof ByteBuffer) { + ByteBuffer byteBuffer = (ByteBuffer) record; + if (byteBuffer.hasArray()) { + return byteBuffer.array(); + } + byte[] bytes = new byte[byteBuffer.position()]; + byteBuffer.flip(); + byteBuffer.get(bytes); + return bytes; + } else if (record instanceof ByteBuf) { + ByteBuf byteBuf = (ByteBuf) record; + if (byteBuf.hasArray()) { + return byteBuf.array(); + } + byte[] bytes = new byte[byteBuf.readableBytes()]; + byteBuf.readBytes(bytes); + return bytes; + } else { + try { + return (byte[]) record; + } catch (Exception e) { + throw new PrestoException(NOT_SUPPORTED, "Unsupported type " + record.getClass().getName()); + } + } + } + private void checkFieldType(int field, Class expected) { Class actual = getType(field).getJavaType(); checkArgument(actual == expected, "Expected field %s to be type %s but is %s", field, expected, actual); } - @VisibleForTesting - SchemaHandler getSchemaHandler() { - return this.schemaHandler; - } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSet.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSet.java index cb186abe9da50..f67d3b12594e0 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSet.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSet.java @@ -18,14 +18,15 @@ */ package org.apache.pulsar.sql.presto; -import static java.util.Objects.requireNonNull; - import com.google.common.collect.ImmutableList; import io.prestosql.spi.connector.RecordCursor; import io.prestosql.spi.connector.RecordSet; import io.prestosql.spi.type.Type; + import java.util.List; +import static java.util.Objects.requireNonNull; + /** * Implementation of a record set. */ @@ -36,8 +37,10 @@ public class PulsarRecordSet implements RecordSet { private final PulsarSplit pulsarSplit; private final PulsarConnectorConfig pulsarConnectorConfig; + private PulsarDispatchingRowDecoderFactory decoderFactory; + public PulsarRecordSet(PulsarSplit split, List columnHandles, PulsarConnectorConfig - pulsarConnectorConfig) { + pulsarConnectorConfig, PulsarDispatchingRowDecoderFactory decoderFactory) { requireNonNull(split, "split is null"); this.columnHandles = requireNonNull(columnHandles, "column handles is null"); ImmutableList.Builder types = ImmutableList.builder(); @@ -49,6 +52,8 @@ public PulsarRecordSet(PulsarSplit split, List columnHandles this.pulsarSplit = split; this.pulsarConnectorConfig = pulsarConnectorConfig; + + this.decoderFactory =decoderFactory; } @@ -59,6 +64,6 @@ public List getColumnTypes() { @Override public RecordCursor cursor() { - return new PulsarRecordCursor(this.columnHandles, this.pulsarSplit, this.pulsarConnectorConfig); + return new PulsarRecordCursor(this.columnHandles, this.pulsarSplit, this.pulsarConnectorConfig,this.decoderFactory); } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSetProvider.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSetProvider.java index 80b93b495b627..9613f3049afc2 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSetProvider.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSetProvider.java @@ -18,17 +18,13 @@ */ package org.apache.pulsar.sql.presto; -import static java.util.Objects.requireNonNull; - import com.google.common.collect.ImmutableList; -import io.prestosql.spi.connector.ColumnHandle; -import io.prestosql.spi.connector.ConnectorRecordSetProvider; -import io.prestosql.spi.connector.ConnectorSession; -import io.prestosql.spi.connector.ConnectorSplit; -import io.prestosql.spi.connector.ConnectorTransactionHandle; -import io.prestosql.spi.connector.RecordSet; -import java.util.List; +import io.prestosql.spi.connector.*; + import javax.inject.Inject; +import java.util.List; + +import static java.util.Objects.requireNonNull; /** * Implementation of the provider for record sets. @@ -37,8 +33,11 @@ public class PulsarRecordSetProvider implements ConnectorRecordSetProvider { private final PulsarConnectorConfig pulsarConnectorConfig; + private final PulsarDispatchingRowDecoderFactory decoderFactory; + @Inject - public PulsarRecordSetProvider(PulsarConnectorConfig pulsarConnectorConfig) { + public PulsarRecordSetProvider(PulsarConnectorConfig pulsarConnectorConfig, PulsarDispatchingRowDecoderFactory decoderFactory) { + this.decoderFactory = requireNonNull(decoderFactory, "decoderFactory is null"); this.pulsarConnectorConfig = requireNonNull(pulsarConnectorConfig, "pulsarConnectorConfig is null"); } @@ -54,6 +53,6 @@ public RecordSet getRecordSet(ConnectorTransactionHandle transactionHandle, Conn handles.add((PulsarColumnHandle) handle); } - return new PulsarRecordSet(pulsarSplit, handles.build(), this.pulsarConnectorConfig); + return new PulsarRecordSet(pulsarSplit, handles.build(), this.pulsarConnectorConfig, decoderFactory); } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java new file mode 100644 index 0000000000000..e02ff611f1c71 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto; + +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.RowDecoder; +import io.prestosql.spi.connector.ColumnMetadata; +import org.apache.pulsar.common.schema.SchemaInfo; + +import java.util.List; +import java.util.Set; + +/** + * Pulsar RowDecoderFactory interface + */ +public interface PulsarRowDecoderFactory { + + /** + * extract ColumnMetadata from pulsar SchemaInfo and HandleKeyValueType + * + * @param schemaInfo + * @param handleKeyValueType + * @return + */ + public List extractColumnMetadata(SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType); + + /** + * createRowDecoder RowDecoder by pulsar SchemaInfo and column DecoderColumnHandles + * + * @param schemaInfo + * @param columns + * @return + */ + public RowDecoder createRowDecoder(SchemaInfo schemaInfo, Set columns); + +} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSchemaHandlers.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSchemaHandlers.java deleted file mode 100644 index 144f795a5c7c7..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSchemaHandlers.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - - -import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; - -import io.prestosql.spi.PrestoException; -import java.util.List; - -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; - -class PulsarSchemaHandlers { - - static SchemaHandler newPulsarSchemaHandler(TopicName topicName, - PulsarConnectorConfig pulsarConnectorConfig, - SchemaInfo schemaInfo, - List columnHandles) throws RuntimeException{ - if (schemaInfo.getType().isPrimitive()) { - return new PulsarPrimitiveSchemaHandler(schemaInfo); - } else if (schemaInfo.getType().isStruct()) { - try { - switch (schemaInfo.getType()) { - case JSON: - return new JSONSchemaHandler(columnHandles); - case AVRO: - return new AvroSchemaHandler(topicName, pulsarConnectorConfig, schemaInfo, columnHandles); - default: - throw new PrestoException(NOT_SUPPORTED, "Not supported schema type: " + schemaInfo.getType()); - } - } catch (PulsarClientException e) { - throw new RuntimeException( - new Throwable("PulsarAdmin gets version schema fail, topicName : " - + topicName.toString(), e)); - } - } else if (schemaInfo.getType().equals(SchemaType.KEY_VALUE)) { - return new KeyValueSchemaHandler(topicName, pulsarConnectorConfig, schemaInfo, columnHandles); - } else { - throw new PrestoException( - NOT_SUPPORTED, - "Schema `" + schemaInfo.getType() + "` is not supported by presto yet : " + schemaInfo); - } - - } - - static SchemaInfo defaultSchema() { - return Schema.BYTES.getSchemaInfo(); - } -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java index 7602c8ea84e42..c10aff399b5a6 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java @@ -18,41 +18,17 @@ */ package org.apache.pulsar.sql.presto; -import static com.google.common.base.Preconditions.checkArgument; -import static io.prestosql.spi.StandardErrorCode.QUERY_REJECTED; -import static java.util.Objects.requireNonNull; -import static org.apache.bookkeeper.mledger.ManagedCursor.FindPositionConstraint.SearchAllAvailableEntries; -import static org.apache.pulsar.sql.presto.PulsarConnectorUtils.restoreNamespaceDelimiterIfNeeded; - import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Predicate; import io.airlift.log.Logger; import io.prestosql.spi.PrestoException; -import io.prestosql.spi.connector.ColumnHandle; -import io.prestosql.spi.connector.ConnectorSession; -import io.prestosql.spi.connector.ConnectorSplitManager; -import io.prestosql.spi.connector.ConnectorSplitSource; -import io.prestosql.spi.connector.ConnectorTableLayoutHandle; -import io.prestosql.spi.connector.ConnectorTransactionHandle; -import io.prestosql.spi.connector.FixedSplitSource; +import io.prestosql.spi.connector.*; import io.prestosql.spi.predicate.Domain; import io.prestosql.spi.predicate.Range; import io.prestosql.spi.predicate.TupleDomain; -import java.io.IOException; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import javax.inject.Inject; import lombok.Data; -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.ManagedLedgerConfig; -import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.bookkeeper.mledger.ManagedLedgerFactory; -import org.apache.bookkeeper.mledger.ReadOnlyCursor; +import org.apache.bookkeeper.mledger.*; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -64,6 +40,17 @@ import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.schema.SchemaInfo; +import javax.inject.Inject; +import java.io.IOException; +import java.sql.Timestamp; +import java.util.*; + +import static com.google.common.base.Preconditions.checkArgument; +import static io.prestosql.spi.StandardErrorCode.QUERY_REJECTED; +import static java.util.Objects.requireNonNull; +import static org.apache.bookkeeper.mledger.ManagedCursor.FindPositionConstraint.SearchAllAvailableEntries; +import static org.apache.pulsar.sql.presto.PulsarConnectorUtils.restoreNamespaceDelimiterIfNeeded; + /** * The class helping to manage splits. */ @@ -117,7 +104,7 @@ public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHand String.format("Failed to get pulsar topic schema for topic %s/%s: Unauthorized", namespace, tableHandle.getTableName())); } else if (e.getStatusCode() == 404) { - schemaInfo = PulsarSchemaHandlers.defaultSchema(); + schemaInfo = PulsarSqlSchemaInfoProvider.defaultSchema(); } else { throw new RuntimeException("Failed to get pulsar topic schema for topic " + String.format("%s/%s", namespace, tableHandle.getTableName()) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java index c88d7e76b535f..453219ffa8603 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java @@ -18,20 +18,12 @@ */ package org.apache.pulsar.sql.presto; -import static java.util.concurrent.CompletableFuture.completedFuture; - import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; - -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.schema.SchemaInfoProvider; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.protocol.schema.BytesSchemaVersion; @@ -40,6 +32,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import static java.util.concurrent.CompletableFuture.completedFuture; + /** * Multi version schema info provider for Pulsar SQL leverage guava cache. @@ -60,7 +60,7 @@ public SchemaInfo load(BytesSchemaVersion schemaVersion) throws Exception { } }); - PulsarSqlSchemaInfoProvider(TopicName topicName, PulsarAdmin pulsarAdmin) { + public PulsarSqlSchemaInfoProvider(TopicName topicName, PulsarAdmin pulsarAdmin) { this.topicName = topicName; this.pulsarAdmin = pulsarAdmin; } @@ -101,4 +101,9 @@ private SchemaInfo loadSchema(BytesSchemaVersion bytesSchemaVersion) throws Puls .getSchemaInfo(topicName.toString(), ByteBuffer.wrap(bytesSchemaVersion.get()).getLong()); } + + public static SchemaInfo defaultSchema(){ + return Schema.BYTES.getSchemaInfo(); + } + } \ No newline at end of file diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/SchemaHandler.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/SchemaHandler.java deleted file mode 100644 index 050eec98576ca..0000000000000 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/SchemaHandler.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import io.netty.buffer.ByteBuf; - -/** - * This interface defines the methods to work with schemas. - */ -public interface SchemaHandler { - - default Object deserialize(ByteBuf payload) { - return null; - } - - // if schemaHandler don't support multi version, we will use method deserialize(ByteBuf payload) - default Object deserialize(ByteBuf byteBuf, byte[] schemaVersion) { - return deserialize(byteBuf); - } - - // if SchemaHandler don't support key value multi version - // we will use method deserialize(ByteBuf byteBuf, byte[] schemaVersion) - default Object deserialize(ByteBuf keyPayload, ByteBuf dataPayload, byte[] schemaVersion) { - return deserialize(dataPayload, schemaVersion); - } - - Object extractField(int index, Object currentRecord); - -} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java new file mode 100644 index 0000000000000..f86bc9d70d376 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java @@ -0,0 +1,347 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder.avro; + +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableSet; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.block.BlockBuilder; +import io.prestosql.spi.type.*; +import io.prestosql.spi.type.RowType.Field; +import org.apache.avro.generic.GenericEnumSymbol; +import org.apache.avro.generic.GenericFixed; +import org.apache.avro.generic.GenericRecord; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static io.airlift.slice.Slices.utf8Slice; +import static io.prestosql.decoder.DecoderErrorCode.DECODER_CONVERSION_NOT_SUPPORTED; +import static io.prestosql.spi.StandardErrorCode.GENERIC_USER_ERROR; +import static io.prestosql.spi.type.Varchars.truncateToLength; +import static java.lang.Float.floatToIntBits; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +/** + * copy from {@link io.prestosql.decoder.avro.AvroColumnDecoder} (presto-record-decoder-345) with A little bit pulsar's extension: + * + * 1) support TimestampType.TIMESTAMP | DateType.DATE | TimeType.TIME + */ +public class PulsarAvroColumnDecoder { + private static final Set SUPPORTED_PRIMITIVE_TYPES = ImmutableSet.of( + BooleanType.BOOLEAN, + TinyintType.TINYINT, + SmallintType.SMALLINT, + IntegerType.INTEGER, + BigintType.BIGINT, + RealType.REAL, + DoubleType.DOUBLE, + TimestampType.TIMESTAMP, + DateType.DATE, + TimeType.TIME, + VarbinaryType.VARBINARY); + + private final Type columnType; + private final String columnMapping; + private final String columnName; + + public PulsarAvroColumnDecoder(DecoderColumnHandle columnHandle) { + try { + requireNonNull(columnHandle, "columnHandle is null"); + this.columnType = columnHandle.getType(); + this.columnMapping = columnHandle.getMapping(); + + this.columnName = columnHandle.getName(); + checkArgument(!columnHandle.isInternal(), "unexpected internal column '%s'", columnName); + checkArgument(columnHandle.getFormatHint() == null, "unexpected format hint '%s' defined for column '%s'", columnHandle.getFormatHint(), columnName); + checkArgument(columnHandle.getDataFormat() == null, "unexpected data format '%s' defined for column '%s'", columnHandle.getDataFormat(), columnName); + checkArgument(columnHandle.getMapping() != null, "mapping not defined for column '%s'", columnName); + + checkArgument(isSupportedType(columnType), "Unsupported column type '%s' for column '%s'", columnType, columnName); + } catch (IllegalArgumentException e) { + throw new PrestoException(GENERIC_USER_ERROR, e); + } + } + + private boolean isSupportedType(Type type) { + if (isSupportedPrimitive(type)) { + return true; + } + + if (type instanceof ArrayType) { + checkArgument(type.getTypeParameters().size() == 1, "expecting exactly one type parameter for array"); + return isSupportedType(type.getTypeParameters().get(0)); + } + + if (type instanceof MapType) { + List typeParameters = type.getTypeParameters(); + checkArgument(typeParameters.size() == 2, "expecting exactly two type parameters for map"); + checkArgument(typeParameters.get(0) instanceof VarcharType, "Unsupported column type '%s' for map key", typeParameters.get(0)); + return isSupportedType(type.getTypeParameters().get(1)); + } + + if (type instanceof RowType) { + for (Type fieldType : type.getTypeParameters()) { + if (!isSupportedType(fieldType)) { + return false; + } + } + return true; + } + return false; + } + + private boolean isSupportedPrimitive(Type type) { + return type instanceof VarcharType || SUPPORTED_PRIMITIVE_TYPES.contains(type); + } + + public FieldValueProvider decodeField(GenericRecord avroRecord) { + Object avroColumnValue = locateNode(avroRecord, columnMapping); + return new ObjectValueProvider(avroColumnValue, columnType, columnName); + } + + private static Object locateNode(GenericRecord element, String columnMapping) { + Object value = element; + for (String pathElement : Splitter.on('/').omitEmptyStrings().split(columnMapping)) { + if (value == null) { + return null; + } + value = ((GenericRecord) value).get(pathElement); + } + return value; + } + + private static class ObjectValueProvider + extends FieldValueProvider { + private final Object value; + private final Type columnType; + private final String columnName; + + public ObjectValueProvider(Object value, Type columnType, String columnName) { + this.value = value; + this.columnType = columnType; + this.columnName = columnName; + } + + @Override + public boolean isNull() { + return value == null; + } + + @Override + public double getDouble() { + if (value instanceof Double || value instanceof Float) { + return ((Number) value).doubleValue(); + } + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), columnType, columnName)); + } + + @Override + public boolean getBoolean() { + if (value instanceof Boolean) { + return (Boolean) value; + } + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), columnType, columnName)); + } + + @Override + public long getLong() { + if (value instanceof Long || value instanceof Integer) { + return ((Number) value).longValue(); + } + + // columnType + + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), columnType, columnName)); + } + + @Override + public Slice getSlice() { + return PulsarAvroColumnDecoder.getSlice(value, columnType, columnName); + } + + @Override + public Block getBlock() { + return serializeObject(null, value, columnType, columnName); + } + } + + private static Slice getSlice(Object value, Type type, String columnName) { + if (type instanceof VarcharType && (value instanceof CharSequence || value instanceof GenericEnumSymbol)) { + return truncateToLength(utf8Slice(value.toString()), type); + } + + if (type instanceof VarbinaryType) { + if (value instanceof ByteBuffer) { + return Slices.wrappedBuffer((ByteBuffer) value); + } else if (value instanceof GenericFixed) { + return Slices.wrappedBuffer(((GenericFixed) value).bytes()); + } + } + + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), type, columnName)); + } + + private static Block serializeObject(BlockBuilder builder, Object value, Type type, String columnName) { + if (type instanceof ArrayType) { + return serializeList(builder, value, type, columnName); + } + if (type instanceof MapType) { + return serializeMap(builder, value, type, columnName); + } + if (type instanceof RowType) { + return serializeRow(builder, value, type, columnName); + } + serializePrimitive(builder, value, type, columnName); + return null; + } + + private static Block serializeList(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { + if (value == null) { + checkState(parentBlockBuilder != null, "parentBlockBuilder is null"); + parentBlockBuilder.appendNull(); + return null; + } + List list = (List) value; + List typeParameters = type.getTypeParameters(); + Type elementType = typeParameters.get(0); + + BlockBuilder blockBuilder = elementType.createBlockBuilder(null, list.size()); + for (Object element : list) { + serializeObject(blockBuilder, element, elementType, columnName); + } + if (parentBlockBuilder != null) { + type.writeObject(parentBlockBuilder, blockBuilder.build()); + return null; + } + return blockBuilder.build(); + } + + private static void serializePrimitive(BlockBuilder blockBuilder, Object value, Type type, String columnName) { + requireNonNull(blockBuilder, "parent blockBuilder is null"); + + if (value == null) { + blockBuilder.appendNull(); + return; + } + + if (type instanceof BooleanType) { + type.writeBoolean(blockBuilder, (Boolean) value); + return; + } + + if ((value instanceof Integer || value instanceof Long) && (type instanceof BigintType || type instanceof IntegerType || type instanceof SmallintType || type instanceof TinyintType)) { + type.writeLong(blockBuilder, ((Number) value).longValue()); + return; + } + + if (type instanceof DoubleType) { + type.writeDouble(blockBuilder, (Double) value); + return; + } + + if (type instanceof RealType) { + type.writeLong(blockBuilder, floatToIntBits((Float) value)); + return; + } + + if (type instanceof VarcharType || type instanceof VarbinaryType) { + type.writeSlice(blockBuilder, getSlice(value, type, columnName)); + return; + } + + if (type instanceof TimestampType) { + type.writeLong(blockBuilder, (Long) value); + return; + } + + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), type, columnName)); + } + + private static Block serializeMap(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { + if (value == null) { + checkState(parentBlockBuilder != null, "parentBlockBuilder is null"); + parentBlockBuilder.appendNull(); + return null; + } + + Map map = (Map) value; + List typeParameters = type.getTypeParameters(); + Type keyType = typeParameters.get(0); + Type valueType = typeParameters.get(1); + + BlockBuilder blockBuilder; + if (parentBlockBuilder != null) { + blockBuilder = parentBlockBuilder; + } else { + blockBuilder = type.createBlockBuilder(null, 1); + } + + BlockBuilder entryBuilder = blockBuilder.beginBlockEntry(); + for (Map.Entry entry : map.entrySet()) { + if (entry.getKey() != null) { + keyType.writeSlice(entryBuilder, truncateToLength(utf8Slice(entry.getKey().toString()), keyType)); + serializeObject(entryBuilder, entry.getValue(), valueType, columnName); + } + } + blockBuilder.closeEntry(); + + if (parentBlockBuilder == null) { + return blockBuilder.getObject(0, Block.class); + } + return null; + } + + private static Block serializeRow(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { + if (value == null) { + checkState(parentBlockBuilder != null, "parent block builder is null"); + parentBlockBuilder.appendNull(); + return null; + } + + BlockBuilder blockBuilder; + if (parentBlockBuilder != null) { + blockBuilder = parentBlockBuilder; + } else { + blockBuilder = type.createBlockBuilder(null, 1); + } + BlockBuilder singleRowBuilder = blockBuilder.beginBlockEntry(); + GenericRecord record = (GenericRecord) value; + List fields = ((RowType) type).getFields(); + for (Field field : fields) { + checkState(field.getName().isPresent(), "field name not found"); + serializeObject(singleRowBuilder, record.get(field.getName().get()), field.getType(), columnName); + } + blockBuilder.closeEntry(); + if (parentBlockBuilder == null) { + return blockBuilder.getObject(0, Block.class); + } + return null; + } +} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java new file mode 100644 index 0000000000000..c110986f75f88 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder.avro; + +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; +import io.prestosql.decoder.RowDecoder; +import io.prestosql.spi.PrestoException; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DecoderFactory; + +import javax.annotation.Nullable; +import java.io.ByteArrayInputStream; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.base.Functions.identity; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static java.util.Objects.requireNonNull; + +/** + * Refer to {@link io.prestosql.decoder.avro.AvroRowDecoder} + */ +public class PulsarAvroRowDecoder implements RowDecoder { + + public static final String NAME = "avro"; + private final DatumReader avroRecordReader; + private final Map columnDecoders; + + public PulsarAvroRowDecoder(DatumReader avroRecordReader, Set columns) { + this.avroRecordReader = requireNonNull(avroRecordReader, "avroRecordReader is null"); + requireNonNull(columns, "columns is null"); + columnDecoders = columns.stream() + .collect(toImmutableMap(identity(), this::createColumnDecoder)); + } + + private PulsarAvroColumnDecoder createColumnDecoder(DecoderColumnHandle columnHandle) { + return new PulsarAvroColumnDecoder(columnHandle); + } + + @Override + public Optional> decodeRow(byte[] data, @Nullable Map dataMap) { + GenericRecord avroRecord; + try { + avroRecord = avroRecordReader.read(null, DecoderFactory.get().binaryDecoder(new ByteArrayInputStream(data), null)); + } catch (Exception e) { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Decoding Avro record failed.", e); + } + return Optional.of(columnDecoders.entrySet().stream() + .collect(toImmutableMap( + Map.Entry::getKey, + entry -> entry.getValue().decodeField(avroRecord)))); + } + + +} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java new file mode 100644 index 0000000000000..d0de64560ed38 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder.avro; + +import com.google.common.collect.ImmutableList; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.RowDecoder; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.connector.ColumnMetadata; +import io.prestosql.spi.type.*; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.SchemaParseException; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.reflect.ReflectDatumReader; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.sql.presto.PulsarColumnHandle; +import org.apache.pulsar.sql.presto.PulsarColumnMetadata; +import org.apache.pulsar.sql.presto.PulsarRowDecoderFactory; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.prestosql.spi.type.DateType.DATE; +import static io.prestosql.spi.type.TimeType.TIME; +import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType; +import static java.lang.String.format; +import static java.util.stream.Collectors.toList; + +/** + * PulsarRowDecoderFactory for {@link org.apache.pulsar.common.schema.SchemaType#AVRO}. + */ +public class PulsarAvroRowDecoderFactory implements PulsarRowDecoderFactory { + + private TypeManager typeManager; + + public PulsarAvroRowDecoderFactory(TypeManager typeManager) { + this.typeManager = typeManager; + } + + @Override + public RowDecoder createRowDecoder(SchemaInfo schemaInfo, Set columns) { + byte[] dataSchema = schemaInfo.getSchema(); + Schema parsedSchema = (new Schema.Parser()).parse(new String(dataSchema, StandardCharsets.UTF_8)); + GenericDatumReader datumReader = new ReflectDatumReader<>(parsedSchema); + return new PulsarAvroRowDecoder(datumReader, columns); + } + + @Override + public List extractColumnMetadata(SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + String schemaJson = new String(schemaInfo.getSchema()); + if (StringUtils.isBlank(schemaJson)) { + throw new PrestoException(NOT_SUPPORTED, "Topic " + + " does not have a valid schema"); + } + Schema schema; + try { + schema = GenericJsonSchema.of(schemaInfo).getAvroSchema(); + } catch (SchemaParseException ex) { + throw new PrestoException(NOT_SUPPORTED, "Topic " + + " does not have a valid schema"); + } + + //TODO : check schema cyclic definitions witch may case java.lang.StackOverflowError + + return schema.getFields().stream() + .map(field -> + new PulsarColumnMetadata(field.name(), parseAvroPrestoType(field.name(), field.schema()), field.schema().toString(), null, false, false, + handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(field.name(), null, null)) + + ).collect(toList()); + } + + /** + * parse AvroPrestoType , avro LogicalType supported. + * @param fieldname + * @param schema + * @return + */ + private Type parseAvroPrestoType(String fieldname, Schema schema) { + Schema.Type type = schema.getType(); + LogicalType logicalType = schema.getLogicalType(); + switch (type) { + case STRING: + case ENUM: + return createUnboundedVarcharType(); + case NULL: + throw new UnsupportedOperationException(format("field '%s' NULL Type code should not be reached ļ¼Œplease check the schema or report the bug.", fieldname)); + case FIXED: + case BYTES: + //TODO: support decimal logicalType + return VarbinaryType.VARBINARY; + case INT: + if (logicalType == LogicalTypes.timeMillis()) { + return TIME; + } else if (logicalType == LogicalTypes.date()) { + return DATE; + } + return IntegerType.INTEGER; + case LONG: + if (logicalType == LogicalTypes.timestampMillis()) { + return TimestampType.TIMESTAMP; + } + //TODO: support timestamp_microseconds logicalType : https://github.com/prestosql/presto/issues/1284 + return BigintType.BIGINT; + case FLOAT: + return RealType.REAL; + case DOUBLE: + return DoubleType.DOUBLE; + case BOOLEAN: + return BooleanType.BOOLEAN; + case ARRAY: + return new ArrayType(parseAvroPrestoType(fieldname, schema.getElementType())); + case MAP: + //The key for an Avro map must be a string + TypeSignature valueType = parseAvroPrestoType(fieldname, schema.getValueType()).getTypeSignature(); + return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(valueType))); + case RECORD: + if (schema.getFields().size() > 0) { + return RowType.from(schema.getFields().stream() + .map(field -> new RowType.Field(Optional.of(field.name()), parseAvroPrestoType(field.name(), field.schema()))) + .collect(toImmutableList())); + } else { + throw new UnsupportedOperationException(format("field '%s' of record Type has no fields, please check avro schema definition. ", fieldname)); + } + case UNION: + for (Schema nestType : schema.getTypes()) { + if (nestType.getType() != Schema.Type.NULL) { + return parseAvroPrestoType(fieldname, nestType); + } + } + throw new UnsupportedOperationException(format("field '%s' of UNION type must contains not null type ", fieldname)); + default: + throw new UnsupportedOperationException(format("Cannot convert from Schema type '%s' (%s) to Presto type", schema.getType(), schema.getFullName())); + } + } +} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java new file mode 100644 index 0000000000000..97c62ce111739 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java @@ -0,0 +1,387 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableList; +import io.airlift.slice.Slice; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; +import io.prestosql.decoder.json.JsonFieldDecoder; +import io.prestosql.decoder.json.JsonRowDecoderFactory; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.block.BlockBuilder; +import io.prestosql.spi.type.*; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static io.airlift.slice.Slices.utf8Slice; +import static io.prestosql.decoder.DecoderErrorCode.DECODER_CONVERSION_NOT_SUPPORTED; +import static io.prestosql.spi.type.BigintType.BIGINT; +import static io.prestosql.spi.type.BooleanType.BOOLEAN; +import static io.prestosql.spi.type.DoubleType.DOUBLE; +import static io.prestosql.spi.type.IntegerType.INTEGER; +import static io.prestosql.spi.type.SmallintType.SMALLINT; +import static io.prestosql.spi.type.TinyintType.TINYINT; +import static io.prestosql.spi.type.Varchars.isVarcharType; +import static io.prestosql.spi.type.Varchars.truncateToLength; +import static java.lang.Double.parseDouble; +import static java.lang.Float.floatToIntBits; +import static java.lang.Long.parseLong; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +/** + * copy from {@link io.prestosql.decoder.json.DefaultJsonFieldDecoder} (presto-record-decoder-345) with some pulsar's extension : + * 1) support array + * 2) support map + * 3) support row + */ +public class PulsarJsonFieldDecoder + implements JsonFieldDecoder { + private final DecoderColumnHandle columnHandle; + private final long minValue; + private final long maxValue; + + public PulsarJsonFieldDecoder(DecoderColumnHandle columnHandle) { + this.columnHandle = requireNonNull(columnHandle, "columnHandle is null"); + if (!isSupportedType(columnHandle.getType())) { + JsonRowDecoderFactory.throwUnsupportedColumnType(columnHandle); + } + + if (columnHandle.getType() == TINYINT) { + minValue = Byte.MIN_VALUE; + maxValue = Byte.MAX_VALUE; + } else if (columnHandle.getType() == SMALLINT) { + minValue = Short.MIN_VALUE; + maxValue = Short.MAX_VALUE; + } else if (columnHandle.getType() == INTEGER) { + minValue = Integer.MIN_VALUE; + maxValue = Integer.MAX_VALUE; + } else if (columnHandle.getType() == BIGINT) { + minValue = Long.MIN_VALUE; + maxValue = Long.MAX_VALUE; + } else { + // those values will not be used if column type is not one of mentioned above + minValue = Long.MAX_VALUE; + maxValue = Long.MIN_VALUE; + } + } + + private boolean isSupportedType(Type type) { + if (isVarcharType(type)) { + return true; + } + if (ImmutableList.of( + BIGINT, + INTEGER, + SMALLINT, + TINYINT, + BOOLEAN, + DOUBLE + ).contains(type)) { + return true; + } + + if (type instanceof ArrayType) { + checkArgument(type.getTypeParameters().size() == 1, "expecting exactly one type parameter for array"); + return isSupportedType(type.getTypeParameters().get(0)); + } + if (type instanceof MapType) { + List typeParameters = type.getTypeParameters(); + checkArgument(typeParameters.size() == 2, "expecting exactly two type parameters for map"); + return isSupportedType(type.getTypeParameters().get(0)) && isSupportedType(type.getTypeParameters().get(1)); + } + + if (type instanceof RowType) { + for (Type fieldType : type.getTypeParameters()) { + if (!isSupportedType(fieldType)) { + return false; + } + } + return true; + } + + return false; + } + + @Override + public FieldValueProvider decode(JsonNode value) { + return new JsonValueProvider(value, columnHandle, minValue, maxValue); + } + + public static class JsonValueProvider + extends FieldValueProvider { + private final JsonNode value; + private final DecoderColumnHandle columnHandle; + private final long minValue; + private final long maxValue; + + public JsonValueProvider(JsonNode value, DecoderColumnHandle columnHandle, long minValue, long maxValue) { + this.value = value; + this.columnHandle = columnHandle; + this.minValue = minValue; + this.maxValue = maxValue; + } + + @Override + public final boolean isNull() { + return value.isMissingNode() || value.isNull(); + } + + @Override + public boolean getBoolean() { + if (value.isValueNode()) { + return value.asBoolean(); + } + throw new PrestoException( + DECODER_CONVERSION_NOT_SUPPORTED, + format("could not parse non-value node as '%s' for column '%s'", columnHandle.getType(), columnHandle.getName())); + } + + @Override + public long getLong() { + try { + long longValue; + if (value.isIntegralNumber() && !value.isBigInteger()) { + longValue = value.longValue(); + if (longValue >= minValue && longValue <= maxValue) { + return longValue; + } + } else if (value.isValueNode()) { + longValue = parseLong(value.asText()); + if (longValue >= minValue && longValue <= maxValue) { + return longValue; + } + } + } catch (NumberFormatException ignore) { + // ignore + } + throw new PrestoException( + DECODER_CONVERSION_NOT_SUPPORTED, + format("could not parse value '%s' as '%s' for column '%s'", value.asText(), columnHandle.getType(), columnHandle.getName())); + } + + @Override + public double getDouble() { + try { + if (value.isNumber()) { + return value.doubleValue(); + } + if (value.isValueNode()) { + return parseDouble(value.asText()); + } + } catch (NumberFormatException ignore) { + // ignore + } + throw new PrestoException( + DECODER_CONVERSION_NOT_SUPPORTED, + format("could not parse value '%s' as '%s' for column '%s'", value.asText(), columnHandle.getType(), columnHandle.getName())); + } + + @Override + public Slice getSlice() { + String textValue = value.isValueNode() ? value.asText() : value.toString(); + Slice slice = utf8Slice(textValue); + if (isVarcharType(columnHandle.getType())) { + slice = truncateToLength(slice, columnHandle.getType()); + } + return slice; + } + + @Override + public Block getBlock() { + return serializeObject(null, value, columnHandle.getType(), columnHandle.getName()); + } + + + private static Slice getSlice(Object value, Type type, String columnName) { + String textValue; + if (type instanceof JsonNode) { + JsonNode jsonNode = (JsonNode) value; + textValue = jsonNode.isValueNode() ? jsonNode.asText() : jsonNode.toString(); + } else { + textValue = value.toString(); + } + + Slice slice = utf8Slice(textValue); + if (isVarcharType(type)) { + slice = truncateToLength(slice, type); + } + return slice; + } + + + private Block serializeObject(BlockBuilder builder, Object value, Type type, String columnName) { + if (type instanceof ArrayType) { + return serializeList(builder, value, type, columnName); + } + if (type instanceof MapType) { + return serializeMap(builder, value, type, columnName); + } + if (type instanceof RowType) { + return serializeRow(builder, value, type, columnName); + } + serializePrimitive(builder, value, type, columnName); + return null; + } + + private Block serializeList(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { + if (value == null) { + checkState(parentBlockBuilder != null, "parentBlockBuilder is null"); + parentBlockBuilder.appendNull(); + return null; + } + + com.fasterxml.jackson.databind.node.ArrayNode arrayNode = (com.fasterxml.jackson.databind.node.ArrayNode) value; + + Iterator jsonNodeIterator = arrayNode.elements(); + + // List list = (List) value; + List typeParameters = type.getTypeParameters(); + Type elementType = typeParameters.get(0); + + BlockBuilder blockBuilder = elementType.createBlockBuilder(null, arrayNode.size()); + + while (jsonNodeIterator.hasNext()) { + Object element = jsonNodeIterator.next(); + serializeObject(blockBuilder, element, elementType, columnName); + } + + if (parentBlockBuilder != null) { + type.writeObject(parentBlockBuilder, blockBuilder.build()); + return null; + } + return blockBuilder.build(); + } + + private void serializePrimitive(BlockBuilder blockBuilder, Object value, Type type, String columnName) { + requireNonNull(blockBuilder, "parent blockBuilder is null"); + + if (value == null) { + blockBuilder.appendNull(); + return; + } + + if (type instanceof BooleanType) { + type.writeBoolean(blockBuilder, (Boolean) value); + return; + } + + if ((value instanceof Integer || value instanceof Long) && (type instanceof BigintType || type instanceof IntegerType || type instanceof SmallintType || type instanceof TinyintType)) { + type.writeLong(blockBuilder, ((Number) value).longValue()); + return; + } + + if (type instanceof DoubleType) { + type.writeDouble(blockBuilder, (Double) value); + return; + } + + if (type instanceof RealType) { + type.writeLong(blockBuilder, floatToIntBits((Float) value)); + return; + } + + if (type instanceof VarcharType || type instanceof VarbinaryType) { + type.writeSlice(blockBuilder, getSlice(value, type, columnName)); + return; + } + + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), type, columnName)); + } + + private Block serializeMap(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { + if (value == null) { + checkState(parentBlockBuilder != null, "parentBlockBuilder is null"); + parentBlockBuilder.appendNull(); + return null; + } + + com.fasterxml.jackson.databind.node.ObjectNode objectNode = (com.fasterxml.jackson.databind.node.ObjectNode) value; + + List typeParameters = type.getTypeParameters(); + Type keyType = typeParameters.get(0); + Type valueType = typeParameters.get(1); + + BlockBuilder blockBuilder; + if (parentBlockBuilder != null) { + blockBuilder = parentBlockBuilder; + } else { + blockBuilder = type.createBlockBuilder(null, 1); + } + + BlockBuilder entryBuilder = blockBuilder.beginBlockEntry(); + + Iterator> fields = objectNode.fields(); + while (fields.hasNext()) { + Map.Entry entry = fields.next(); + if (entry.getKey() != null) { + keyType.writeSlice(entryBuilder, truncateToLength(utf8Slice(entry.getKey().toString()), keyType)); + serializeObject(entryBuilder, entry.getValue(), valueType, columnName); + } + } + + + blockBuilder.closeEntry(); + + if (parentBlockBuilder == null) { + return blockBuilder.getObject(0, Block.class); + } + return null; + } + + + private Block serializeRow(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { + if (value == null) { + checkState(parentBlockBuilder != null, "parent block builder is null"); + parentBlockBuilder.appendNull(); + return null; + } + + BlockBuilder blockBuilder; + if (parentBlockBuilder != null) { + blockBuilder = parentBlockBuilder; + } else { + blockBuilder = type.createBlockBuilder(null, 1); + } + BlockBuilder singleRowBuilder = blockBuilder.beginBlockEntry(); + // GenericRecord record = (GenericRecord) value; + com.fasterxml.jackson.databind.node.ObjectNode objectNode = (com.fasterxml.jackson.databind.node.ObjectNode) value; + + List fields = ((RowType) type).getFields(); + + for (RowType.Field field : fields) { + checkState(field.getName().isPresent(), "field name not found"); + serializeObject(singleRowBuilder, objectNode.get(field.getName().get()), field.getType(), columnName); + } + blockBuilder.closeEntry(); + if (parentBlockBuilder == null) { + return blockBuilder.getObject(0, Block.class); + } + return null; + } + + } +} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java new file mode 100644 index 0000000000000..fadfeee0e52df --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.MissingNode; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableMap; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; +import io.prestosql.decoder.RowDecoder; +import io.prestosql.decoder.json.JsonFieldDecoder; + +import javax.annotation.Nullable; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +public class PulsarJsonRowDecoder implements RowDecoder { + + private final ObjectMapper objectMapper; + private final Map fieldDecoders; + + public PulsarJsonRowDecoder(ObjectMapper objectMapper, Map fieldDecoders) { + this.objectMapper = requireNonNull(objectMapper, "objectMapper is null"); + this.fieldDecoders = ImmutableMap.copyOf(fieldDecoders); + } + + @Override + public Optional> decodeRow(byte[] data, + @Nullable Map dataMap) { + JsonNode tree; + try { + tree = objectMapper.readTree(data); + } catch (Exception e) { + return Optional.empty(); + } + + Map decodedRow = new HashMap<>(); + for (Map.Entry entry : fieldDecoders.entrySet()) { + DecoderColumnHandle columnHandle = entry.getKey(); + JsonFieldDecoder decoder = entry.getValue(); + JsonNode node = locateNode(tree, columnHandle); + decodedRow.put(columnHandle, decoder.decode(node)); + } + return Optional.of(decodedRow); + } + + private static JsonNode locateNode(JsonNode tree, DecoderColumnHandle columnHandle) { + String mapping = columnHandle.getMapping(); + checkState(mapping != null, "No mapping for %s", columnHandle.getName()); + JsonNode currentNode = tree; + for (String pathElement : Splitter.on('/').omitEmptyStrings().split(mapping)) { + if (!currentNode.has(pathElement)) { + return MissingNode.getInstance(); + } + currentNode = currentNode.path(pathElement); + } + return currentNode; + } +} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java new file mode 100644 index 0000000000000..4e248745766c0 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder.json; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.RowDecoder; +import io.prestosql.decoder.json.JsonFieldDecoder; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.connector.ColumnMetadata; +import io.prestosql.spi.type.*; +import org.apache.avro.Schema; +import org.apache.avro.SchemaParseException; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.sql.presto.PulsarColumnHandle; +import org.apache.pulsar.sql.presto.PulsarColumnMetadata; +import org.apache.pulsar.sql.presto.PulsarRowDecoderFactory; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType; +import static java.lang.String.format; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toList; + +/** + * PulsarRowDecoderFactory for {@link org.apache.pulsar.common.schema.SchemaType#JSON}. + */ +public class PulsarJsonRowDecoderFactory implements PulsarRowDecoderFactory { + + private TypeManager typeManager; + + public PulsarJsonRowDecoderFactory(TypeManager typeManager) { + this.typeManager = typeManager; + } + + @Override + public RowDecoder createRowDecoder(SchemaInfo schemaInfo, Set columns) { + ObjectMapper objectMapper = new ObjectMapper(); + Map fieldDecoders = columns.stream().collect(toImmutableMap(identity(), PulsarJsonFieldDecoder::new)); + return new PulsarJsonRowDecoder(objectMapper, fieldDecoders); + } + + @Override + public List extractColumnMetadata(SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + String schemaJson = new String(schemaInfo.getSchema()); + if (StringUtils.isBlank(schemaJson)) { + throw new PrestoException(NOT_SUPPORTED, "Topic " + + " does not have a valid schema"); + } + Schema schema; + try { + schema = GenericJsonSchema.of(schemaInfo).getAvroSchema(); + } catch (SchemaParseException ex) { + throw new PrestoException(NOT_SUPPORTED, "Topic " + + " does not have a valid schema"); + } + + return schema.getFields().stream() + .map(field -> + new PulsarColumnMetadata(field.name(), parseJsonPrestoType(field.name(), field.schema()), field.schema().toString(), null, false, false, + handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(field.name(), null, null)) + + ).collect(toList()); + } + + /** + * parse JsonPrestoType , + * warnning : Json only support json native primitive type which ignore Schema logicalType. + * + * @param fieldname + * @param schema + * @return + */ + private Type parseJsonPrestoType(String fieldname, Schema schema) { + Schema.Type type = schema.getType(); + switch (type) { + case STRING: + case ENUM: + return createUnboundedVarcharType(); + case NULL: + throw new UnsupportedOperationException(format("field '%s' NULL Type code should not be reached ļ¼Œplease check the schema or report the bug.", fieldname)); + case FIXED: + case BYTES: + return VarbinaryType.VARBINARY; + case INT: + return IntegerType.INTEGER; + case LONG: + return BigintType.BIGINT; + case FLOAT: + return RealType.REAL; + case DOUBLE: + return DoubleType.DOUBLE; + case BOOLEAN: + return BooleanType.BOOLEAN; + case ARRAY: + return new ArrayType(parseJsonPrestoType(fieldname, schema.getElementType())); + case MAP: + //The key for an Avro map must be a string + TypeSignature valueType = parseJsonPrestoType(fieldname, schema.getValueType()).getTypeSignature(); + return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(valueType))); + case RECORD: + if (schema.getFields().size() > 0) { + return RowType.from(schema.getFields().stream() + .map(field -> new RowType.Field(Optional.of(field.name()), parseJsonPrestoType(field.name(), field.schema()))) + .collect(toImmutableList())); + } else { + throw new UnsupportedOperationException(format("field '%s' of record Type has no fields, please check avro schema definition. ", fieldname)); + } + case UNION: + for (Schema nestType : schema.getTypes()) { + if (nestType.getType() != Schema.Type.NULL) { + return parseJsonPrestoType(fieldname, nestType); + } + } + throw new UnsupportedOperationException(format("field '%s' of UNION type must contains not null type ", fieldname)); + default: + throw new UnsupportedOperationException(format("Cannot convert from Schema type '%s' (%s) to Presto type", schema.getType(), schema.getFullName())); + } + } +} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java new file mode 100644 index 0000000000000..bd5f9ce19f198 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder.primitive; + +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; +import io.prestosql.decoder.RowDecoder; +import io.prestosql.spi.type.*; + +import javax.annotation.Nullable; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static io.prestosql.decoder.FieldValueProviders.*; +import static org.apache.pulsar.sql.presto.PulsarFieldValueProviders.doubleValueProvider; + +public class PulsarPrimitiveRowDecoder implements RowDecoder { + + private final DecoderColumnHandle columnHandle; + + public PulsarPrimitiveRowDecoder(DecoderColumnHandle column) { + this.columnHandle = column; + } + + @Override + public Optional> decodeRow(byte[] data, @Nullable Map dataMap) { + Map primitiveColumn = new HashMap(); + Type type = columnHandle.getType(); + if (type instanceof BooleanType) { + primitiveColumn.put(columnHandle, booleanValueProvider(Boolean.valueOf(new String(data)))); + } else if (type instanceof TinyintType || type instanceof SmallintType || type instanceof IntegerType || type instanceof BigintType) { + primitiveColumn.put(columnHandle, longValueProvider(Long.valueOf(new String(data)))); + } else if (type instanceof RealType || type instanceof DoubleType) { + primitiveColumn.put(columnHandle, doubleValueProvider(Double.valueOf(new String(data)))); + } else if (type instanceof VarbinaryType || type instanceof VarcharType) { + primitiveColumn.put(columnHandle, bytesValueProvider(data)); + } else if (type instanceof DateType || type instanceof TimeType || type instanceof TimestampType) { + primitiveColumn.put(columnHandle, longValueProvider(Long.valueOf(new String(data)))); + } else { + primitiveColumn.put(columnHandle, bytesValueProvider(data)); + } + return Optional.of(primitiveColumn); + } + +} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java new file mode 100644 index 0000000000000..261b41c3f9b3d --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder.primitive; + +import io.airlift.log.Logger; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.RowDecoder; +import io.prestosql.spi.connector.ColumnMetadata; +import io.prestosql.spi.type.*; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; +import org.apache.pulsar.sql.presto.PulsarColumnHandle; +import org.apache.pulsar.sql.presto.PulsarColumnMetadata; +import org.apache.pulsar.sql.presto.PulsarRowDecoderFactory; + +import java.util.Arrays; +import java.util.List; +import java.util.Set; + +public class PulsarPrimitiveRowDecoderFactory implements PulsarRowDecoderFactory { + + private static final Logger log = Logger.get(PulsarPrimitiveRowDecoderFactory.class); + + @Override + public RowDecoder createRowDecoder(SchemaInfo schemaInfo, Set columns) { + if (columns.size() == 1) { + return new PulsarPrimitiveRowDecoder(columns.iterator().next()); + } else { + throw new RuntimeException("Primitive type must has only one ColumnHandle "); + } + } + + @Override + public List extractColumnMetadata(SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + ColumnMetadata valueColumn = new PulsarColumnMetadata( + PulsarColumnMetadata.getColumnName(handleKeyValueType, "__value__"), + parsePrimitivePrestoType("__value__", schemaInfo.getType()), + "The value of the message with primitive type schema", null, false, false, + handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo("__value__", null, null)); + return Arrays.asList(valueColumn); + } + + + private Type parsePrimitivePrestoType(String fieldName, SchemaType pulsarType) { + switch (pulsarType) { + case BOOLEAN: + return BooleanType.BOOLEAN; + case INT8: + return TinyintType.TINYINT; + case INT16: + return SmallintType.SMALLINT; + case INT32: + return IntegerType.INTEGER; + case INT64: + return BigintType.BIGINT; + case FLOAT: + return RealType.REAL; + case DOUBLE: + return DoubleType.DOUBLE; + case NONE: + case BYTES: + return VarbinaryType.VARBINARY; + case STRING: + return VarcharType.VARCHAR; + case DATE: + return DateType.DATE; + case TIME: + return TimeType.TIME; + case TIMESTAMP: + return TimestampType.TIMESTAMP; + default: + log.error("Cannot convert type: %s", pulsarType); + return null; + } + + } +} From f94e77072a9ea25c521a1daf31780862f5c4bd12 Mon Sep 17 00:00:00 2001 From: wangguowei Date: Mon, 9 Nov 2020 14:41:28 +0800 Subject: [PATCH 02/19] commit unit-tests --- pulsar-sql/presto-pulsar/pom.xml | 20 + .../PulsarDispatchingRowDecoderFactory.java | 16 +- .../sql/presto/PulsarFieldValueProviders.java | 4 +- .../pulsar/sql/presto/PulsarMetadata.java | 2 +- .../pulsar/sql/presto/PulsarRecordCursor.java | 106 ++-- .../pulsar/sql/presto/PulsarRowDecoder.java | 41 ++ .../sql/presto/PulsarRowDecoderFactory.java | 8 +- .../decoder/avro/PulsarAvroColumnDecoder.java | 10 +- .../decoder/avro/PulsarAvroRowDecoder.java | 34 +- .../avro/PulsarAvroRowDecoderFactory.java | 23 +- .../decoder/json/PulsarJsonFieldDecoder.java | 155 +++--- .../decoder/json/PulsarJsonRowDecoder.java | 59 ++- .../json/PulsarJsonRowDecoderFactory.java | 45 +- .../primitive/PulsarPrimitiveRowDecoder.java | 15 +- .../PulsarPrimitiveRowDecoderFactory.java | 10 +- .../sql/presto/TestAvroSchemaHandler.java | 127 ----- .../sql/presto/TestPulsarConnector.java | 485 ++++-------------- .../sql/presto/TestPulsarConnectorConfig.java | 2 +- .../TestPulsarKeyValueSchemaHandler.java | 353 ------------- .../pulsar/sql/presto/TestPulsarMetadata.java | 56 +- .../TestPulsarPrimitiveSchemaHandler.java | 164 ------ .../sql/presto/TestPulsarRecordCursor.java | 96 +--- .../presto/decoder/AbstractDecoderTester.java | 96 ++++ .../presto/decoder/DecoderTestMessage.java | 71 +++ .../sql/presto/decoder/DecoderTestUtil.java | 126 +++++ .../decoder/avro/AvroDecoderTestUtil.java | 199 +++++++ .../presto/decoder/avro/TestAvroDecoder.java | 273 ++++++++++ .../decoder/json/JsonDecoderTestUtil.java | 199 +++++++ .../presto/decoder/json/TestJsonDecoder.java | 268 ++++++++++ 29 files changed, 1660 insertions(+), 1403 deletions(-) create mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoder.java delete mode 100644 pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestAvroSchemaHandler.java delete mode 100644 pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarKeyValueSchemaHandler.java delete mode 100644 pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarPrimitiveSchemaHandler.java create mode 100644 pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java create mode 100644 pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java create mode 100644 pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java create mode 100644 pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/AvroDecoderTestUtil.java create mode 100644 pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java create mode 100644 pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/JsonDecoderTestUtil.java create mode 100644 pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index 82883aa678247..89f83960a6d71 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -115,6 +115,26 @@ ${presto.version} + + ${project.groupId} + pulsar-client-original + ${project.version} + + + + io.prestosql + presto-main + ${presto.version} + test + + + + io.prestosql + presto-testing + ${presto.version} + test + + diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java index 62a71d031e10c..bc68124d3ebbd 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java @@ -21,9 +21,9 @@ import com.google.inject.Inject; import io.airlift.log.Logger; import io.prestosql.decoder.DecoderColumnHandle; -import io.prestosql.decoder.RowDecoder; import io.prestosql.spi.connector.ColumnMetadata; import io.prestosql.spi.type.TypeManager; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.sql.presto.decoder.avro.PulsarAvroRowDecoderFactory; @@ -48,15 +48,14 @@ public PulsarDispatchingRowDecoderFactory(TypeManager typeManager) { this.typeManager = typeManager; } - public RowDecoder createRowDecoder(SchemaInfo schemaInfo, Set columns) { + public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, Set columns) { PulsarRowDecoderFactory rowDecoderFactory = createDecoderFactory(schemaInfo); - return rowDecoderFactory.createRowDecoder(schemaInfo, columns); + return rowDecoderFactory.createRowDecoder(topicName,schemaInfo, columns); } - - public List extractColumnMetadata(SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { PulsarRowDecoderFactory rowDecoderFactory = createDecoderFactory(schemaInfo); - return rowDecoderFactory.extractColumnMetadata(schemaInfo, handleKeyValueType); + return rowDecoderFactory.extractColumnMetadata(topicName,schemaInfo, handleKeyValueType); } private PulsarRowDecoderFactory createDecoderFactory(SchemaInfo schemaInfo) { @@ -71,7 +70,8 @@ private PulsarRowDecoderFactory createDecoderFactory(SchemaInfo schemaInfo) { } } - - + public TypeManager getTypeManager() { + return typeManager; + } } \ No newline at end of file diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarFieldValueProviders.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarFieldValueProviders.java index 18e0e080ad6ab..215380b23c9c3 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarFieldValueProviders.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarFieldValueProviders.java @@ -20,6 +20,9 @@ import io.prestosql.decoder.FieldValueProvider; +/** + * custom FieldValueProvider for Pulsar + */ public class PulsarFieldValueProviders { public static FieldValueProvider doubleValueProvider(double value) { @@ -57,5 +60,4 @@ public boolean isNull() { }; } - } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java index 45b073be1f221..ea4a45e32e6eb 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java @@ -325,7 +325,7 @@ List getPulsarColumnsFromSchema(TopicName topicName, ImmutableList.Builder builder = ImmutableList.builder(); - builder.addAll(decoderFactory.extractColumnMetadata(schemaInfo,handleKeyValueType)); + builder.addAll(decoderFactory.extractColumnMetadata(topicName, schemaInfo, handleKeyValueType)); if (withInternalColumns) { PulsarInternalColumn.getInternalFields() diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index a073390d7dac6..29d52eaeec9ba 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -23,11 +23,8 @@ import com.google.common.annotations.VisibleForTesting; import io.airlift.log.Logger; import io.airlift.slice.Slice; -import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; -import io.prestosql.decoder.RowDecoder; -import io.prestosql.spi.PrestoException; import io.prestosql.spi.block.Block; import io.prestosql.spi.connector.ColumnHandle; import io.prestosql.spi.connector.RecordCursor; @@ -47,7 +44,6 @@ import org.jctools.queues.SpscArrayQueue; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -59,7 +55,6 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet; import static io.prestosql.decoder.FieldValueProviders.bytesValueProvider; import static io.prestosql.decoder.FieldValueProviders.longValueProvider; -import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; /** * Implementation of a cursor to read records. @@ -92,10 +87,9 @@ public class PulsarRecordCursor implements RecordCursor { private int partition = -1; - private PulsarSqlSchemaInfoProvider schemaInfoProvider; - - private FieldValueProvider[] currentRowValues = null; + private PulsarSqlSchemaInfoProvider schemaInfoProvider; + private FieldValueProvider[] currentRowValues = null; PulsarDispatchingRowDecoderFactory decoderFactory; @@ -134,10 +128,11 @@ public PulsarRecordCursor(List columnHandles, PulsarSplit pu // Exposed for testing purposes PulsarRecordCursor(List columnHandles, PulsarSplit pulsarSplit, PulsarConnectorConfig pulsarConnectorConfig, ManagedLedgerFactory managedLedgerFactory, ManagedLedgerConfig managedLedgerConfig, - PulsarConnectorMetricsTracker pulsarConnectorMetricsTracker) { + PulsarConnectorMetricsTracker pulsarConnectorMetricsTracker, PulsarDispatchingRowDecoderFactory decoderFactory) { this.splitSize = pulsarSplit.getSplitSize(); initialize(columnHandles, pulsarSplit, pulsarConnectorConfig, managedLedgerFactory, managedLedgerConfig, pulsarConnectorMetricsTracker); + this.decoderFactory = decoderFactory; } private void initialize(List columnHandles, PulsarSplit pulsarSplit, PulsarConnectorConfig @@ -203,6 +198,11 @@ public Type getType(int field) { return columnHandles.get(field).getType(); } + @VisibleForTesting + public void setPulsarSqlSchemaInfoProvider(PulsarSqlSchemaInfoProvider schemaInfoProvider) { + this.schemaInfoProvider = schemaInfoProvider; + } + @VisibleForTesting class DeserializeEntries implements Runnable { @@ -437,14 +437,14 @@ public boolean advanceNextPosition() { if(schemaInfo.getType().equals(SchemaType.KEY_VALUE) ){ - RowDecoder keyDecoder = decoderFactory.createRowDecoder( + PulsarRowDecoder keyDecoder = decoderFactory.createRowDecoder(topicName, schemaInfo, columnHandles.stream() .filter(col -> !col.isInternal()) .filter(col -> PulsarColumnHandle.HandleKeyValueType.KEY.equals(col.getHandleKeyValueType())) .collect(toImmutableSet())); - RowDecoder messageDecoder = decoderFactory.createRowDecoder( + PulsarRowDecoder messageDecoder = decoderFactory.createRowDecoder(topicName, schemaInfo, columnHandles.stream() .filter(col -> !col.isInternal()) @@ -453,51 +453,48 @@ public boolean advanceNextPosition() { Optional> decodedKey; if (this.currentMessage.getKeyBytes().isPresent()) { - ByteBuf keyByteBuf = this.currentMessage.getKeyBytes().get(); - decodedKey = keyDecoder.decodeRow(keyByteBuf.array(),null); + decodedKey = keyDecoder.decodeRow(this.currentMessage.getKeyBytes().get()); decodedKey.ifPresent(currentRowValuesMap::putAll); } - - Optional> decodedValue = messageDecoder.decodeRow(toBytes(this.currentMessage.getData()),null); + Optional> decodedValue = messageDecoder.decodeRow(this.currentMessage.getData()); decodedValue.ifPresent(currentRowValuesMap::putAll); }else { - RowDecoder messageDecoder = decoderFactory.createRowDecoder( + PulsarRowDecoder messageDecoder = decoderFactory.createRowDecoder(topicName, schemaInfo, columnHandles.stream() .filter(col -> !col.isInternal()) .filter(col -> PulsarColumnHandle.HandleKeyValueType.NONE.equals(col.getHandleKeyValueType())) .collect(toImmutableSet())); - byte[] data = toBytes(this.currentMessage.getData()); - Optional> decodedValue = messageDecoder.decodeRow(data, null); + Optional> decodedValue = messageDecoder.decodeRow(this.currentMessage.getData()); decodedValue.ifPresent(currentRowValuesMap::putAll); } for (DecoderColumnHandle columnHandle : columnHandles) { if (columnHandle.isInternal()) { - if(PulsarInternalColumn.PARTITION.getName().equals(columnHandle.getName())){ - currentRowValuesMap.put(columnHandle, longValueProvider(this.partition)); - }else if(PulsarInternalColumn.EVENT_TIME.getName().equals(columnHandle.getName())){ - currentRowValuesMap.put(columnHandle, PulsarFieldValueProviders.timeValueProvider(this.currentMessage.getEventTime(), this.currentMessage.getPublishTime() == 0)); - }else if(PulsarInternalColumn.PUBLISH_TIME.getName().equals(columnHandle.getName())){ - currentRowValuesMap.put(columnHandle, PulsarFieldValueProviders.timeValueProvider(this.currentMessage.getPublishTime(), this.currentMessage.getPublishTime() == 0)); - }else if(PulsarInternalColumn.MESSAGE_ID.getName().equals(columnHandle.getName())){ - currentRowValuesMap.put(columnHandle, bytesValueProvider(this.currentMessage.getMessageId().toString().getBytes())); - }else if(PulsarInternalColumn.SEQUENCE_ID.getName().equals(columnHandle.getName())){ - currentRowValuesMap.put(columnHandle, longValueProvider(this.currentMessage.getSequenceId())); - }else if(PulsarInternalColumn.PRODUCER_NAME.getName().equals(columnHandle.getName())){ - currentRowValuesMap.put(columnHandle, bytesValueProvider(this.currentMessage.getProducerName().getBytes())); - }else if(PulsarInternalColumn.KEY.getName().equals(columnHandle.getName())){ - String key = this.currentMessage.getKey().orElse(null); - currentRowValuesMap.put(columnHandle, bytesValueProvider(key ==null? null:key.getBytes())); - }else if(PulsarInternalColumn.PROPERTIES.getName().equals(columnHandle.getName())){ - try { - currentRowValuesMap.put(columnHandle, bytesValueProvider(new ObjectMapper().writeValueAsBytes(this.currentMessage.getProperties()))); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }else { - throw new IllegalArgumentException("unknown internal field " + columnHandle.getName()); + if (PulsarInternalColumn.PARTITION.getName().equals(columnHandle.getName())) { + currentRowValuesMap.put(columnHandle, longValueProvider(this.partition)); + } else if (PulsarInternalColumn.EVENT_TIME.getName().equals(columnHandle.getName())) { + currentRowValuesMap.put(columnHandle, PulsarFieldValueProviders.timeValueProvider(this.currentMessage.getEventTime(), this.currentMessage.getPublishTime() == 0)); + } else if (PulsarInternalColumn.PUBLISH_TIME.getName().equals(columnHandle.getName())) { + currentRowValuesMap.put(columnHandle, PulsarFieldValueProviders.timeValueProvider(this.currentMessage.getPublishTime(), this.currentMessage.getPublishTime() == 0)); + } else if (PulsarInternalColumn.MESSAGE_ID.getName().equals(columnHandle.getName())) { + currentRowValuesMap.put(columnHandle, bytesValueProvider(this.currentMessage.getMessageId().toString().getBytes())); + } else if (PulsarInternalColumn.SEQUENCE_ID.getName().equals(columnHandle.getName())) { + currentRowValuesMap.put(columnHandle, longValueProvider(this.currentMessage.getSequenceId())); + } else if (PulsarInternalColumn.PRODUCER_NAME.getName().equals(columnHandle.getName())) { + currentRowValuesMap.put(columnHandle, bytesValueProvider(this.currentMessage.getProducerName().getBytes())); + } else if (PulsarInternalColumn.KEY.getName().equals(columnHandle.getName())) { + String key = this.currentMessage.getKey().orElse(null); + currentRowValuesMap.put(columnHandle, bytesValueProvider(key == null ? null : key.getBytes())); + } else if (PulsarInternalColumn.PROPERTIES.getName().equals(columnHandle.getName())) { + try { + currentRowValuesMap.put(columnHandle, bytesValueProvider(new ObjectMapper().writeValueAsBytes(this.currentMessage.getProperties()))); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); } + } else { + throw new IllegalArgumentException("unknown internal field " + columnHandle.getName()); + } } } for (int i = 0; i < columnHandles.size(); i++) { @@ -588,33 +585,6 @@ public void close() { } - private byte[] toBytes(Object record) { - if (record instanceof ByteBuffer) { - ByteBuffer byteBuffer = (ByteBuffer) record; - if (byteBuffer.hasArray()) { - return byteBuffer.array(); - } - byte[] bytes = new byte[byteBuffer.position()]; - byteBuffer.flip(); - byteBuffer.get(bytes); - return bytes; - } else if (record instanceof ByteBuf) { - ByteBuf byteBuf = (ByteBuf) record; - if (byteBuf.hasArray()) { - return byteBuf.array(); - } - byte[] bytes = new byte[byteBuf.readableBytes()]; - byteBuf.readBytes(bytes); - return bytes; - } else { - try { - return (byte[]) record; - } catch (Exception e) { - throw new PrestoException(NOT_SUPPORTED, "Unsupported type " + record.getClass().getName()); - } - } - } - private void checkFieldType(int field, Class expected) { Class actual = getType(field).getJavaType(); checkArgument(actual == expected, "Expected field %s to be type %s but is %s", field, expected, actual); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoder.java new file mode 100644 index 0000000000000..fa53e847f44c5 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoder.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto; + +import io.netty.buffer.ByteBuf; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; + +import java.util.Map; +import java.util.Optional; + +/** + * RowDecoder interface for Pulsar + */ +public interface PulsarRowDecoder { + + /** + * decode byteBuf to Map + * + * @param byteBuf + * @return + */ + public Optional> decodeRow(ByteBuf byteBuf); + +} diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java index e02ff611f1c71..ff49edfdb814e 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java @@ -19,15 +19,15 @@ package org.apache.pulsar.sql.presto; import io.prestosql.decoder.DecoderColumnHandle; -import io.prestosql.decoder.RowDecoder; import io.prestosql.spi.connector.ColumnMetadata; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; import java.util.List; import java.util.Set; /** - * Pulsar RowDecoderFactory interface + * Pulsar customized RowDecoderFactory interface */ public interface PulsarRowDecoderFactory { @@ -38,7 +38,7 @@ public interface PulsarRowDecoderFactory { * @param handleKeyValueType * @return */ - public List extractColumnMetadata(SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType); + public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType); /** * createRowDecoder RowDecoder by pulsar SchemaInfo and column DecoderColumnHandles @@ -47,6 +47,6 @@ public interface PulsarRowDecoderFactory { * @param columns * @return */ - public RowDecoder createRowDecoder(SchemaInfo schemaInfo, Set columns); + public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, Set columns); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java index f86bc9d70d376..fcfb37e307b07 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java @@ -49,9 +49,10 @@ import static java.util.Objects.requireNonNull; /** - * copy from {@link io.prestosql.decoder.avro.AvroColumnDecoder} (presto-record-decoder-345) with A little bit pulsar's extension: + * copy from {@link io.prestosql.decoder.avro.AvroColumnDecoder} (presto-record-decoder-345) with A little bit pulsar's extensions: * - * 1) support TimestampType.TIMESTAMP | DateType.DATE | TimeType.TIME + * 1) support TIMESTAMP,DATE,TIME + * 2) support RealType */ public class PulsarAvroColumnDecoder { private static final Set SUPPORTED_PRIMITIVE_TYPES = ImmutableSet.of( @@ -76,7 +77,6 @@ public PulsarAvroColumnDecoder(DecoderColumnHandle columnHandle) { requireNonNull(columnHandle, "columnHandle is null"); this.columnType = columnHandle.getType(); this.columnMapping = columnHandle.getMapping(); - this.columnName = columnHandle.getName(); checkArgument(!columnHandle.isInternal(), "unexpected internal column '%s'", columnName); checkArgument(columnHandle.getFormatHint() == null, "unexpected format hint '%s' defined for column '%s'", columnHandle.getFormatHint(), columnName); @@ -176,7 +176,9 @@ public long getLong() { return ((Number) value).longValue(); } - // columnType + if (columnType instanceof RealType) { + return floatToIntBits((Float) value); + } throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), columnType, columnName)); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java index c110986f75f88..9e94195074a6a 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java @@ -18,16 +18,16 @@ */ package org.apache.pulsar.sql.presto.decoder.avro; +import io.airlift.log.Logger; +import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; -import io.prestosql.decoder.RowDecoder; import io.prestosql.spi.PrestoException; import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.DecoderFactory; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; +import org.apache.pulsar.sql.presto.PulsarRowDecoder; -import javax.annotation.Nullable; -import java.io.ByteArrayInputStream; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -40,15 +40,15 @@ /** * Refer to {@link io.prestosql.decoder.avro.AvroRowDecoder} */ -public class PulsarAvroRowDecoder implements RowDecoder { +public class PulsarAvroRowDecoder implements PulsarRowDecoder { - public static final String NAME = "avro"; - private final DatumReader avroRecordReader; + private static final Logger log = Logger.get(PulsarAvroRowDecoderFactory.class); + + private final GenericAvroSchema genericAvroSchema; private final Map columnDecoders; - public PulsarAvroRowDecoder(DatumReader avroRecordReader, Set columns) { - this.avroRecordReader = requireNonNull(avroRecordReader, "avroRecordReader is null"); - requireNonNull(columns, "columns is null"); + public PulsarAvroRowDecoder(GenericAvroSchema genericAvroSchema, Set columns) { + this.genericAvroSchema = requireNonNull(genericAvroSchema, "genericAvroSchema is null"); columnDecoders = columns.stream() .collect(toImmutableMap(identity(), this::createColumnDecoder)); } @@ -57,11 +57,17 @@ private PulsarAvroColumnDecoder createColumnDecoder(DecoderColumnHandle columnHa return new PulsarAvroColumnDecoder(columnHandle); } + /** + * decode ByteBuf by {@link org.apache.pulsar.client.api.schema.GenericSchema}. + * @param byteBuf + * @return + */ @Override - public Optional> decodeRow(byte[] data, @Nullable Map dataMap) { + public Optional> decodeRow(ByteBuf byteBuf) { GenericRecord avroRecord; try { - avroRecord = avroRecordReader.read(null, DecoderFactory.get().binaryDecoder(new ByteArrayInputStream(data), null)); + GenericAvroRecord record = (GenericAvroRecord) genericAvroSchema.decode(byteBuf); + avroRecord = record.getAvroRecord(); } catch (Exception e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, "Decoding Avro record failed.", e); } @@ -70,6 +76,4 @@ public Optional> decodeRow(byte[] d Map.Entry::getKey, entry -> entry.getValue().decodeField(avroRecord)))); } - - } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java index d0de64560ed38..d2b46d87e596e 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java @@ -20,7 +20,6 @@ import com.google.common.collect.ImmutableList; import io.prestosql.decoder.DecoderColumnHandle; -import io.prestosql.decoder.RowDecoder; import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ColumnMetadata; import io.prestosql.spi.type.*; @@ -28,17 +27,16 @@ import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.SchemaParseException; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.reflect.ReflectDatumReader; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.sql.presto.PulsarColumnHandle; import org.apache.pulsar.sql.presto.PulsarColumnMetadata; +import org.apache.pulsar.sql.presto.PulsarRowDecoder; import org.apache.pulsar.sql.presto.PulsarRowDecoderFactory; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Optional; import java.util.Set; @@ -63,29 +61,26 @@ public PulsarAvroRowDecoderFactory(TypeManager typeManager) { } @Override - public RowDecoder createRowDecoder(SchemaInfo schemaInfo, Set columns) { - byte[] dataSchema = schemaInfo.getSchema(); - Schema parsedSchema = (new Schema.Parser()).parse(new String(dataSchema, StandardCharsets.UTF_8)); - GenericDatumReader datumReader = new ReflectDatumReader<>(parsedSchema); - return new PulsarAvroRowDecoder(datumReader, columns); + public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, Set columns) { + return new PulsarAvroRowDecoder((GenericAvroSchema) GenericAvroSchema.of(schemaInfo), columns); } @Override - public List extractColumnMetadata(SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { String schemaJson = new String(schemaInfo.getSchema()); if (StringUtils.isBlank(schemaJson)) { throw new PrestoException(NOT_SUPPORTED, "Topic " - + " does not have a valid schema"); + + topicName.toString() + " does not have a valid schema"); } Schema schema; try { schema = GenericJsonSchema.of(schemaInfo).getAvroSchema(); } catch (SchemaParseException ex) { throw new PrestoException(NOT_SUPPORTED, "Topic " - + " does not have a valid schema"); + + topicName.toString() + " does not have a valid schema"); } - //TODO : check schema cyclic definitions witch may case java.lang.StackOverflowError + //TODO : check schema cyclic definitions which may case java.lang.StackOverflowError return schema.getFields().stream() .map(field -> diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java index 97c62ce111739..cfb8732742c6c 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java @@ -19,6 +19,8 @@ package org.apache.pulsar.sql.presto.decoder.json; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slice; import io.prestosql.decoder.DecoderColumnHandle; @@ -29,6 +31,7 @@ import io.prestosql.spi.block.Block; import io.prestosql.spi.block.BlockBuilder; import io.prestosql.spi.type.*; +import javafx.util.Pair; import java.util.Iterator; import java.util.List; @@ -40,23 +43,30 @@ import static io.prestosql.decoder.DecoderErrorCode.DECODER_CONVERSION_NOT_SUPPORTED; import static io.prestosql.spi.type.BigintType.BIGINT; import static io.prestosql.spi.type.BooleanType.BOOLEAN; +import static io.prestosql.spi.type.DateType.DATE; import static io.prestosql.spi.type.DoubleType.DOUBLE; import static io.prestosql.spi.type.IntegerType.INTEGER; +import static io.prestosql.spi.type.RealType.REAL; import static io.prestosql.spi.type.SmallintType.SMALLINT; +import static io.prestosql.spi.type.TimeType.TIME; +import static io.prestosql.spi.type.TimestampType.TIMESTAMP; import static io.prestosql.spi.type.TinyintType.TINYINT; import static io.prestosql.spi.type.Varchars.isVarcharType; import static io.prestosql.spi.type.Varchars.truncateToLength; import static java.lang.Double.parseDouble; import static java.lang.Float.floatToIntBits; +import static java.lang.Float.parseFloat; import static java.lang.Long.parseLong; import static java.lang.String.format; import static java.util.Objects.requireNonNull; /** - * copy from {@link io.prestosql.decoder.json.DefaultJsonFieldDecoder} (presto-record-decoder-345) with some pulsar's extension : + * copy from {@link io.prestosql.decoder.json.DefaultJsonFieldDecoder} (presto-record-decoder-345) with some pulsar's extensions : * 1) support array * 2) support map * 3) support row + * 3) support TIMESTAMP,DATE,TIME + * 4) support Real */ public class PulsarJsonFieldDecoder implements JsonFieldDecoder { @@ -69,23 +79,24 @@ public PulsarJsonFieldDecoder(DecoderColumnHandle columnHandle) { if (!isSupportedType(columnHandle.getType())) { JsonRowDecoderFactory.throwUnsupportedColumnType(columnHandle); } + Pair range = getNumRangeByType(columnHandle.getType()); + minValue = range.getKey(); + maxValue = range.getValue(); - if (columnHandle.getType() == TINYINT) { - minValue = Byte.MIN_VALUE; - maxValue = Byte.MAX_VALUE; - } else if (columnHandle.getType() == SMALLINT) { - minValue = Short.MIN_VALUE; - maxValue = Short.MAX_VALUE; - } else if (columnHandle.getType() == INTEGER) { - minValue = Integer.MIN_VALUE; - maxValue = Integer.MAX_VALUE; - } else if (columnHandle.getType() == BIGINT) { - minValue = Long.MIN_VALUE; - maxValue = Long.MAX_VALUE; + } + + private static Pair getNumRangeByType(Type type) { + if (type == TINYINT) { + return new Pair((long) Byte.MIN_VALUE, (long) Byte.MAX_VALUE); + } else if (type == SMALLINT) { + return new Pair((long) Short.MIN_VALUE, (long) Short.MAX_VALUE); + } else if (type == INTEGER) { + return new Pair((long) Integer.MIN_VALUE, (long) Integer.MAX_VALUE); + } else if (type == BIGINT) { + return new Pair((long) Long.MIN_VALUE, (long) Long.MAX_VALUE); } else { // those values will not be used if column type is not one of mentioned above - minValue = Long.MAX_VALUE; - maxValue = Long.MIN_VALUE; + return new Pair(Long.MIN_VALUE, Long.MAX_VALUE); } } @@ -99,7 +110,11 @@ private boolean isSupportedType(Type type) { SMALLINT, TINYINT, BOOLEAN, - DOUBLE + DOUBLE, + TIMESTAMP, + DATE, + TIME, + REAL ).contains(type)) { return true; } @@ -152,17 +167,45 @@ public final boolean isNull() { @Override public boolean getBoolean() { + return getBoolean(value,columnHandle.getType(),columnHandle.getName()); + } + + @Override + public long getLong() { + return getLong(value,columnHandle.getType(),columnHandle.getName(),minValue,maxValue); + } + + @Override + public double getDouble() { + return getDouble(value,columnHandle.getType(),columnHandle.getName()); + } + + @Override + public Slice getSlice() { + return getSlice(value,columnHandle.getType(),columnHandle.getName()); + } + + @Override + public Block getBlock() { + return serializeObject(null, value, columnHandle.getType(), columnHandle.getName()); + } + + + public static boolean getBoolean(JsonNode value, Type type, String columnName) { if (value.isValueNode()) { return value.asBoolean(); } throw new PrestoException( DECODER_CONVERSION_NOT_SUPPORTED, - format("could not parse non-value node as '%s' for column '%s'", columnHandle.getType(), columnHandle.getName())); + format("could not parse non-value node as '%s' for column '%s'", type, columnName)); } - @Override - public long getLong() { + public static long getLong(JsonNode value, Type type, String columnName,long minValue,long maxValue) { try { + if (type instanceof RealType) { + return floatToIntBits((Float) parseFloat(value.asText())); + } + long longValue; if (value.isIntegralNumber() && !value.isBigInteger()) { longValue = value.longValue(); @@ -180,11 +223,10 @@ public long getLong() { } throw new PrestoException( DECODER_CONVERSION_NOT_SUPPORTED, - format("could not parse value '%s' as '%s' for column '%s'", value.asText(), columnHandle.getType(), columnHandle.getName())); + format("could not parse value '%s' as '%s' for column '%s'", value.asText(), type, columnName)); } - @Override - public double getDouble() { + public static double getDouble(JsonNode value, Type type, String columnName) { try { if (value.isNumber()) { return value.doubleValue(); @@ -197,34 +239,12 @@ public double getDouble() { } throw new PrestoException( DECODER_CONVERSION_NOT_SUPPORTED, - format("could not parse value '%s' as '%s' for column '%s'", value.asText(), columnHandle.getType(), columnHandle.getName())); - } + format("could not parse value '%s' as '%s' for column '%s'", value.asText(), type, columnName)); - @Override - public Slice getSlice() { - String textValue = value.isValueNode() ? value.asText() : value.toString(); - Slice slice = utf8Slice(textValue); - if (isVarcharType(columnHandle.getType())) { - slice = truncateToLength(slice, columnHandle.getType()); - } - return slice; - } - - @Override - public Block getBlock() { - return serializeObject(null, value, columnHandle.getType(), columnHandle.getName()); } - - private static Slice getSlice(Object value, Type type, String columnName) { - String textValue; - if (type instanceof JsonNode) { - JsonNode jsonNode = (JsonNode) value; - textValue = jsonNode.isValueNode() ? jsonNode.asText() : jsonNode.toString(); - } else { - textValue = value.toString(); - } - + private static Slice getSlice(JsonNode value, Type type, String columnName) { + String textValue = value.isValueNode() ? value.asText() : value.toString(); Slice slice = utf8Slice(textValue); if (isVarcharType(type)) { slice = truncateToLength(slice, type); @@ -254,15 +274,14 @@ private Block serializeList(BlockBuilder parentBlockBuilder, Object value, Type return null; } - com.fasterxml.jackson.databind.node.ArrayNode arrayNode = (com.fasterxml.jackson.databind.node.ArrayNode) value; + checkState(value instanceof ArrayNode, "Json array node must is ArrayNode type"); - Iterator jsonNodeIterator = arrayNode.elements(); + Iterator jsonNodeIterator = ((ArrayNode)value).elements(); - // List list = (List) value; List typeParameters = type.getTypeParameters(); Type elementType = typeParameters.get(0); - BlockBuilder blockBuilder = elementType.createBlockBuilder(null, arrayNode.size()); + BlockBuilder blockBuilder = elementType.createBlockBuilder(null, ((ArrayNode) value).size()); while (jsonNodeIterator.hasNext()) { Object element = jsonNodeIterator.next(); @@ -276,8 +295,14 @@ private Block serializeList(BlockBuilder parentBlockBuilder, Object value, Type return blockBuilder.build(); } - private void serializePrimitive(BlockBuilder blockBuilder, Object value, Type type, String columnName) { + private void serializePrimitive(BlockBuilder blockBuilder, Object node, Type type, String columnName) { requireNonNull(blockBuilder, "parent blockBuilder is null"); + JsonNode value; + if (node instanceof JsonNode) { + value = (JsonNode) node; + } else { + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("primitive object of '%s' as '%s' for column '%s' cann't convert to JsonNode", node.getClass(), type, columnName)); + } if (value == null) { blockBuilder.appendNull(); @@ -285,22 +310,18 @@ private void serializePrimitive(BlockBuilder blockBuilder, Object value, Type ty } if (type instanceof BooleanType) { - type.writeBoolean(blockBuilder, (Boolean) value); + type.writeBoolean(blockBuilder, getBoolean(value, type, columnName)); return; } - if ((value instanceof Integer || value instanceof Long) && (type instanceof BigintType || type instanceof IntegerType || type instanceof SmallintType || type instanceof TinyintType)) { - type.writeLong(blockBuilder, ((Number) value).longValue()); + if (type instanceof RealType || type instanceof BigintType || type instanceof IntegerType || type instanceof SmallintType || type instanceof TinyintType || type instanceof TimestampType || type instanceof TimeType || type instanceof DateType) { + Pair numRange = getNumRangeByType(type); + type.writeLong(blockBuilder, getLong(value, type, columnName, numRange.getKey(), numRange.getValue())); return; } if (type instanceof DoubleType) { - type.writeDouble(blockBuilder, (Double) value); - return; - } - - if (type instanceof RealType) { - type.writeLong(blockBuilder, floatToIntBits((Float) value)); + type.writeDouble(blockBuilder, getDouble(value, type, columnName)); return; } @@ -318,8 +339,7 @@ private Block serializeMap(BlockBuilder parentBlockBuilder, Object value, Type t parentBlockBuilder.appendNull(); return null; } - - com.fasterxml.jackson.databind.node.ObjectNode objectNode = (com.fasterxml.jackson.databind.node.ObjectNode) value; + checkState(value instanceof ObjectNode, "Json map node must is ObjectNode type"); List typeParameters = type.getTypeParameters(); Type keyType = typeParameters.get(0); @@ -334,7 +354,7 @@ private Block serializeMap(BlockBuilder parentBlockBuilder, Object value, Type t BlockBuilder entryBuilder = blockBuilder.beginBlockEntry(); - Iterator> fields = objectNode.fields(); + Iterator> fields = ((ObjectNode)value).fields(); while (fields.hasNext()) { Map.Entry entry = fields.next(); if (entry.getKey() != null) { @@ -343,7 +363,6 @@ private Block serializeMap(BlockBuilder parentBlockBuilder, Object value, Type t } } - blockBuilder.closeEntry(); if (parentBlockBuilder == null) { @@ -367,14 +386,14 @@ private Block serializeRow(BlockBuilder parentBlockBuilder, Object value, Type t blockBuilder = type.createBlockBuilder(null, 1); } BlockBuilder singleRowBuilder = blockBuilder.beginBlockEntry(); - // GenericRecord record = (GenericRecord) value; - com.fasterxml.jackson.databind.node.ObjectNode objectNode = (com.fasterxml.jackson.databind.node.ObjectNode) value; List fields = ((RowType) type).getFields(); + checkState(value instanceof ObjectNode, "Json row node must is ObjectNode type"); + for (RowType.Field field : fields) { checkState(field.getName().isPresent(), "field name not found"); - serializeObject(singleRowBuilder, objectNode.get(field.getName().get()), field.getType(), columnName); + serializeObject(singleRowBuilder, ((ObjectNode) value).get(field.getName().get()), field.getType(), columnName); } blockBuilder.closeEntry(); if (parentBlockBuilder == null) { diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java index fadfeee0e52df..1b77bf45fdbf5 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java @@ -19,51 +19,35 @@ package org.apache.pulsar.sql.presto.decoder.json; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.MissingNode; import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableMap; +import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; -import io.prestosql.decoder.RowDecoder; import io.prestosql.decoder.json.JsonFieldDecoder; +import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; +import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; +import org.apache.pulsar.sql.presto.PulsarRowDecoder; -import javax.annotation.Nullable; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.Set; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static java.util.Objects.requireNonNull; +import static java.util.function.Function.identity; -public class PulsarJsonRowDecoder implements RowDecoder { +public class PulsarJsonRowDecoder implements PulsarRowDecoder { - private final ObjectMapper objectMapper; private final Map fieldDecoders; - public PulsarJsonRowDecoder(ObjectMapper objectMapper, Map fieldDecoders) { - this.objectMapper = requireNonNull(objectMapper, "objectMapper is null"); - this.fieldDecoders = ImmutableMap.copyOf(fieldDecoders); - } - - @Override - public Optional> decodeRow(byte[] data, - @Nullable Map dataMap) { - JsonNode tree; - try { - tree = objectMapper.readTree(data); - } catch (Exception e) { - return Optional.empty(); - } + private final GenericJsonSchema genericJsonSchema; - Map decodedRow = new HashMap<>(); - for (Map.Entry entry : fieldDecoders.entrySet()) { - DecoderColumnHandle columnHandle = entry.getKey(); - JsonFieldDecoder decoder = entry.getValue(); - JsonNode node = locateNode(tree, columnHandle); - decodedRow.put(columnHandle, decoder.decode(node)); - } - return Optional.of(decodedRow); + public PulsarJsonRowDecoder(GenericJsonSchema genericJsonSchema, Set columns) { + this.genericJsonSchema = requireNonNull(genericJsonSchema, "genericJsonSchema is null"); + this.fieldDecoders = columns.stream().collect(toImmutableMap(identity(), PulsarJsonFieldDecoder::new)); } private static JsonNode locateNode(JsonNode tree, DecoderColumnHandle columnHandle) { @@ -78,4 +62,23 @@ private static JsonNode locateNode(JsonNode tree, DecoderColumnHandle columnHand } return currentNode; } + + /** + * decode ByteBuf by {@link org.apache.pulsar.client.api.schema.GenericSchema}. + * @param byteBuf + * @return + */ + @Override + public Optional> decodeRow(ByteBuf byteBuf) { + GenericJsonRecord record = (GenericJsonRecord) genericJsonSchema.decode(byteBuf); + JsonNode tree = record.getJsonNode(); + Map decodedRow = new HashMap<>(); + for (Map.Entry entry : fieldDecoders.entrySet()) { + DecoderColumnHandle columnHandle = entry.getKey(); + JsonFieldDecoder decoder = entry.getValue(); + JsonNode node = locateNode(tree, columnHandle); + decodedRow.put(columnHandle, decoder.decode(node)); + } + return Optional.of(decodedRow); + } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java index 4e248745766c0..5194f6c6fd9e0 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java @@ -18,34 +18,33 @@ */ package org.apache.pulsar.sql.presto.decoder.json; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import io.prestosql.decoder.DecoderColumnHandle; -import io.prestosql.decoder.RowDecoder; -import io.prestosql.decoder.json.JsonFieldDecoder; import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ColumnMetadata; import io.prestosql.spi.type.*; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.SchemaParseException; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.sql.presto.PulsarColumnHandle; import org.apache.pulsar.sql.presto.PulsarColumnMetadata; import org.apache.pulsar.sql.presto.PulsarRowDecoderFactory; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.ImmutableMap.toImmutableMap; import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.prestosql.spi.type.DateType.DATE; +import static io.prestosql.spi.type.TimeType.TIME; import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType; import static java.lang.String.format; -import static java.util.function.Function.identity; import static java.util.stream.Collectors.toList; /** @@ -60,27 +59,27 @@ public PulsarJsonRowDecoderFactory(TypeManager typeManager) { } @Override - public RowDecoder createRowDecoder(SchemaInfo schemaInfo, Set columns) { - ObjectMapper objectMapper = new ObjectMapper(); - Map fieldDecoders = columns.stream().collect(toImmutableMap(identity(), PulsarJsonFieldDecoder::new)); - return new PulsarJsonRowDecoder(objectMapper, fieldDecoders); + public PulsarJsonRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, Set columns) { + return new PulsarJsonRowDecoder((GenericJsonSchema) GenericJsonSchema.of(schemaInfo), columns); } @Override - public List extractColumnMetadata(SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { String schemaJson = new String(schemaInfo.getSchema()); if (StringUtils.isBlank(schemaJson)) { throw new PrestoException(NOT_SUPPORTED, "Topic " - + " does not have a valid schema"); + + topicName.toString() + " does not have a valid schema"); } Schema schema; try { schema = GenericJsonSchema.of(schemaInfo).getAvroSchema(); } catch (SchemaParseException ex) { throw new PrestoException(NOT_SUPPORTED, "Topic " - + " does not have a valid schema"); + + topicName.toString() + " does not have a valid schema"); } + //TODO : check schema cyclic definitions which may case java.lang.StackOverflowError + return schema.getFields().stream() .map(field -> new PulsarColumnMetadata(field.name(), parseJsonPrestoType(field.name(), field.schema()), field.schema().toString(), null, false, false, @@ -89,16 +88,10 @@ public List extractColumnMetadata(SchemaInfo schemaInfo, PulsarC ).collect(toList()); } - /** - * parse JsonPrestoType , - * warnning : Json only support json native primitive type which ignore Schema logicalType. - * - * @param fieldname - * @param schema - * @return - */ + private Type parseJsonPrestoType(String fieldname, Schema schema) { Schema.Type type = schema.getType(); + LogicalType logicalType = schema.getLogicalType(); switch (type) { case STRING: case ENUM: @@ -109,8 +102,16 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { case BYTES: return VarbinaryType.VARBINARY; case INT: + if (logicalType == LogicalTypes.timeMillis()) { + return TIME; + } else if (logicalType == LogicalTypes.date()) { + return DATE; + } return IntegerType.INTEGER; case LONG: + if (logicalType == LogicalTypes.timestampMillis()) { + return TimestampType.TIMESTAMP; + } return BigintType.BIGINT; case FLOAT: return RealType.REAL; @@ -121,7 +122,7 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { case ARRAY: return new ArrayType(parseJsonPrestoType(fieldname, schema.getElementType())); case MAP: - //The key for an Avro map must be a string + //The key for an avro map must be a string TypeSignature valueType = parseJsonPrestoType(fieldname, schema.getValueType()).getTypeSignature(); return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(valueType))); case RECORD: diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java index bd5f9ce19f198..80d2c5e16746a 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java @@ -18,12 +18,12 @@ */ package org.apache.pulsar.sql.presto.decoder.primitive; +import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; -import io.prestosql.decoder.RowDecoder; import io.prestosql.spi.type.*; +import org.apache.pulsar.sql.presto.PulsarRowDecoder; -import javax.annotation.Nullable; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -31,7 +31,7 @@ import static io.prestosql.decoder.FieldValueProviders.*; import static org.apache.pulsar.sql.presto.PulsarFieldValueProviders.doubleValueProvider; -public class PulsarPrimitiveRowDecoder implements RowDecoder { +public class PulsarPrimitiveRowDecoder implements PulsarRowDecoder { private final DecoderColumnHandle columnHandle; @@ -40,15 +40,19 @@ public PulsarPrimitiveRowDecoder(DecoderColumnHandle column) { } @Override - public Optional> decodeRow(byte[] data, @Nullable Map dataMap) { + public Optional> decodeRow(ByteBuf byteBuf) { + byte[] data = new byte[byteBuf.readableBytes()]; + byteBuf.readBytes(data); Map primitiveColumn = new HashMap(); Type type = columnHandle.getType(); if (type instanceof BooleanType) { primitiveColumn.put(columnHandle, booleanValueProvider(Boolean.valueOf(new String(data)))); } else if (type instanceof TinyintType || type instanceof SmallintType || type instanceof IntegerType || type instanceof BigintType) { primitiveColumn.put(columnHandle, longValueProvider(Long.valueOf(new String(data)))); - } else if (type instanceof RealType || type instanceof DoubleType) { + } else if (type instanceof DoubleType) { primitiveColumn.put(columnHandle, doubleValueProvider(Double.valueOf(new String(data)))); + } else if (type instanceof RealType) { + primitiveColumn.put(columnHandle, longValueProvider(Float.floatToIntBits((Float.valueOf(new String(data)))))); } else if (type instanceof VarbinaryType || type instanceof VarcharType) { primitiveColumn.put(columnHandle, bytesValueProvider(data)); } else if (type instanceof DateType || type instanceof TimeType || type instanceof TimestampType) { @@ -58,5 +62,4 @@ public Optional> decodeRow(byte[] d } return Optional.of(primitiveColumn); } - } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java index 261b41c3f9b3d..3aa2505e540f3 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java @@ -20,13 +20,14 @@ import io.airlift.log.Logger; import io.prestosql.decoder.DecoderColumnHandle; -import io.prestosql.decoder.RowDecoder; import io.prestosql.spi.connector.ColumnMetadata; import io.prestosql.spi.type.*; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.sql.presto.PulsarColumnHandle; import org.apache.pulsar.sql.presto.PulsarColumnMetadata; +import org.apache.pulsar.sql.presto.PulsarRowDecoder; import org.apache.pulsar.sql.presto.PulsarRowDecoderFactory; import java.util.Arrays; @@ -38,7 +39,7 @@ public class PulsarPrimitiveRowDecoderFactory implements PulsarRowDecoderFactory private static final Logger log = Logger.get(PulsarPrimitiveRowDecoderFactory.class); @Override - public RowDecoder createRowDecoder(SchemaInfo schemaInfo, Set columns) { + public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, Set columns) { if (columns.size() == 1) { return new PulsarPrimitiveRowDecoder(columns.iterator().next()); } else { @@ -47,7 +48,7 @@ public RowDecoder createRowDecoder(SchemaInfo schemaInfo, Set extractColumnMetadata(SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { ColumnMetadata valueColumn = new PulsarColumnMetadata( PulsarColumnMetadata.getColumnName(handleKeyValueType, "__value__"), parsePrimitivePrestoType("__value__", schemaInfo.getType()), @@ -56,7 +57,6 @@ public List extractColumnMetadata(SchemaInfo schemaInfo, PulsarC return Arrays.asList(valueColumn); } - private Type parsePrimitivePrestoType(String fieldName, SchemaType pulsarType) { switch (pulsarType) { case BOOLEAN: @@ -85,7 +85,7 @@ private Type parsePrimitivePrestoType(String fieldName, SchemaType pulsarType) { case TIMESTAMP: return TimestampType.TIMESTAMP; default: - log.error("Cannot convert type: %s", pulsarType); + log.error("Cannot convert type: %s for %s", pulsarType, fieldName); return null; } diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestAvroSchemaHandler.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestAvroSchemaHandler.java deleted file mode 100644 index 7b150ea5d252d..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestAvroSchemaHandler.java +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import io.netty.buffer.ByteBufAllocator; -import io.prestosql.spi.type.BigintType; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.avro.Schema; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.reflect.ReflectData; -import org.apache.avro.reflect.ReflectDatumWriter; -import org.apache.pulsar.client.api.schema.SchemaDefinition; -import org.apache.pulsar.client.impl.schema.StructSchema; -import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord; -import org.apache.pulsar.common.api.raw.RawMessage; -import org.apache.pulsar.common.schema.SchemaType; -import org.testng.Assert; -import org.testng.annotations.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static java.util.concurrent.CompletableFuture.completedFuture; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.when; - -import static org.mockito.Mockito.mock; - -@Slf4j -public class TestAvroSchemaHandler { - - @Data - public static class Foo1 { - String field1; - - Bar bar; - } - @Data - public static class Foo2 { - String field1; - - String field2; - - Bar bar; - } - - @Data static class Bar { - String field1; - - String field2; - } - - @Test - public void testAvroSchemaHandler() throws IOException { - List columnHandles = new ArrayList(); - RawMessage message = mock(RawMessage.class); - Schema schema1 = ReflectData.AllowNull.get().getSchema(Foo1.class); - PulsarSqlSchemaInfoProvider pulsarSqlSchemaInfoProvider = mock(PulsarSqlSchemaInfoProvider.class); - AvroSchemaHandler avroSchemaHandler = new AvroSchemaHandler(pulsarSqlSchemaInfoProvider, - StructSchema.parseSchemaInfo(SchemaDefinition.builder().withPojo(Foo2.class).build(), SchemaType.AVRO), columnHandles); - byte[] schemaVersion = new byte[8]; - for (int i = 0 ; i<8; i++) { - schemaVersion[i] = 0; - } - ReflectDatumWriter writer; - BinaryEncoder encoder = null; - ByteArrayOutputStream byteArrayOutputStream; - byteArrayOutputStream = new ByteArrayOutputStream(); - encoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, encoder); - writer = new ReflectDatumWriter<>(schema1); - Foo1 foo1 = new Foo1(); - foo1.setField1("value1"); - foo1.setBar(new Bar()); - foo1.getBar().setField1("value1"); - writer.write(foo1, encoder); - encoder.flush(); - when(message.getSchemaVersion()).thenReturn(schemaVersion); - byte[] bytes =byteArrayOutputStream.toByteArray(); - - when(message.getData()).thenReturn(ByteBufAllocator.DEFAULT - .buffer(bytes.length, bytes.length).writeBytes(byteArrayOutputStream.toByteArray())); - when(pulsarSqlSchemaInfoProvider.getSchemaByVersion(any())) - .thenReturn(completedFuture(StructSchema.parseSchemaInfo(SchemaDefinition.builder() - .withPojo(Foo1.class).build(), SchemaType.AVRO))); - - Object object = ((GenericAvroRecord)avroSchemaHandler.deserialize(message.getData(), - message.getSchemaVersion())).getField("field1"); - Assert.assertEquals(foo1.field1, (String)object); - String[] fields = new String[2]; - fields[0] = "bar"; - fields[1] = "field1"; - PulsarColumnHandle pulsarColumnHandle = new PulsarColumnHandle("test", - "bar.field1", - BigintType.BIGINT, - true, - true, - fields, - new Integer[5], - null); - columnHandles.add(pulsarColumnHandle); - when(message.getData()).thenReturn(ByteBufAllocator.DEFAULT - .buffer(bytes.length, bytes.length).writeBytes(byteArrayOutputStream.toByteArray())); - object = avroSchemaHandler.extractField(0, avroSchemaHandler.deserialize(message.getData(), - message.getSchemaVersion())); - Assert.assertEquals(foo1.bar.field1, (String)object); - } -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java index 2487cd53bd043..15fafa4165a54 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java @@ -21,78 +21,49 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.airlift.log.Logger; import io.netty.buffer.ByteBuf; +import io.prestosql.spi.connector.ColumnMetadata; +import io.prestosql.spi.connector.ConnectorContext; import io.prestosql.spi.predicate.TupleDomain; -import io.prestosql.spi.type.BigintType; -import io.prestosql.spi.type.BooleanType; -import io.prestosql.spi.type.DoubleType; -import io.prestosql.spi.type.IntegerType; -import io.prestosql.spi.type.RealType; -import io.prestosql.spi.type.Type; -import io.prestosql.spi.type.VarcharType; -import org.apache.bookkeeper.mledger.AsyncCallbacks; -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.ManagedLedgerConfig; -import org.apache.bookkeeper.mledger.ManagedLedgerFactory; -import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.ReadOnlyCursor; +import io.prestosql.testing.TestingConnectorContext; +import org.apache.bookkeeper.mledger.*; import org.apache.bookkeeper.mledger.impl.EntryImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.mledger.impl.ReadOnlyCursorImpl; import org.apache.bookkeeper.mledger.proto.MLDataFormats; +import org.apache.bookkeeper.stats.NullStatsProvider; import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.admin.Namespaces; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.admin.Schemas; -import org.apache.pulsar.client.admin.Tenants; -import org.apache.pulsar.client.admin.Topics; +import org.apache.pulsar.client.admin.*; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.client.impl.schema.JSONSchema; -import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.api.proto.PulsarApi; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; -import javax.ws.rs.ClientErrorException; -import javax.ws.rs.core.Response; -import org.apache.bookkeeper.stats.NullStatsProvider; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; +import javax.ws.rs.ClientErrorException; +import javax.ws.rs.core.Response; import java.time.LocalDate; import java.time.LocalTime; import java.time.ZoneId; import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; -import static io.prestosql.spi.type.DateType.DATE; -import static io.prestosql.spi.type.TimeType.TIME; -import static io.prestosql.spi.type.TimestampType.TIMESTAMP; import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyInt; -import static org.mockito.Mockito.anyString; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; +import static org.testng.Assert.assertNotNull; public abstract class TestPulsarConnector { @@ -110,16 +81,20 @@ public abstract class TestPulsarConnector { protected Map pulsarRecordCursors = new HashMap<>(); + protected static PulsarDispatchingRowDecoderFactory dispatchingRowDecoderFactory; + protected final static PulsarConnectorId pulsarConnectorId = new PulsarConnectorId("test-connector"); - protected static List topicNames; - protected static List partitionedTopicNames; + protected static List topicNames; + protected static List partitionedTopicNames; protected static Map partitionedTopicsToPartitions; protected static Map topicsToSchemas; protected static Map topicsToNumEntries; private final static ObjectMapper objectMapper = new ObjectMapper(); + protected static List fooFieldNames = new ArrayList<>(); + protected static final NamespaceName NAMESPACE_NAME_1 = NamespaceName.get("tenant-1", "ns-1"); protected static final NamespaceName NAMESPACE_NAME_2 = NamespaceName.get("tenant-1", "ns-2"); protected static final NamespaceName NAMESPACE_NAME_3 = NamespaceName.get("tenant-2", "ns-1"); @@ -150,10 +125,6 @@ public abstract class TestPulsarConnector { public static class Foo { - public static class Bar { - public int field1; - } - public enum TestEnum { TEST_ENUM_1, TEST_ENUM_2, @@ -179,28 +150,13 @@ public enum TestEnum { public static class Bar { public Integer field1; public String field2; - public Boo test; public float field3; - public Boo test2; } - public static class Boo { - public Double field4; - public Boolean field5; - public long field6; - // for test cyclic definitions - public Foo foo; - public Boo boo; - public Bar bar; - // different namespace with same classname should work though - public Foo.Bar foobar; - } - protected static Map fooTypes; - protected static List fooColumnHandles; + protected static Map> topicsToColumnHandles = new HashMap<>(); + protected static Map splits; - protected static Map fooFieldNames; - protected static Map fooPositionIndices; protected static Map> fooFunctions; static { @@ -246,30 +202,6 @@ public static class Boo { topicsToSchemas.put(PARTITIONED_TOPIC_5.getSchemaName(), Schema.JSON(TestPulsarMetadata.Foo.class).getSchemaInfo()); topicsToSchemas.put(PARTITIONED_TOPIC_6.getSchemaName(), Schema.JSON(TestPulsarMetadata.Foo.class).getSchemaInfo()); - fooTypes = new HashMap<>(); - fooTypes.put("field1", IntegerType.INTEGER); - fooTypes.put("field2", VarcharType.VARCHAR); - fooTypes.put("field3", RealType.REAL); - fooTypes.put("field4", DoubleType.DOUBLE); - fooTypes.put("field5", BooleanType.BOOLEAN); - fooTypes.put("field6", BigintType.BIGINT); - fooTypes.put("timestamp", TIMESTAMP); - fooTypes.put("time", TIME); - fooTypes.put("date", DATE); - fooTypes.put("bar.field1", IntegerType.INTEGER); - fooTypes.put("bar.field2", VarcharType.VARCHAR); - fooTypes.put("bar.test.field4", DoubleType.DOUBLE); - fooTypes.put("bar.test.field5", BooleanType.BOOLEAN); - fooTypes.put("bar.test.field6", BigintType.BIGINT); - fooTypes.put("bar.test.foobar.field1", IntegerType.INTEGER); - fooTypes.put("bar.field3", RealType.REAL); - fooTypes.put("bar.test2.field4", DoubleType.DOUBLE); - fooTypes.put("bar.test2.field5", BooleanType.BOOLEAN); - fooTypes.put("bar.test2.field6", BigintType.BIGINT); - fooTypes.put("bar.test2.foobar.field1", IntegerType.INTEGER); - // Enums currently map to VARCHAR - fooTypes.put("field7", VarcharType.VARCHAR); - topicsToNumEntries = new HashMap<>(); topicsToNumEntries.put(TOPIC_1.getSchemaName(), 1233L); topicsToNumEntries.put(TOPIC_2.getSchemaName(), 0L); @@ -286,267 +218,38 @@ public static class Boo { topicsToNumEntries.put(PARTITIONED_TOPIC_5.getSchemaName(), 800L); topicsToNumEntries.put(PARTITIONED_TOPIC_6.getSchemaName(), 1L); - fooFieldNames = new HashMap<>(); - fooPositionIndices = new HashMap<>(); - fooColumnHandles = new LinkedList<>(); - - String[] fieldNames1 = {"field1"}; - Integer[] positionIndices1 = {0}; - fooFieldNames.put("field1", fieldNames1); - fooPositionIndices.put("field1", positionIndices1); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "field1", - fooTypes.get("field1"), - false, - false, - fooFieldNames.get("field1"), - fooPositionIndices.get("field1"), null)); - - - String[] fieldNames2 = {"field2"}; - Integer[] positionIndices2 = {1}; - fooFieldNames.put("field2", fieldNames2); - fooPositionIndices.put("field2", positionIndices2); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "field2", - fooTypes.get("field2"), - false, - false, - fieldNames2, - positionIndices2, null)); - - String[] fieldNames3 = {"field3"}; - Integer[] positionIndices3 = {2}; - fooFieldNames.put("field3", fieldNames3); - fooPositionIndices.put("field3", positionIndices3); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "field3", - fooTypes.get("field3"), - false, - false, - fieldNames3, - positionIndices3, null)); - - String[] fieldNames4 = {"field4"}; - Integer[] positionIndices4 = {3}; - fooFieldNames.put("field4", fieldNames4); - fooPositionIndices.put("field4", positionIndices4); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "field4", - fooTypes.get("field4"), - false, - false, - fieldNames4, - positionIndices4, null)); - - - String[] fieldNames5 = {"field5"}; - Integer[] positionIndices5 = {4}; - fooFieldNames.put("field5", fieldNames5); - fooPositionIndices.put("field5", positionIndices5); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "field5", - fooTypes.get("field5"), - false, - false, - fieldNames5, - positionIndices5, null)); - - String[] fieldNames6 = {"field6"}; - Integer[] positionIndices6 = {5}; - fooFieldNames.put("field6", fieldNames6); - fooPositionIndices.put("field6", positionIndices6); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "field6", - fooTypes.get("field6"), - false, - false, - fieldNames6, - positionIndices6, null)); - - String[] fieldNames7 = {"timestamp"}; - Integer[] positionIndices7 = {6}; - fooFieldNames.put("timestamp", fieldNames7); - fooPositionIndices.put("timestamp", positionIndices7); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "timestamp", - fooTypes.get("timestamp"), - false, - false, - fieldNames7, - positionIndices7, null)); - - String[] fieldNames8 = {"time"}; - Integer[] positionIndices8 = {7}; - fooFieldNames.put("time", fieldNames8); - fooPositionIndices.put("time", positionIndices8); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "time", - fooTypes.get("time"), - false, - false, - fieldNames8, - positionIndices8, null)); - - String[] fieldNames9 = {"date"}; - Integer[] positionIndices9 = {8}; - fooFieldNames.put("date", fieldNames9); - fooPositionIndices.put("date", positionIndices9); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "date", - fooTypes.get("date"), - false, - false, - fieldNames9, - positionIndices9, null)); - - String[] bar_fieldNames1 = {"bar", "field1"}; - Integer[] bar_positionIndices1 = {9, 0}; - fooFieldNames.put("bar.field1", bar_fieldNames1); - fooPositionIndices.put("bar.field1", bar_positionIndices1); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "bar.field1", - fooTypes.get("bar.field1"), - false, - false, - bar_fieldNames1, - bar_positionIndices1, null)); - - String[] bar_fieldNames2 = {"bar", "field2"}; - Integer[] bar_positionIndices2 = {9, 1}; - fooFieldNames.put("bar.field2", bar_fieldNames2); - fooPositionIndices.put("bar.field2", bar_positionIndices2); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "bar.field2", - fooTypes.get("bar.field2"), - false, - false, - bar_fieldNames2, - bar_positionIndices2, null)); - - String[] bar_test_fieldNames4 = {"bar", "test", "field4"}; - Integer[] bar_test_positionIndices4 = {9, 2, 0}; - fooFieldNames.put("bar.test.field4", bar_test_fieldNames4); - fooPositionIndices.put("bar.test.field4", bar_test_positionIndices4); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "bar.test.field4", - fooTypes.get("bar.test.field4"), - false, - false, - bar_test_fieldNames4, - bar_test_positionIndices4, null)); - - String[] bar_test_fieldNames5 = {"bar", "test", "field5"}; - Integer[] bar_test_positionIndices5 = {9, 2, 1}; - fooFieldNames.put("bar.test.field5", bar_test_fieldNames5); - fooPositionIndices.put("bar.test.field5", bar_test_positionIndices5); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "bar.test.field5", - fooTypes.get("bar.test.field5"), - false, - false, - bar_test_fieldNames5, - bar_test_positionIndices5, null)); - - String[] bar_test_fieldNames6 = {"bar", "test", "field6"}; - Integer[] bar_test_positionIndices6 = {9, 2, 2}; - fooFieldNames.put("bar.test.field6", bar_test_fieldNames6); - fooPositionIndices.put("bar.test.field6", bar_test_positionIndices6); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "bar.test.field6", - fooTypes.get("bar.test.field6"), - false, - false, - bar_test_fieldNames6, - bar_test_positionIndices6, null)); - - String[] bar_test_foobar_fieldNames1 = {"bar", "test", "foobar", "field1"}; - Integer[] bar_test_foobar_positionIndices1 = {9, 2, 6, 0}; - fooFieldNames.put("bar.test.foobar.field1", bar_test_foobar_fieldNames1); - fooPositionIndices.put("bar.test.foobar.field1", bar_test_foobar_positionIndices1); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "bar.test.foobar.field1", - fooTypes.get("bar.test.foobar.field1"), - false, - false, - bar_test_foobar_fieldNames1, - bar_test_foobar_positionIndices1, null)); - - String[] bar_field3 = {"bar", "field3"}; - Integer[] bar_positionIndices3 = {9, 3}; - fooFieldNames.put("bar.field3", bar_field3); - fooPositionIndices.put("bar.field3", bar_positionIndices3); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "bar.field3", - fooTypes.get("bar.field3"), - false, - false, - bar_field3, - bar_positionIndices3, null)); - - String[] bar_test2_fieldNames4 = {"bar", "test2", "field4"}; - Integer[] bar_test2_positionIndices4 = {9, 4, 0}; - fooFieldNames.put("bar.test2.field4", bar_test2_fieldNames4); - fooPositionIndices.put("bar.test2.field4", bar_test2_positionIndices4); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "bar.test2.field4", - fooTypes.get("bar.test2.field4"), - false, - false, - bar_test2_fieldNames4, - bar_test2_positionIndices4, null)); - - String[] bar_test2_fieldNames5 = {"bar", "test2", "field5"}; - Integer[] bar_test2_positionIndices5 = {9, 4, 1}; - fooFieldNames.put("bar.test2.field5", bar_test2_fieldNames5); - fooPositionIndices.put("bar.test2.field5", bar_test2_positionIndices5); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "bar.test2.field5", - fooTypes.get("bar.test2.field5"), - false, - false, - bar_test2_fieldNames5, - bar_test2_positionIndices5, null)); - - String[] bar_test2_fieldNames6 = {"bar", "test2", "field6"}; - Integer[] bar_test2_positionIndices6 = {9, 4, 2}; - fooFieldNames.put("bar.test2.field6", bar_test2_fieldNames6); - fooPositionIndices.put("bar.test2.field6", bar_test2_positionIndices6); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "bar.test2.field6", - fooTypes.get("bar.test2.field6"), - false, - false, - bar_test2_fieldNames6, - bar_test2_positionIndices6, null)); - - String[] bar_test2_foobar_fieldNames1 = {"bar", "test2", "foobar", "field1"}; - Integer[] bar_test2_foobar_positionIndices1 = {9, 4, 6, 0}; - fooFieldNames.put("bar.test2.foobar.field1", bar_test2_foobar_fieldNames1); - fooPositionIndices.put("bar.test2.foobar.field1", bar_test2_foobar_positionIndices1); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "bar.test2.foobar.field1", - fooTypes.get("bar.test2.foobar.field1"), - false, - false, - bar_test2_foobar_fieldNames1, - bar_test2_foobar_positionIndices1, null)); - - String[] fieldNames10 = {"field7"}; - Integer[] positionIndices10 = {10}; - fooFieldNames.put("field7", fieldNames10); - fooPositionIndices.put("field7", positionIndices10); - fooColumnHandles.add(new PulsarColumnHandle(pulsarConnectorId.toString(), - "field7", - fooTypes.get("field7"), - false, - false, - fieldNames10, - positionIndices10, null)); - - fooColumnHandles.addAll(PulsarInternalColumn.getInternalFields().stream() - .map(pulsarInternalColumn -> pulsarInternalColumn.getColumnHandle(pulsarConnectorId.toString(), false)) - .collect(Collectors.toList())); + + fooFieldNames.add("field1"); + fooFieldNames.add("field2"); + fooFieldNames.add("field3"); + fooFieldNames.add("field4"); + fooFieldNames.add("field5"); + fooFieldNames.add("field6"); + fooFieldNames.add("timestamp"); + fooFieldNames.add("time"); + fooFieldNames.add("date"); + fooFieldNames.add("bar"); + fooFieldNames.add("field7"); + + + ConnectorContext prestoConnectorContext = new TestingConnectorContext(); + dispatchingRowDecoderFactory = new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager()); + + topicsToColumnHandles.put(PARTITIONED_TOPIC_1, getColumnColumnHandles(PARTITIONED_TOPIC_1,topicsToSchemas.get(PARTITIONED_TOPIC_1.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); + topicsToColumnHandles.put(PARTITIONED_TOPIC_2, getColumnColumnHandles(PARTITIONED_TOPIC_2,topicsToSchemas.get(PARTITIONED_TOPIC_2.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); + topicsToColumnHandles.put(PARTITIONED_TOPIC_3, getColumnColumnHandles(PARTITIONED_TOPIC_3,topicsToSchemas.get(PARTITIONED_TOPIC_3.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); + topicsToColumnHandles.put(PARTITIONED_TOPIC_4, getColumnColumnHandles(PARTITIONED_TOPIC_4,topicsToSchemas.get(PARTITIONED_TOPIC_4.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); + topicsToColumnHandles.put(PARTITIONED_TOPIC_5, getColumnColumnHandles(PARTITIONED_TOPIC_5,topicsToSchemas.get(PARTITIONED_TOPIC_5.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); + topicsToColumnHandles.put(PARTITIONED_TOPIC_6, getColumnColumnHandles(PARTITIONED_TOPIC_6,topicsToSchemas.get(PARTITIONED_TOPIC_6.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); + + + topicsToColumnHandles.put(TOPIC_1, getColumnColumnHandles(TOPIC_1,topicsToSchemas.get(TOPIC_1.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); + topicsToColumnHandles.put(TOPIC_2, getColumnColumnHandles(TOPIC_2,topicsToSchemas.get(TOPIC_2.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); + topicsToColumnHandles.put(TOPIC_3, getColumnColumnHandles(TOPIC_3,topicsToSchemas.get(TOPIC_3.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); + topicsToColumnHandles.put(TOPIC_4, getColumnColumnHandles(TOPIC_4,topicsToSchemas.get(TOPIC_4.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); + topicsToColumnHandles.put(TOPIC_5, getColumnColumnHandles(TOPIC_5,topicsToSchemas.get(TOPIC_5.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); + topicsToColumnHandles.put(TOPIC_6, getColumnColumnHandles(TOPIC_6,topicsToSchemas.get(TOPIC_6.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); + splits = new HashMap<>(); @@ -554,6 +257,7 @@ public static class Boo { allTopics.addAll(topicNames); allTopics.addAll(partitionedTopicNames); + for (TopicName topicName : allTopics) { if (topicsToSchemas.containsKey(topicName.getSchemaName())) { splits.put(topicName, new PulsarSplit(0, pulsarConnectorId.toString(), @@ -590,15 +294,6 @@ public static class Boo { fooFunctions.put("bar.field2", integer -> integer % 2 == 0 ? null : String.valueOf(integer + 2)); fooFunctions.put("bar.field3", integer -> integer + 3.0f); - fooFunctions.put("bar.test.field4", integer -> integer + 1.0); - fooFunctions.put("bar.test.field5", integer -> (integer + 1) % 2 == 0); - fooFunctions.put("bar.test.field6", integer -> integer + 10L); - fooFunctions.put("bar.test.foobar.field1", integer -> integer % 3); - - fooFunctions.put("bar.test2.field4", integer -> integer + 2.0); - fooFunctions.put("bar.test2.field5", integer -> (integer + 1) % 32 == 0); - fooFunctions.put("bar.test2.field6", integer -> integer + 15L); - fooFunctions.put("bar.test2.foobar.field1", integer -> integer % 3); fooFunctions.put("field7", integer -> Foo.TestEnum.values()[integer % Foo.TestEnum.values().length]); } catch (Throwable e) { @@ -607,6 +302,47 @@ public static class Boo { } } + + /** + * Parse PulsarColumnMetadata to PulsarColumnHandle Util + * @param schemaInfo + * @param handleKeyValueType + * @param includeInternalColumn + * @param dispatchingRowDecoderFactory + * @return + */ + public static List getColumnColumnHandles(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType, boolean includeInternalColumn, PulsarDispatchingRowDecoderFactory dispatchingRowDecoderFactory) { + List columnHandles = new ArrayList<>(); + List columnMetadata = dispatchingRowDecoderFactory.extractColumnMetadata(topicName,schemaInfo, handleKeyValueType); + + columnMetadata.forEach(column -> { + PulsarColumnMetadata pulsarColumnMetadata = (PulsarColumnMetadata) column; + columnHandles.add(new PulsarColumnHandle( + pulsarConnectorId.toString(), + pulsarColumnMetadata.getNameWithCase(), + pulsarColumnMetadata.getType(), + pulsarColumnMetadata.isHidden(), + pulsarColumnMetadata.isInternal(), + pulsarColumnMetadata.getDecoderExtraInfo().getMapping(), + pulsarColumnMetadata.getDecoderExtraInfo().getDataFormat(), pulsarColumnMetadata.getDecoderExtraInfo().getFormatHint(), + pulsarColumnMetadata.getHandleKeyValueType())); + + }); + + if (includeInternalColumn) { + columnHandles.addAll(PulsarInternalColumn.getInternalFields().stream() + .map(pulsarInternalColumn -> pulsarInternalColumn.getColumnHandle(pulsarConnectorId.toString(), false)) + .collect(Collectors.toList())); + } + + return columnHandles; + } + + public static PulsarConnectorId getPulsarConnectorId() { + assertNotNull(pulsarConnectorId); + return pulsarConnectorId; + } + private static List getTopicEntries(String topicSchemaName) { List entries = new LinkedList<>(); @@ -765,7 +501,7 @@ public SchemaInfo answer(InvocationOnMock invocationOnMock) throws Throwable { doReturn(schemas).when(pulsarAdmin).schemas(); doReturn(pulsarAdmin).when(this.pulsarConnectorConfig).getPulsarAdmin(); - this.pulsarMetadata = new PulsarMetadata(pulsarConnectorId, this.pulsarConnectorConfig); + this.pulsarMetadata = new PulsarMetadata(pulsarConnectorId, this.pulsarConnectorConfig, dispatchingRowDecoderFactory); this.pulsarSplitManager = Mockito.spy(new PulsarSplitManager(pulsarConnectorId, this.pulsarConnectorConfig)); ManagedLedgerFactory managedLedgerFactory = mock(ManagedLedgerFactory.class); @@ -821,36 +557,14 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { new Thread(new Runnable() { @Override public void run() { - List < Entry > entries = new LinkedList<>(); + List entries = new LinkedList<>(); for (int i = 0; i < readEntries; i++) { - Foo.Bar foobar = new Foo.Bar(); - foobar.field1 = (int) fooFunctions.get("bar.test.foobar.field1").apply(count); - - Boo boo1 = new Boo(); - boo1.field4 = (double) fooFunctions.get("bar.test.field4").apply(count); - boo1.field5 = (boolean) fooFunctions.get("bar.test.field5").apply(count); - boo1.field6 = (long) fooFunctions.get("bar.test.field6").apply(count); - boo1.foo = new Foo(); - boo1.boo = null; - boo1.bar = new Bar(); - boo1.foobar = foobar; - - Boo boo2 = new Boo(); - boo2.field4 = (double) fooFunctions.get("bar.test2.field4").apply(count); - boo2.field5 = (boolean) fooFunctions.get("bar.test2.field5").apply(count); - boo2.field6 = (long) fooFunctions.get("bar.test2.field6").apply(count); - boo2.foo = new Foo(); - boo2.boo = boo1; - boo2.bar = new Bar(); - boo2.foobar = foobar; - TestPulsarConnector.Bar bar = new TestPulsarConnector.Bar(); bar.field1 = fooFunctions.get("bar.field1").apply(count) == null ? null : (int) fooFunctions.get("bar.field1").apply(count); bar.field2 = fooFunctions.get("bar.field2").apply(count) == null ? null : (String) fooFunctions.get("bar.field2").apply(count); bar.field3 = (float) fooFunctions.get("bar.field3").apply(count); - bar.test = boo1; - bar.test2 = count % 2 == 0 ? null : boo2; + Foo foo = new Foo(); foo.field1 = (int) fooFunctions.get("field1").apply(count); @@ -946,11 +660,10 @@ public Long answer(InvocationOnMock invocationOnMock) throws Throwable { when(PulsarConnectorCache.instance.getManagedLedgerFactory()).thenReturn(managedLedgerFactory); for (Map.Entry split : splits.entrySet()) { - PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor( - fooColumnHandles, split.getValue(), + topicsToColumnHandles.get(split.getKey()), split.getValue(), pulsarConnectorConfig, managedLedgerFactory, new ManagedLedgerConfig(), - new PulsarConnectorMetricsTracker(new NullStatsProvider()))); + new PulsarConnectorMetricsTracker(new NullStatsProvider()),dispatchingRowDecoderFactory)); this.pulsarRecordCursors.put(split.getKey(), pulsarRecordCursor); } } diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnectorConfig.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnectorConfig.java index 0b2609af5178c..247189085e84a 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnectorConfig.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnectorConfig.java @@ -94,7 +94,7 @@ public void testGetOffloadPolices() throws Exception { Assert.assertNotNull(offloadPolicies); Assert.assertEquals(offloadPolicies.getManagedLedgerOffloadDriver(), managedLedgerOffloadDriver); Assert.assertEquals(offloadPolicies.getOffloadersDirectory(), offloaderDirectory); - Assert.assertEquals(offloadPolicies.getManagedLedgerOffloadMaxThreads(), managedLedgerOffloadMaxThreads); + Assert.assertEquals((int) offloadPolicies.getManagedLedgerOffloadMaxThreads(), (int) managedLedgerOffloadMaxThreads); Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadBucket(), bucket); Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadRegion(), region); Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadServiceEndpoint(), endpoint); diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarKeyValueSchemaHandler.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarKeyValueSchemaHandler.java deleted file mode 100644 index 1fe881ee3999e..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarKeyValueSchemaHandler.java +++ /dev/null @@ -1,353 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import com.fasterxml.jackson.databind.ObjectMapper; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.prestosql.spi.connector.ColumnMetadata; -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.schema.KeyValueSchema; -import org.apache.pulsar.client.impl.schema.KeyValueSchemaInfo; -import org.apache.pulsar.common.api.raw.RawMessage; -import org.apache.pulsar.common.api.raw.RawMessageImpl; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.KeyValue; -import org.apache.pulsar.common.schema.KeyValueEncodingType; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.mockito.Mockito; -import org.testng.Assert; -import org.testng.annotations.Test; - -import static org.mockito.Mockito.mock; - - -/** - * Unit test for KeyValueSchemaHandler - */ -@Slf4j -public class TestPulsarKeyValueSchemaHandler { - - private final static ObjectMapper objectMapper = new ObjectMapper(); - - private Schema> schema1 = - Schema.KeyValue(Schema.STRING, Schema.INT32, KeyValueEncodingType.INLINE); - - private Schema> schema2 = - Schema.KeyValue(Schema.STRING, Schema.JSON(Foo.class), KeyValueEncodingType.INLINE); - - private Schema> schema3 = - Schema.KeyValue(Schema.AVRO(Boo.class), Schema.INT64, KeyValueEncodingType.SEPARATED); - - private Schema> schema4 = - Schema.KeyValue(Schema.JSON(Boo.class), Schema.AVRO(Foo.class), KeyValueEncodingType.SEPARATED); - - private final static TopicName topicName = TopicName.get("persistent://public/default/kv-test"); - - private final static Foo foo; - - private final static Boo boo; - - private final Integer KEY_FIELD_NAME_PREFIX_LENGTH = PulsarColumnMetadata.KEY_SCHEMA_COLUMN_PREFIX.length(); - - static { - foo = new Foo(); - foo.field1 = "field1-value"; - foo.field2 = 20; - - boo = new Boo(); - boo.field1 = "field1-value"; - boo.field2 = true; - boo.field3 = 10.2; - } - - - @Test - public void testSchema1() throws IOException { - final String keyData = "test-key"; - final Integer valueData = 10; - List columnMetadataList = - PulsarMetadata.getPulsarColumns(topicName, schema1.getSchemaInfo(), - true, null); - int keyCount = 0; - int valueCount = 0; - for (ColumnMetadata columnMetadata : columnMetadataList) { - PulsarColumnMetadata pulsarColumnMetadata = (PulsarColumnMetadata) columnMetadata; - if (pulsarColumnMetadata.isKey()) { - keyCount++; - } else if (pulsarColumnMetadata.isValue()) { - valueCount++; - } - } - Assert.assertEquals(keyCount, 1); - Assert.assertEquals(valueCount, 1); - - List columnHandleList = getColumnHandlerList(columnMetadataList); - - KeyValueSchemaHandler keyValueSchemaHandler = - new KeyValueSchemaHandler(null, null,schema1.getSchemaInfo(), columnHandleList); - - RawMessageImpl message = mock(RawMessageImpl.class); - Mockito.when(message.getData()).thenReturn( - Unpooled.wrappedBuffer(schema1.encode(new KeyValue<>(keyData, valueData))) - ); - - KeyValue byteBufKeyValue = getKeyValueByteBuf(message, schema1); - Object object = keyValueSchemaHandler.deserialize(byteBufKeyValue.getKey(), byteBufKeyValue.getValue(), null); - Assert.assertEquals(keyValueSchemaHandler.extractField(0, object), keyData); - Assert.assertEquals(keyValueSchemaHandler.extractField(1, object), valueData); - } - - @Test - public void testSchema2() throws IOException { - final String keyData = "test-key"; - - List columnMetadataList = - PulsarMetadata.getPulsarColumns(topicName, schema2.getSchemaInfo(), - true, null); - int keyCount = 0; - int valueCount = 0; - for (ColumnMetadata columnMetadata : columnMetadataList) { - PulsarColumnMetadata pulsarColumnMetadata = (PulsarColumnMetadata) columnMetadata; - if (pulsarColumnMetadata.isKey()) { - keyCount++; - } else if (pulsarColumnMetadata.isValue()) { - valueCount++; - } - } - Assert.assertEquals(keyCount, 1); - Assert.assertEquals(valueCount, 2); - - List columnHandleList = getColumnHandlerList(columnMetadataList); - - RawMessage message = mock(RawMessage.class); - Mockito.when(message.getData()).thenReturn( - Unpooled.wrappedBuffer(schema2.encode(new KeyValue<>(keyData, foo))) - ); - - - KeyValueSchemaHandler keyValueSchemaHandler = - new KeyValueSchemaHandler(null, null, schema2.getSchemaInfo(), columnHandleList); - - KeyValue byteBufKeyValue = getKeyValueByteBuf(message, schema2); - Object object = keyValueSchemaHandler.deserialize(byteBufKeyValue.getKey(), byteBufKeyValue.getValue(), null); - Assert.assertEquals(keyValueSchemaHandler.extractField(0, object), keyData); - Assert.assertEquals(keyValueSchemaHandler.extractField(1, object), - foo.getValue(columnHandleList.get(1).getName())); - Assert.assertEquals(keyValueSchemaHandler.extractField(2, object), - foo.getValue(columnHandleList.get(2).getName())); - } - - @Test - public void testSchema3() throws IOException { - final Boo boo = new Boo(); - boo.field1 = "field1-value"; - boo.field2 = true; - boo.field3 = 10.2; - final Long valueData = 999999L; - - List columnMetadataList = - PulsarMetadata.getPulsarColumns(topicName, schema3.getSchemaInfo(), - true, null); - int keyCount = 0; - int valueCount = 0; - for (ColumnMetadata columnMetadata : columnMetadataList) { - PulsarColumnMetadata pulsarColumnMetadata = (PulsarColumnMetadata) columnMetadata; - if (pulsarColumnMetadata.isKey()) { - keyCount++; - } else if (pulsarColumnMetadata.isValue()) { - valueCount++; - } - } - Assert.assertEquals(keyCount, 3); - Assert.assertEquals(valueCount, 1); - - List columnHandleList = getColumnHandlerList(columnMetadataList); - - PulsarSqlSchemaInfoProvider pulsarSqlSchemaInfoProvider = mock(PulsarSqlSchemaInfoProvider.class); - - KeyValue kvSchemaInfo = - KeyValueSchemaInfo.decodeKeyValueSchemaInfo(schema3.getSchemaInfo()); - - AvroSchemaHandler avroSchemaHandler = - new AvroSchemaHandler(pulsarSqlSchemaInfoProvider, kvSchemaInfo.getKey(), columnHandleList); - PulsarPrimitiveSchemaHandler pulsarPrimitiveSchemaHandler = - new PulsarPrimitiveSchemaHandler(kvSchemaInfo.getValue()); - KeyValueSchemaHandler keyValueSchemaHandler = - new KeyValueSchemaHandler(avroSchemaHandler, pulsarPrimitiveSchemaHandler, columnHandleList); - - RawMessage message = mock(RawMessage.class); - Mockito.when(message.getKeyBytes()).thenReturn( - Optional.of(Unpooled.wrappedBuffer( - ((KeyValueSchema) schema3).getKeySchema().encode(boo) - )) - ); - Mockito.when(message.getData()).thenReturn( - Unpooled.wrappedBuffer(schema3.encode(new KeyValue<>(boo, valueData))) - ); - - KeyValue byteBufKeyValue = getKeyValueByteBuf(message, schema3); - Integer a = 1; - Object object = keyValueSchemaHandler.deserialize(byteBufKeyValue.getKey(), byteBufKeyValue.getValue(), null); - Assert.assertEquals(keyValueSchemaHandler.extractField(0, object).toString(), - boo.getValue(columnHandleList.get(0).getName().substring(KEY_FIELD_NAME_PREFIX_LENGTH))); - Assert.assertEquals(keyValueSchemaHandler.extractField(1, object), - boo.getValue(columnHandleList.get(1).getName().substring(KEY_FIELD_NAME_PREFIX_LENGTH))); - Assert.assertEquals(keyValueSchemaHandler.extractField(2, object), - boo.getValue(columnHandleList.get(2).getName().substring(KEY_FIELD_NAME_PREFIX_LENGTH))); - Assert.assertEquals(keyValueSchemaHandler.extractField(3, object), valueData); - } - - @Test - public void testSchema4() throws IOException { - List columnMetadataList = - PulsarMetadata.getPulsarColumns(topicName, schema4.getSchemaInfo(), - true, null); - int keyCount = 0; - int valueCount = 0; - for (ColumnMetadata columnMetadata : columnMetadataList) { - PulsarColumnMetadata pulsarColumnMetadata = (PulsarColumnMetadata) columnMetadata; - if (pulsarColumnMetadata.isKey()) { - keyCount++; - } else if (pulsarColumnMetadata.isValue()) { - valueCount++; - } - } - Assert.assertEquals(keyCount, 3); - Assert.assertEquals(valueCount, 2); - - List columnHandleList = getColumnHandlerList(columnMetadataList); - - PulsarSqlSchemaInfoProvider pulsarSqlSchemaInfoProvider = mock(PulsarSqlSchemaInfoProvider.class); - - KeyValue kvSchemaInfo = - KeyValueSchemaInfo.decodeKeyValueSchemaInfo(schema4.getSchemaInfo()); - - AvroSchemaHandler avroSchemaHandler = - new AvroSchemaHandler(pulsarSqlSchemaInfoProvider, kvSchemaInfo.getValue(), columnHandleList); - JSONSchemaHandler jsonSchemaHandler = new JSONSchemaHandler(columnHandleList); - KeyValueSchemaHandler keyValueSchemaHandler = - new KeyValueSchemaHandler(jsonSchemaHandler, avroSchemaHandler, columnHandleList); - - - RawMessage message = mock(RawMessage.class); - Mockito.when(message.getKeyBytes()).thenReturn( - Optional.of(Unpooled.wrappedBuffer( - ((KeyValueSchema) schema4).getKeySchema().encode(boo) - )) - ); - Mockito.when(message.getData()).thenReturn( - Unpooled.wrappedBuffer(schema4.encode(new KeyValue<>(boo, foo))) - ); - - KeyValue byteBufKeyValue = getKeyValueByteBuf(message, schema4); - Object object = keyValueSchemaHandler.deserialize(byteBufKeyValue.getKey(), byteBufKeyValue.getValue(), null); - Assert.assertEquals(keyValueSchemaHandler.extractField(0, object).toString(), - boo.getValue(columnHandleList.get(0).getName().substring(KEY_FIELD_NAME_PREFIX_LENGTH))); - Assert.assertEquals(keyValueSchemaHandler.extractField(1, object), - boo.getValue(columnHandleList.get(1).getName().substring(KEY_FIELD_NAME_PREFIX_LENGTH))); - Assert.assertEquals(keyValueSchemaHandler.extractField(2, object), - boo.getValue(columnHandleList.get(2).getName().substring(KEY_FIELD_NAME_PREFIX_LENGTH))); - Assert.assertEquals(keyValueSchemaHandler.extractField(3, object).toString(), - foo.getValue(columnHandleList.get(3).getName())); - Assert.assertEquals(keyValueSchemaHandler.extractField(4, object).toString(), - foo.getValue(columnHandleList.get(4).getName()) + ""); - } - - private List getColumnHandlerList(List columnMetadataList) { - List columnHandleList = new LinkedList<>(); - - columnMetadataList.forEach(columnMetadata -> { - PulsarColumnMetadata pulsarColumnMetadata = (PulsarColumnMetadata) columnMetadata; - PulsarColumnHandle pulsarColumnHandle = new PulsarColumnHandle( - "connectorId", - pulsarColumnMetadata.getNameWithCase(), - pulsarColumnMetadata.getType(), - pulsarColumnMetadata.isHidden(), - pulsarColumnMetadata.isInternal(), - pulsarColumnMetadata.getFieldNames(), - pulsarColumnMetadata.getPositionIndices(), - pulsarColumnMetadata.getHandleKeyValueType()); - columnHandleList.add(pulsarColumnHandle); - }); - - return columnHandleList; - } - - public KeyValue getKeyValueByteBuf(RawMessage message, Schema schema) { - KeyValueEncodingType encodingType = KeyValueSchemaInfo.decodeKeyValueEncodingType(schema.getSchemaInfo()); - ByteBuf keyByteBuf = null; - if (Objects.equals(KeyValueEncodingType.SEPARATED, encodingType)) { - if (message.getKeyBytes().isPresent()) { - keyByteBuf = message.getKeyBytes().get(); - } else { - keyByteBuf = null; - } - } else { - keyByteBuf = null; - } - return new KeyValue<>(keyByteBuf, Unpooled.wrappedBuffer(message.getData())); - } - - @Data - static class Foo { - private String field1; - private Integer field2; - - public Object getValue(String fieldName) { - switch (fieldName) { - case "field1": - return field1; - case "field2": - return field2 == null ? null : new Long(field2); - default: - return null; - } - } - } - - @Data - static class Boo { - private String field1; - private Boolean field2; - private Double field3; - - public Object getValue(String fieldName) { - switch (fieldName) { - case "field1": - return field1; - case "field2": - return field2; - case "field3": - return field3 == null ? null : field3.doubleValue(); - default: - return null; - } - } - - } - -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarMetadata.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarMetadata.java index f137c8a59dbf9..6e4b0defb581b 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarMetadata.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarMetadata.java @@ -20,44 +20,23 @@ import io.airlift.log.Logger; import io.prestosql.spi.PrestoException; -import io.prestosql.spi.connector.ColumnHandle; -import io.prestosql.spi.connector.ColumnMetadata; -import io.prestosql.spi.connector.ConnectorSession; -import io.prestosql.spi.connector.ConnectorTableHandle; -import io.prestosql.spi.connector.ConnectorTableMetadata; -import io.prestosql.spi.connector.SchemaTableName; -import io.prestosql.spi.connector.SchemaTablePrefix; -import io.prestosql.spi.connector.TableNotFoundException; -import java.util.Optional; -import org.apache.avro.Schema; +import io.prestosql.spi.connector.*; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; -import javax.ws.rs.ClientErrorException; -import javax.ws.rs.core.Response; - import org.apache.pulsar.common.schema.SchemaType; import org.testng.annotations.Test; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import javax.ws.rs.ClientErrorException; +import javax.ws.rs.core.Response; +import java.util.*; import java.util.stream.Collectors; import static io.prestosql.spi.StandardErrorCode.NOT_FOUND; import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; +import static org.mockito.Mockito.*; +import static org.testng.Assert.*; public class TestPulsarMetadata extends TestPulsarConnector { @@ -116,6 +95,8 @@ public void testGetTableMetadata(String delimiter) { topic.getLocalName() ); + List fooColumnHandles = topicsToColumnHandles.get(topic); + ConnectorTableMetadata tableMetadata = this.pulsarMetadata.getTableMetadata(mock(ConnectorSession.class), pulsarTableHandle); @@ -124,7 +105,7 @@ public void testGetTableMetadata(String delimiter) { assertEquals(tableMetadata.getColumns().size(), fooColumnHandles.size()); - List fieldNames = new LinkedList<>(fooFieldNames.keySet()); + List fieldNames = new LinkedList<>(fooFieldNames); for (PulsarInternalColumn internalField : PulsarInternalColumn.getInternalFields()) { fieldNames.add(internalField.getName()); @@ -286,7 +267,7 @@ public void testGetColumnHandles(String delimiter) { Map columnHandleMap = new HashMap<>(this.pulsarMetadata.getColumnHandles(mock(ConnectorSession.class), pulsarTableHandle)); - List fieldNames = new LinkedList<>(fooFieldNames.keySet()); + List fieldNames = new LinkedList<>(fooFieldNames); for (PulsarInternalColumn internalField : PulsarInternalColumn.getInternalFields()) { fieldNames.add(internalField.getName()); @@ -300,13 +281,8 @@ public void testGetColumnHandles(String delimiter) { assertEquals(pulsarColumnHandle, pulsarInternalColumn.getColumnHandle(pulsarConnectorId.toString(), false)); } else { - Schema schema = new Schema.Parser().parse(new String(topicsToSchemas.get(TOPIC_1.getSchemaName()) - .getSchema())); assertEquals(pulsarColumnHandle.getConnectorId(), pulsarConnectorId.toString()); assertEquals(pulsarColumnHandle.getName(), field); - assertEquals(pulsarColumnHandle.getPositionIndices(), fooPositionIndices.get(field)); - assertEquals(pulsarColumnHandle.getFieldNames(), fooFieldNames.get(field)); - assertEquals(pulsarColumnHandle.getType(), fooTypes.get(field)); assertFalse(pulsarColumnHandle.isHidden()); } columnHandleMap.remove(field); @@ -326,9 +302,9 @@ public void testListTableColumns(String delimiter) { = tableColumnsMap.get(new SchemaTableName(TOPIC_1.getNamespace(), TOPIC_1.getLocalName())); assertNotNull(columnMetadataList); assertEquals(columnMetadataList.size(), - fooColumnHandles.size()); + topicsToColumnHandles.get(TOPIC_1).size()); - List fieldNames = new LinkedList<>(fooFieldNames.keySet()); + List fieldNames = new LinkedList<>(fooFieldNames); for (PulsarInternalColumn internalField : PulsarInternalColumn.getInternalFields()) { fieldNames.add(internalField.getName()); @@ -349,9 +325,9 @@ public void testListTableColumns(String delimiter) { columnMetadataList = tableColumnsMap.get(new SchemaTableName(TOPIC_2.getNamespace(), TOPIC_2.getLocalName())); assertNotNull(columnMetadataList); assertEquals(columnMetadataList.size(), - fooColumnHandles.size()); + topicsToColumnHandles.get(TOPIC_2).size()); - fieldNames = new LinkedList<>(fooFieldNames.keySet()); + fieldNames = new LinkedList<>(fooFieldNames); for (PulsarInternalColumn internalField : PulsarInternalColumn.getInternalFields()) { fieldNames.add(internalField.getName()); @@ -378,9 +354,9 @@ public void testListTableColumns(String delimiter) { columnMetadataList = tableColumnsMap.get(new SchemaTableName(TOPIC_4.getNamespace(), TOPIC_4.getLocalName())); assertNotNull(columnMetadataList); assertEquals(columnMetadataList.size(), - fooColumnHandles.size()); + topicsToColumnHandles.get(TOPIC_4).size()); - fieldNames = new LinkedList<>(fooFieldNames.keySet()); + fieldNames = new LinkedList<>(fooFieldNames); for (PulsarInternalColumn internalField : PulsarInternalColumn.getInternalFields()) { fieldNames.add(internalField.getName()); diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarPrimitiveSchemaHandler.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarPrimitiveSchemaHandler.java deleted file mode 100644 index fc330779db051..0000000000000 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarPrimitiveSchemaHandler.java +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.sql.presto; - -import io.netty.buffer.ByteBufAllocator; -import io.prestosql.spi.connector.ColumnMetadata; -import lombok.extern.slf4j.Slf4j; - -import org.apache.pulsar.client.impl.schema.BooleanSchema; -import org.apache.pulsar.client.impl.schema.ByteSchema; -import org.apache.pulsar.client.impl.schema.BytesSchema; -import org.apache.pulsar.client.impl.schema.DateSchema; -import org.apache.pulsar.client.impl.schema.DoubleSchema; -import org.apache.pulsar.client.impl.schema.FloatSchema; -import org.apache.pulsar.client.impl.schema.IntSchema; -import org.apache.pulsar.client.impl.schema.LongSchema; -import org.apache.pulsar.client.impl.schema.ShortSchema; -import org.apache.pulsar.client.impl.schema.StringSchema; -import org.apache.pulsar.client.impl.schema.TimeSchema; -import org.apache.pulsar.client.impl.schema.TimestampSchema; -import org.apache.pulsar.common.api.raw.RawMessage; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; -import org.testng.Assert; -import org.testng.annotations.Test; - -import java.sql.Time; -import java.sql.Timestamp; -import java.util.Date; -import java.util.List; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@Slf4j -public class TestPulsarPrimitiveSchemaHandler { - - private static final TopicName stringTopicName = TopicName.get("persistent", "tenant-1", "ns-1", "topic-1"); - @Test - public void testPulsarPrimitiveSchemaHandler() { - PulsarPrimitiveSchemaHandler pulsarPrimitiveSchemaHandler; - RawMessage rawMessage = mock(RawMessage.class); - SchemaInfo schemaInfoInt8 = SchemaInfo.builder().type(SchemaType.INT8).build(); - pulsarPrimitiveSchemaHandler = new PulsarPrimitiveSchemaHandler(schemaInfoInt8); - byte int8Value = 1; - when(rawMessage.getData()).thenReturn(ByteBufAllocator.DEFAULT.buffer().writeBytes(ByteSchema.of().encode(int8Value))); - Assert.assertEquals(int8Value, (byte)pulsarPrimitiveSchemaHandler.deserialize(rawMessage.getData())); - - SchemaInfo schemaInfoInt16 = SchemaInfo.builder().type(SchemaType.INT16).build(); - pulsarPrimitiveSchemaHandler = new PulsarPrimitiveSchemaHandler(schemaInfoInt16); - short int16Value = 2; - when(rawMessage.getData()).thenReturn(ByteBufAllocator.DEFAULT.buffer().writeBytes(ShortSchema.of().encode(int16Value))); - Assert.assertEquals(int16Value, pulsarPrimitiveSchemaHandler.deserialize(rawMessage.getData())); - - SchemaInfo schemaInfoInt32 = SchemaInfo.builder().type(SchemaType.INT32).build(); - pulsarPrimitiveSchemaHandler = new PulsarPrimitiveSchemaHandler(schemaInfoInt32); - int int32Value = 2; - when(rawMessage.getData()).thenReturn(ByteBufAllocator.DEFAULT.buffer().writeBytes(IntSchema.of().encode(int32Value))); - Assert.assertEquals(int32Value, pulsarPrimitiveSchemaHandler.deserialize(rawMessage.getData())); - - SchemaInfo schemaInfoInt64 = SchemaInfo.builder().type(SchemaType.INT64).build(); - pulsarPrimitiveSchemaHandler = new PulsarPrimitiveSchemaHandler(schemaInfoInt64); - long int64Value = 2; - when(rawMessage.getData()).thenReturn(ByteBufAllocator.DEFAULT.buffer().writeBytes(LongSchema.of().encode(int64Value))); - Assert.assertEquals(int64Value, pulsarPrimitiveSchemaHandler.deserialize(rawMessage.getData())); - - SchemaInfo schemaInfoString = SchemaInfo.builder().type(SchemaType.STRING).build(); - pulsarPrimitiveSchemaHandler = new PulsarPrimitiveSchemaHandler(schemaInfoString); - String stringValue = "test"; - when(rawMessage.getData()).thenReturn(ByteBufAllocator.DEFAULT.buffer().writeBytes(StringSchema.utf8().encode(stringValue))); - Assert.assertEquals(stringValue, pulsarPrimitiveSchemaHandler.deserialize(rawMessage.getData())); - - SchemaInfo schemaInfoFloat = SchemaInfo.builder().type(SchemaType.FLOAT).build(); - pulsarPrimitiveSchemaHandler = new PulsarPrimitiveSchemaHandler(schemaInfoFloat); - float floatValue = 0.2f; - when(rawMessage.getData()).thenReturn(ByteBufAllocator.DEFAULT.buffer().writeBytes(FloatSchema.of().encode(floatValue))); - Assert.assertEquals(floatValue, pulsarPrimitiveSchemaHandler.deserialize(rawMessage.getData())); - - SchemaInfo schemaInfoDouble = SchemaInfo.builder().type(SchemaType.DOUBLE).build(); - pulsarPrimitiveSchemaHandler = new PulsarPrimitiveSchemaHandler(schemaInfoDouble); - double doubleValue = 0.22d; - when(rawMessage.getData()).thenReturn(ByteBufAllocator.DEFAULT.buffer().writeBytes(DoubleSchema.of().encode(doubleValue))); - Assert.assertEquals(doubleValue, pulsarPrimitiveSchemaHandler.deserialize(rawMessage.getData())); - - SchemaInfo schemaInfoBoolean = SchemaInfo.builder().type(SchemaType.BOOLEAN).build(); - pulsarPrimitiveSchemaHandler = new PulsarPrimitiveSchemaHandler(schemaInfoBoolean); - boolean booleanValue = true; - when(rawMessage.getData()).thenReturn(ByteBufAllocator.DEFAULT.buffer().writeBytes(BooleanSchema.of().encode(booleanValue))); - Assert.assertEquals(booleanValue, pulsarPrimitiveSchemaHandler.deserialize(rawMessage.getData())); - - SchemaInfo schemaInfoBytes = SchemaInfo.builder().type(SchemaType.BYTES).build(); - pulsarPrimitiveSchemaHandler = new PulsarPrimitiveSchemaHandler(schemaInfoBytes); - byte[] bytesValue = new byte[1]; - bytesValue[0] = 1; - when(rawMessage.getData()).thenReturn(ByteBufAllocator.DEFAULT.buffer().writeBytes(BytesSchema.of().encode(bytesValue))); - Assert.assertEquals(bytesValue, pulsarPrimitiveSchemaHandler.deserialize(rawMessage.getData())); - - SchemaInfo schemaInfoDate = SchemaInfo.builder().type(SchemaType.DATE).build(); - pulsarPrimitiveSchemaHandler = new PulsarPrimitiveSchemaHandler(schemaInfoDate); - Date dateValue = new Date(System.currentTimeMillis()); - when(rawMessage.getData()).thenReturn(ByteBufAllocator.DEFAULT.buffer().writeBytes(DateSchema.of().encode(dateValue))); - Object dateDeserializeValue = pulsarPrimitiveSchemaHandler.deserialize(rawMessage.getData()); - Assert.assertEquals(dateValue.getTime(), dateDeserializeValue); - - SchemaInfo schemaInfoTime = SchemaInfo.builder().type(SchemaType.TIME).build(); - pulsarPrimitiveSchemaHandler = new PulsarPrimitiveSchemaHandler(schemaInfoTime); - Time timeValue = new Time(System.currentTimeMillis()); - when(rawMessage.getData()).thenReturn(ByteBufAllocator.DEFAULT.buffer().writeBytes(TimeSchema.of().encode(timeValue))); - Object timeDeserializeValue = pulsarPrimitiveSchemaHandler.deserialize(rawMessage.getData()); - Assert.assertEquals(timeValue.getTime(), timeDeserializeValue); - - SchemaInfo schemaInfoTimestamp = SchemaInfo.builder().type(SchemaType.TIMESTAMP).build(); - pulsarPrimitiveSchemaHandler = new PulsarPrimitiveSchemaHandler(schemaInfoTimestamp); - Timestamp timestampValue = new Timestamp(System.currentTimeMillis()); - when(rawMessage.getData()).thenReturn(ByteBufAllocator.DEFAULT.buffer().writeBytes(TimestampSchema.of().encode(timestampValue))); - Object timestampDeserializeValue = pulsarPrimitiveSchemaHandler.deserialize(rawMessage.getData()); - Assert.assertEquals(timestampValue.getTime(), timestampDeserializeValue); - } - - @Test - public void testNewPulsarPrimitiveSchemaHandler() { - RawMessage rawMessage = mock(RawMessage.class); - SchemaHandler schemaHandler = PulsarSchemaHandlers.newPulsarSchemaHandler( - null, - null, - StringSchema.utf8().getSchemaInfo(), - null); - - String stringValue = "test"; - when(rawMessage.getData()).thenReturn(ByteBufAllocator.DEFAULT.buffer().writeBytes(StringSchema.utf8().encode(stringValue))); - - Object deserializeValue = schemaHandler.deserialize(rawMessage.getData()); - Assert.assertEquals(stringValue, (String)deserializeValue); - Assert.assertEquals(stringValue, (String)deserializeValue); - - } - - @Test - public void testNewColumnMetadata() { - List columnMetadataList = PulsarMetadata.getPulsarColumns(stringTopicName, - StringSchema.utf8().getSchemaInfo(), false, null); - Assert.assertEquals(columnMetadataList.size(), 1); - ColumnMetadata columnMetadata = columnMetadataList.get(0); - Assert.assertEquals("__value__", columnMetadata.getName()); - Assert.assertEquals("varchar", columnMetadata.getType().toString()); - } -} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java index 1e28fb91900c9..6be688a16065a 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java @@ -19,16 +19,10 @@ package org.apache.pulsar.sql.presto; import io.airlift.log.Logger; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; +import io.prestosql.spi.type.RowType; import org.apache.pulsar.common.naming.TopicName; -import org.mockito.Mock; -import org.mockito.Mockito; import org.testng.annotations.Test; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.nio.ByteBuffer; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -38,8 +32,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; public class TestPulsarRecordCursor extends TestPulsarConnector { @@ -53,15 +45,13 @@ public void testTopics() throws Exception { log.info("!------ topic %s ------!", entry.getKey()); setup(); + + List fooColumnHandles = topicsToColumnHandles.get(entry.getKey()); PulsarRecordCursor pulsarRecordCursor = entry.getValue(); - SchemaHandler schemaHandler = pulsarRecordCursor.getSchemaHandler(); PulsarSqlSchemaInfoProvider pulsarSqlSchemaInfoProvider = mock(PulsarSqlSchemaInfoProvider.class); - if (schemaHandler instanceof AvroSchemaHandler) { - AvroSchemaHandler avroSchemaHandler = (AvroSchemaHandler) schemaHandler; - avroSchemaHandler.getSchema().setSchemaInfoProvider(pulsarSqlSchemaInfoProvider); - when(pulsarSqlSchemaInfoProvider.getSchemaByVersion(any())).thenReturn(completedFuture(avroSchemaHandler.getSchemaInfo())); - } + when(pulsarSqlSchemaInfoProvider.getSchemaByVersion(any())).thenReturn(completedFuture(topicsToSchemas.get(entry.getKey().getSchemaName()))); + pulsarRecordCursor.setPulsarSqlSchemaInfoProvider(pulsarSqlSchemaInfoProvider); TopicName topicName = entry.getKey(); @@ -99,40 +89,10 @@ public void testTopics() throws Exception { } else if (fooColumnHandles.get(i).getName().equals("date")) { pulsarRecordCursor.getLong(i); columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("bar.field1")) { - assertEquals(pulsarRecordCursor.getLong(i), ((Integer) fooFunctions.get("bar.field1").apply(count)).longValue()); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("bar.field2")) { - assertEquals(pulsarRecordCursor.getSlice(i).getBytes(), ((String) fooFunctions.get("bar.field2").apply(count)).getBytes()); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("bar.field3")) { - assertEquals(pulsarRecordCursor.getLong(i), Float.floatToIntBits(((Float) fooFunctions.get("bar.field3").apply(count)).floatValue())); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("bar.test.field4")) { - assertEquals(pulsarRecordCursor.getDouble(i), ((Double) fooFunctions.get("bar.test.field4").apply(count)).doubleValue()); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("bar.test.field5")) { - assertEquals(pulsarRecordCursor.getBoolean(i), ((Boolean) fooFunctions.get("bar.test.field5").apply(count)).booleanValue()); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("bar.test.field6")) { - assertEquals(pulsarRecordCursor.getLong(i), ((Long) fooFunctions.get("bar.test.field6").apply(count)).longValue()); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("bar.test.foobar.field1")) { - assertEquals(pulsarRecordCursor.getLong(i), ((Integer) fooFunctions.get("bar.test.foobar.field1").apply(count)).longValue()); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("bar.test2.field4")) { - assertEquals(pulsarRecordCursor.getDouble(i), ((Double) fooFunctions.get("bar.test2.field4").apply(count)).doubleValue()); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("bar.test2.field5")) { - assertEquals(pulsarRecordCursor.getBoolean(i), ((Boolean) fooFunctions.get("bar.test2.field5").apply(count)).booleanValue()); + } else if (fooColumnHandles.get(i).getName().equals("bar")) { + assertTrue(fooColumnHandles.get(i).getType() instanceof RowType); columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("bar.test2.field6")) { - assertEquals(pulsarRecordCursor.getLong(i), ((Long) fooFunctions.get("bar.test2.field6").apply(count)).longValue()); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("bar.test2.foobar.field1")) { - assertEquals(pulsarRecordCursor.getLong(i), ((Integer) fooFunctions.get("bar.test2.foobar.field1").apply(count)).longValue()); - columnsSeen.add(fooColumnHandles.get(i).getName()); - } else if (fooColumnHandles.get(i).getName().equals("field7")) { + }else if (fooColumnHandles.get(i).getName().equals("field7")) { assertEquals(pulsarRecordCursor.getSlice(i).getBytes(), fooFunctions.get("field7").apply(count).toString().getBytes()); columnsSeen.add(fooColumnHandles.get(i).getName()); } else { @@ -152,44 +112,4 @@ public void testTopics() throws Exception { } } - @Test - public void testRecordToBytes() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { - PulsarRecordCursor pulsarRecordCursor = Mockito.mock(PulsarRecordCursor.class); - Method method = PulsarRecordCursor.class.getDeclaredMethod("toBytes", Object.class); - method.setAccessible(true); - - final String msg = "Hello!"; - - byte[] bytes = msg.getBytes(); - Object obj = method.invoke(pulsarRecordCursor, bytes); - assertNotNull(obj); - assertEquals(new String((byte[]) obj), msg); - - ByteBuffer byteBuffer1 = ByteBuffer.wrap(msg.getBytes()); - assertTrue(byteBuffer1.hasArray()); - obj = method.invoke(pulsarRecordCursor, byteBuffer1); - assertNotNull(obj); - assertEquals(new String((byte[]) obj), msg); - - ByteBuffer byteBuffer2 = ByteBuffer.allocateDirect(msg.getBytes().length); - byteBuffer2.put(msg.getBytes()); - assertFalse(byteBuffer2.hasArray()); - obj = method.invoke(pulsarRecordCursor, byteBuffer2); - assertNotNull(obj); - assertEquals(new String((byte[]) obj), msg); - - ByteBuf byteBuf1 = Unpooled.wrappedBuffer(msg.getBytes()); - assertTrue(byteBuf1.hasArray()); - obj = method.invoke(pulsarRecordCursor, byteBuf1); - assertNotNull(obj); - assertEquals(new String((byte[]) obj), msg); - - ByteBuf byteBuf2 = Unpooled.directBuffer(); - byteBuf2.writeBytes(msg.getBytes()); - assertFalse(byteBuf2.hasArray()); - obj = method.invoke(pulsarRecordCursor, byteBuf2); - assertNotNull(obj); - assertEquals(new String((byte[]) obj), msg); - } - } diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java new file mode 100644 index 0000000000000..44e6453baf598 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder; + +import io.airlift.slice.Slice; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.connector.ConnectorContext; +import io.prestosql.spi.type.Type; +import io.prestosql.testing.TestingConnectorContext; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.sql.presto.PulsarColumnHandle; +import org.apache.pulsar.sql.presto.PulsarDispatchingRowDecoderFactory; +import org.apache.pulsar.sql.presto.PulsarRowDecoder; + +import java.util.List; +import java.util.Map; + +import static org.testng.Assert.assertNotNull; + +/** + * Abstract superclass for TestXXDecoder (e.g. TestAvroDecoder 态TestJsonDecoder) + */ +public abstract class AbstractDecoderTester { + + protected PulsarDispatchingRowDecoderFactory decoderFactory; + protected SchemaInfo schemaInfo; + protected TopicName topicName; + protected List pulsarColumnHandle; + protected PulsarRowDecoder pulsarRowDecoder; + protected DecoderTestUtil decoderTestUtil; + + protected void init() { + ConnectorContext prestoConnectorContext = new TestingConnectorContext(); + decoderFactory = new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager()); + topicName = TopicName.get("persistent", NamespaceName.get("tenant-1", "ns-1"), "topic-1"); + } + + protected void checkArrayValues(Block block, Type type, Object value) { + decoderTestUtil.checkArrayValues(block, type, value); + } + + protected void checkMapValues(Block block, Type type, Object value) { + decoderTestUtil.checkMapValues(block, type, value); + } + + protected void checkRowValues(Block block, Type type, Object value) { + decoderTestUtil.checkRowValues(block, type, value); + } + + protected void checkValue(Map decodedRow, DecoderColumnHandle handle, Slice value) { + decoderTestUtil.checkValue(decodedRow, handle, value); + } + + protected void checkValue(Map decodedRow, DecoderColumnHandle handle, String value) { + decoderTestUtil.checkValue(decodedRow, handle, value); + } + + protected void checkValue(Map decodedRow, DecoderColumnHandle handle, long value) { + decoderTestUtil.checkValue(decodedRow, handle, value); + } + + protected void checkValue(Map decodedRow, DecoderColumnHandle handle, double value) { + decoderTestUtil.checkValue(decodedRow, handle, value); + } + + protected void checkValue(Map decodedRow, DecoderColumnHandle handle, boolean value) { + decoderTestUtil.checkValue(decodedRow, handle, value); + } + + protected Block getBlock(Map decodedRow, DecoderColumnHandle handle) { + FieldValueProvider provider = decodedRow.get(handle); + assertNotNull(provider); + return provider.getBlock(); + } + +} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java new file mode 100644 index 0000000000000..4b95e881cd809 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder; + +import java.util.List; +import java.util.Map; + +public class DecoderTestMessage { + + public static enum TestEnum { + TEST_ENUM_1, + TEST_ENUM_2, + TEST_ENUM_3 + } + + public int intField; + public String stringField; + public float floatField; + public double doubleField; + public boolean booleanField; + public long longField; + @org.apache.avro.reflect.AvroSchema("{ \"type\": \"long\", \"logicalType\": \"timestamp-millis\" }") + public long timestampField; + @org.apache.avro.reflect.AvroSchema("{ \"type\": \"int\", \"logicalType\": \"time-millis\" }") + public int timeField; + @org.apache.avro.reflect.AvroSchema("{ \"type\": \"int\", \"logicalType\": \"date\" }") + public int dateField; + public TestRow rowField; + public TestEnum enumField; + + public List arrayField; + public Map mapField; + public CompositeRow compositeRow; + + public static class TestRow { + public String stringField; + public int intField; + public NestedRow nestedRow; + } + + + public static class NestedRow { + public String stringField; + public long longField; + } + + + public static class CompositeRow { + public String stringField; + public List arrayField; + public Map mapField; + public NestedRow nestedRow; + public Map> structedField; + } +} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java new file mode 100644 index 0000000000000..d0fdd59578eb3 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder; + +import io.airlift.slice.Slice; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.type.ArrayType; +import io.prestosql.spi.type.MapType; +import io.prestosql.spi.type.RowType; +import io.prestosql.spi.type.Type; + +import java.util.Map; + +import static io.prestosql.testing.TestingConnectorSession.SESSION; +import static org.testng.Assert.*; + +/** + * Abstract util superclass for XXDecoderTestUtil (e.g. AvroDecoderTestUtil 态JsonDecoderTestUtil) + */ +public abstract class DecoderTestUtil { + + protected DecoderTestUtil() { + + } + + public abstract void checkArrayValues(Block block, Type type, Object value); + + public abstract void checkMapValues(Block block, Type type, Object value); + + public abstract void checkRowValues(Block block, Type type, Object value); + + public abstract void checkPrimitiveValue(Object actual, Object expected); + + + public void checkField(Block actualBlock, Type type, int position, Object expectedValue) { + assertNotNull(type, "Type is null"); + assertNotNull(actualBlock, "actualBlock is null"); + assertTrue(!actualBlock.isNull(position)); + assertNotNull(expectedValue, "expectedValue is null"); + + if (type instanceof ArrayType) { + checkArrayValues(actualBlock.getObject(position, Block.class), type, expectedValue); + } else if (type instanceof MapType) { + checkMapValues(actualBlock.getObject(position, Block.class), type, expectedValue); + } else if (type instanceof RowType) { + checkRowValues(actualBlock.getObject(position, Block.class), type, expectedValue); + } else { + checkPrimitiveValue(getObjectValue(type, actualBlock, position), expectedValue); + } + } + + + public boolean isIntegralType(Object value) { + return value instanceof Long + || value instanceof Integer + || value instanceof Short + || value instanceof Byte; + } + + public boolean isRealType(Object value) { + return value instanceof Float || value instanceof Double; + } + + + public Object getObjectValue(Type type, Block block, int position) { + if (block.isNull(position)) { + return null; + } + return type.getObjectValue(SESSION, block, position); + } + + + public void checkValue(Map decodedRow, DecoderColumnHandle handle, Slice value) { + FieldValueProvider provider = decodedRow.get(handle); + assertNotNull(provider); + assertEquals(provider.getSlice(), value); + } + + public void checkValue(Map decodedRow, DecoderColumnHandle handle, String value) { + FieldValueProvider provider = decodedRow.get(handle); + assertNotNull(provider); + assertEquals(provider.getSlice().toStringUtf8(), value); + } + + public void checkValue(Map decodedRow, DecoderColumnHandle handle, long value) { + FieldValueProvider provider = decodedRow.get(handle); + assertNotNull(provider); + assertEquals(provider.getLong(), value); + } + + public void checkValue(Map decodedRow, DecoderColumnHandle handle, double value) { + FieldValueProvider provider = decodedRow.get(handle); + assertNotNull(provider); + assertEquals(provider.getDouble(), value, 0.0001); + } + + public void checkValue(Map decodedRow, DecoderColumnHandle handle, boolean value) { + FieldValueProvider provider = decodedRow.get(handle); + assertNotNull(provider); + assertEquals(provider.getBoolean(), value); + } + + public void checkIsNull(Map decodedRow, DecoderColumnHandle handle) { + FieldValueProvider provider = decodedRow.get(handle); + assertNotNull(provider); + assertTrue(provider.isNull()); + } +} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/AvroDecoderTestUtil.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/AvroDecoderTestUtil.java new file mode 100644 index 0000000000000..f11360f03e4bd --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/AvroDecoderTestUtil.java @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder.avro; + +import io.prestosql.spi.block.Block; +import io.prestosql.spi.type.*; +import org.apache.avro.generic.GenericEnumSymbol; +import org.apache.avro.generic.GenericFixed; +import org.apache.avro.generic.GenericRecord; +import org.apache.pulsar.sql.presto.decoder.DecoderTestUtil; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; + +import static io.prestosql.spi.type.VarcharType.VARCHAR; +import static java.lang.String.format; +import static org.testng.Assert.*; + +/** + * TestUtil for AvroDecoder + */ +public class AvroDecoderTestUtil extends DecoderTestUtil { + public AvroDecoderTestUtil() { + super(); + } + + public void checkPrimitiveValue(Object actual, Object expected) { + if (actual == null || expected == null) { + assertNull(expected); + assertNull(actual); + } else if (actual instanceof CharSequence) { + assertTrue(expected instanceof CharSequence || expected instanceof GenericEnumSymbol); + assertEquals(actual.toString(), expected.toString()); + } else if (actual instanceof SqlVarbinary) { + if (expected instanceof GenericFixed) { + assertEquals(((SqlVarbinary) actual).getBytes(), ((GenericFixed) expected).bytes()); + } else if (expected instanceof ByteBuffer) { + assertEquals(((SqlVarbinary) actual).getBytes(), ((ByteBuffer) expected).array()); + } else { + fail(format("Unexpected value type %s", actual.getClass())); + } + } else if (isIntegralType(actual) && isIntegralType(expected)) { + assertEquals(((Number) actual).longValue(), ((Number) expected).longValue()); + } else if (isRealType(actual) && isRealType(expected)) { + assertEquals(((Number) actual).doubleValue(), ((Number) expected).doubleValue()); + } else { + assertEquals(actual, expected); + } + } + + + public void checkArrayValues(Block block, Type type, Object value) { + assertNotNull(type, "Type is null"); + assertTrue(type instanceof ArrayType, "Unexpected type"); + assertNotNull(block, "Block is null"); + assertNotNull(value, "Value is null"); + + List list = (List) value; + + assertEquals(block.getPositionCount(), list.size()); + Type elementType = ((ArrayType) type).getElementType(); + if (elementType instanceof ArrayType) { + for (int index = 0; index < block.getPositionCount(); index++) { + if (block.isNull(index)) { + assertNull(list.get(index)); + continue; + } + Block arrayBlock = block.getObject(index, Block.class); + checkArrayValues(arrayBlock, elementType, list.get(index)); + } + } else if (elementType instanceof MapType) { + for (int index = 0; index < block.getPositionCount(); index++) { + if (block.isNull(index)) { + assertNull(list.get(index)); + continue; + } + Block mapBlock = block.getObject(index, Block.class); + checkMapValues(mapBlock, elementType, list.get(index)); + } + } else if (elementType instanceof RowType) { + for (int index = 0; index < block.getPositionCount(); index++) { + if (block.isNull(index)) { + assertNull(list.get(index)); + continue; + } + Block rowBlock = block.getObject(index, Block.class); + checkRowValues(rowBlock, elementType, list.get(index)); + } + } else { + for (int index = 0; index < block.getPositionCount(); index++) { + checkPrimitiveValue(getObjectValue(elementType, block, index), list.get(index)); + } + } + } + + /** + * fix key as org.apache.avro.util.Utf8 + * + * @param block + * @param type + * @param value + */ + public void checkMapValues(Block block, Type type, Object value) { + assertNotNull(type, "Type is null"); + assertTrue(type instanceof MapType, "Unexpected type"); + assertTrue(((MapType) type).getKeyType() instanceof VarcharType, "Unexpected key type"); + assertNotNull(block, "Block is null"); + assertNotNull(value, "Value is null"); + + + Map expected = (Map) value; + + assertEquals(block.getPositionCount(), expected.size() * 2); + Type valueType = ((MapType) type).getValueType(); + if (valueType instanceof ArrayType) { + for (int index = 0; index < block.getPositionCount(); index += 2) { + String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); + assertTrue(expected.keySet().stream().anyMatch(e -> e.toString().equals(actualKey))); + if (block.isNull(index + 1)) { + assertNull(expected.get(actualKey)); + continue; + } + Block arrayBlock = block.getObject(index + 1, Block.class); + Object keyValue = expected.entrySet().stream().filter(e -> e.getKey().toString().equals(actualKey)).findFirst().get().getValue(); + checkArrayValues(arrayBlock, valueType, keyValue); + } + } else if (valueType instanceof MapType) { + for (int index = 0; index < block.getPositionCount(); index += 2) { + String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); + assertTrue(expected.keySet().stream().anyMatch(e -> e.toString().equals(actualKey))); + if (block.isNull(index + 1)) { + assertNull(expected.get(actualKey)); + continue; + } + Block mapBlock = block.getObject(index + 1, Block.class); + Object keyValue = expected.entrySet().stream().filter(e -> e.getKey().toString().equals(actualKey)).findFirst().get().getValue(); + checkMapValues(mapBlock, valueType, keyValue); + } + } else if (valueType instanceof RowType) { + for (int index = 0; index < block.getPositionCount(); index += 2) { + String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); + assertTrue(expected.keySet().stream().anyMatch(e -> e.toString().equals(actualKey))); + if (block.isNull(index + 1)) { + assertNull(expected.get(actualKey)); + continue; + } + Block rowBlock = block.getObject(index + 1, Block.class); + Object keyValue = expected.entrySet().stream().filter(e -> e.getKey().toString().equals(actualKey)).findFirst().get().getValue(); + checkRowValues(rowBlock, valueType, keyValue); + } + } else { + for (int index = 0; index < block.getPositionCount(); index += 2) { + String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); + assertTrue(expected.keySet().stream().anyMatch(e -> e.toString().equals(actualKey))); + Object keyValue = expected.entrySet().stream().filter(e -> e.getKey().toString().equals(actualKey)).findFirst().get().getValue(); + checkPrimitiveValue(getObjectValue(valueType, block, index + 1), keyValue); + } + } + } + + public void checkRowValues(Block block, Type type, Object value) { + assertNotNull(type, "Type is null"); + assertTrue(type instanceof RowType, "Unexpected type"); + assertNotNull(block, "Block is null"); + assertNotNull(value, "Value is null"); + + GenericRecord record = (GenericRecord) value; + RowType rowType = (RowType) type; + assertEquals(record.getSchema().getFields().size(), rowType.getFields().size(), "Avro field size mismatch"); + assertEquals(block.getPositionCount(), rowType.getFields().size(), "Presto type field size mismatch"); + for (int fieldIndex = 0; fieldIndex < rowType.getFields().size(); fieldIndex++) { + RowType.Field rowField = rowType.getFields().get(fieldIndex); + Object expectedValue = record.get(rowField.getName().get()); + if (block.isNull(fieldIndex)) { + assertNull(expectedValue); + continue; + } + checkField(block, rowField.getType(), fieldIndex, expectedValue); + } + } + +} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java new file mode 100644 index 0000000000000..9e3a38e8eb6d3 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java @@ -0,0 +1,273 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder.avro; + +import com.google.common.collect.ImmutableList; +import io.netty.buffer.ByteBuf; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; +import io.prestosql.spi.type.*; +import org.apache.pulsar.client.impl.schema.AvroSchema; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; +import org.apache.pulsar.sql.presto.PulsarColumnHandle; +import org.apache.pulsar.sql.presto.decoder.AbstractDecoderTester; +import org.apache.pulsar.sql.presto.decoder.DecoderTestMessage; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.util.*; + +import static io.prestosql.spi.type.BigintType.BIGINT; +import static io.prestosql.spi.type.BooleanType.BOOLEAN; +import static io.prestosql.spi.type.DoubleType.DOUBLE; +import static io.prestosql.spi.type.IntegerType.INTEGER; +import static io.prestosql.spi.type.RealType.REAL; +import static io.prestosql.spi.type.VarcharType.VARCHAR; +import static java.lang.Float.floatToIntBits; +import static org.apache.pulsar.sql.presto.TestPulsarConnector.getColumnColumnHandles; +import static org.apache.pulsar.sql.presto.TestPulsarConnector.getPulsarConnectorId; +import static org.testng.Assert.assertTrue; + +public class TestAvroDecoder extends AbstractDecoderTester { + + private AvroSchema schema; + + @BeforeMethod + public void init() { + super.init(); + schema = AvroSchema.of(DecoderTestMessage.class); + schemaInfo = schema.getSchemaInfo(); + pulsarColumnHandle = getColumnColumnHandles(topicName, schemaInfo, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); + pulsarRowDecoder = decoderFactory.createRowDecoder(topicName, schemaInfo, new HashSet<>(pulsarColumnHandle)); + decoderTestUtil = new AvroDecoderTestUtil(); + assertTrue(pulsarRowDecoder instanceof PulsarAvroRowDecoder); + } + + @Test + public void testPrimitiveType() { + DecoderTestMessage message = new DecoderTestMessage(); + message.stringField = "message_1"; + message.intField = 22; + message.floatField = 2.2f; + message.doubleField = 22.20D; + message.booleanField = true; + message.longField = 222L; + message.timestampField = System.currentTimeMillis(); + message.enumField = DecoderTestMessage.TestEnum.TEST_ENUM_1; + + LocalTime now = LocalTime.now(ZoneId.systemDefault()); + message.timeField = now.toSecondOfDay() * 1000; + + LocalDate localDate = LocalDate.now(); + LocalDate epoch = LocalDate.ofEpochDay(0); + message.dateField = Math.toIntExact(ChronoUnit.DAYS.between(epoch, localDate)); + + ByteBuf payload = io.netty.buffer.Unpooled + .copiedBuffer(schema.encode(message)); + Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); + + PulsarColumnHandle stringFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "stringField", VARCHAR, false, false, "stringField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, stringFieldColumnHandle, message.stringField); + + PulsarColumnHandle intFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "intField", INTEGER, false, false, "intField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, intFieldColumnHandle, message.intField); + + PulsarColumnHandle floatFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "floatField", REAL, false, false, "floatField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, floatFieldColumnHandle, floatToIntBits(message.floatField)); + + PulsarColumnHandle doubleFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "doubleField", DOUBLE, false, false, "doubleField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, doubleFieldColumnHandle, message.doubleField); + + PulsarColumnHandle booleanFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "booleanField", BOOLEAN, false, false, "booleanField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, booleanFieldColumnHandle, message.booleanField); + + PulsarColumnHandle longFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "longField", BIGINT, false, false, "longField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, longFieldColumnHandle, message.longField); + + PulsarColumnHandle enumFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "enumField", VARCHAR, false, false, "enumField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, enumFieldColumnHandle, message.enumField.toString()); + + } + + @Test + public void testRow() { + DecoderTestMessage message = new DecoderTestMessage(); + message.stringField = "message_2"; + DecoderTestMessage.TestRow testRow = new DecoderTestMessage.TestRow(); + message.rowField = testRow; + testRow.intField = 22; + testRow.stringField = "message_2_testRow"; + DecoderTestMessage.NestedRow nestedRow = new DecoderTestMessage.NestedRow(); + nestedRow.longField = 222L; + nestedRow.stringField = "message_2_nestedRow"; + testRow.nestedRow = nestedRow; + + byte[] bytes = schema.encode(message); + ByteBuf payload = io.netty.buffer.Unpooled + .copiedBuffer(bytes); + GenericAvroRecord genericRecord = (GenericAvroRecord) GenericAvroSchema.of(schemaInfo).decode(bytes); + Object fieldValue = genericRecord.getAvroRecord().get("rowField"); + + Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); + RowType columnType = RowType.from(ImmutableList.builder() + .add(RowType.field("stringField", VARCHAR)) + .add(RowType.field("intField", INTEGER)) + .add(RowType.field("nestedRow", RowType.from(ImmutableList.builder() + .add(RowType.field("stringField", VARCHAR)) + .add(RowType.field("longField", BIGINT)) + .build()))) + .build()); + + PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "rowField", columnType, false, false, "rowField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + + checkRowValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); + } + + @Test + public void testArray() { + DecoderTestMessage message = new DecoderTestMessage(); + message.arrayField = Arrays.asList("message_1", "message_2", "message_3"); + + byte[] bytes = schema.encode(message); + ByteBuf payload = io.netty.buffer.Unpooled + .copiedBuffer(bytes); + GenericAvroRecord genericRecord = (GenericAvroRecord) GenericAvroSchema.of(schemaInfo).decode(bytes); + Object fieldValue = genericRecord.getAvroRecord().get("arrayField"); + Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); + + ArrayType columnType = new ArrayType(VARCHAR); + PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "arrayField", columnType, false, false, "arrayField", + null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + + checkArrayValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); + } + + @Test + public void testMap() { + + DecoderTestMessage message = new DecoderTestMessage(); + message.mapField = new HashMap() {{ + put("key1", 2L); + put("key2", 22L); + }}; + + byte[] bytes = schema.encode(message); + ByteBuf payload = io.netty.buffer.Unpooled + .copiedBuffer(bytes); + GenericAvroRecord genericRecord = (GenericAvroRecord) GenericAvroSchema.of(schemaInfo).decode(bytes); + Object fieldValue = genericRecord.getAvroRecord().get("mapField"); + + Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); + Type columnType = decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, + ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), + TypeSignatureParameter.typeParameter(BigintType.BIGINT.getTypeSignature()))); + + PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "mapField", columnType, false, false, + "mapField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkMapValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); + } + + @Test + public void testCompositeType() { + DecoderTestMessage message = new DecoderTestMessage(); + + DecoderTestMessage.NestedRow nestedRow = new DecoderTestMessage.NestedRow(); + nestedRow.longField = 222L; + nestedRow.stringField = "message_2_nestedRow"; + + DecoderTestMessage.CompositeRow compositeRow = new DecoderTestMessage.CompositeRow(); + DecoderTestMessage.NestedRow nestedRow1 = new DecoderTestMessage.NestedRow(); + nestedRow1.longField = 2; + nestedRow1.stringField = "nestedRow_1"; + DecoderTestMessage.NestedRow nestedRow2 = new DecoderTestMessage.NestedRow(); + nestedRow2.longField = 2; + nestedRow2.stringField = "nestedRow_2"; + compositeRow.arrayField = Arrays.asList(nestedRow1, nestedRow2); + compositeRow.stringField = "compositeRow_1"; + + compositeRow.mapField = new HashMap() {{ + put("key1", nestedRow1); + put("key2", nestedRow2); + }}; + compositeRow.nestedRow = nestedRow; + + new HashMap() {{ + put("key1_1", 2L); + put("key1_2", 22L); + }}; + compositeRow.structedField = new HashMap>() {{ + put("key2_1", Arrays.asList(2L, 3L)); + put("key2_2", Arrays.asList(2L, 3L)); + put("key2_3", Arrays.asList(2L, 3L)); + }}; + + + message.compositeRow = compositeRow; + byte[] bytes = schema.encode(message); + ByteBuf payload = io.netty.buffer.Unpooled + .copiedBuffer(bytes); + GenericAvroRecord genericRecord = (GenericAvroRecord) GenericAvroSchema.of(schemaInfo).decode(bytes); + Object fieldValue = genericRecord.getAvroRecord().get("compositeRow"); + + Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); + + RowType columnType = RowType.from(ImmutableList.builder() + .add(RowType.field("stringField", VARCHAR)) + .add(RowType.field("arrayField", new ArrayType( + RowType.from(ImmutableList.builder() + .add(RowType.field("stringField", VARCHAR)) + .add(RowType.field("longField", BIGINT)) + .build())))) + .add(RowType.field("mapField", decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, + ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), + TypeSignatureParameter.typeParameter(RowType.from(ImmutableList.builder() + .add(RowType.field("stringField", VARCHAR)) + .add(RowType.field("longField", BIGINT)) + .build()).getTypeSignature()) + )))) + .add(RowType.field("nestedRow", RowType.from(ImmutableList.builder() + .add(RowType.field("stringField", VARCHAR)) + .add(RowType.field("longField", BIGINT)) + .build()))) + .add(RowType.field("structedField", + decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, + ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), + TypeSignatureParameter.typeParameter(new ArrayType(BIGINT).getTypeSignature()))))) + .build()); + + PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "compositeRow", columnType, false, false, "compositeRow", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + + checkRowValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); + } +} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/JsonDecoderTestUtil.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/JsonDecoderTestUtil.java new file mode 100644 index 0000000000000..f522a787b9fa6 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/JsonDecoderTestUtil.java @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Iterators; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.type.*; +import org.apache.pulsar.sql.presto.decoder.DecoderTestUtil; + +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; + +import static io.prestosql.spi.type.VarcharType.VARCHAR; +import static java.lang.String.format; +import static org.testng.Assert.*; + +/** + * + * TestUtil for JsonDecoder + */ +public class JsonDecoderTestUtil extends DecoderTestUtil { + + public JsonDecoderTestUtil() { + super(); + } + + @Override + public void checkPrimitiveValue(Object actual, Object expected) { + assertTrue(expected instanceof JsonNode); + if (actual == null || null == expected) { + assertNull(expected); + assertNull(actual); + } else if (actual instanceof CharSequence) { + assertEquals(actual.toString(), ((JsonNode) expected).asText()); + } else if (actual instanceof SqlVarbinary) { + try { + assertEquals(((SqlVarbinary) actual).getBytes(), ((JsonNode) expected).binaryValue()); + } catch (IOException e) { + fail(format("JsonNode %s formate binary Value failed", ((JsonNode) expected).getNodeType().name())); + } + } else if (isIntegralType(actual)) { + assertEquals(((Number) actual).longValue(), ((JsonNode) expected).asLong()); + } else if (isRealType(actual)) { + assertEquals(((Number) actual).doubleValue(), ((JsonNode) expected).asDouble()); + } else { + assertEquals(actual, expected); + } + } + + @Override + public void checkMapValues(Block block, Type type, Object value) { + assertNotNull(type, "Type is null"); + assertTrue(type instanceof MapType, "Unexpected type"); + assertTrue(((MapType) type).getKeyType() instanceof VarcharType, "Unexpected key type"); + assertNotNull(block, "Block is null"); + assertNotNull(value, "Value is null"); + + assertTrue(value instanceof ObjectNode, "map node isn't ObjectNode type"); + + ObjectNode expected = (ObjectNode) value; + + Iterator> fields = expected.fields(); + + assertEquals(block.getPositionCount(), Iterators.size(expected.fields()) * 2); + Type valueType = ((MapType) type).getValueType(); + if (valueType instanceof ArrayType) { + for (int index = 0; index < block.getPositionCount(); index += 2) { + String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); + assertTrue(Iterators.any(fields, entry -> entry.getKey().equals(actualKey))); + if (block.isNull(index + 1)) { + assertNull(expected.get(actualKey)); + continue; + } + Block arrayBlock = block.getObject(index + 1, Block.class); + checkArrayValues(arrayBlock, valueType, expected.get(actualKey)); + } + } else if (valueType instanceof MapType) { + for (int index = 0; index < block.getPositionCount(); index += 2) { + String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); + assertTrue(Iterators.any(fields, entry -> entry.getKey().equals(actualKey))); + if (block.isNull(index + 1)) { + assertNull(expected.get(actualKey)); + continue; + } + Block mapBlock = block.getObject(index + 1, Block.class); + checkMapValues(mapBlock, valueType, expected.get(actualKey)); + } + } else if (valueType instanceof RowType) { + for (int index = 0; index < block.getPositionCount(); index += 2) { + String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); + assertTrue(Iterators.any(fields, entry -> entry.getKey().equals(actualKey))); + + if (block.isNull(index + 1)) { + assertNull(expected.get(actualKey)); + continue; + } + Block rowBlock = block.getObject(index + 1, Block.class); + checkRowValues(rowBlock, valueType, expected.get(actualKey)); + } + } else { + for (int index = 0; index < block.getPositionCount(); index += 2) { + String actualKey = VARCHAR.getSlice(block, index).toStringUtf8(); + Map.Entry entry = Iterators.tryFind(fields, e -> e.getKey().equals(actualKey)).get(); + assertNotNull(entry); + assertNotNull(entry.getKey()); + checkPrimitiveValue(getObjectValue(valueType, block, index + 1), entry.getValue()); + } + } + } + + @Override + public void checkRowValues(Block block, Type type, Object value) { + assertNotNull(type, "Type is null"); + assertTrue(type instanceof RowType, "Unexpected type"); + assertNotNull(block, "Block is null"); + assertNotNull(value, "Value is null"); + + ObjectNode record = (ObjectNode) value; + RowType rowType = (RowType) type; + assertEquals(Iterators.size(record.fields()), rowType.getFields().size(), "Json field size mismatch"); + assertEquals(block.getPositionCount(), rowType.getFields().size(), "Presto type field size mismatch"); + for (int fieldIndex = 0; fieldIndex < rowType.getFields().size(); fieldIndex++) { + RowType.Field rowField = rowType.getFields().get(fieldIndex); + Object expectedValue = record.get(rowField.getName().get()); + if (block.isNull(fieldIndex)) { + assertNull(expectedValue); + continue; + } + checkField(block, rowField.getType(), fieldIndex, expectedValue); + } + } + + @Override + public void checkArrayValues(Block block, Type type, Object value) { + assertNotNull(type, "Type is null"); + assertTrue(type instanceof ArrayType, "Unexpected type"); + assertNotNull(block, "Block is null"); + assertNotNull(value, "Value is null"); + + assertTrue(value instanceof ArrayNode, "Array node isn't ArrayNode type"); + ArrayNode arrayNode = (ArrayNode) value; + + assertEquals(block.getPositionCount(), arrayNode.size()); + Type elementType = ((ArrayType) type).getElementType(); + if (elementType instanceof ArrayType) { + for (int index = 0; index < block.getPositionCount(); index++) { + if (block.isNull(index)) { + assertNull(arrayNode.get(index)); + continue; + } + Block arrayBlock = block.getObject(index, Block.class); + checkArrayValues(arrayBlock, elementType, arrayNode.get(index)); + } + } else if (elementType instanceof MapType) { + for (int index = 0; index < block.getPositionCount(); index++) { + if (block.isNull(index)) { + assertNull(arrayNode.get(index)); + continue; + } + Block mapBlock = block.getObject(index, Block.class); + checkMapValues(mapBlock, elementType, arrayNode.get(index)); + } + } else if (elementType instanceof RowType) { + for (int index = 0; index < block.getPositionCount(); index++) { + if (block.isNull(index)) { + assertNull(arrayNode.get(index)); + continue; + } + Block rowBlock = block.getObject(index, Block.class); + checkRowValues(rowBlock, elementType, arrayNode.get(index)); + } + } else { + for (int index = 0; index < block.getPositionCount(); index++) { + checkPrimitiveValue(getObjectValue(elementType, block, index), arrayNode.get(index)); + } + } + } + +} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java new file mode 100644 index 0000000000000..ace6ca56e887b --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java @@ -0,0 +1,268 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder.json; + +import com.google.common.collect.ImmutableList; +import io.netty.buffer.ByteBuf; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; +import io.prestosql.spi.type.*; +import org.apache.pulsar.client.impl.schema.JSONSchema; +import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; +import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; +import org.apache.pulsar.sql.presto.PulsarColumnHandle; +import org.apache.pulsar.sql.presto.decoder.AbstractDecoderTester; +import org.apache.pulsar.sql.presto.decoder.DecoderTestMessage; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.util.*; + +import static io.prestosql.spi.type.BigintType.BIGINT; +import static io.prestosql.spi.type.BooleanType.BOOLEAN; +import static io.prestosql.spi.type.DoubleType.DOUBLE; +import static io.prestosql.spi.type.IntegerType.INTEGER; +import static io.prestosql.spi.type.RealType.REAL; +import static io.prestosql.spi.type.VarcharType.VARCHAR; +import static java.lang.Float.floatToIntBits; +import static org.apache.pulsar.sql.presto.TestPulsarConnector.getColumnColumnHandles; +import static org.apache.pulsar.sql.presto.TestPulsarConnector.getPulsarConnectorId; +import static org.testng.Assert.assertTrue; + +public class TestJsonDecoder extends AbstractDecoderTester { + + private JSONSchema schema; + + @BeforeMethod + public void init() { + super.init(); + schema = JSONSchema.of(DecoderTestMessage.class); + schemaInfo = schema.getSchemaInfo(); + pulsarColumnHandle = getColumnColumnHandles(topicName, schemaInfo, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); + pulsarRowDecoder = decoderFactory.createRowDecoder(topicName, schemaInfo, new HashSet<>(pulsarColumnHandle)); + decoderTestUtil = new JsonDecoderTestUtil(); + assertTrue(pulsarRowDecoder instanceof PulsarJsonRowDecoder); + } + + @Test + public void testPrimitiveType() { + DecoderTestMessage message = new DecoderTestMessage(); + message.stringField = "message_1"; + message.intField = 22; + message.floatField = 2.2f; + message.doubleField = 22.20D; + message.booleanField = true; + message.longField = 222L; + message.timestampField = System.currentTimeMillis(); + message.enumField = DecoderTestMessage.TestEnum.TEST_ENUM_2; + + LocalTime now = LocalTime.now(ZoneId.systemDefault()); + message.timeField = now.toSecondOfDay() * 1000; + + LocalDate localDate = LocalDate.now(); + LocalDate epoch = LocalDate.ofEpochDay(0); + message.dateField = Math.toIntExact(ChronoUnit.DAYS.between(epoch, localDate)); + + ByteBuf payload = io.netty.buffer.Unpooled + .copiedBuffer(schema.encode(message)); + Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); + + PulsarColumnHandle stringFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "stringField", VARCHAR, false, false, "stringField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, stringFieldColumnHandle, message.stringField); + + PulsarColumnHandle intFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "intField", INTEGER, false, false, "intField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, intFieldColumnHandle, message.intField); + + PulsarColumnHandle floatFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "floatField", REAL, false, false, "floatField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, floatFieldColumnHandle, floatToIntBits(message.floatField)); + + PulsarColumnHandle doubleFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "doubleField", DOUBLE, false, false, "doubleField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, doubleFieldColumnHandle, message.doubleField); + + PulsarColumnHandle booleanFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "booleanField", BOOLEAN, false, false, "booleanField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, booleanFieldColumnHandle, message.booleanField); + + PulsarColumnHandle longFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "longField", BIGINT, false, false, "longField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, longFieldColumnHandle, message.longField); + + PulsarColumnHandle enumFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "enumField", VARCHAR, false, false, "enumField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, enumFieldColumnHandle, message.enumField.toString()); + + } + + @Test + public void testArray() { + DecoderTestMessage message = new DecoderTestMessage(); + message.arrayField = Arrays.asList("message_1", "message_2", "message_3"); + + byte[] bytes = schema.encode(message); + ByteBuf payload = io.netty.buffer.Unpooled + .copiedBuffer(bytes); + + GenericJsonRecord genericRecord = (GenericJsonRecord) GenericJsonSchema.of(schemaInfo).decode(bytes); + Object fieldValue = genericRecord.getJsonNode().get("arrayField"); + Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); + + ArrayType columnType = new ArrayType(VARCHAR); + PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "arrayField", columnType, false, false, "arrayField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + + checkArrayValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); + } + + @Test + public void testRow() { + DecoderTestMessage message = new DecoderTestMessage(); + message.stringField = "message_2"; + DecoderTestMessage.TestRow testRow = new DecoderTestMessage.TestRow(); + message.rowField = testRow; + testRow.intField = 22; + testRow.stringField = "message_2_testRow"; + DecoderTestMessage.NestedRow nestedRow = new DecoderTestMessage.NestedRow(); + nestedRow.longField = 222L; + nestedRow.stringField = "message_2_nestedRow"; + testRow.nestedRow = nestedRow; + + byte[] bytes = schema.encode(message); + ByteBuf payload = io.netty.buffer.Unpooled + .copiedBuffer(bytes); + + GenericJsonRecord genericRecord = (GenericJsonRecord) GenericJsonSchema.of(schemaInfo).decode(bytes); + Object fieldValue = genericRecord.getJsonNode().get("rowField"); + Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); + + RowType columnType = RowType.from(ImmutableList.builder() + .add(RowType.field("stringField", VARCHAR)) + .add(RowType.field("intField", INTEGER)) + .add(RowType.field("nestedRow", RowType.from(ImmutableList.builder() + .add(RowType.field("stringField", VARCHAR)) + .add(RowType.field("longField", BIGINT)) + .build()))) + .build()); + + PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "rowField", columnType, false, false, "rowField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + + checkRowValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); + + } + + @Test + public void testMap() { + DecoderTestMessage message = new DecoderTestMessage(); + message.mapField = new HashMap() {{ + put("key1", 2L); + put("key2", 22L); + }}; + + byte[] bytes = schema.encode(message); + ByteBuf payload = io.netty.buffer.Unpooled + .copiedBuffer(bytes); + + GenericJsonRecord genericRecord = (GenericJsonRecord) GenericJsonSchema.of(schemaInfo).decode(bytes); + Object fieldValue = genericRecord.getJsonNode().get("mapField"); + Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); + + Type columnType = decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(BigintType.BIGINT.getTypeSignature()))); + PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "mapField", columnType, false, false, "mapField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + + checkMapValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); + + } + + @Test + public void testCompositeType() { + DecoderTestMessage message = new DecoderTestMessage(); + + DecoderTestMessage.NestedRow nestedRow = new DecoderTestMessage.NestedRow(); + nestedRow.longField = 222L; + nestedRow.stringField = "message_2_nestedRow"; + + DecoderTestMessage.CompositeRow compositeRow = new DecoderTestMessage.CompositeRow(); + DecoderTestMessage.NestedRow nestedRow1 = new DecoderTestMessage.NestedRow(); + nestedRow1.longField = 2; + nestedRow1.stringField = "nestedRow_1"; + DecoderTestMessage.NestedRow nestedRow2 = new DecoderTestMessage.NestedRow(); + nestedRow2.longField = 2; + nestedRow2.stringField = "nestedRow_2"; + compositeRow.arrayField = Arrays.asList(nestedRow1, nestedRow2); + compositeRow.stringField = "compositeRow_1"; + + compositeRow.mapField = new HashMap() {{ + put("key1", nestedRow1); + put("key2", nestedRow2); + }}; + compositeRow.nestedRow = nestedRow; + new HashMap() {{ + put("key1_1", 2L); + put("key1_2", 22L); + }}; + compositeRow.structedField = new HashMap>() {{ + put("key2_1", Arrays.asList(2L, 3L)); + put("key2_2", Arrays.asList(2L, 3L)); + put("key2_3", Arrays.asList(2L, 3L)); + }}; + message.compositeRow = compositeRow; + + byte[] bytes = schema.encode(message); + ByteBuf payload = io.netty.buffer.Unpooled + .copiedBuffer(bytes); + GenericJsonRecord genericRecord = (GenericJsonRecord) GenericJsonSchema.of(schemaInfo).decode(bytes); + Object fieldValue = genericRecord.getJsonNode().get("compositeRow"); + Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); + + RowType columnType = RowType.from(ImmutableList.builder() + .add(RowType.field("stringField", VARCHAR)) + .add(RowType.field("arrayField", new ArrayType( + RowType.from(ImmutableList.builder() + .add(RowType.field("stringField", VARCHAR)) + .add(RowType.field("longField", BIGINT)) + .build())))) + .add(RowType.field("mapField", decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, + ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), + TypeSignatureParameter.typeParameter(RowType.from(ImmutableList.builder() + .add(RowType.field("stringField", VARCHAR)) + .add(RowType.field("longField", BIGINT)) + .build()).getTypeSignature()) + )))) + .add(RowType.field("nestedRow", RowType.from(ImmutableList.builder() + .add(RowType.field("stringField", VARCHAR)) + .add(RowType.field("longField", BIGINT)) + .build()))) + .add(RowType.field("structedField", + decoderFactory.getTypeManager().getParameterizedType(StandardTypes.MAP, + ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), + TypeSignatureParameter.typeParameter(new ArrayType(BIGINT).getTypeSignature()))))) + .build()); + + PulsarColumnHandle columnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "compositeRow", columnType, false, false, "compositeRow", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + + checkRowValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); + } + +} From 4d2755b254bb3e3f0e5759d726ee2a4fe2d68e08 Mon Sep 17 00:00:00 2001 From: wangguowei Date: Mon, 9 Nov 2020 19:40:03 +0800 Subject: [PATCH 03/19] fix Pair bug --- .../presto/decoder/json/PulsarJsonFieldDecoder.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java index cfb8732742c6c..04c434cd0a8ff 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java @@ -31,7 +31,7 @@ import io.prestosql.spi.block.Block; import io.prestosql.spi.block.BlockBuilder; import io.prestosql.spi.type.*; -import javafx.util.Pair; +import org.apache.commons.lang3.tuple.Pair; import java.util.Iterator; import java.util.List; @@ -87,16 +87,16 @@ public PulsarJsonFieldDecoder(DecoderColumnHandle columnHandle) { private static Pair getNumRangeByType(Type type) { if (type == TINYINT) { - return new Pair((long) Byte.MIN_VALUE, (long) Byte.MAX_VALUE); + return Pair.of((long) Byte.MIN_VALUE, (long) Byte.MAX_VALUE); } else if (type == SMALLINT) { - return new Pair((long) Short.MIN_VALUE, (long) Short.MAX_VALUE); + return Pair.of((long) Short.MIN_VALUE, (long) Short.MAX_VALUE); } else if (type == INTEGER) { - return new Pair((long) Integer.MIN_VALUE, (long) Integer.MAX_VALUE); + return Pair.of((long) Integer.MIN_VALUE, (long) Integer.MAX_VALUE); } else if (type == BIGINT) { - return new Pair((long) Long.MIN_VALUE, (long) Long.MAX_VALUE); + return Pair.of((long) Long.MIN_VALUE, (long) Long.MAX_VALUE); } else { // those values will not be used if column type is not one of mentioned above - return new Pair(Long.MIN_VALUE, Long.MAX_VALUE); + return Pair.of(Long.MIN_VALUE, Long.MAX_VALUE); } } From 98d5ea3970e6cd32525bfd22b9723826808ec595 Mon Sep 17 00:00:00 2001 From: wangguowei Date: Tue, 10 Nov 2020 20:37:16 +0800 Subject: [PATCH 04/19] codestyle fix --- .../pulsar/sql/presto/PulsarColumnHandle.java | 20 ++++---- .../sql/presto/PulsarColumnMetadata.java | 6 +-- .../sql/presto/PulsarConnectorModule.java | 8 ++-- .../sql/presto/PulsarInternalColumn.java | 38 ++++++++------- .../pulsar/sql/presto/PulsarMetadata.java | 35 +++++++------- .../pulsar/sql/presto/PulsarRecordCursor.java | 48 +++++++++++-------- .../pulsar/sql/presto/PulsarRowDecoder.java | 6 +-- .../sql/presto/decoder/avro/package-info.java | 22 +++++++++ .../decoder/json/PulsarJsonFieldDecoder.java | 44 ++++++++++------- .../sql/presto/decoder/json/package-info.java | 22 +++++++++ .../primitive/PulsarPrimitiveRowDecoder.java | 15 ++++-- .../PulsarPrimitiveRowDecoderFactory.java | 17 ++++--- .../decoder/primitive/package-info.java | 22 +++++++++ 13 files changed, 200 insertions(+), 103 deletions(-) create mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/package-info.java create mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/package-info.java create mode 100644 pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/package-info.java diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnHandle.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnHandle.java index e751d7a878b3e..0beebb7d80f1f 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnHandle.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnHandle.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.sql.presto; +import static java.util.Objects.requireNonNull; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -27,8 +29,6 @@ import java.util.Objects; -import static java.util.Objects.requireNonNull; - /** * This class represents the basic information about a presto column. */ @@ -60,16 +60,16 @@ public class PulsarColumnHandle implements DecoderColumnHandle { private HandleKeyValueType handleKeyValueType; /** - * {@link org.apache.pulsar.sql.presto.PulsarColumnMetadata.DecoderExtraInfo#mapping} + * {@link org.apache.pulsar.sql.presto.PulsarColumnMetadata.DecoderExtraInfo#mapping}. */ private String mapping; /** - * {@link org.apache.pulsar.sql.presto.PulsarColumnMetadata.DecoderExtraInfo#dataFormat} + * {@link org.apache.pulsar.sql.presto.PulsarColumnMetadata.DecoderExtraInfo#dataFormat}. */ private String dataFormat; /** - * {@link org.apache.pulsar.sql.presto.PulsarColumnMetadata.DecoderExtraInfo#formatHint} + * {@link org.apache.pulsar.sql.presto.PulsarColumnMetadata.DecoderExtraInfo#formatHint}. */ private String formatHint; @@ -78,7 +78,6 @@ public class PulsarColumnHandle implements DecoderColumnHandle { */ public enum HandleKeyValueType { /** - * * The handle not for keyValue schema. */ NONE, @@ -108,9 +107,9 @@ public PulsarColumnHandle( this.type = requireNonNull(type, "type is null"); this.hidden = hidden; this.internal = internal; - this.mapping=mapping; - this.dataFormat=dataFormat; - this.formatHint=formatHint; + this.mapping = mapping; + this.dataFormat = dataFormat; + this.formatHint = formatHint; if (handleKeyValueType == null) { this.handleKeyValueType = HandleKeyValueType.NONE; } else { @@ -174,7 +173,8 @@ public boolean isValue() { } ColumnMetadata getColumnMetadata() { - return new PulsarColumnMetadata(name, type, null, null, hidden, internal, handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(mapping, dataFormat, formatHint)); + return new PulsarColumnMetadata(name, type, null, null, hidden, + internal, handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(mapping, dataFormat, formatHint)); } @Override diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java index c126c98ebf05a..be2e437e1b6e4 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java @@ -34,8 +34,7 @@ public class PulsarColumnMetadata extends ColumnMetadata { private PulsarColumnHandle.HandleKeyValueType handleKeyValueType; public final static String KEY_SCHEMA_COLUMN_PREFIX = "__key."; - - private DecoderExtraInfo decoderExtraInfo ; + private DecoderExtraInfo decoderExtraInfo; public PulsarColumnMetadata(String name, Type type, String comment, String extraInfo, boolean hidden, boolean isInternal, @@ -123,7 +122,8 @@ public int hashCode() { /** - * Decoder ExtraInfo for {@link org.apache.pulsar.sql.presto.PulsarColumnHandle} used by {@link io.prestosql.decoder.RowDecoder} + * Decoder ExtraInfo for {@link org.apache.pulsar.sql.presto.PulsarColumnHandle} + * used by {@link io.prestosql.decoder.RowDecoder}. */ public static class DecoderExtraInfo { diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorModule.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorModule.java index ff2a638fd3b22..901a85e3e2db4 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorModule.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorModule.java @@ -18,6 +18,10 @@ */ package org.apache.pulsar.sql.presto; +import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.airlift.json.JsonBinder.jsonBinder; +import static java.util.Objects.requireNonNull; + import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; import com.google.inject.Binder; @@ -30,10 +34,6 @@ import javax.inject.Inject; -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonBinder.jsonBinder; -import static java.util.Objects.requireNonNull; - /** * This class defines binding of classes in the Presto connector. */ diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java index a8ebb7b602bb7..f90b0896cb52b 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java @@ -20,7 +20,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import io.prestosql.spi.type.*; +import io.prestosql.spi.type.BigintType; +import io.prestosql.spi.type.IntegerType; +import io.prestosql.spi.type.TimestampType; +import io.prestosql.spi.type.VarcharType; +import io.prestosql.spi.type.Type; import java.util.Map; import java.util.Set; @@ -36,32 +40,32 @@ public class PulsarInternalColumn { - public static final PulsarInternalColumn PARTITION = new PulsarInternalColumn("__partition__", IntegerType.INTEGER, - "The partition number which the message belongs to"); + public static final PulsarInternalColumn PARTITION = new PulsarInternalColumn("__partition__", + IntegerType.INTEGER, "The partition number which the message belongs to"); - public static final PulsarInternalColumn EVENT_TIME = new PulsarInternalColumn("__event_time__", TimestampType - .TIMESTAMP, "Application defined timestamp in milliseconds of when the event occurred"); + public static final PulsarInternalColumn EVENT_TIME = new PulsarInternalColumn("__event_time__", + TimestampType.TIMESTAMP, "Application defined timestamp in milliseconds of when the event occurred"); public static final PulsarInternalColumn PUBLISH_TIME = new PulsarInternalColumn("__publish_time__", TimestampType.TIMESTAMP, "The timestamp in milliseconds of when event as published"); - public static final PulsarInternalColumn MESSAGE_ID = new PulsarInternalColumn("__message_id__", VarcharType.VARCHAR, - "The message ID of the message used to generate this row"); + public static final PulsarInternalColumn MESSAGE_ID = new PulsarInternalColumn("__message_id__", + VarcharType.VARCHAR,"The message ID of the message used to generate this row"); - public static final PulsarInternalColumn SEQUENCE_ID = new PulsarInternalColumn("__sequence_id__", BigintType.BIGINT, - "The sequence ID of the message used to generate this row"); + public static final PulsarInternalColumn SEQUENCE_ID = new PulsarInternalColumn("__sequence_id__", + BigintType.BIGINT,"The sequence ID of the message used to generate this row"); - public static final PulsarInternalColumn PRODUCER_NAME = new PulsarInternalColumn("__producer_name__", VarcharType - .VARCHAR, "The name of the producer that publish the message used to generate this row"); + public static final PulsarInternalColumn PRODUCER_NAME = new PulsarInternalColumn("__producer_name__", + VarcharType.VARCHAR, "The name of the producer that publish the message used to generate this row"); - public static final PulsarInternalColumn KEY = new PulsarInternalColumn("__key__", VarcharType.VARCHAR, "The partition key " - + "for the topic"); + public static final PulsarInternalColumn KEY = new PulsarInternalColumn("__key__", + VarcharType.VARCHAR, "The partition key for the topic"); - public static final PulsarInternalColumn PROPERTIES = new PulsarInternalColumn("__properties__", VarcharType.VARCHAR, - "User defined properties"); + public static final PulsarInternalColumn PROPERTIES = new PulsarInternalColumn("__properties__", + VarcharType.VARCHAR, "User defined properties"); - private static Set internalFields = ImmutableSet.of(PARTITION, EVENT_TIME, PUBLISH_TIME, MESSAGE_ID, SEQUENCE_ID, PRODUCER_NAME, KEY, - PROPERTIES); + private static Set internalFields = ImmutableSet.of(PARTITION, EVENT_TIME, PUBLISH_TIME, + MESSAGE_ID, SEQUENCE_ID, PRODUCER_NAME, KEY, PROPERTIES); private final String name; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java index ea4a45e32e6eb..c2dff11a38596 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java @@ -18,6 +18,14 @@ */ package org.apache.pulsar.sql.presto; +import static io.prestosql.spi.StandardErrorCode.NOT_FOUND; +import static io.prestosql.spi.StandardErrorCode.QUERY_REJECTED; +import static java.util.Objects.requireNonNull; +import static org.apache.pulsar.sql.presto.PulsarConnectorUtils.restoreNamespaceDelimiterIfNeeded; +import static org.apache.pulsar.sql.presto.PulsarConnectorUtils.rewriteNamespaceDelimiterIfNeeded; +import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertColumnHandle; +import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertTableHandle; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airlift.log.Logger; @@ -37,13 +45,6 @@ import java.util.*; import java.util.stream.Collectors; -import static io.prestosql.spi.StandardErrorCode.NOT_FOUND; -import static io.prestosql.spi.StandardErrorCode.QUERY_REJECTED; -import static java.util.Objects.requireNonNull; -import static org.apache.pulsar.sql.presto.PulsarConnectorUtils.restoreNamespaceDelimiterIfNeeded; -import static org.apache.pulsar.sql.presto.PulsarConnectorUtils.rewriteNamespaceDelimiterIfNeeded; -import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertColumnHandle; -import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertTableHandle; /** * This connector helps to work with metadata. @@ -62,7 +63,8 @@ public class PulsarMetadata implements ConnectorMetadata { private static final Logger log = Logger.get(PulsarMetadata.class); @Inject - public PulsarMetadata(PulsarConnectorId connectorId, PulsarConnectorConfig pulsarConnectorConfig, PulsarDispatchingRowDecoderFactory decoderFactory) { + public PulsarMetadata(PulsarConnectorId connectorId, PulsarConnectorConfig pulsarConnectorConfig, + PulsarDispatchingRowDecoderFactory decoderFactory) { this.decoderFactory = decoderFactory; this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); this.pulsarConnectorConfig = pulsarConnectorConfig; @@ -188,7 +190,8 @@ public Map getColumnHandles(ConnectorSession session, Conn pulsarColumnMetadata.isHidden(), pulsarColumnMetadata.isInternal(), pulsarColumnMetadata.getDecoderExtraInfo().getMapping(), - pulsarColumnMetadata.getDecoderExtraInfo().getDataFormat(),pulsarColumnMetadata.getDecoderExtraInfo().getFormatHint(), + pulsarColumnMetadata.getDecoderExtraInfo().getDataFormat(), + pulsarColumnMetadata.getDecoderExtraInfo().getFormatHint(), pulsarColumnMetadata.getHandleKeyValueType()); columnHandles.put( @@ -306,13 +309,13 @@ List getPulsarColumns(TopicName topicName, SchemaInfo schemaInfo, boolean withInternalColumns, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { - SchemaType schemaType = schemaInfo.getType(); - if (schemaType.isStruct()|| schemaType.isPrimitive()) { - return getPulsarColumnsFromSchema(topicName, schemaInfo, withInternalColumns, handleKeyValueType); - } else if (schemaType.equals(SchemaType.KEY_VALUE)) { - return getPulsarColumnsFromKeyValueSchema(topicName, schemaInfo, withInternalColumns); - } else { - throw new IllegalArgumentException("Unsupported schema : " + schemaInfo); + SchemaType schemaType = schemaInfo.getType(); + if (schemaType.isStruct() || schemaType.isPrimitive()) { + return getPulsarColumnsFromSchema(topicName, schemaInfo, withInternalColumns, handleKeyValueType); + } else if (schemaType.equals(SchemaType.KEY_VALUE)) { + return getPulsarColumnsFromKeyValueSchema(topicName, schemaInfo, withInternalColumns); + } else { + throw new IllegalArgumentException("Unsupported schema : " + schemaInfo); } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index 29d52eaeec9ba..6d2661db97212 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -18,6 +18,19 @@ */ package org.apache.pulsar.sql.presto; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static io.prestosql.decoder.FieldValueProviders.bytesValueProvider; +import static io.prestosql.decoder.FieldValueProviders.longValueProvider; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicLong; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; @@ -43,18 +56,6 @@ import org.jctools.queues.MessagePassingQueue; import org.jctools.queues.SpscArrayQueue; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicLong; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static io.prestosql.decoder.FieldValueProviders.bytesValueProvider; -import static io.prestosql.decoder.FieldValueProviders.longValueProvider; /** * Implementation of a cursor to read records. @@ -96,7 +97,8 @@ public class PulsarRecordCursor implements RecordCursor { private static final Logger log = Logger.get(PulsarRecordCursor.class); public PulsarRecordCursor(List columnHandles, PulsarSplit pulsarSplit, - PulsarConnectorConfig pulsarConnectorConfig, PulsarDispatchingRowDecoderFactory decoderFactory) { + PulsarConnectorConfig pulsarConnectorConfig, + PulsarDispatchingRowDecoderFactory decoderFactory) { this.splitSize = pulsarSplit.getSplitSize(); // Set start time for split this.startTime = System.nanoTime(); @@ -127,7 +129,7 @@ public PulsarRecordCursor(List columnHandles, PulsarSplit pu // Exposed for testing purposes PulsarRecordCursor(List columnHandles, PulsarSplit pulsarSplit, PulsarConnectorConfig - pulsarConnectorConfig, ManagedLedgerFactory managedLedgerFactory, ManagedLedgerConfig managedLedgerConfig, + pulsarConnectorConfig, ManagedLedgerFactory managedLedgerFactory, ManagedLedgerConfig managedLedgerConfig, PulsarConnectorMetricsTracker pulsarConnectorMetricsTracker, PulsarDispatchingRowDecoderFactory decoderFactory) { this.splitSize = pulsarSplit.getSplitSize(); initialize(columnHandles, pulsarSplit, pulsarConnectorConfig, managedLedgerFactory, managedLedgerConfig, @@ -154,7 +156,7 @@ private void initialize(List columnHandles, PulsarSplit puls this.pulsarConnectorConfig = pulsarConnectorConfig; try { - this.schemaInfoProvider = new PulsarSqlSchemaInfoProvider( this.topicName, pulsarConnectorConfig.getPulsarAdmin()); + this.schemaInfoProvider = new PulsarSqlSchemaInfoProvider(this.topicName, pulsarConnectorConfig.getPulsarAdmin()); } catch (PulsarClientException e) { log.error(e, "Failed to init Pulsar SchemaInfo Provider"); throw new RuntimeException(e); @@ -435,7 +437,7 @@ public boolean advanceNextPosition() { Map currentRowValuesMap = new HashMap<>(); - if(schemaInfo.getType().equals(SchemaType.KEY_VALUE) ){ + if (schemaInfo.getType().equals(SchemaType.KEY_VALUE)) { PulsarRowDecoder keyDecoder = decoderFactory.createRowDecoder(topicName, schemaInfo, @@ -474,11 +476,14 @@ public boolean advanceNextPosition() { if (PulsarInternalColumn.PARTITION.getName().equals(columnHandle.getName())) { currentRowValuesMap.put(columnHandle, longValueProvider(this.partition)); } else if (PulsarInternalColumn.EVENT_TIME.getName().equals(columnHandle.getName())) { - currentRowValuesMap.put(columnHandle, PulsarFieldValueProviders.timeValueProvider(this.currentMessage.getEventTime(), this.currentMessage.getPublishTime() == 0)); + currentRowValuesMap.put(columnHandle, PulsarFieldValueProviders.timeValueProvider( + this.currentMessage.getEventTime(), this.currentMessage.getPublishTime() == 0)); } else if (PulsarInternalColumn.PUBLISH_TIME.getName().equals(columnHandle.getName())) { - currentRowValuesMap.put(columnHandle, PulsarFieldValueProviders.timeValueProvider(this.currentMessage.getPublishTime(), this.currentMessage.getPublishTime() == 0)); + currentRowValuesMap.put(columnHandle, PulsarFieldValueProviders.timeValueProvider( + this.currentMessage.getPublishTime(), this.currentMessage.getPublishTime() == 0)); } else if (PulsarInternalColumn.MESSAGE_ID.getName().equals(columnHandle.getName())) { - currentRowValuesMap.put(columnHandle, bytesValueProvider(this.currentMessage.getMessageId().toString().getBytes())); + currentRowValuesMap.put(columnHandle, bytesValueProvider( + this.currentMessage.getMessageId().toString().getBytes())); } else if (PulsarInternalColumn.SEQUENCE_ID.getName().equals(columnHandle.getName())) { currentRowValuesMap.put(columnHandle, longValueProvider(this.currentMessage.getSequenceId())); } else if (PulsarInternalColumn.PRODUCER_NAME.getName().equals(columnHandle.getName())) { @@ -488,7 +493,8 @@ public boolean advanceNextPosition() { currentRowValuesMap.put(columnHandle, bytesValueProvider(key == null ? null : key.getBytes())); } else if (PulsarInternalColumn.PROPERTIES.getName().equals(columnHandle.getName())) { try { - currentRowValuesMap.put(columnHandle, bytesValueProvider(new ObjectMapper().writeValueAsBytes(this.currentMessage.getProperties()))); + currentRowValuesMap.put(columnHandle, bytesValueProvider( + new ObjectMapper().writeValueAsBytes(this.currentMessage.getProperties()))); } catch (JsonProcessingException e) { throw new RuntimeException(e); } @@ -545,7 +551,7 @@ public Object getObject(int field) { @Override public boolean isNull(int field) { - FieldValueProvider provider= currentRowValues[field]; + FieldValueProvider provider = currentRowValues[field]; return provider == null || provider.isNull(); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoder.java index fa53e847f44c5..602ec37f3fc50 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoder.java @@ -26,16 +26,16 @@ import java.util.Optional; /** - * RowDecoder interface for Pulsar + * RowDecoder interface for Pulsar. */ public interface PulsarRowDecoder { /** - * decode byteBuf to Map + * decode byteBuf to Column FieldValueProvider. * * @param byteBuf * @return */ - public Optional> decodeRow(ByteBuf byteBuf); + Optional> decodeRow(ByteBuf byteBuf); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/package-info.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/package-info.java new file mode 100644 index 0000000000000..57bf297819fa7 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * This package contains decoder for SchemaType.AVRO. + */ +package org.apache.pulsar.sql.presto.decoder.avro; \ No newline at end of file diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java index 04c434cd0a8ff..850c009cb01f3 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java @@ -61,12 +61,13 @@ import static java.util.Objects.requireNonNull; /** - * copy from {@link io.prestosql.decoder.json.DefaultJsonFieldDecoder} (presto-record-decoder-345) with some pulsar's extensions : - * 1) support array - * 2) support map - * 3) support row - * 3) support TIMESTAMP,DATE,TIME - * 4) support Real + * copy from {@link io.prestosql.decoder.json.DefaultJsonFieldDecoder} (presto-record-decoder-345) + * with some pulsar's extensions: + * 1) support array. + * 2) support map. + * 3) support row. + * 3) support TIMESTAMP,DATE,TIME. + * 4) support Real. */ public class PulsarJsonFieldDecoder implements JsonFieldDecoder { @@ -146,6 +147,9 @@ public FieldValueProvider decode(JsonNode value) { return new JsonValueProvider(value, columnHandle, minValue, maxValue); } + /** + * JsonValueProvider. + */ public static class JsonValueProvider extends FieldValueProvider { private final JsonNode value; @@ -167,22 +171,22 @@ public final boolean isNull() { @Override public boolean getBoolean() { - return getBoolean(value,columnHandle.getType(),columnHandle.getName()); + return getBoolean(value, columnHandle.getType(), columnHandle.getName()); } @Override public long getLong() { - return getLong(value,columnHandle.getType(),columnHandle.getName(),minValue,maxValue); + return getLong(value, columnHandle.getType(), columnHandle.getName(), minValue, maxValue); } @Override public double getDouble() { - return getDouble(value,columnHandle.getType(),columnHandle.getName()); + return getDouble(value, columnHandle.getType(), columnHandle.getName()); } @Override public Slice getSlice() { - return getSlice(value,columnHandle.getType(),columnHandle.getName()); + return getSlice(value, columnHandle.getType(), columnHandle.getName()); } @Override @@ -200,7 +204,7 @@ public static boolean getBoolean(JsonNode value, Type type, String columnName) { format("could not parse non-value node as '%s' for column '%s'", type, columnName)); } - public static long getLong(JsonNode value, Type type, String columnName,long minValue,long maxValue) { + public static long getLong(JsonNode value, Type type, String columnName, long minValue, long maxValue) { try { if (type instanceof RealType) { return floatToIntBits((Float) parseFloat(value.asText())); @@ -252,7 +256,6 @@ private static Slice getSlice(JsonNode value, Type type, String columnName) { return slice; } - private Block serializeObject(BlockBuilder builder, Object value, Type type, String columnName) { if (type instanceof ArrayType) { return serializeList(builder, value, type, columnName); @@ -276,7 +279,7 @@ private Block serializeList(BlockBuilder parentBlockBuilder, Object value, Type checkState(value instanceof ArrayNode, "Json array node must is ArrayNode type"); - Iterator jsonNodeIterator = ((ArrayNode)value).elements(); + Iterator jsonNodeIterator = ((ArrayNode) value).elements(); List typeParameters = type.getTypeParameters(); Type elementType = typeParameters.get(0); @@ -301,7 +304,8 @@ private void serializePrimitive(BlockBuilder blockBuilder, Object node, Type typ if (node instanceof JsonNode) { value = (JsonNode) node; } else { - throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("primitive object of '%s' as '%s' for column '%s' cann't convert to JsonNode", node.getClass(), type, columnName)); + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, + format("primitive object of '%s' as '%s' for column '%s' cann't convert to JsonNode", node.getClass(), type, columnName)); } if (value == null) { @@ -314,7 +318,9 @@ private void serializePrimitive(BlockBuilder blockBuilder, Object node, Type typ return; } - if (type instanceof RealType || type instanceof BigintType || type instanceof IntegerType || type instanceof SmallintType || type instanceof TinyintType || type instanceof TimestampType || type instanceof TimeType || type instanceof DateType) { + if (type instanceof RealType || type instanceof BigintType || type instanceof IntegerType || type instanceof SmallintType + || type instanceof TinyintType || type instanceof TimestampType || type instanceof TimeType + || type instanceof DateType) { Pair numRange = getNumRangeByType(type); type.writeLong(blockBuilder, getLong(value, type, columnName, numRange.getKey(), numRange.getValue())); return; @@ -330,7 +336,8 @@ private void serializePrimitive(BlockBuilder blockBuilder, Object node, Type typ return; } - throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), type, columnName)); + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, + format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), type, columnName)); } private Block serializeMap(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { @@ -354,7 +361,7 @@ private Block serializeMap(BlockBuilder parentBlockBuilder, Object value, Type t BlockBuilder entryBuilder = blockBuilder.beginBlockEntry(); - Iterator> fields = ((ObjectNode)value).fields(); + Iterator> fields = ((ObjectNode) value).fields(); while (fields.hasNext()) { Map.Entry entry = fields.next(); if (entry.getKey() != null) { @@ -393,7 +400,8 @@ private Block serializeRow(BlockBuilder parentBlockBuilder, Object value, Type t for (RowType.Field field : fields) { checkState(field.getName().isPresent(), "field name not found"); - serializeObject(singleRowBuilder, ((ObjectNode) value).get(field.getName().get()), field.getType(), columnName); + serializeObject(singleRowBuilder, ((ObjectNode) value).get(field.getName().get()), + field.getType(), columnName); } blockBuilder.closeEntry(); if (parentBlockBuilder == null) { diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/package-info.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/package-info.java new file mode 100644 index 0000000000000..a8d8b67aa037f --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * This package contains decoder for SchemaType.JSON. + */ +package org.apache.pulsar.sql.presto.decoder.json; \ No newline at end of file diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java index 80d2c5e16746a..fa4b4ab424255 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java @@ -18,19 +18,23 @@ */ package org.apache.pulsar.sql.presto.decoder.primitive; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; + import io.prestosql.spi.type.*; import org.apache.pulsar.sql.presto.PulsarRowDecoder; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - import static io.prestosql.decoder.FieldValueProviders.*; import static org.apache.pulsar.sql.presto.PulsarFieldValueProviders.doubleValueProvider; +/** + * Primitive Schema PulsarRowDecoder. + */ public class PulsarPrimitiveRowDecoder implements PulsarRowDecoder { private final DecoderColumnHandle columnHandle; @@ -47,7 +51,8 @@ public Optional> decodeRow(ByteBuf Type type = columnHandle.getType(); if (type instanceof BooleanType) { primitiveColumn.put(columnHandle, booleanValueProvider(Boolean.valueOf(new String(data)))); - } else if (type instanceof TinyintType || type instanceof SmallintType || type instanceof IntegerType || type instanceof BigintType) { + } else if (type instanceof TinyintType || type instanceof SmallintType || type instanceof IntegerType + || type instanceof BigintType) { primitiveColumn.put(columnHandle, longValueProvider(Long.valueOf(new String(data)))); } else if (type instanceof DoubleType) { primitiveColumn.put(columnHandle, doubleValueProvider(Double.valueOf(new String(data)))); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java index 3aa2505e540f3..ac46354df9af0 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java @@ -18,6 +18,10 @@ */ package org.apache.pulsar.sql.presto.decoder.primitive; +import java.util.Arrays; +import java.util.List; +import java.util.Set; + import io.airlift.log.Logger; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.connector.ColumnMetadata; @@ -30,10 +34,9 @@ import org.apache.pulsar.sql.presto.PulsarRowDecoder; import org.apache.pulsar.sql.presto.PulsarRowDecoderFactory; -import java.util.Arrays; -import java.util.List; -import java.util.Set; - +/** + * Primitive Schema PulsarRowDecoderFactory. + */ public class PulsarPrimitiveRowDecoderFactory implements PulsarRowDecoderFactory { private static final Logger log = Logger.get(PulsarPrimitiveRowDecoderFactory.class); @@ -48,12 +51,14 @@ public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaI } @Override - public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, + PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { ColumnMetadata valueColumn = new PulsarColumnMetadata( PulsarColumnMetadata.getColumnName(handleKeyValueType, "__value__"), parsePrimitivePrestoType("__value__", schemaInfo.getType()), "The value of the message with primitive type schema", null, false, false, - handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo("__value__", null, null)); + handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo("__value__", + null, null)); return Arrays.asList(valueColumn); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/package-info.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/package-info.java new file mode 100644 index 0000000000000..7ceecfbb28adb --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * This package contains decoder for SchemaType of SchemaType.isPrimitive() return true. + */ +package org.apache.pulsar.sql.presto.decoder.primitive; \ No newline at end of file From 9dd57e0288f3e11b4e3d314737b54b57b4f319e1 Mon Sep 17 00:00:00 2001 From: wangguowei Date: Tue, 10 Nov 2020 21:31:50 +0800 Subject: [PATCH 05/19] codestyle fix v2 --- .../sql/presto/PulsarInternalColumn.java | 22 ++++----- .../pulsar/sql/presto/PulsarMetadata.java | 28 ++++++++--- .../pulsar/sql/presto/PulsarRecordCursor.java | 16 +++++-- .../sql/presto/PulsarRecordSetProvider.java | 17 ++++--- .../decoder/avro/PulsarAvroColumnDecoder.java | 47 ++++++++++++------- .../avro/PulsarAvroRowDecoderFactory.java | 40 +++++++++++----- .../decoder/json/PulsarJsonFieldDecoder.java | 47 ++++++++++--------- .../decoder/json/PulsarJsonRowDecoder.java | 13 +++-- .../json/PulsarJsonRowDecoderFactory.java | 44 ++++++++++------- .../primitive/PulsarPrimitiveRowDecoder.java | 21 +++++++-- .../PulsarPrimitiveRowDecoderFactory.java | 14 +++++- 11 files changed, 204 insertions(+), 105 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java index f90b0896cb52b..068deafadfe85 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java @@ -18,21 +18,21 @@ */ package org.apache.pulsar.sql.presto; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; +import static java.util.Objects.requireNonNull; + +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import io.prestosql.spi.type.Type; import io.prestosql.spi.type.BigintType; import io.prestosql.spi.type.IntegerType; import io.prestosql.spi.type.TimestampType; import io.prestosql.spi.type.VarcharType; -import io.prestosql.spi.type.Type; - -import java.util.Map; -import java.util.Set; -import java.util.function.Consumer; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Strings.isNullOrEmpty; -import static java.util.Objects.requireNonNull; /** * This abstract class represents internal columns. @@ -50,10 +50,10 @@ public class PulsarInternalColumn { TimestampType.TIMESTAMP, "The timestamp in milliseconds of when event as published"); public static final PulsarInternalColumn MESSAGE_ID = new PulsarInternalColumn("__message_id__", - VarcharType.VARCHAR,"The message ID of the message used to generate this row"); + VarcharType.VARCHAR, "The message ID of the message used to generate this row"); public static final PulsarInternalColumn SEQUENCE_ID = new PulsarInternalColumn("__sequence_id__", - BigintType.BIGINT,"The sequence ID of the message used to generate this row"); + BigintType.BIGINT, "The sequence ID of the message used to generate this row"); public static final PulsarInternalColumn PRODUCER_NAME = new PulsarInternalColumn("__producer_name__", VarcharType.VARCHAR, "The name of the producer that publish the message used to generate this row"); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java index c2dff11a38596..90412426fa3a5 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java @@ -26,11 +26,32 @@ import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertColumnHandle; import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertTableHandle; +import javax.inject.Inject; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airlift.log.Logger; import io.prestosql.spi.PrestoException; -import io.prestosql.spi.connector.*; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.ColumnMetadata; +import io.prestosql.spi.connector.ConnectorMetadata; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.connector.ConnectorTableHandle; +import io.prestosql.spi.connector.ConnectorTableLayout; +import io.prestosql.spi.connector.ConnectorTableLayoutHandle; +import io.prestosql.spi.connector.ConnectorTableLayoutResult; +import io.prestosql.spi.connector.ConnectorTableMetadata; +import io.prestosql.spi.connector.Constraint; +import io.prestosql.spi.connector.SchemaTableName; +import io.prestosql.spi.connector.SchemaTablePrefix; +import io.prestosql.spi.connector.TableNotFoundException; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -41,11 +62,6 @@ import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; - - /** * This connector helps to work with metadata. */ diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index 6d2661db97212..24b784019fe92 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -23,6 +23,9 @@ import static io.prestosql.decoder.FieldValueProviders.bytesValueProvider; import static io.prestosql.decoder.FieldValueProviders.longValueProvider; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + import java.io.IOException; import java.util.HashMap; import java.util.List; @@ -31,8 +34,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicLong; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import io.airlift.log.Logger; import io.airlift.slice.Slice; @@ -42,7 +43,13 @@ import io.prestosql.spi.connector.ColumnHandle; import io.prestosql.spi.connector.RecordCursor; import io.prestosql.spi.type.Type; -import org.apache.bookkeeper.mledger.*; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.ManagedLedgerFactory; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.ReadOnlyCursor; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.mledger.impl.ReadOnlyCursorImpl; import org.apache.pulsar.client.api.PulsarClientException; @@ -130,7 +137,8 @@ public PulsarRecordCursor(List columnHandles, PulsarSplit pu // Exposed for testing purposes PulsarRecordCursor(List columnHandles, PulsarSplit pulsarSplit, PulsarConnectorConfig pulsarConnectorConfig, ManagedLedgerFactory managedLedgerFactory, ManagedLedgerConfig managedLedgerConfig, - PulsarConnectorMetricsTracker pulsarConnectorMetricsTracker, PulsarDispatchingRowDecoderFactory decoderFactory) { + PulsarConnectorMetricsTracker pulsarConnectorMetricsTracker, + PulsarDispatchingRowDecoderFactory decoderFactory) { this.splitSize = pulsarSplit.getSplitSize(); initialize(columnHandles, pulsarSplit, pulsarConnectorConfig, managedLedgerFactory, managedLedgerConfig, pulsarConnectorMetricsTracker); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSetProvider.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSetProvider.java index 9613f3049afc2..1f8bf7b7e6846 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSetProvider.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSetProvider.java @@ -18,13 +18,17 @@ */ package org.apache.pulsar.sql.presto; -import com.google.common.collect.ImmutableList; -import io.prestosql.spi.connector.*; +import static java.util.Objects.requireNonNull; -import javax.inject.Inject; +import com.google.common.collect.ImmutableList; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.ConnectorRecordSetProvider; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.connector.ConnectorSplit; +import io.prestosql.spi.connector.ConnectorTransactionHandle; +import io.prestosql.spi.connector.RecordSet; import java.util.List; - -import static java.util.Objects.requireNonNull; +import javax.inject.Inject; /** * Implementation of the provider for record sets. @@ -36,7 +40,8 @@ public class PulsarRecordSetProvider implements ConnectorRecordSetProvider { private final PulsarDispatchingRowDecoderFactory decoderFactory; @Inject - public PulsarRecordSetProvider(PulsarConnectorConfig pulsarConnectorConfig, PulsarDispatchingRowDecoderFactory decoderFactory) { + public PulsarRecordSetProvider(PulsarConnectorConfig pulsarConnectorConfig, + PulsarDispatchingRowDecoderFactory decoderFactory) { this.decoderFactory = requireNonNull(decoderFactory, "decoderFactory is null"); this.pulsarConnectorConfig = requireNonNull(pulsarConnectorConfig, "pulsarConnectorConfig is null"); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java index fcfb37e307b07..262a27f9a8cdf 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java @@ -18,6 +18,21 @@ */ package org.apache.pulsar.sql.presto.decoder.avro; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static io.airlift.slice.Slices.utf8Slice; +import static io.prestosql.decoder.DecoderErrorCode.DECODER_CONVERSION_NOT_SUPPORTED; +import static io.prestosql.spi.StandardErrorCode.GENERIC_USER_ERROR; +import static io.prestosql.spi.type.Varchars.truncateToLength; +import static java.lang.Float.floatToIntBits; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Set; + import com.google.common.base.Splitter; import com.google.common.collect.ImmutableSet; import io.airlift.slice.Slice; @@ -27,27 +42,27 @@ import io.prestosql.spi.PrestoException; import io.prestosql.spi.block.Block; import io.prestosql.spi.block.BlockBuilder; -import io.prestosql.spi.type.*; +import io.prestosql.spi.type.ArrayType; +import io.prestosql.spi.type.BigintType; +import io.prestosql.spi.type.BooleanType; +import io.prestosql.spi.type.DateType; +import io.prestosql.spi.type.DoubleType; +import io.prestosql.spi.type.IntegerType; +import io.prestosql.spi.type.MapType; +import io.prestosql.spi.type.RealType; +import io.prestosql.spi.type.RowType; import io.prestosql.spi.type.RowType.Field; +import io.prestosql.spi.type.SmallintType; +import io.prestosql.spi.type.TimeType; +import io.prestosql.spi.type.TimestampType; +import io.prestosql.spi.type.TinyintType; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.VarbinaryType; +import io.prestosql.spi.type.VarcharType; import org.apache.avro.generic.GenericEnumSymbol; import org.apache.avro.generic.GenericFixed; import org.apache.avro.generic.GenericRecord; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static io.airlift.slice.Slices.utf8Slice; -import static io.prestosql.decoder.DecoderErrorCode.DECODER_CONVERSION_NOT_SUPPORTED; -import static io.prestosql.spi.StandardErrorCode.GENERIC_USER_ERROR; -import static io.prestosql.spi.type.Varchars.truncateToLength; -import static java.lang.Float.floatToIntBits; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - /** * copy from {@link io.prestosql.decoder.avro.AvroColumnDecoder} (presto-record-decoder-345) with A little bit pulsar's extensions: * diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java index d2b46d87e596e..d4bd91158675f 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java @@ -18,11 +18,37 @@ */ package org.apache.pulsar.sql.presto.decoder.avro; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.prestosql.spi.type.DateType.DATE; +import static io.prestosql.spi.type.TimeType.TIME; +import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType; +import static java.lang.String.format; +import static java.util.stream.Collectors.toList; + +import java.util.List; +import java.util.Optional; +import java.util.Set; + import com.google.common.collect.ImmutableList; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ColumnMetadata; -import io.prestosql.spi.type.*; +import io.prestosql.spi.type.ArrayType; +import io.prestosql.spi.type.BigintType; +import io.prestosql.spi.type.BooleanType; +import io.prestosql.spi.type.DoubleType; +import io.prestosql.spi.type.IntegerType; +import io.prestosql.spi.type.RealType; +import io.prestosql.spi.type.RowType; +import io.prestosql.spi.type.StandardTypes; +import io.prestosql.spi.type.TimestampType; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.TypeManager; +import io.prestosql.spi.type.TypeSignature; +import io.prestosql.spi.type.TypeSignatureParameter; +import io.prestosql.spi.type.VarbinaryType; +import io.prestosql.spi.type.VarcharType; import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; @@ -37,18 +63,6 @@ import org.apache.pulsar.sql.presto.PulsarRowDecoder; import org.apache.pulsar.sql.presto.PulsarRowDecoderFactory; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.prestosql.spi.type.DateType.DATE; -import static io.prestosql.spi.type.TimeType.TIME; -import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType; -import static java.lang.String.format; -import static java.util.stream.Collectors.toList; - /** * PulsarRowDecoderFactory for {@link org.apache.pulsar.common.schema.SchemaType#AVRO}. */ diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java index 850c009cb01f3..4c7549f2ddcb1 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java @@ -18,25 +18,6 @@ */ package org.apache.pulsar.sql.presto.decoder.json; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableList; -import io.airlift.slice.Slice; -import io.prestosql.decoder.DecoderColumnHandle; -import io.prestosql.decoder.FieldValueProvider; -import io.prestosql.decoder.json.JsonFieldDecoder; -import io.prestosql.decoder.json.JsonRowDecoderFactory; -import io.prestosql.spi.PrestoException; -import io.prestosql.spi.block.Block; -import io.prestosql.spi.block.BlockBuilder; -import io.prestosql.spi.type.*; -import org.apache.commons.lang3.tuple.Pair; - -import java.util.Iterator; -import java.util.List; -import java.util.Map; - import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static io.airlift.slice.Slices.utf8Slice; @@ -60,6 +41,26 @@ import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableList; +import io.airlift.slice.Slice; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; +import io.prestosql.decoder.json.JsonFieldDecoder; +import io.prestosql.decoder.json.JsonRowDecoderFactory; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.block.BlockBuilder; +import io.prestosql.spi.type.*; +import org.apache.commons.lang3.tuple.Pair; + + /** * copy from {@link io.prestosql.decoder.json.DefaultJsonFieldDecoder} (presto-record-decoder-345) * with some pulsar's extensions: @@ -305,7 +306,8 @@ private void serializePrimitive(BlockBuilder blockBuilder, Object node, Type typ value = (JsonNode) node; } else { throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, - format("primitive object of '%s' as '%s' for column '%s' cann't convert to JsonNode", node.getClass(), type, columnName)); + format("primitive object of '%s' as '%s' for column '%s' cann't convert to JsonNode", + node.getClass(), type, columnName)); } if (value == null) { @@ -318,8 +320,9 @@ private void serializePrimitive(BlockBuilder blockBuilder, Object node, Type typ return; } - if (type instanceof RealType || type instanceof BigintType || type instanceof IntegerType || type instanceof SmallintType - || type instanceof TinyintType || type instanceof TimestampType || type instanceof TimeType + if (type instanceof RealType || type instanceof BigintType || type instanceof + IntegerType || type instanceof SmallintType || type instanceof + TinyintType || type instanceof TimestampType || type instanceof TimeType || type instanceof DateType) { Pair numRange = getNumRangeByType(type); type.writeLong(blockBuilder, getLong(value, type, columnName, numRange.getKey(), numRange.getValue())); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java index 1b77bf45fdbf5..780dbd79dacf4 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java @@ -18,6 +18,11 @@ */ package org.apache.pulsar.sql.presto.decoder.json; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.util.Objects.requireNonNull; +import static java.util.function.Function.identity; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.MissingNode; import com.google.common.base.Splitter; @@ -34,11 +39,9 @@ import java.util.Optional; import java.util.Set; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static java.util.Objects.requireNonNull; -import static java.util.function.Function.identity; - +/** + * Json PulsarRowDecoder. + */ public class PulsarJsonRowDecoder implements PulsarRowDecoder { private final Map fieldDecoders; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java index 5194f6c6fd9e0..ce303e3e64273 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java @@ -18,6 +18,18 @@ */ package org.apache.pulsar.sql.presto.decoder.json; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.prestosql.spi.type.DateType.DATE; +import static io.prestosql.spi.type.TimeType.TIME; +import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType; +import static java.lang.String.format; +import static java.util.stream.Collectors.toList; + +import java.util.List; +import java.util.Optional; +import java.util.Set; + import com.google.common.collect.ImmutableList; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.PrestoException; @@ -35,17 +47,6 @@ import org.apache.pulsar.sql.presto.PulsarColumnMetadata; import org.apache.pulsar.sql.presto.PulsarRowDecoderFactory; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.prestosql.spi.type.DateType.DATE; -import static io.prestosql.spi.type.TimeType.TIME; -import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType; -import static java.lang.String.format; -import static java.util.stream.Collectors.toList; /** * PulsarRowDecoderFactory for {@link org.apache.pulsar.common.schema.SchemaType#JSON}. @@ -59,12 +60,14 @@ public PulsarJsonRowDecoderFactory(TypeManager typeManager) { } @Override - public PulsarJsonRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, Set columns) { + public PulsarJsonRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, + Set columns) { return new PulsarJsonRowDecoder((GenericJsonSchema) GenericJsonSchema.of(schemaInfo), columns); } @Override - public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, + PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { String schemaJson = new String(schemaInfo.getSchema()); if (StringUtils.isBlank(schemaJson)) { throw new PrestoException(NOT_SUPPORTED, "Topic " @@ -82,7 +85,8 @@ public List extractColumnMetadata(TopicName topicName, SchemaInf return schema.getFields().stream() .map(field -> - new PulsarColumnMetadata(field.name(), parseJsonPrestoType(field.name(), field.schema()), field.schema().toString(), null, false, false, + new PulsarColumnMetadata(field.name(), parseJsonPrestoType(field.name(), field.schema()), + field.schema().toString(), null, false, false, handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(field.name(), null, null)) ).collect(toList()); @@ -97,7 +101,8 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { case ENUM: return createUnboundedVarcharType(); case NULL: - throw new UnsupportedOperationException(format("field '%s' NULL Type code should not be reached ļ¼Œplease check the schema or report the bug.", fieldname)); + throw new UnsupportedOperationException(format("field '%s' NULL Type code should not be reached ļ¼Œ" + + "please check the schema or report the bug.", fieldname)); case FIXED: case BYTES: return VarbinaryType.VARBINARY; @@ -124,14 +129,16 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { case MAP: //The key for an avro map must be a string TypeSignature valueType = parseJsonPrestoType(fieldname, schema.getValueType()).getTypeSignature(); - return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(valueType))); + return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter. + typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(valueType))); case RECORD: if (schema.getFields().size() > 0) { return RowType.from(schema.getFields().stream() .map(field -> new RowType.Field(Optional.of(field.name()), parseJsonPrestoType(field.name(), field.schema()))) .collect(toImmutableList())); } else { - throw new UnsupportedOperationException(format("field '%s' of record Type has no fields, please check avro schema definition. ", fieldname)); + throw new UnsupportedOperationException(format("field '%s' of record Type has no fields, " + + "please check avro schema definition. ", fieldname)); } case UNION: for (Schema nestType : schema.getTypes()) { @@ -141,7 +148,8 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { } throw new UnsupportedOperationException(format("field '%s' of UNION type must contains not null type ", fieldname)); default: - throw new UnsupportedOperationException(format("Cannot convert from Schema type '%s' (%s) to Presto type", schema.getType(), schema.getFullName())); + throw new UnsupportedOperationException(format("Cannot convert from Schema type '%s' (%s) to " + + "Presto type", schema.getType(), schema.getFullName())); } } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java index fa4b4ab424255..cb4c9755ea1d8 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java @@ -18,6 +18,11 @@ */ package org.apache.pulsar.sql.presto.decoder.primitive; +import static io.prestosql.decoder.FieldValueProviders.booleanValueProvider; +import static io.prestosql.decoder.FieldValueProviders.bytesValueProvider; +import static io.prestosql.decoder.FieldValueProviders.longValueProvider; +import static org.apache.pulsar.sql.presto.PulsarFieldValueProviders.doubleValueProvider; + import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -26,11 +31,21 @@ import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; -import io.prestosql.spi.type.*; +import io.prestosql.spi.type.BigintType; +import io.prestosql.spi.type.BooleanType; +import io.prestosql.spi.type.DateType; +import io.prestosql.spi.type.DoubleType; +import io.prestosql.spi.type.IntegerType; +import io.prestosql.spi.type.RealType; +import io.prestosql.spi.type.SmallintType; +import io.prestosql.spi.type.TimeType; +import io.prestosql.spi.type.TimestampType; +import io.prestosql.spi.type.TinyintType; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.VarbinaryType; +import io.prestosql.spi.type.VarcharType; import org.apache.pulsar.sql.presto.PulsarRowDecoder; -import static io.prestosql.decoder.FieldValueProviders.*; -import static org.apache.pulsar.sql.presto.PulsarFieldValueProviders.doubleValueProvider; /** * Primitive Schema PulsarRowDecoder. diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java index ac46354df9af0..71d886af73315 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java @@ -25,7 +25,19 @@ import io.airlift.log.Logger; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.connector.ColumnMetadata; -import io.prestosql.spi.type.*; +import io.prestosql.spi.type.BigintType; +import io.prestosql.spi.type.BooleanType; +import io.prestosql.spi.type.DateType; +import io.prestosql.spi.type.DoubleType; +import io.prestosql.spi.type.IntegerType; +import io.prestosql.spi.type.RealType; +import io.prestosql.spi.type.SmallintType; +import io.prestosql.spi.type.TimeType; +import io.prestosql.spi.type.TimestampType; +import io.prestosql.spi.type.TinyintType; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.VarbinaryType; +import io.prestosql.spi.type.VarcharType; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; From 66be5b29d66ef73384a525cb36f6792e72926513 Mon Sep 17 00:00:00 2001 From: wangguowei Date: Tue, 10 Nov 2020 22:25:13 +0800 Subject: [PATCH 06/19] codestyle fix v3 --- .../pulsar/sql/presto/PulsarColumnHandle.java | 3 +- .../sql/presto/PulsarColumnMetadata.java | 3 +- .../PulsarDispatchingRowDecoderFactory.java | 15 +++-- .../sql/presto/PulsarFieldValueProviders.java | 11 ++-- .../sql/presto/PulsarInternalColumn.java | 5 +- .../pulsar/sql/presto/PulsarMetadata.java | 6 +- .../pulsar/sql/presto/PulsarRecordCursor.java | 25 +++++--- .../pulsar/sql/presto/PulsarRecordSet.java | 9 +-- .../sql/presto/PulsarRowDecoderFactory.java | 15 +++-- .../pulsar/sql/presto/PulsarSplitManager.java | 39 ++++++++----- .../presto/PulsarSqlSchemaInfoProvider.java | 17 +++--- .../decoder/avro/PulsarAvroColumnDecoder.java | 57 ++++++++++++------- .../decoder/avro/PulsarAvroRowDecoder.java | 19 +++---- .../avro/PulsarAvroRowDecoderFactory.java | 28 ++++++--- .../decoder/json/PulsarJsonFieldDecoder.java | 10 ++-- .../decoder/json/PulsarJsonRowDecoder.java | 11 ++-- .../json/PulsarJsonRowDecoderFactory.java | 39 ++++++++++--- .../primitive/PulsarPrimitiveRowDecoder.java | 8 +-- .../PulsarPrimitiveRowDecoderFactory.java | 2 +- 19 files changed, 197 insertions(+), 125 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnHandle.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnHandle.java index 0beebb7d80f1f..0ba520e7f7799 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnHandle.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnHandle.java @@ -174,7 +174,8 @@ public boolean isValue() { ColumnMetadata getColumnMetadata() { return new PulsarColumnMetadata(name, type, null, null, hidden, - internal, handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(mapping, dataFormat, formatHint)); + internal, handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo( + mapping, dataFormat, formatHint)); } @Override diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java index be2e437e1b6e4..57aad3b39af6d 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java @@ -38,7 +38,8 @@ public class PulsarColumnMetadata extends ColumnMetadata { public PulsarColumnMetadata(String name, Type type, String comment, String extraInfo, boolean hidden, boolean isInternal, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType, DecoderExtraInfo decoderExtraInfo) { + PulsarColumnHandle.HandleKeyValueType handleKeyValueType, + DecoderExtraInfo decoderExtraInfo) { super(name, type, comment, extraInfo, hidden); this.nameWithCase = name; this.isInternal = isInternal; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java index bc68124d3ebbd..f12306f55523b 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java @@ -19,6 +19,10 @@ package org.apache.pulsar.sql.presto; import com.google.inject.Inject; + +import java.util.List; +import java.util.Set; + import io.airlift.log.Logger; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.connector.ColumnMetadata; @@ -30,9 +34,7 @@ import org.apache.pulsar.sql.presto.decoder.json.PulsarJsonRowDecoderFactory; import org.apache.pulsar.sql.presto.decoder.primitive.PulsarPrimitiveRowDecoderFactory; -import java.util.List; -import java.util.Set; - +import static java.lang.String.format; /** * dispatcher RowDecoderFactory for {@link org.apache.pulsar.common.schema.SchemaType}. @@ -50,12 +52,12 @@ public PulsarDispatchingRowDecoderFactory(TypeManager typeManager) { public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, Set columns) { PulsarRowDecoderFactory rowDecoderFactory = createDecoderFactory(schemaInfo); - return rowDecoderFactory.createRowDecoder(topicName,schemaInfo, columns); + return rowDecoderFactory.createRowDecoder(topicName, schemaInfo, columns); } public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { PulsarRowDecoderFactory rowDecoderFactory = createDecoderFactory(schemaInfo); - return rowDecoderFactory.extractColumnMetadata(topicName,schemaInfo, handleKeyValueType); + return rowDecoderFactory.extractColumnMetadata(topicName, schemaInfo, handleKeyValueType); } private PulsarRowDecoderFactory createDecoderFactory(SchemaInfo schemaInfo) { @@ -66,7 +68,8 @@ private PulsarRowDecoderFactory createDecoderFactory(SchemaInfo schemaInfo) { } else if (schemaInfo.getType().isPrimitive()) { return new PulsarPrimitiveRowDecoderFactory(); } else { - throw new RuntimeException("unsupported typd ..."); + throw new RuntimeException(format("'%s' is unsupported type '%s'", schemaInfo.getName(), + schemaInfo.getType())); } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarFieldValueProviders.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarFieldValueProviders.java index 215380b23c9c3..c1ef3c8cc6c5c 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarFieldValueProviders.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarFieldValueProviders.java @@ -21,7 +21,7 @@ import io.prestosql.decoder.FieldValueProvider; /** - * custom FieldValueProvider for Pulsar + * custom FieldValueProvider for Pulsar. */ public class PulsarFieldValueProviders { @@ -40,8 +40,7 @@ public boolean isNull() { } /** - * FieldValueProvider for Time (Data,Timstamp etc.) with indicate Null instead of longValueProvider - * + * FieldValueProvider for Time (Data,Timstamp etc.) with indicate Null instead of longValueProvider. * @param value * @param isNull * @return @@ -49,9 +48,9 @@ public boolean isNull() { public static FieldValueProvider timeValueProvider(long value, boolean isNull) { return new FieldValueProvider() { @Override - public long getLong() { - return value; - } + public long getLong() { + return value; + } @Override public boolean isNull() { diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java index 068deafadfe85..46d6b6ea482bb 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java @@ -22,12 +22,11 @@ import static com.google.common.base.Strings.isNullOrEmpty; import static java.util.Objects.requireNonNull; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import java.util.Map; import java.util.Set; import java.util.function.Consumer; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import io.prestosql.spi.type.Type; import io.prestosql.spi.type.BigintType; import io.prestosql.spi.type.IntegerType; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java index 90412426fa3a5..34149938299cb 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java @@ -26,6 +26,9 @@ import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertColumnHandle; import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertTableHandle; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.log.Logger; import javax.inject.Inject; import java.util.HashMap; import java.util.LinkedList; @@ -35,9 +38,6 @@ import java.util.Set; import java.util.stream.Collectors; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ColumnHandle; import io.prestosql.spi.connector.ColumnMetadata; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index 24b784019fe92..e1b345475b7a2 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -23,6 +23,7 @@ import static io.prestosql.decoder.FieldValueProviders.bytesValueProvider; import static io.prestosql.decoder.FieldValueProviders.longValueProvider; +import com.google.common.annotations.VisibleForTesting; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -34,7 +35,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicLong; -import com.google.common.annotations.VisibleForTesting; import io.airlift.log.Logger; import io.airlift.slice.Slice; import io.prestosql.decoder.DecoderColumnHandle; @@ -164,7 +164,8 @@ private void initialize(List columnHandles, PulsarSplit puls this.pulsarConnectorConfig = pulsarConnectorConfig; try { - this.schemaInfoProvider = new PulsarSqlSchemaInfoProvider(this.topicName, pulsarConnectorConfig.getPulsarAdmin()); + this.schemaInfoProvider = new PulsarSqlSchemaInfoProvider(this.topicName, + pulsarConnectorConfig.getPulsarAdmin()); } catch (PulsarClientException e) { log.error(e, "Failed to init Pulsar SchemaInfo Provider"); throw new RuntimeException(e); @@ -451,14 +452,16 @@ public boolean advanceNextPosition() { schemaInfo, columnHandles.stream() .filter(col -> !col.isInternal()) - .filter(col -> PulsarColumnHandle.HandleKeyValueType.KEY.equals(col.getHandleKeyValueType())) + .filter(col -> PulsarColumnHandle.HandleKeyValueType.KEY + .equals(col.getHandleKeyValueType())) .collect(toImmutableSet())); PulsarRowDecoder messageDecoder = decoderFactory.createRowDecoder(topicName, schemaInfo, columnHandles.stream() .filter(col -> !col.isInternal()) - .filter(col -> PulsarColumnHandle.HandleKeyValueType.VALUE.equals(col.getHandleKeyValueType())) + .filter(col -> PulsarColumnHandle.HandleKeyValueType.VALUE + .equals(col.getHandleKeyValueType())) .collect(toImmutableSet())); Optional> decodedKey; @@ -466,16 +469,19 @@ public boolean advanceNextPosition() { decodedKey = keyDecoder.decodeRow(this.currentMessage.getKeyBytes().get()); decodedKey.ifPresent(currentRowValuesMap::putAll); } - Optional> decodedValue = messageDecoder.decodeRow(this.currentMessage.getData()); + Optional> decodedValue = + messageDecoder.decodeRow(this.currentMessage.getData()); decodedValue.ifPresent(currentRowValuesMap::putAll); - }else { + } else { PulsarRowDecoder messageDecoder = decoderFactory.createRowDecoder(topicName, schemaInfo, columnHandles.stream() .filter(col -> !col.isInternal()) - .filter(col -> PulsarColumnHandle.HandleKeyValueType.NONE.equals(col.getHandleKeyValueType())) + .filter(col -> PulsarColumnHandle.HandleKeyValueType.NONE + .equals(col.getHandleKeyValueType())) .collect(toImmutableSet())); - Optional> decodedValue = messageDecoder.decodeRow(this.currentMessage.getData()); + Optional> decodedValue = + messageDecoder.decodeRow(this.currentMessage.getData()); decodedValue.ifPresent(currentRowValuesMap::putAll); } @@ -495,7 +501,8 @@ public boolean advanceNextPosition() { } else if (PulsarInternalColumn.SEQUENCE_ID.getName().equals(columnHandle.getName())) { currentRowValuesMap.put(columnHandle, longValueProvider(this.currentMessage.getSequenceId())); } else if (PulsarInternalColumn.PRODUCER_NAME.getName().equals(columnHandle.getName())) { - currentRowValuesMap.put(columnHandle, bytesValueProvider(this.currentMessage.getProducerName().getBytes())); + currentRowValuesMap.put(columnHandle, + bytesValueProvider(this.currentMessage.getProducerName().getBytes())); } else if (PulsarInternalColumn.KEY.getName().equals(columnHandle.getName())) { String key = this.currentMessage.getKey().orElse(null); currentRowValuesMap.put(columnHandle, bytesValueProvider(key == null ? null : key.getBytes())); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSet.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSet.java index f67d3b12594e0..fe1a8c538b8b9 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSet.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordSet.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.sql.presto; +import static java.util.Objects.requireNonNull; + import com.google.common.collect.ImmutableList; import io.prestosql.spi.connector.RecordCursor; import io.prestosql.spi.connector.RecordSet; @@ -25,8 +27,6 @@ import java.util.List; -import static java.util.Objects.requireNonNull; - /** * Implementation of a record set. */ @@ -53,7 +53,7 @@ public PulsarRecordSet(PulsarSplit split, List columnHandles this.pulsarConnectorConfig = pulsarConnectorConfig; - this.decoderFactory =decoderFactory; + this.decoderFactory = decoderFactory; } @@ -64,6 +64,7 @@ public List getColumnTypes() { @Override public RecordCursor cursor() { - return new PulsarRecordCursor(this.columnHandles, this.pulsarSplit, this.pulsarConnectorConfig,this.decoderFactory); + return new PulsarRecordCursor(this.columnHandles, this.pulsarSplit, + this.pulsarConnectorConfig, this.decoderFactory); } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java index ff49edfdb814e..62838c399d7af 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java @@ -18,35 +18,34 @@ */ package org.apache.pulsar.sql.presto; +import java.util.List; +import java.util.Set; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.connector.ColumnMetadata; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; -import java.util.List; -import java.util.Set; - /** - * Pulsar customized RowDecoderFactory interface + * Pulsar customized RowDecoderFactory interface. */ public interface PulsarRowDecoderFactory { /** - * extract ColumnMetadata from pulsar SchemaInfo and HandleKeyValueType + * extract ColumnMetadata from pulsar SchemaInfo and HandleKeyValueType. * * @param schemaInfo * @param handleKeyValueType * @return */ - public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType); + List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType); /** - * createRowDecoder RowDecoder by pulsar SchemaInfo and column DecoderColumnHandles + * createRowDecoder RowDecoder by pulsar SchemaInfo and column DecoderColumnHandles. * * @param schemaInfo * @param columns * @return */ - public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, Set columns); + PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, Set columns); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java index c10aff399b5a6..b982eef9496d3 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java @@ -18,17 +18,41 @@ */ package org.apache.pulsar.sql.presto; +import static com.google.common.base.Preconditions.checkArgument; +import static io.prestosql.spi.StandardErrorCode.QUERY_REJECTED; +import static java.util.Objects.requireNonNull; +import static org.apache.bookkeeper.mledger.ManagedCursor.FindPositionConstraint.SearchAllAvailableEntries; +import static org.apache.pulsar.sql.presto.PulsarConnectorUtils.restoreNamespaceDelimiterIfNeeded; + import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Predicate; import io.airlift.log.Logger; import io.prestosql.spi.PrestoException; -import io.prestosql.spi.connector.*; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.connector.ConnectorSplitManager; +import io.prestosql.spi.connector.ConnectorSplitSource; +import io.prestosql.spi.connector.ConnectorTableLayoutHandle; +import io.prestosql.spi.connector.ConnectorTransactionHandle; +import io.prestosql.spi.connector.FixedSplitSource; import io.prestosql.spi.predicate.Domain; import io.prestosql.spi.predicate.Range; import io.prestosql.spi.predicate.TupleDomain; +import java.io.IOException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import javax.inject.Inject; import lombok.Data; -import org.apache.bookkeeper.mledger.*; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.ManagedLedgerFactory; +import org.apache.bookkeeper.mledger.ReadOnlyCursor; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -40,17 +64,6 @@ import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.schema.SchemaInfo; -import javax.inject.Inject; -import java.io.IOException; -import java.sql.Timestamp; -import java.util.*; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.prestosql.spi.StandardErrorCode.QUERY_REJECTED; -import static java.util.Objects.requireNonNull; -import static org.apache.bookkeeper.mledger.ManagedCursor.FindPositionConstraint.SearchAllAvailableEntries; -import static org.apache.pulsar.sql.presto.PulsarConnectorUtils.restoreNamespaceDelimiterIfNeeded; - /** * The class helping to manage splits. */ diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java index 453219ffa8603..fe9b6198ea086 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java @@ -18,9 +18,18 @@ */ package org.apache.pulsar.sql.presto; +import static java.util.concurrent.CompletableFuture.completedFuture; + import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Schema; @@ -32,14 +41,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - -import static java.util.concurrent.CompletableFuture.completedFuture; - /** * Multi version schema info provider for Pulsar SQL leverage guava cache. diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java index 262a27f9a8cdf..cc6bdfd9c3e11 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java @@ -28,13 +28,13 @@ import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableSet; import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import java.util.Set; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableSet; import io.airlift.slice.Slice; import io.airlift.slice.Slices; import io.prestosql.decoder.DecoderColumnHandle; @@ -64,8 +64,8 @@ import org.apache.avro.generic.GenericRecord; /** - * copy from {@link io.prestosql.decoder.avro.AvroColumnDecoder} (presto-record-decoder-345) with A little bit pulsar's extensions: - * + * copy from {@link io.prestosql.decoder.avro.AvroColumnDecoder} (presto-record-decoder-345) + * with A little bit pulsar's extensions. * 1) support TIMESTAMP,DATE,TIME * 2) support RealType */ @@ -93,12 +93,16 @@ public PulsarAvroColumnDecoder(DecoderColumnHandle columnHandle) { this.columnType = columnHandle.getType(); this.columnMapping = columnHandle.getMapping(); this.columnName = columnHandle.getName(); - checkArgument(!columnHandle.isInternal(), "unexpected internal column '%s'", columnName); - checkArgument(columnHandle.getFormatHint() == null, "unexpected format hint '%s' defined for column '%s'", columnHandle.getFormatHint(), columnName); - checkArgument(columnHandle.getDataFormat() == null, "unexpected data format '%s' defined for column '%s'", columnHandle.getDataFormat(), columnName); - checkArgument(columnHandle.getMapping() != null, "mapping not defined for column '%s'", columnName); - - checkArgument(isSupportedType(columnType), "Unsupported column type '%s' for column '%s'", columnType, columnName); + checkArgument(!columnHandle.isInternal(), + "unexpected internal column '%s'", columnName); + checkArgument(columnHandle.getFormatHint() == null, + "unexpected format hint '%s' defined for column '%s'", columnHandle.getFormatHint(), columnName); + checkArgument(columnHandle.getDataFormat() == null, + "unexpected data format '%s' defined for column '%s'", columnHandle.getDataFormat(), columnName); + checkArgument(columnHandle.getMapping() != null, + "mapping not defined for column '%s'", columnName); + checkArgument(isSupportedType(columnType), + "Unsupported column type '%s' for column '%s'", columnType, columnName); } catch (IllegalArgumentException e) { throw new PrestoException(GENERIC_USER_ERROR, e); } @@ -110,14 +114,17 @@ private boolean isSupportedType(Type type) { } if (type instanceof ArrayType) { - checkArgument(type.getTypeParameters().size() == 1, "expecting exactly one type parameter for array"); + checkArgument(type.getTypeParameters().size() == 1, + "expecting exactly one type parameter for array"); return isSupportedType(type.getTypeParameters().get(0)); } if (type instanceof MapType) { List typeParameters = type.getTypeParameters(); - checkArgument(typeParameters.size() == 2, "expecting exactly two type parameters for map"); - checkArgument(typeParameters.get(0) instanceof VarcharType, "Unsupported column type '%s' for map key", typeParameters.get(0)); + checkArgument(typeParameters.size() == 2, + "expecting exactly two type parameters for map"); + checkArgument(typeParameters.get(0) instanceof VarcharType, + "Unsupported column type '%s' for map key", typeParameters.get(0)); return isSupportedType(type.getTypeParameters().get(1)); } @@ -174,7 +181,9 @@ public double getDouble() { if (value instanceof Double || value instanceof Float) { return ((Number) value).doubleValue(); } - throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), columnType, columnName)); + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, + format("cannot decode object of '%s' as '%s' for column '%s'", + value.getClass(), columnType, columnName)); } @Override @@ -182,7 +191,9 @@ public boolean getBoolean() { if (value instanceof Boolean) { return (Boolean) value; } - throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), columnType, columnName)); + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, + format("cannot decode object of '%s' as '%s' for column '%s'", + value.getClass(), columnType, columnName)); } @Override @@ -195,7 +206,9 @@ public long getLong() { return floatToIntBits((Float) value); } - throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), columnType, columnName)); + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, + format("cannot decode object of '%s' as '%s' for column '%s'", + value.getClass(), columnType, columnName)); } @Override @@ -222,7 +235,9 @@ private static Slice getSlice(Object value, Type type, String columnName) { } } - throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), type, columnName)); + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, + format("cannot decode object of '%s' as '%s' for column '%s'", + value.getClass(), type, columnName)); } private static Block serializeObject(BlockBuilder builder, Object value, Type type, String columnName) { @@ -273,7 +288,9 @@ private static void serializePrimitive(BlockBuilder blockBuilder, Object value, return; } - if ((value instanceof Integer || value instanceof Long) && (type instanceof BigintType || type instanceof IntegerType || type instanceof SmallintType || type instanceof TinyintType)) { + if ((value instanceof Integer || value instanceof Long) + && (type instanceof BigintType || type instanceof IntegerType + || type instanceof SmallintType || type instanceof TinyintType)) { type.writeLong(blockBuilder, ((Number) value).longValue()); return; } @@ -298,7 +315,9 @@ private static void serializePrimitive(BlockBuilder blockBuilder, Object value, return; } - throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), type, columnName)); + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, + format("cannot decode object of '%s' as '%s' for column '%s'", + value.getClass(), type, columnName)); } private static Block serializeMap(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) { diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java index 9e94195074a6a..cdf6119a75718 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java @@ -18,6 +18,14 @@ */ package org.apache.pulsar.sql.presto.decoder.avro; +import static com.google.common.base.Functions.identity; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static java.util.Objects.requireNonNull; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; import io.airlift.log.Logger; import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; @@ -28,17 +36,8 @@ import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; import org.apache.pulsar.sql.presto.PulsarRowDecoder; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -import static com.google.common.base.Functions.identity; -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static java.util.Objects.requireNonNull; - /** - * Refer to {@link io.prestosql.decoder.avro.AvroRowDecoder} + * Refer to {@link io.prestosql.decoder.avro.AvroRowDecoder}. */ public class PulsarAvroRowDecoder implements PulsarRowDecoder { diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java index d4bd91158675f..699d7c12ad390 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java @@ -26,11 +26,11 @@ import static java.lang.String.format; import static java.util.stream.Collectors.toList; +import com.google.common.collect.ImmutableList; import java.util.List; import java.util.Optional; import java.util.Set; -import com.google.common.collect.ImmutableList; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ColumnMetadata; @@ -80,7 +80,8 @@ public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaI } @Override - public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, + PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { String schemaJson = new String(schemaInfo.getSchema()); if (StringUtils.isBlank(schemaJson)) { throw new PrestoException(NOT_SUPPORTED, "Topic " @@ -98,8 +99,10 @@ public List extractColumnMetadata(TopicName topicName, SchemaInf return schema.getFields().stream() .map(field -> - new PulsarColumnMetadata(field.name(), parseAvroPrestoType(field.name(), field.schema()), field.schema().toString(), null, false, false, - handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(field.name(), null, null)) + new PulsarColumnMetadata(field.name(), parseAvroPrestoType( + field.name(), field.schema()), field.schema().toString(), null, false, false, + handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(field.name(), + null, null)) ).collect(toList()); } @@ -118,7 +121,9 @@ private Type parseAvroPrestoType(String fieldname, Schema schema) { case ENUM: return createUnboundedVarcharType(); case NULL: - throw new UnsupportedOperationException(format("field '%s' NULL Type code should not be reached ļ¼Œplease check the schema or report the bug.", fieldname)); + throw new UnsupportedOperationException( + format("field '%s' NULL Type code should not be reached ļ¼Œ" + + "please check the schema or report the bug.", fieldname)); case FIXED: case BYTES: //TODO: support decimal logicalType @@ -147,14 +152,17 @@ private Type parseAvroPrestoType(String fieldname, Schema schema) { case MAP: //The key for an Avro map must be a string TypeSignature valueType = parseAvroPrestoType(fieldname, schema.getValueType()).getTypeSignature(); - return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(valueType))); + return typeManager.getParameterizedType(StandardTypes.MAP, + ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), + TypeSignatureParameter.typeParameter(valueType))); case RECORD: if (schema.getFields().size() > 0) { return RowType.from(schema.getFields().stream() .map(field -> new RowType.Field(Optional.of(field.name()), parseAvroPrestoType(field.name(), field.schema()))) .collect(toImmutableList())); } else { - throw new UnsupportedOperationException(format("field '%s' of record Type has no fields, please check avro schema definition. ", fieldname)); + throw new UnsupportedOperationException(format( + "field '%s' of record Type has no fields, please check avro schema definition. ", fieldname)); } case UNION: for (Schema nestType : schema.getTypes()) { @@ -162,9 +170,11 @@ private Type parseAvroPrestoType(String fieldname, Schema schema) { return parseAvroPrestoType(fieldname, nestType); } } - throw new UnsupportedOperationException(format("field '%s' of UNION type must contains not null type ", fieldname)); + throw new UnsupportedOperationException(format( + "field '%s' of UNION type must contains not null type ", fieldname)); default: - throw new UnsupportedOperationException(format("Cannot convert from Schema type '%s' (%s) to Presto type", schema.getType(), schema.getFullName())); + throw new UnsupportedOperationException(format( + "Cannot convert from Schema type '%s' (%s) to Presto type", schema.getType(), schema.getFullName())); } } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java index 4c7549f2ddcb1..2ba78ba740a55 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java @@ -41,6 +41,7 @@ import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import com.google.common.collect.ImmutableList; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -48,7 +49,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableList; import io.airlift.slice.Slice; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; @@ -320,10 +320,10 @@ private void serializePrimitive(BlockBuilder blockBuilder, Object node, Type typ return; } - if (type instanceof RealType || type instanceof BigintType || type instanceof - IntegerType || type instanceof SmallintType || type instanceof - TinyintType || type instanceof TimestampType || type instanceof TimeType - || type instanceof DateType) { + if (type instanceof RealType || type instanceof BigintType || + type instanceof IntegerType || type instanceof SmallintType || + type instanceof TinyintType || type instanceof TimestampType || + type instanceof TimeType || type instanceof DateType) { Pair numRange = getNumRangeByType(type); type.writeLong(blockBuilder, getLong(value, type, columnName, numRange.getKey(), numRange.getValue())); return; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java index 780dbd79dacf4..ada1eb2ce3943 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java @@ -23,9 +23,13 @@ import static java.util.Objects.requireNonNull; import static java.util.function.Function.identity; +import com.google.common.base.Splitter; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.MissingNode; -import com.google.common.base.Splitter; import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; @@ -34,11 +38,6 @@ import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; import org.apache.pulsar.sql.presto.PulsarRowDecoder; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - /** * Json PulsarRowDecoder. */ diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java index ce303e3e64273..6e306600b86dd 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java @@ -26,15 +26,29 @@ import static java.lang.String.format; import static java.util.stream.Collectors.toList; +import com.google.common.collect.ImmutableList; import java.util.List; import java.util.Optional; import java.util.Set; -import com.google.common.collect.ImmutableList; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ColumnMetadata; -import io.prestosql.spi.type.*; +import io.prestosql.spi.type.ArrayType; +import io.prestosql.spi.type.BigintType; +import io.prestosql.spi.type.BooleanType; +import io.prestosql.spi.type.DoubleType; +import io.prestosql.spi.type.IntegerType; +import io.prestosql.spi.type.RealType; +import io.prestosql.spi.type.RowType; +import io.prestosql.spi.type.StandardTypes; +import io.prestosql.spi.type.TimestampType; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.TypeManager; +import io.prestosql.spi.type.TypeSignature; +import io.prestosql.spi.type.TypeSignatureParameter; +import io.prestosql.spi.type.VarbinaryType; +import io.prestosql.spi.type.VarcharType; import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; @@ -87,7 +101,8 @@ public List extractColumnMetadata(TopicName topicName, SchemaInf .map(field -> new PulsarColumnMetadata(field.name(), parseJsonPrestoType(field.name(), field.schema()), field.schema().toString(), null, false, false, - handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(field.name(), null, null)) + handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo( + field.name(), null, null)) ).collect(toList()); } @@ -101,7 +116,8 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { case ENUM: return createUnboundedVarcharType(); case NULL: - throw new UnsupportedOperationException(format("field '%s' NULL Type code should not be reached ļ¼Œ" + + throw new UnsupportedOperationException(format( + "field '%s' NULL Type code should not be reached ļ¼Œ" + "please check the schema or report the bug.", fieldname)); case FIXED: case BYTES: @@ -130,14 +146,17 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { //The key for an avro map must be a string TypeSignature valueType = parseJsonPrestoType(fieldname, schema.getValueType()).getTypeSignature(); return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter. - typeParameter(VarcharType.VARCHAR.getTypeSignature()), TypeSignatureParameter.typeParameter(valueType))); + typeParameter(VarcharType.VARCHAR.getTypeSignature()), + TypeSignatureParameter.typeParameter(valueType))); case RECORD: if (schema.getFields().size() > 0) { return RowType.from(schema.getFields().stream() - .map(field -> new RowType.Field(Optional.of(field.name()), parseJsonPrestoType(field.name(), field.schema()))) + .map(field -> new RowType.Field(Optional.of(field.name()), + parseJsonPrestoType(field.name(), field.schema()))) .collect(toImmutableList())); } else { - throw new UnsupportedOperationException(format("field '%s' of record Type has no fields, " + + throw new UnsupportedOperationException(format( + "field '%s' of record Type has no fields, " + "please check avro schema definition. ", fieldname)); } case UNION: @@ -146,9 +165,11 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { return parseJsonPrestoType(fieldname, nestType); } } - throw new UnsupportedOperationException(format("field '%s' of UNION type must contains not null type ", fieldname)); + throw new UnsupportedOperationException(format( + "field '%s' of UNION type must contains not null type ", fieldname)); default: - throw new UnsupportedOperationException(format("Cannot convert from Schema type '%s' (%s) to " + + throw new UnsupportedOperationException(format( + "Cannot convert from Schema type '%s' (%s) to " + "Presto type", schema.getType(), schema.getFullName())); } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java index cb4c9755ea1d8..9f93211615151 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java @@ -23,11 +23,10 @@ import static io.prestosql.decoder.FieldValueProviders.longValueProvider; import static org.apache.pulsar.sql.presto.PulsarFieldValueProviders.doubleValueProvider; +import io.netty.buffer.ByteBuf; import java.util.HashMap; import java.util.Map; import java.util.Optional; - -import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; @@ -62,7 +61,7 @@ public PulsarPrimitiveRowDecoder(DecoderColumnHandle column) { public Optional> decodeRow(ByteBuf byteBuf) { byte[] data = new byte[byteBuf.readableBytes()]; byteBuf.readBytes(data); - Map primitiveColumn = new HashMap(); + Map primitiveColumn = new HashMap<>(); Type type = columnHandle.getType(); if (type instanceof BooleanType) { primitiveColumn.put(columnHandle, booleanValueProvider(Boolean.valueOf(new String(data)))); @@ -72,7 +71,8 @@ public Optional> decodeRow(ByteBuf } else if (type instanceof DoubleType) { primitiveColumn.put(columnHandle, doubleValueProvider(Double.valueOf(new String(data)))); } else if (type instanceof RealType) { - primitiveColumn.put(columnHandle, longValueProvider(Float.floatToIntBits((Float.valueOf(new String(data)))))); + primitiveColumn.put(columnHandle, longValueProvider( + Float.floatToIntBits((Float.valueOf(new String(data)))))); } else if (type instanceof VarbinaryType || type instanceof VarcharType) { primitiveColumn.put(columnHandle, bytesValueProvider(data)); } else if (type instanceof DateType || type instanceof TimeType || type instanceof TimestampType) { diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java index 71d886af73315..4fa796973d4f0 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java @@ -18,11 +18,11 @@ */ package org.apache.pulsar.sql.presto.decoder.primitive; +import io.airlift.log.Logger; import java.util.Arrays; import java.util.List; import java.util.Set; -import io.airlift.log.Logger; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.connector.ColumnMetadata; import io.prestosql.spi.type.BigintType; From e61a37d292f6d67b1613cf943c00e162ac9f6601 Mon Sep 17 00:00:00 2001 From: wangguowei Date: Tue, 10 Nov 2020 22:36:44 +0800 Subject: [PATCH 07/19] codestyle fix v4 --- .../pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java index cdf6119a75718..e6683934db340 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java @@ -41,8 +41,6 @@ */ public class PulsarAvroRowDecoder implements PulsarRowDecoder { - private static final Logger log = Logger.get(PulsarAvroRowDecoderFactory.class); - private final GenericAvroSchema genericAvroSchema; private final Map columnDecoders; From 8f9360003a2972b6b009b45a29eaaddbae42acc1 Mon Sep 17 00:00:00 2001 From: wangguowei Date: Tue, 10 Nov 2020 23:20:46 +0800 Subject: [PATCH 08/19] codestyle fix v5 --- .../PulsarDispatchingRowDecoderFactory.java | 14 +++++------ .../sql/presto/PulsarInternalColumn.java | 8 +++--- .../pulsar/sql/presto/PulsarMetadata.java | 17 ++++++------- .../pulsar/sql/presto/PulsarRecordCursor.java | 18 ++++++------- .../sql/presto/PulsarRowDecoderFactory.java | 12 ++++----- .../decoder/avro/PulsarAvroColumnDecoder.java | 4 +-- .../avro/PulsarAvroRowDecoderFactory.java | 14 +++++++---- .../decoder/json/PulsarJsonFieldDecoder.java | 25 +++++++++++++++---- .../decoder/json/PulsarJsonRowDecoder.java | 5 ++-- .../json/PulsarJsonRowDecoderFactory.java | 12 ++++----- .../primitive/PulsarPrimitiveRowDecoder.java | 4 +-- 11 files changed, 75 insertions(+), 58 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java index f12306f55523b..bd5d3bc6a95e0 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java @@ -18,12 +18,12 @@ */ package org.apache.pulsar.sql.presto; -import com.google.inject.Inject; +import static java.lang.String.format; +import com.google.inject.Inject; +import io.airlift.log.Logger; import java.util.List; import java.util.Set; - -import io.airlift.log.Logger; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.connector.ColumnMetadata; import io.prestosql.spi.type.TypeManager; @@ -34,8 +34,6 @@ import org.apache.pulsar.sql.presto.decoder.json.PulsarJsonRowDecoderFactory; import org.apache.pulsar.sql.presto.decoder.primitive.PulsarPrimitiveRowDecoderFactory; -import static java.lang.String.format; - /** * dispatcher RowDecoderFactory for {@link org.apache.pulsar.common.schema.SchemaType}. */ @@ -50,12 +48,14 @@ public PulsarDispatchingRowDecoderFactory(TypeManager typeManager) { this.typeManager = typeManager; } - public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, Set columns) { + public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, + Set columns) { PulsarRowDecoderFactory rowDecoderFactory = createDecoderFactory(schemaInfo); return rowDecoderFactory.createRowDecoder(topicName, schemaInfo, columns); } - public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, + PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { PulsarRowDecoderFactory rowDecoderFactory = createDecoderFactory(schemaInfo); return rowDecoderFactory.extractColumnMetadata(topicName, schemaInfo, handleKeyValueType); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java index 46d6b6ea482bb..05c3e72b89ee0 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java @@ -24,14 +24,14 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import java.util.Map; -import java.util.Set; -import java.util.function.Consumer; -import io.prestosql.spi.type.Type; import io.prestosql.spi.type.BigintType; import io.prestosql.spi.type.IntegerType; import io.prestosql.spi.type.TimestampType; +import io.prestosql.spi.type.Type; import io.prestosql.spi.type.VarcharType; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; /** * This abstract class represents internal columns. diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java index 34149938299cb..1dc7f1bae0eac 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java @@ -29,15 +29,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airlift.log.Logger; -import javax.inject.Inject; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ColumnHandle; import io.prestosql.spi.connector.ColumnMetadata; @@ -52,6 +43,14 @@ import io.prestosql.spi.connector.SchemaTableName; import io.prestosql.spi.connector.SchemaTablePrefix; import io.prestosql.spi.connector.TableNotFoundException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import javax.inject.Inject; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index e1b345475b7a2..17c9ff245bbc1 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -23,18 +23,9 @@ import static io.prestosql.decoder.FieldValueProviders.bytesValueProvider; import static io.prestosql.decoder.FieldValueProviders.longValueProvider; -import com.google.common.annotations.VisibleForTesting; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicLong; - +import com.google.common.annotations.VisibleForTesting; import io.airlift.log.Logger; import io.airlift.slice.Slice; import io.prestosql.decoder.DecoderColumnHandle; @@ -43,6 +34,13 @@ import io.prestosql.spi.connector.ColumnHandle; import io.prestosql.spi.connector.RecordCursor; import io.prestosql.spi.type.Type; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicLong; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java index 62838c399d7af..c2a2b73471e51 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRowDecoderFactory.java @@ -18,10 +18,10 @@ */ package org.apache.pulsar.sql.presto; -import java.util.List; -import java.util.Set; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.connector.ColumnMetadata; +import java.util.List; +import java.util.Set; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; @@ -32,20 +32,20 @@ public interface PulsarRowDecoderFactory { /** * extract ColumnMetadata from pulsar SchemaInfo and HandleKeyValueType. - * * @param schemaInfo * @param handleKeyValueType * @return */ - List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType); + List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, + PulsarColumnHandle.HandleKeyValueType handleKeyValueType); /** * createRowDecoder RowDecoder by pulsar SchemaInfo and column DecoderColumnHandles. - * * @param schemaInfo * @param columns * @return */ - PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, Set columns); + PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, + Set columns); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java index cc6bdfd9c3e11..5b11c73e3b7c0 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java @@ -28,6 +28,8 @@ import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableSet; import java.nio.ByteBuffer; @@ -35,8 +37,6 @@ import java.util.Map; import java.util.Set; -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; import io.prestosql.spi.PrestoException; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java index 699d7c12ad390..a2456a942eb4f 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java @@ -75,7 +75,8 @@ public PulsarAvroRowDecoderFactory(TypeManager typeManager) { } @Override - public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, Set columns) { + public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, + Set columns) { return new PulsarAvroRowDecoder((GenericAvroSchema) GenericAvroSchema.of(schemaInfo), columns); } @@ -122,7 +123,7 @@ private Type parseAvroPrestoType(String fieldname, Schema schema) { return createUnboundedVarcharType(); case NULL: throw new UnsupportedOperationException( - format("field '%s' NULL Type code should not be reached ļ¼Œ" + + format("field '%s' NULL Type code should not be reached," + "please check the schema or report the bug.", fieldname)); case FIXED: case BYTES: @@ -158,11 +159,13 @@ private Type parseAvroPrestoType(String fieldname, Schema schema) { case RECORD: if (schema.getFields().size() > 0) { return RowType.from(schema.getFields().stream() - .map(field -> new RowType.Field(Optional.of(field.name()), parseAvroPrestoType(field.name(), field.schema()))) + .map(field -> new RowType.Field(Optional.of(field.name()), + parseAvroPrestoType(field.name(), field.schema()))) .collect(toImmutableList())); } else { throw new UnsupportedOperationException(format( - "field '%s' of record Type has no fields, please check avro schema definition. ", fieldname)); + "field '%s' of record Type has no fields, " + + "please check avro schema definition. ", fieldname)); } case UNION: for (Schema nestType : schema.getTypes()) { @@ -174,7 +177,8 @@ private Type parseAvroPrestoType(String fieldname, Schema schema) { "field '%s' of UNION type must contains not null type ", fieldname)); default: throw new UnsupportedOperationException(format( - "Cannot convert from Schema type '%s' (%s) to Presto type", schema.getType(), schema.getFullName())); + "Cannot convert from Schema type '%s' (%s) to Presto type", + schema.getType(), schema.getFullName())); } } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java index 2ba78ba740a55..af81bba4c5361 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java @@ -57,7 +57,22 @@ import io.prestosql.spi.PrestoException; import io.prestosql.spi.block.Block; import io.prestosql.spi.block.BlockBuilder; -import io.prestosql.spi.type.*; +import io.prestosql.spi.type.ArrayType; +import io.prestosql.spi.type.BigintType; +import io.prestosql.spi.type.BooleanType; +import io.prestosql.spi.type.DateType; +import io.prestosql.spi.type.DoubleType; +import io.prestosql.spi.type.IntegerType; +import io.prestosql.spi.type.MapType; +import io.prestosql.spi.type.RealType; +import io.prestosql.spi.type.RowType; +import io.prestosql.spi.type.SmallintType; +import io.prestosql.spi.type.TimeType; +import io.prestosql.spi.type.TimestampType; +import io.prestosql.spi.type.TinyintType; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.VarbinaryType; +import io.prestosql.spi.type.VarcharType; import org.apache.commons.lang3.tuple.Pair; @@ -320,10 +335,10 @@ private void serializePrimitive(BlockBuilder blockBuilder, Object node, Type typ return; } - if (type instanceof RealType || type instanceof BigintType || - type instanceof IntegerType || type instanceof SmallintType || - type instanceof TinyintType || type instanceof TimestampType || - type instanceof TimeType || type instanceof DateType) { + if (type instanceof RealType || type instanceof BigintType + || type instanceof IntegerType || type instanceof SmallintType + || type instanceof TinyintType || type instanceof TimestampType + || type instanceof TimeType || type instanceof DateType) { Pair numRange = getNumRangeByType(type); type.writeLong(blockBuilder, getLong(value, type, columnName, numRange.getKey(), numRange.getValue())); return; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java index ada1eb2ce3943..5f4ab7c77b2a3 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java @@ -23,13 +23,14 @@ import static java.util.Objects.requireNonNull; import static java.util.function.Function.identity; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.MissingNode; import com.google.common.base.Splitter; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.Set; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.MissingNode; + import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java index 6e306600b86dd..68712647ff528 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java @@ -117,8 +117,8 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { return createUnboundedVarcharType(); case NULL: throw new UnsupportedOperationException(format( - "field '%s' NULL Type code should not be reached ļ¼Œ" + - "please check the schema or report the bug.", fieldname)); + "field '%s' NULL Type code should not be reached ļ¼Œ" + + "please check the schema or report the bug.", fieldname)); case FIXED: case BYTES: return VarbinaryType.VARBINARY; @@ -156,8 +156,8 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { .collect(toImmutableList())); } else { throw new UnsupportedOperationException(format( - "field '%s' of record Type has no fields, " + - "please check avro schema definition. ", fieldname)); + "field '%s' of record Type has no fields, " + + "please check avro schema definition. ", fieldname)); } case UNION: for (Schema nestType : schema.getTypes()) { @@ -169,8 +169,8 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { "field '%s' of UNION type must contains not null type ", fieldname)); default: throw new UnsupportedOperationException(format( - "Cannot convert from Schema type '%s' (%s) to " + - "Presto type", schema.getType(), schema.getFullName())); + "Cannot convert from Schema type '%s' (%s) to Presto type", + schema.getType(), schema.getFullName())); } } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java index 9f93211615151..a2a66cf95c326 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java @@ -23,12 +23,12 @@ import static io.prestosql.decoder.FieldValueProviders.longValueProvider; import static org.apache.pulsar.sql.presto.PulsarFieldValueProviders.doubleValueProvider; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; import io.netty.buffer.ByteBuf; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import io.prestosql.decoder.DecoderColumnHandle; -import io.prestosql.decoder.FieldValueProvider; import io.prestosql.spi.type.BigintType; import io.prestosql.spi.type.BooleanType; From 203dc3646426973a354d8ff35eab35cfecf5664a Mon Sep 17 00:00:00 2001 From: wangguowei Date: Wed, 11 Nov 2020 10:56:26 +0800 Subject: [PATCH 09/19] codeStyle fix v6 --- .../sql/presto/PulsarDispatchingRowDecoderFactory.java | 8 ++++++-- .../apache/pulsar/sql/presto/PulsarRecordCursor.java | 3 +-- .../presto/decoder/avro/PulsarAvroColumnDecoder.java | 8 ++++---- .../sql/presto/decoder/avro/PulsarAvroRowDecoder.java | 1 - .../decoder/avro/PulsarAvroRowDecoderFactory.java | 10 ++++++---- .../presto/decoder/json/PulsarJsonFieldDecoder.java | 9 ++++++--- .../sql/presto/decoder/json/PulsarJsonRowDecoder.java | 5 ++++- .../decoder/primitive/PulsarPrimitiveRowDecoder.java | 8 +++++--- .../primitive/PulsarPrimitiveRowDecoderFactory.java | 7 +++++-- 9 files changed, 37 insertions(+), 22 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java index bd5d3bc6a95e0..9fc6f99452910 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarDispatchingRowDecoderFactory.java @@ -21,12 +21,16 @@ import static java.lang.String.format; import com.google.inject.Inject; + import io.airlift.log.Logger; -import java.util.List; -import java.util.Set; + import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.connector.ColumnMetadata; import io.prestosql.spi.type.TypeManager; + +import java.util.List; +import java.util.Set; + import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index 17c9ff245bbc1..8e9f8023dece9 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -550,8 +550,7 @@ public Slice getSlice(int field) { return getFieldValueProvider(field, Slice.class).getSlice(); } - private FieldValueProvider getFieldValueProvider(int fieldIndex, Class expectedType) - { + private FieldValueProvider getFieldValueProvider(int fieldIndex, Class expectedType) { checkArgument(fieldIndex < columnHandles.size(), "Invalid field index"); checkFieldType(fieldIndex, expectedType); return currentRowValues[fieldIndex]; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java index 5b11c73e3b7c0..61c1eb77b5ffb 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java @@ -28,15 +28,15 @@ import static java.lang.String.format; import static java.util.Objects.requireNonNull; -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableSet; import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import java.util.Set; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableSet; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; import io.prestosql.spi.PrestoException; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java index e6683934db340..199a920b18d95 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import io.airlift.log.Logger; import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java index a2456a942eb4f..fa6a869532f35 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java @@ -32,6 +32,7 @@ import java.util.Set; import io.prestosql.decoder.DecoderColumnHandle; + import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ColumnMetadata; import io.prestosql.spi.type.ArrayType; @@ -49,6 +50,7 @@ import io.prestosql.spi.type.TypeSignatureParameter; import io.prestosql.spi.type.VarbinaryType; import io.prestosql.spi.type.VarcharType; + import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; @@ -123,8 +125,8 @@ private Type parseAvroPrestoType(String fieldname, Schema schema) { return createUnboundedVarcharType(); case NULL: throw new UnsupportedOperationException( - format("field '%s' NULL Type code should not be reached," + - "please check the schema or report the bug.", fieldname)); + format("field '%s' NULL Type code should not be reached," + + "please check the schema or report the bug.", fieldname)); case FIXED: case BYTES: //TODO: support decimal logicalType @@ -164,8 +166,8 @@ private Type parseAvroPrestoType(String fieldname, Schema schema) { .collect(toImmutableList())); } else { throw new UnsupportedOperationException(format( - "field '%s' of record Type has no fields, " + - "please check avro schema definition. ", fieldname)); + "field '%s' of record Type has no fields, " + + "please check avro schema definition. ", fieldname)); } case UNION: for (Schema nestType : schema.getTypes()) { diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java index af81bba4c5361..0a38a629506f3 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java @@ -41,14 +41,16 @@ import static java.lang.String.format; import static java.util.Objects.requireNonNull; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableList; + import java.util.Iterator; import java.util.List; import java.util.Map; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import io.airlift.slice.Slice; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; @@ -73,6 +75,7 @@ import io.prestosql.spi.type.Type; import io.prestosql.spi.type.VarbinaryType; import io.prestosql.spi.type.VarcharType; + import org.apache.commons.lang3.tuple.Pair; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java index 5f4ab7c77b2a3..a38851c0dea2f 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java @@ -26,15 +26,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.MissingNode; import com.google.common.base.Splitter; + import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.Set; -import io.netty.buffer.ByteBuf; + import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; import io.prestosql.decoder.json.JsonFieldDecoder; +import io.netty.buffer.ByteBuf; + import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; import org.apache.pulsar.sql.presto.PulsarRowDecoder; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java index a2a66cf95c326..7a1d6db27d3e1 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java @@ -23,13 +23,14 @@ import static io.prestosql.decoder.FieldValueProviders.longValueProvider; import static org.apache.pulsar.sql.presto.PulsarFieldValueProviders.doubleValueProvider; -import io.prestosql.decoder.DecoderColumnHandle; -import io.prestosql.decoder.FieldValueProvider; -import io.netty.buffer.ByteBuf; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import io.netty.buffer.ByteBuf; + +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; import io.prestosql.spi.type.BigintType; import io.prestosql.spi.type.BooleanType; import io.prestosql.spi.type.DateType; @@ -43,6 +44,7 @@ import io.prestosql.spi.type.Type; import io.prestosql.spi.type.VarbinaryType; import io.prestosql.spi.type.VarcharType; + import org.apache.pulsar.sql.presto.PulsarRowDecoder; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java index 4fa796973d4f0..31988cd712744 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java @@ -18,11 +18,12 @@ */ package org.apache.pulsar.sql.presto.decoder.primitive; -import io.airlift.log.Logger; + import java.util.Arrays; import java.util.List; import java.util.Set; +import io.airlift.log.Logger; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.connector.ColumnMetadata; import io.prestosql.spi.type.BigintType; @@ -38,6 +39,7 @@ import io.prestosql.spi.type.Type; import io.prestosql.spi.type.VarbinaryType; import io.prestosql.spi.type.VarcharType; + import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; @@ -54,7 +56,8 @@ public class PulsarPrimitiveRowDecoderFactory implements PulsarRowDecoderFactory private static final Logger log = Logger.get(PulsarPrimitiveRowDecoderFactory.class); @Override - public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, Set columns) { + public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, + Set columns) { if (columns.size() == 1) { return new PulsarPrimitiveRowDecoder(columns.iterator().next()); } else { From aa636b9bd30cfefbff43ae65c02a3cdcccb1c7fe Mon Sep 17 00:00:00 2001 From: wangguowei Date: Wed, 11 Nov 2020 15:21:05 +0800 Subject: [PATCH 10/19] codeStyle fix v7 --- .../presto/decoder/avro/PulsarAvroColumnDecoder.java | 11 ++++++----- .../presto/decoder/avro/PulsarAvroRowDecoder.java | 7 ++++--- .../decoder/avro/PulsarAvroRowDecoderFactory.java | 9 ++++----- .../presto/decoder/json/PulsarJsonFieldDecoder.java | 10 +++------- .../presto/decoder/json/PulsarJsonRowDecoder.java | 12 ++++-------- .../decoder/json/PulsarJsonRowDecoderFactory.java | 8 +++----- .../decoder/primitive/PulsarPrimitiveRowDecoder.java | 10 ++++------ .../primitive/PulsarPrimitiveRowDecoderFactory.java | 9 +++------ .../sql/presto/decoder/avro/AvroDecoderTestUtil.java | 6 +++--- 9 files changed, 34 insertions(+), 48 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java index 61c1eb77b5ffb..99f1fc773cfbf 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java @@ -28,13 +28,8 @@ import static java.lang.String.format; import static java.util.Objects.requireNonNull; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; -import java.util.Set; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableSet; - import io.airlift.slice.Slice; import io.airlift.slice.Slices; import io.prestosql.decoder.DecoderColumnHandle; @@ -59,6 +54,12 @@ import io.prestosql.spi.type.Type; import io.prestosql.spi.type.VarbinaryType; import io.prestosql.spi.type.VarcharType; +import java.nio.ByteBuffer; + +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.avro.generic.GenericEnumSymbol; import org.apache.avro.generic.GenericFixed; import org.apache.avro.generic.GenericRecord; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java index 199a920b18d95..b99da9aea90e6 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java @@ -23,13 +23,14 @@ import static io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static java.util.Objects.requireNonNull; -import java.util.Map; -import java.util.Optional; -import java.util.Set; import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; import io.prestosql.spi.PrestoException; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + import org.apache.avro.generic.GenericRecord; import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord; import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java index fa6a869532f35..f56f376011b50 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java @@ -27,12 +27,7 @@ import static java.util.stream.Collectors.toList; import com.google.common.collect.ImmutableList; -import java.util.List; -import java.util.Optional; -import java.util.Set; - import io.prestosql.decoder.DecoderColumnHandle; - import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ColumnMetadata; import io.prestosql.spi.type.ArrayType; @@ -51,6 +46,10 @@ import io.prestosql.spi.type.VarbinaryType; import io.prestosql.spi.type.VarcharType; +import java.util.List; +import java.util.Optional; +import java.util.Set; + import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java index 0a38a629506f3..ed925df5392c9 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java @@ -41,16 +41,10 @@ import static java.lang.String.format; import static java.util.Objects.requireNonNull; - import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableList; - -import java.util.Iterator; -import java.util.List; -import java.util.Map; - import io.airlift.slice.Slice; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; @@ -75,10 +69,12 @@ import io.prestosql.spi.type.Type; import io.prestosql.spi.type.VarbinaryType; import io.prestosql.spi.type.VarcharType; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import org.apache.commons.lang3.tuple.Pair; - /** * copy from {@link io.prestosql.decoder.json.DefaultJsonFieldDecoder} (presto-record-decoder-345) * with some pulsar's extensions: diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java index a38851c0dea2f..c516bc08ceb29 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java @@ -26,18 +26,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.MissingNode; import com.google.common.base.Splitter; - +import io.netty.buffer.ByteBuf; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; +import io.prestosql.decoder.json.JsonFieldDecoder; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.Set; - - -import io.prestosql.decoder.DecoderColumnHandle; -import io.prestosql.decoder.FieldValueProvider; -import io.prestosql.decoder.json.JsonFieldDecoder; -import io.netty.buffer.ByteBuf; - import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; import org.apache.pulsar.sql.presto.PulsarRowDecoder; diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java index 68712647ff528..fc622c6d20fc2 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java @@ -27,10 +27,6 @@ import static java.util.stream.Collectors.toList; import com.google.common.collect.ImmutableList; -import java.util.List; -import java.util.Optional; -import java.util.Set; - import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ColumnMetadata; @@ -49,6 +45,9 @@ import io.prestosql.spi.type.TypeSignatureParameter; import io.prestosql.spi.type.VarbinaryType; import io.prestosql.spi.type.VarcharType; +import java.util.List; +import java.util.Optional; +import java.util.Set; import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; @@ -61,7 +60,6 @@ import org.apache.pulsar.sql.presto.PulsarColumnMetadata; import org.apache.pulsar.sql.presto.PulsarRowDecoderFactory; - /** * PulsarRowDecoderFactory for {@link org.apache.pulsar.common.schema.SchemaType#JSON}. */ diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java index 7a1d6db27d3e1..182b01bf15511 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java @@ -23,12 +23,7 @@ import static io.prestosql.decoder.FieldValueProviders.longValueProvider; import static org.apache.pulsar.sql.presto.PulsarFieldValueProviders.doubleValueProvider; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - import io.netty.buffer.ByteBuf; - import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; import io.prestosql.spi.type.BigintType; @@ -45,8 +40,11 @@ import io.prestosql.spi.type.VarbinaryType; import io.prestosql.spi.type.VarcharType; -import org.apache.pulsar.sql.presto.PulsarRowDecoder; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import org.apache.pulsar.sql.presto.PulsarRowDecoder; /** * Primitive Schema PulsarRowDecoder. diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java index 31988cd712744..00a0e9c4a3d6d 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java @@ -18,11 +18,6 @@ */ package org.apache.pulsar.sql.presto.decoder.primitive; - -import java.util.Arrays; -import java.util.List; -import java.util.Set; - import io.airlift.log.Logger; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.connector.ColumnMetadata; @@ -39,7 +34,9 @@ import io.prestosql.spi.type.Type; import io.prestosql.spi.type.VarbinaryType; import io.prestosql.spi.type.VarcharType; - +import java.util.Arrays; +import java.util.List; +import java.util.Set; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/AvroDecoderTestUtil.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/AvroDecoderTestUtil.java index f11360f03e4bd..521807f80f0df 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/AvroDecoderTestUtil.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/AvroDecoderTestUtil.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY From 6cfe40eef12dfba3350f9982dc61d547328db896 Mon Sep 17 00:00:00 2001 From: wangguowei Date: Wed, 11 Nov 2020 16:18:08 +0800 Subject: [PATCH 11/19] add License --- pulsar-sql/presto-distribution/LICENSE | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index b1475ee0e247d..83a8107f2d206 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -384,6 +384,7 @@ The Apache Software License, Version 2.0 - presto-parser-332.jar - presto-plugin-toolkit-332.jar - presto-spi-332.jar + - presto-record-decoder-332.jar * RocksDB JNI - rocksdbjni-6.10.2.jar * SnakeYAML From 592130b1635f1eba8e83c90825d7b326e16534ad Mon Sep 17 00:00:00 2001 From: wangguowei Date: Thu, 12 Nov 2020 14:08:52 +0800 Subject: [PATCH 12/19] add snappy license --- pulsar-sql/presto-distribution/LICENSE | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 83a8107f2d206..a5c969a76fab9 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -430,6 +430,8 @@ The Apache Software License, Version 2.0 - commons-logging-1.2.jar * GSON - gson-2.8.2.jar + * Snappy + - snappy-java-1.1.7.3.jar * Jackson - jackson-module-parameter-names-2.10.0.jar - jackson-module-parameter-names-2.11.1.jar From 7c5f93ce2e89894e985c5d986982dd34b54ba228 Mon Sep 17 00:00:00 2001 From: wangguowei Date: Mon, 16 Nov 2020 14:49:41 +0800 Subject: [PATCH 13/19] codeStyle fix --- .../sql/presto/PulsarColumnMetadata.java | 8 ++++---- .../decoder/avro/PulsarAvroColumnDecoder.java | 7 ++++--- .../decoder/avro/PulsarAvroRowDecoder.java | 3 ++- .../avro/PulsarAvroRowDecoderFactory.java | 18 ++++++------------ .../decoder/json/PulsarJsonFieldDecoder.java | 18 ++++++++++-------- .../json/PulsarJsonRowDecoderFactory.java | 12 ++++++------ .../PulsarPrimitiveRowDecoderFactory.java | 4 ++-- 7 files changed, 34 insertions(+), 36 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java index 57aad3b39af6d..c7b43d86d2075 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java @@ -123,7 +123,7 @@ public int hashCode() { /** - * Decoder ExtraInfo for {@link org.apache.pulsar.sql.presto.PulsarColumnHandle} + * Decoder extra info for {@link org.apache.pulsar.sql.presto.PulsarColumnHandle} * used by {@link io.prestosql.decoder.RowDecoder}. */ public static class DecoderExtraInfo { @@ -136,11 +136,11 @@ public DecoderExtraInfo(String mapping, String dataFormat, String formatHint) { public DecoderExtraInfo() {} - //equals ColumnName in general, may used as alias or embedded field future + //equals ColumnName in general, may used as alias or embedded field in future. private String mapping; - //reserved dataFormat for used by RowDecoder + //reserved dataFormat used by RowDecoder. private String dataFormat; - //reserved formatHint for used by RowDecoder + //reserved formatHint used by RowDecoder. private String formatHint; public String getMapping() { diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java index 99f1fc773cfbf..3b7e293557923 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java @@ -65,10 +65,11 @@ import org.apache.avro.generic.GenericRecord; /** - * copy from {@link io.prestosql.decoder.avro.AvroColumnDecoder} (presto-record-decoder-345) + * Copy from {@link io.prestosql.decoder.avro.AvroColumnDecoder} (presto-record-decoder-345) * with A little bit pulsar's extensions. - * 1) support TIMESTAMP,DATE,TIME - * 2) support RealType + * 1) support {@link io.prestosql.spi.type.TimestampType},{@link io.prestosql.spi.type.DateType}DATE, + * * {@link io.prestosql.spi.type.TimeType}. + * 2) support {@link io.prestosql.spi.type.RealType}. */ public class PulsarAvroColumnDecoder { private static final Set SUPPORTED_PRIMITIVE_TYPES = ImmutableSet.of( diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java index b99da9aea90e6..b7c946a5408cc 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoder.java @@ -66,7 +66,8 @@ public Optional> decodeRow(ByteBuf GenericAvroRecord record = (GenericAvroRecord) genericAvroSchema.decode(byteBuf); avroRecord = record.getAvroRecord(); } catch (Exception e) { - throw new PrestoException(GENERIC_INTERNAL_ERROR, "Decoding Avro record failed.", e); + e.printStackTrace(); + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Decoding avro record failed.", e); } return Optional.of(columnDecoders.entrySet().stream() .collect(toImmutableMap( diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java index f56f376011b50..3459f91f27371 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java @@ -109,12 +109,6 @@ public List extractColumnMetadata(TopicName topicName, SchemaInf ).collect(toList()); } - /** - * parse AvroPrestoType , avro LogicalType supported. - * @param fieldname - * @param schema - * @return - */ private Type parseAvroPrestoType(String fieldname, Schema schema) { Schema.Type type = schema.getType(); LogicalType logicalType = schema.getLogicalType(); @@ -124,7 +118,7 @@ private Type parseAvroPrestoType(String fieldname, Schema schema) { return createUnboundedVarcharType(); case NULL: throw new UnsupportedOperationException( - format("field '%s' NULL Type code should not be reached," + format("field '%s' NULL type code should not be reached," + "please check the schema or report the bug.", fieldname)); case FIXED: case BYTES: @@ -152,7 +146,7 @@ private Type parseAvroPrestoType(String fieldname, Schema schema) { case ARRAY: return new ArrayType(parseAvroPrestoType(fieldname, schema.getElementType())); case MAP: - //The key for an Avro map must be a string + //The key for an avro map must be string TypeSignature valueType = parseAvroPrestoType(fieldname, schema.getValueType()).getTypeSignature(); return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(VarcharType.VARCHAR.getTypeSignature()), @@ -165,8 +159,8 @@ private Type parseAvroPrestoType(String fieldname, Schema schema) { .collect(toImmutableList())); } else { throw new UnsupportedOperationException(format( - "field '%s' of record Type has no fields, " - + "please check avro schema definition. ", fieldname)); + "field '%s' of record type has no fields, " + + "please check schema definition. ", fieldname)); } case UNION: for (Schema nestType : schema.getTypes()) { @@ -175,10 +169,10 @@ private Type parseAvroPrestoType(String fieldname, Schema schema) { } } throw new UnsupportedOperationException(format( - "field '%s' of UNION type must contains not null type ", fieldname)); + "field '%s' of UNION type must contains not NULL type.", fieldname)); default: throw new UnsupportedOperationException(format( - "Cannot convert from Schema type '%s' (%s) to Presto type", + "Can't convert from schema type '%s' (%s) to presto type.", schema.getType(), schema.getFullName())); } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java index ed925df5392c9..bd7412738f2ef 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java @@ -76,16 +76,18 @@ import org.apache.commons.lang3.tuple.Pair; /** - * copy from {@link io.prestosql.decoder.json.DefaultJsonFieldDecoder} (presto-record-decoder-345) - * with some pulsar's extensions: - * 1) support array. - * 2) support map. - * 3) support row. - * 3) support TIMESTAMP,DATE,TIME. - * 4) support Real. + * Copy from {@link io.prestosql.decoder.json.DefaultJsonFieldDecoder} (presto-record-decoder-345) + * with some pulsar's extensions. + * 1) support {@link io.prestosql.spi.type.ArrayType}. + * 2) support {@link io.prestosql.spi.type.MapType}. + * 3) support {@link io.prestosql.spi.type.RowType}. + * 4) support {@link io.prestosql.spi.type.TimestampType},{@link io.prestosql.spi.type.DateType}, + * {@link io.prestosql.spi.type.TimeType}. + * 5) support {@link io.prestosql.spi.type.RealType}. */ public class PulsarJsonFieldDecoder implements JsonFieldDecoder { + private final DecoderColumnHandle columnHandle; private final long minValue; private final long maxValue; @@ -413,7 +415,7 @@ private Block serializeRow(BlockBuilder parentBlockBuilder, Object value, Type t List fields = ((RowType) type).getFields(); - checkState(value instanceof ObjectNode, "Json row node must is ObjectNode type"); + checkState(value instanceof ObjectNode, "Json row node must be ObjectNode type"); for (RowType.Field field : fields) { checkState(field.getName().isPresent(), "field name not found"); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java index fc622c6d20fc2..deebb8f4c4fe6 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java @@ -115,7 +115,7 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { return createUnboundedVarcharType(); case NULL: throw new UnsupportedOperationException(format( - "field '%s' NULL Type code should not be reached ļ¼Œ" + "field '%s' NULL type code should not be reached ļ¼Œ" + "please check the schema or report the bug.", fieldname)); case FIXED: case BYTES: @@ -141,7 +141,7 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { case ARRAY: return new ArrayType(parseJsonPrestoType(fieldname, schema.getElementType())); case MAP: - //The key for an avro map must be a string + //The key for an avro map must be string. TypeSignature valueType = parseJsonPrestoType(fieldname, schema.getValueType()).getTypeSignature(); return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter. typeParameter(VarcharType.VARCHAR.getTypeSignature()), @@ -154,8 +154,8 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { .collect(toImmutableList())); } else { throw new UnsupportedOperationException(format( - "field '%s' of record Type has no fields, " - + "please check avro schema definition. ", fieldname)); + "field '%s' of record type has no fields, " + + "please check schema definition. ", fieldname)); } case UNION: for (Schema nestType : schema.getTypes()) { @@ -164,10 +164,10 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { } } throw new UnsupportedOperationException(format( - "field '%s' of UNION type must contains not null type ", fieldname)); + "field '%s' of UNION type must contains not NULL type.", fieldname)); default: throw new UnsupportedOperationException(format( - "Cannot convert from Schema type '%s' (%s) to Presto type", + "Can't convert from schema type '%s' (%s) to presto type.", schema.getType(), schema.getFullName())); } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java index 00a0e9c4a3d6d..84fd59b949bd3 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java @@ -58,7 +58,7 @@ public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaI if (columns.size() == 1) { return new PulsarPrimitiveRowDecoder(columns.iterator().next()); } else { - throw new RuntimeException("Primitive type must has only one ColumnHandle "); + throw new RuntimeException("Primitive type must has only one ColumnHandle."); } } @@ -102,7 +102,7 @@ private Type parsePrimitivePrestoType(String fieldName, SchemaType pulsarType) { case TIMESTAMP: return TimestampType.TIMESTAMP; default: - log.error("Cannot convert type: %s for %s", pulsarType, fieldName); + log.error("Can't convert type: %s for %s", pulsarType, fieldName); return null; } From f0a0c610fddc055075aec310a4f3f0e09b59e7fb Mon Sep 17 00:00:00 2001 From: hnail Date: Tue, 1 Dec 2020 14:00:13 +0800 Subject: [PATCH 14/19] Update pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java Co-authored-by: ran --- .../java/org/apache/pulsar/sql/presto/TestPulsarConnector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java index 15fafa4165a54..e058ec66900c6 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java @@ -85,7 +85,7 @@ public abstract class TestPulsarConnector { protected final static PulsarConnectorId pulsarConnectorId = new PulsarConnectorId("test-connector"); - protected static List topicNames; + protected static List topicNames; protected static List partitionedTopicNames; protected static Map partitionedTopicsToPartitions; protected static Map topicsToSchemas; From bf98970ad442e6d36728b6ec4600642d9dfa6b5c Mon Sep 17 00:00:00 2001 From: hnail Date: Tue, 1 Dec 2020 14:01:17 +0800 Subject: [PATCH 15/19] Update pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java Co-authored-by: ran --- .../java/org/apache/pulsar/sql/presto/TestPulsarConnector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java index e058ec66900c6..31899ad9b47dc 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java @@ -86,7 +86,7 @@ public abstract class TestPulsarConnector { protected final static PulsarConnectorId pulsarConnectorId = new PulsarConnectorId("test-connector"); protected static List topicNames; - protected static List partitionedTopicNames; + protected static List partitionedTopicNames; protected static Map partitionedTopicsToPartitions; protected static Map topicsToSchemas; protected static Map topicsToNumEntries; From dbcf5cd2c43eabbafda3decdd3625690697bd109 Mon Sep 17 00:00:00 2001 From: hnail Date: Tue, 1 Dec 2020 14:01:30 +0800 Subject: [PATCH 16/19] Update pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java Co-authored-by: ran --- .../java/org/apache/pulsar/sql/presto/TestPulsarConnector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java index 31899ad9b47dc..e779765cd94e6 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java @@ -93,7 +93,7 @@ public abstract class TestPulsarConnector { private final static ObjectMapper objectMapper = new ObjectMapper(); - protected static List fooFieldNames = new ArrayList<>(); + protected static List fooFieldNames = new ArrayList<>(); protected static final NamespaceName NAMESPACE_NAME_1 = NamespaceName.get("tenant-1", "ns-1"); protected static final NamespaceName NAMESPACE_NAME_2 = NamespaceName.get("tenant-1", "ns-2"); From 11c0cefd4162a906986ecb89c4b7975395147b31 Mon Sep 17 00:00:00 2001 From: wangguowei Date: Wed, 23 Dec 2020 15:24:03 +0800 Subject: [PATCH 17/19] add keyValue\Primitive schema test && add schema cyclic definition detect --- .../sql/presto/PulsarColumnMetadata.java | 13 + .../sql/presto/PulsarInternalColumn.java | 2 +- .../pulsar/sql/presto/PulsarMetadata.java | 42 ++- .../pulsar/sql/presto/PulsarRecordCursor.java | 71 ++-- .../avro/PulsarAvroRowDecoderFactory.java | 31 +- .../decoder/json/PulsarJsonFieldDecoder.java | 15 +- .../decoder/json/PulsarJsonRowDecoder.java | 3 + .../json/PulsarJsonRowDecoderFactory.java | 30 +- .../primitive/PulsarPrimitiveRowDecoder.java | 54 ++-- .../PulsarPrimitiveRowDecoderFactory.java | 13 +- .../sql/presto/TestPulsarConnector.java | 66 ++-- .../sql/presto/TestPulsarRecordCursor.java | 305 ++++++++++++++++++ .../presto/decoder/AbstractDecoderTester.java | 43 ++- .../presto/decoder/DecoderTestMessage.java | 20 ++ .../sql/presto/decoder/DecoderTestUtil.java | 5 - .../presto/decoder/avro/TestAvroDecoder.java | 19 +- .../presto/decoder/json/TestJsonDecoder.java | 18 +- .../primitive/PrimitiveDecoderTestUtil.java | 56 ++++ .../primitive/TestPrimitiveDecoder.java | 233 +++++++++++++ 19 files changed, 913 insertions(+), 126 deletions(-) create mode 100644 pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/PrimitiveDecoderTestUtil.java create mode 100644 pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/TestPrimitiveDecoder.java diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java index c7b43d86d2075..4ba153d99f722 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarColumnMetadata.java @@ -86,6 +86,7 @@ public String toString() { + "isInternal=" + isInternal + ", nameWithCase='" + nameWithCase + '\'' + ", handleKeyValueType=" + handleKeyValueType + + ", decoderExtraInfo=" + decoderExtraInfo.toString() + '}'; } @@ -109,6 +110,9 @@ public boolean equals(Object o) { if (nameWithCase != null ? !nameWithCase.equals(that.nameWithCase) : that.nameWithCase != null) { return false; } + if (decoderExtraInfo != null ? !decoderExtraInfo.equals(that.decoderExtraInfo) : that.decoderExtraInfo != null) { + return false; + } return Objects.equals(handleKeyValueType, that.handleKeyValueType); } @@ -117,6 +121,7 @@ public int hashCode() { int result = super.hashCode(); result = 31 * result + (isInternal ? 1 : 0); result = 31 * result + (nameWithCase != null ? nameWithCase.hashCode() : 0); + result = 31 * result + (decoderExtraInfo != null ? decoderExtraInfo.hashCode() : 0); result = 31 * result + (handleKeyValueType != null ? handleKeyValueType.hashCode() : 0); return result; } @@ -191,6 +196,14 @@ public boolean equals(Object o) { return Objects.equals(formatHint, that.formatHint); } + @Override + public String toString() { + return "DecoderExtraInfo{" + + "mapping=" + mapping + + ", dataFormat=" + dataFormat + + ", formatHint=" + formatHint + + '}'; + } @Override public int hashCode() { diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java index 05c3e72b89ee0..934097759b6ee 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarInternalColumn.java @@ -99,7 +99,7 @@ PulsarColumnHandle getColumnHandle(String connectorId, boolean hidden) { PulsarColumnMetadata getColumnMetadata(boolean hidden) { return new PulsarColumnMetadata(name, type, comment, null, hidden, true, - PulsarColumnHandle.HandleKeyValueType.NONE, null); + PulsarColumnHandle.HandleKeyValueType.NONE, new PulsarColumnMetadata.DecoderExtraInfo()); } public static Set getInternalFields() { diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java index 1dc7f1bae0eac..0a7dfb8724b63 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarMetadata.java @@ -26,6 +26,7 @@ import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertColumnHandle; import static org.apache.pulsar.sql.presto.PulsarHandleResolver.convertTableHandle; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airlift.log.Logger; @@ -320,37 +321,33 @@ private ConnectorTableMetadata getTableMetadata(SchemaTableName schemaTableName, /** * Convert pulsar schema into presto table metadata. */ - List getPulsarColumns(TopicName topicName, - SchemaInfo schemaInfo, - boolean withInternalColumns, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { - SchemaType schemaType = schemaInfo.getType(); - if (schemaType.isStruct() || schemaType.isPrimitive()) { - return getPulsarColumnsFromSchema(topicName, schemaInfo, withInternalColumns, handleKeyValueType); - } else if (schemaType.equals(SchemaType.KEY_VALUE)) { - return getPulsarColumnsFromKeyValueSchema(topicName, schemaInfo, withInternalColumns); - } else { - throw new IllegalArgumentException("Unsupported schema : " + schemaInfo); + @VisibleForTesting + public List getPulsarColumns(TopicName topicName, + SchemaInfo schemaInfo, + boolean withInternalColumns, + PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + SchemaType schemaType = schemaInfo.getType(); + if (schemaType.isStruct() || schemaType.isPrimitive()) { + return getPulsarColumnsFromSchema(topicName, schemaInfo, withInternalColumns, handleKeyValueType); + } else if (schemaType.equals(SchemaType.KEY_VALUE)) { + return getPulsarColumnsFromKeyValueSchema(topicName, schemaInfo, withInternalColumns); + } else { + throw new IllegalArgumentException("Unsupported schema : " + schemaInfo); } } - - - List getPulsarColumnsFromSchema(TopicName topicName, - SchemaInfo schemaInfo, - boolean withInternalColumns, - PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { - + List getPulsarColumnsFromSchema(TopicName topicName, + SchemaInfo schemaInfo, + boolean withInternalColumns, + PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { ImmutableList.Builder builder = ImmutableList.builder(); - builder.addAll(decoderFactory.extractColumnMetadata(topicName, schemaInfo, handleKeyValueType)); - if (withInternalColumns) { PulsarInternalColumn.getInternalFields() .stream() .forEach(pulsarInternalColumn -> builder.add(pulsarInternalColumn.getColumnMetadata(false))); } - return builder.build(); + return builder.build(); } List getPulsarColumnsFromKeyValueSchema(TopicName topicName, @@ -375,7 +372,4 @@ List getPulsarColumnsFromKeyValueSchema(TopicName topicName, return builder.build(); } - - - } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index 8e9f8023dece9..8c8116f7860b4 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -28,6 +28,7 @@ import com.google.common.annotations.VisibleForTesting; import io.airlift.log.Logger; import io.airlift.slice.Slice; +import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; import io.prestosql.spi.block.Block; @@ -38,7 +39,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicLong; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -51,11 +54,14 @@ import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.mledger.impl.ReadOnlyCursorImpl; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.schema.KeyValueSchemaInfo; import org.apache.pulsar.common.api.raw.MessageParser; import org.apache.pulsar.common.api.raw.RawMessage; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.OffloadPolicies; +import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.jctools.queues.MessagePassingQueue; @@ -95,7 +101,7 @@ public class PulsarRecordCursor implements RecordCursor { private PulsarSqlSchemaInfoProvider schemaInfoProvider; - private FieldValueProvider[] currentRowValues = null; + private FieldValueProvider[] currentRowValues = null; PulsarDispatchingRowDecoderFactory decoderFactory; @@ -445,31 +451,56 @@ public boolean advanceNextPosition() { Map currentRowValuesMap = new HashMap<>(); if (schemaInfo.getType().equals(SchemaType.KEY_VALUE)) { + ByteBuf keyByteBuf; + ByteBuf valueByteBuf; + + KeyValueEncodingType keyValueEncodingType = KeyValueSchemaInfo.decodeKeyValueEncodingType(schemaInfo); + if (Objects.equals(keyValueEncodingType, KeyValueEncodingType.INLINE)) { + ByteBuf dataPayload = this.currentMessage.getData(); + int keyLength = dataPayload.readInt(); + keyByteBuf = dataPayload.readSlice(keyLength); + int valueLength = dataPayload.readInt(); + valueByteBuf = dataPayload.readSlice(valueLength); + } else { + keyByteBuf = this.currentMessage.getKeyBytes().get(); + valueByteBuf = this.currentMessage.getData(); + } - PulsarRowDecoder keyDecoder = decoderFactory.createRowDecoder(topicName, - schemaInfo, - columnHandles.stream() - .filter(col -> !col.isInternal()) - .filter(col -> PulsarColumnHandle.HandleKeyValueType.KEY - .equals(col.getHandleKeyValueType())) - .collect(toImmutableSet())); + KeyValue kvSchemaInfo = KeyValueSchemaInfo.decodeKeyValueSchemaInfo(schemaInfo); + Set keyColumnHandles = columnHandles.stream() + .filter(col -> !col.isInternal()) + .filter(col -> PulsarColumnHandle.HandleKeyValueType.KEY + .equals(col.getHandleKeyValueType())) + .collect(toImmutableSet()); + PulsarRowDecoder keyDecoder = null; + if (keyColumnHandles.size() > 0) { + keyDecoder = decoderFactory.createRowDecoder(topicName, + kvSchemaInfo.getKey(), keyColumnHandles + ); + } - PulsarRowDecoder messageDecoder = decoderFactory.createRowDecoder(topicName, - schemaInfo, - columnHandles.stream() - .filter(col -> !col.isInternal()) - .filter(col -> PulsarColumnHandle.HandleKeyValueType.VALUE - .equals(col.getHandleKeyValueType())) - .collect(toImmutableSet())); + Set valueColumnHandles = columnHandles.stream() + .filter(col -> !col.isInternal()) + .filter(col -> PulsarColumnHandle.HandleKeyValueType.VALUE + .equals(col.getHandleKeyValueType())) + .collect(toImmutableSet()); + PulsarRowDecoder valueDecoder = null; + if (valueColumnHandles.size() > 0) { + valueDecoder = decoderFactory.createRowDecoder(topicName, + kvSchemaInfo.getValue(), + valueColumnHandles); + } Optional> decodedKey; - if (this.currentMessage.getKeyBytes().isPresent()) { - decodedKey = keyDecoder.decodeRow(this.currentMessage.getKeyBytes().get()); + if (keyColumnHandles.size() > 0) { + decodedKey = keyDecoder.decodeRow(keyByteBuf); decodedKey.ifPresent(currentRowValuesMap::putAll); } - Optional> decodedValue = - messageDecoder.decodeRow(this.currentMessage.getData()); - decodedValue.ifPresent(currentRowValuesMap::putAll); + if (valueColumnHandles.size() > 0) { + Optional> decodedValue = + valueDecoder.decodeRow(valueByteBuf); + decodedValue.ifPresent(currentRowValuesMap::putAll); + } } else { PulsarRowDecoder messageDecoder = decoderFactory.createRowDecoder(topicName, schemaInfo, diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java index 3459f91f27371..26c333ab56b82 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java @@ -27,6 +27,7 @@ import static java.util.stream.Collectors.toList; import com.google.common.collect.ImmutableList; +import io.airlift.log.Logger; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ColumnMetadata; @@ -84,6 +85,7 @@ public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaI @Override public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + List columnMetadata; String schemaJson = new String(schemaInfo.getSchema()); if (StringUtils.isBlank(schemaJson)) { throw new PrestoException(NOT_SUPPORTED, "Topic " @@ -97,16 +99,22 @@ public List extractColumnMetadata(TopicName topicName, SchemaInf + topicName.toString() + " does not have a valid schema"); } - //TODO : check schema cyclic definitions which may case java.lang.StackOverflowError - - return schema.getFields().stream() - .map(field -> - new PulsarColumnMetadata(field.name(), parseAvroPrestoType( - field.name(), field.schema()), field.schema().toString(), null, false, false, - handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(field.name(), - null, null)) + try { + columnMetadata = schema.getFields().stream() + .map(field -> + new PulsarColumnMetadata(PulsarColumnMetadata.getColumnName(handleKeyValueType, field.name()), parseAvroPrestoType( + field.name(), field.schema()), field.schema().toString(), null, false, false, + handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(field.name(), + null, null)) - ).collect(toList()); + ).collect(toList()); + }catch (StackOverflowError e){ + log.warn(e, "Topic " + + topicName.toString() + " extractColumnMetadata failed."); + throw new PrestoException(NOT_SUPPORTED, "Topic " + + topicName.toString() + " schema may contains cyclic definitions.",e); + } + return columnMetadata; } private Type parseAvroPrestoType(String fieldname, Schema schema) { @@ -135,7 +143,7 @@ private Type parseAvroPrestoType(String fieldname, Schema schema) { if (logicalType == LogicalTypes.timestampMillis()) { return TimestampType.TIMESTAMP; } - //TODO: support timestamp_microseconds logicalType : https://github.com/prestosql/presto/issues/1284 + //TODO: support timestamp_microseconds logicalType : https://github.com/prestosql/presto/issues/1284 return BigintType.BIGINT; case FLOAT: return RealType.REAL; @@ -176,4 +184,7 @@ private Type parseAvroPrestoType(String fieldname, Schema schema) { schema.getType(), schema.getFullName())); } } + + private static final Logger log = Logger.get(PulsarAvroRowDecoderFactory.class); + } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java index bd7412738f2ef..960d8f42bdf7a 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java @@ -45,6 +45,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableList; +import io.airlift.log.Logger; import io.airlift.slice.Slice; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; @@ -317,7 +318,13 @@ private Block serializeList(BlockBuilder parentBlockBuilder, Object value, Type private void serializePrimitive(BlockBuilder blockBuilder, Object node, Type type, String columnName) { requireNonNull(blockBuilder, "parent blockBuilder is null"); + JsonNode value; + if (node == null) { + blockBuilder.appendNull(); + return; + } + if (node instanceof JsonNode) { value = (JsonNode) node; } else { @@ -326,11 +333,6 @@ private void serializePrimitive(BlockBuilder blockBuilder, Object node, Type typ node.getClass(), type, columnName)); } - if (value == null) { - blockBuilder.appendNull(); - return; - } - if (type instanceof BooleanType) { type.writeBoolean(blockBuilder, getBoolean(value, type, columnName)); return; @@ -430,4 +432,7 @@ private Block serializeRow(BlockBuilder parentBlockBuilder, Object value, Type t } } + + private static final Logger log = Logger.get(PulsarJsonFieldDecoder.class); + } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java index c516bc08ceb29..638a723a7d8af 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoder.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.MissingNode; import com.google.common.base.Splitter; +import io.airlift.log.Logger; import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; @@ -83,4 +84,6 @@ public Optional> decodeRow(ByteBuf } return Optional.of(decodedRow); } + + private static final Logger log = Logger.get(PulsarJsonRowDecoderFactory.class); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java index deebb8f4c4fe6..10a500ba3618e 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java @@ -27,6 +27,7 @@ import static java.util.stream.Collectors.toList; import com.google.common.collect.ImmutableList; +import io.airlift.log.Logger; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ColumnMetadata; @@ -80,11 +81,13 @@ public PulsarJsonRowDecoder createRowDecoder(TopicName topicName, SchemaInfo sch @Override public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { + List columnMetadata; String schemaJson = new String(schemaInfo.getSchema()); if (StringUtils.isBlank(schemaJson)) { throw new PrestoException(NOT_SUPPORTED, "Topic " + topicName.toString() + " does not have a valid schema"); } + Schema schema; try { schema = GenericJsonSchema.of(schemaInfo).getAvroSchema(); @@ -93,16 +96,22 @@ public List extractColumnMetadata(TopicName topicName, SchemaInf + topicName.toString() + " does not have a valid schema"); } - //TODO : check schema cyclic definitions which may case java.lang.StackOverflowError - - return schema.getFields().stream() - .map(field -> - new PulsarColumnMetadata(field.name(), parseJsonPrestoType(field.name(), field.schema()), - field.schema().toString(), null, false, false, - handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo( - field.name(), null, null)) + try { + columnMetadata = schema.getFields().stream() + .map(field -> + new PulsarColumnMetadata(PulsarColumnMetadata.getColumnName(handleKeyValueType, field.name()), parseJsonPrestoType(field.name(), field.schema()), + field.schema().toString(), null, false, false, + handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo( + field.name(), null, null)) - ).collect(toList()); + ).collect(toList()); + } catch (StackOverflowError e) { + log.warn(e, "Topic " + + topicName.toString() + " extractColumnMetadata failed."); + throw new PrestoException(NOT_SUPPORTED, "Topic " + + topicName.toString() + " schema may contains cyclic definitions.", e); + } + return columnMetadata; } @@ -171,4 +180,7 @@ private Type parseJsonPrestoType(String fieldname, Schema schema) { schema.getType(), schema.getFullName())); } } + + private static final Logger log = Logger.get(PulsarJsonRowDecoderFactory.class); + } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java index 182b01bf15511..f2da259dbe700 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoder.java @@ -26,6 +26,7 @@ import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; +import io.prestosql.decoder.FieldValueProviders; import io.prestosql.spi.type.BigintType; import io.prestosql.spi.type.BooleanType; import io.prestosql.spi.type.DateType; @@ -40,10 +41,14 @@ import io.prestosql.spi.type.VarbinaryType; import io.prestosql.spi.type.VarcharType; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import org.apache.pulsar.client.impl.schema.AbstractSchema; import org.apache.pulsar.sql.presto.PulsarRowDecoder; /** @@ -52,33 +57,44 @@ public class PulsarPrimitiveRowDecoder implements PulsarRowDecoder { private final DecoderColumnHandle columnHandle; + private AbstractSchema schema; - public PulsarPrimitiveRowDecoder(DecoderColumnHandle column) { + public PulsarPrimitiveRowDecoder(AbstractSchema schema, DecoderColumnHandle column) { this.columnHandle = column; + this.schema = schema; } @Override public Optional> decodeRow(ByteBuf byteBuf) { - byte[] data = new byte[byteBuf.readableBytes()]; - byteBuf.readBytes(data); + Object value = schema.decode(byteBuf); Map primitiveColumn = new HashMap<>(); - Type type = columnHandle.getType(); - if (type instanceof BooleanType) { - primitiveColumn.put(columnHandle, booleanValueProvider(Boolean.valueOf(new String(data)))); - } else if (type instanceof TinyintType || type instanceof SmallintType || type instanceof IntegerType - || type instanceof BigintType) { - primitiveColumn.put(columnHandle, longValueProvider(Long.valueOf(new String(data)))); - } else if (type instanceof DoubleType) { - primitiveColumn.put(columnHandle, doubleValueProvider(Double.valueOf(new String(data)))); - } else if (type instanceof RealType) { - primitiveColumn.put(columnHandle, longValueProvider( - Float.floatToIntBits((Float.valueOf(new String(data)))))); - } else if (type instanceof VarbinaryType || type instanceof VarcharType) { - primitiveColumn.put(columnHandle, bytesValueProvider(data)); - } else if (type instanceof DateType || type instanceof TimeType || type instanceof TimestampType) { - primitiveColumn.put(columnHandle, longValueProvider(Long.valueOf(new String(data)))); + if (value == null) { + primitiveColumn.put(columnHandle, FieldValueProviders.nullValueProvider()); } else { - primitiveColumn.put(columnHandle, bytesValueProvider(data)); + Type type = columnHandle.getType(); + if (type instanceof BooleanType) { + primitiveColumn.put(columnHandle, booleanValueProvider(Boolean.valueOf((Boolean) value))); + } else if (type instanceof TinyintType || type instanceof SmallintType || type instanceof IntegerType + || type instanceof BigintType) { + primitiveColumn.put(columnHandle, longValueProvider(Long.valueOf(value.toString()))); + } else if (type instanceof DoubleType) { + primitiveColumn.put(columnHandle, doubleValueProvider(Double.valueOf(value.toString()))); + } else if (type instanceof RealType) { + primitiveColumn.put(columnHandle, longValueProvider( + Float.floatToIntBits((Float.valueOf(value.toString()))))); + } else if (type instanceof VarbinaryType) { + primitiveColumn.put(columnHandle, bytesValueProvider((byte[]) value)); + } else if (type instanceof VarcharType) { + primitiveColumn.put(columnHandle, bytesValueProvider(value.toString().getBytes())); + } else if (type instanceof DateType) { + primitiveColumn.put(columnHandle, longValueProvider(((Date) value).getTime())); + } else if (type instanceof TimeType) { + primitiveColumn.put(columnHandle, longValueProvider(((Time) value).getTime())); + } else if (type instanceof TimestampType) { + primitiveColumn.put(columnHandle, longValueProvider(((Timestamp) value).getTime())); + } else { + primitiveColumn.put(columnHandle, bytesValueProvider(value.toString().getBytes())); + } } return Optional.of(primitiveColumn); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java index 84fd59b949bd3..9afddfee36db4 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/primitive/PulsarPrimitiveRowDecoderFactory.java @@ -37,6 +37,8 @@ import java.util.Arrays; import java.util.List; import java.util.Set; +import org.apache.pulsar.client.impl.schema.AbstractSchema; +import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; @@ -52,11 +54,14 @@ public class PulsarPrimitiveRowDecoderFactory implements PulsarRowDecoderFactory private static final Logger log = Logger.get(PulsarPrimitiveRowDecoderFactory.class); + public static final String PRIMITIVE_COLUMN_NAME = "__value__"; + @Override public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaInfo, Set columns) { if (columns.size() == 1) { - return new PulsarPrimitiveRowDecoder(columns.iterator().next()); + return new PulsarPrimitiveRowDecoder((AbstractSchema) AutoConsumeSchema.getSchema(schemaInfo), + columns.iterator().next()); } else { throw new RuntimeException("Primitive type must has only one ColumnHandle."); } @@ -66,10 +71,10 @@ public PulsarRowDecoder createRowDecoder(TopicName topicName, SchemaInfo schemaI public List extractColumnMetadata(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType) { ColumnMetadata valueColumn = new PulsarColumnMetadata( - PulsarColumnMetadata.getColumnName(handleKeyValueType, "__value__"), - parsePrimitivePrestoType("__value__", schemaInfo.getType()), + PulsarColumnMetadata.getColumnName(handleKeyValueType, PRIMITIVE_COLUMN_NAME), + parsePrimitivePrestoType(PRIMITIVE_COLUMN_NAME, schemaInfo.getType()), "The value of the message with primitive type schema", null, false, false, - handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo("__value__", + handleKeyValueType, new PulsarColumnMetadata.DecoderExtraInfo(PRIMITIVE_COLUMN_NAME, null, null)); return Arrays.asList(valueColumn); } diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java index e779765cd94e6..c3eb2e238f0e0 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java @@ -57,12 +57,24 @@ import java.time.LocalTime; import java.time.ZoneId; import java.time.temporal.ChronoUnit; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertNotNull; public abstract class TestPulsarConnector { @@ -235,20 +247,20 @@ public static class Bar { ConnectorContext prestoConnectorContext = new TestingConnectorContext(); dispatchingRowDecoderFactory = new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager()); - topicsToColumnHandles.put(PARTITIONED_TOPIC_1, getColumnColumnHandles(PARTITIONED_TOPIC_1,topicsToSchemas.get(PARTITIONED_TOPIC_1.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); - topicsToColumnHandles.put(PARTITIONED_TOPIC_2, getColumnColumnHandles(PARTITIONED_TOPIC_2,topicsToSchemas.get(PARTITIONED_TOPIC_2.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); - topicsToColumnHandles.put(PARTITIONED_TOPIC_3, getColumnColumnHandles(PARTITIONED_TOPIC_3,topicsToSchemas.get(PARTITIONED_TOPIC_3.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); - topicsToColumnHandles.put(PARTITIONED_TOPIC_4, getColumnColumnHandles(PARTITIONED_TOPIC_4,topicsToSchemas.get(PARTITIONED_TOPIC_4.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); - topicsToColumnHandles.put(PARTITIONED_TOPIC_5, getColumnColumnHandles(PARTITIONED_TOPIC_5,topicsToSchemas.get(PARTITIONED_TOPIC_5.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); - topicsToColumnHandles.put(PARTITIONED_TOPIC_6, getColumnColumnHandles(PARTITIONED_TOPIC_6,topicsToSchemas.get(PARTITIONED_TOPIC_6.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); + topicsToColumnHandles.put(PARTITIONED_TOPIC_1, getColumnColumnHandles(PARTITIONED_TOPIC_1,topicsToSchemas.get(PARTITIONED_TOPIC_1.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); + topicsToColumnHandles.put(PARTITIONED_TOPIC_2, getColumnColumnHandles(PARTITIONED_TOPIC_2,topicsToSchemas.get(PARTITIONED_TOPIC_2.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); + topicsToColumnHandles.put(PARTITIONED_TOPIC_3, getColumnColumnHandles(PARTITIONED_TOPIC_3,topicsToSchemas.get(PARTITIONED_TOPIC_3.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); + topicsToColumnHandles.put(PARTITIONED_TOPIC_4, getColumnColumnHandles(PARTITIONED_TOPIC_4,topicsToSchemas.get(PARTITIONED_TOPIC_4.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); + topicsToColumnHandles.put(PARTITIONED_TOPIC_5, getColumnColumnHandles(PARTITIONED_TOPIC_5,topicsToSchemas.get(PARTITIONED_TOPIC_5.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); + topicsToColumnHandles.put(PARTITIONED_TOPIC_6, getColumnColumnHandles(PARTITIONED_TOPIC_6,topicsToSchemas.get(PARTITIONED_TOPIC_6.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); - topicsToColumnHandles.put(TOPIC_1, getColumnColumnHandles(TOPIC_1,topicsToSchemas.get(TOPIC_1.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); - topicsToColumnHandles.put(TOPIC_2, getColumnColumnHandles(TOPIC_2,topicsToSchemas.get(TOPIC_2.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); - topicsToColumnHandles.put(TOPIC_3, getColumnColumnHandles(TOPIC_3,topicsToSchemas.get(TOPIC_3.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); - topicsToColumnHandles.put(TOPIC_4, getColumnColumnHandles(TOPIC_4,topicsToSchemas.get(TOPIC_4.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); - topicsToColumnHandles.put(TOPIC_5, getColumnColumnHandles(TOPIC_5,topicsToSchemas.get(TOPIC_5.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); - topicsToColumnHandles.put(TOPIC_6, getColumnColumnHandles(TOPIC_6,topicsToSchemas.get(TOPIC_6.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true,dispatchingRowDecoderFactory)); + topicsToColumnHandles.put(TOPIC_1, getColumnColumnHandles(TOPIC_1,topicsToSchemas.get(TOPIC_1.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); + topicsToColumnHandles.put(TOPIC_2, getColumnColumnHandles(TOPIC_2,topicsToSchemas.get(TOPIC_2.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); + topicsToColumnHandles.put(TOPIC_3, getColumnColumnHandles(TOPIC_3,topicsToSchemas.get(TOPIC_3.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); + topicsToColumnHandles.put(TOPIC_4, getColumnColumnHandles(TOPIC_4,topicsToSchemas.get(TOPIC_4.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); + topicsToColumnHandles.put(TOPIC_5, getColumnColumnHandles(TOPIC_5,topicsToSchemas.get(TOPIC_5.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); + topicsToColumnHandles.put(TOPIC_6, getColumnColumnHandles(TOPIC_6,topicsToSchemas.get(TOPIC_6.getSchemaName()), PulsarColumnHandle.HandleKeyValueType.NONE,true)); splits = new HashMap<>(); @@ -311,10 +323,11 @@ public static class Bar { * @param dispatchingRowDecoderFactory * @return */ - public static List getColumnColumnHandles(TopicName topicName, SchemaInfo schemaInfo, PulsarColumnHandle.HandleKeyValueType handleKeyValueType, boolean includeInternalColumn, PulsarDispatchingRowDecoderFactory dispatchingRowDecoderFactory) { + protected static List getColumnColumnHandles(TopicName topicName, SchemaInfo schemaInfo, + PulsarColumnHandle.HandleKeyValueType handleKeyValueType, boolean includeInternalColumn) { List columnHandles = new ArrayList<>(); - List columnMetadata = dispatchingRowDecoderFactory.extractColumnMetadata(topicName,schemaInfo, handleKeyValueType); - + List columnMetadata = mockColumnMetadata().getPulsarColumns(topicName, schemaInfo, + includeInternalColumn, handleKeyValueType); columnMetadata.forEach(column -> { PulsarColumnMetadata pulsarColumnMetadata = (PulsarColumnMetadata) column; columnHandles.add(new PulsarColumnHandle( @@ -328,16 +341,21 @@ public static List getColumnColumnHandles(TopicName topicNam pulsarColumnMetadata.getHandleKeyValueType())); }); - - if (includeInternalColumn) { - columnHandles.addAll(PulsarInternalColumn.getInternalFields().stream() - .map(pulsarInternalColumn -> pulsarInternalColumn.getColumnHandle(pulsarConnectorId.toString(), false)) - .collect(Collectors.toList())); - } - return columnHandles; } + public static PulsarMetadata mockColumnMetadata() { + ConnectorContext prestoConnectorContext = new TestingConnectorContext(); + PulsarConnectorConfig pulsarConnectorConfig = spy(new PulsarConnectorConfig()); + pulsarConnectorConfig.setMaxEntryReadBatchSize(1); + pulsarConnectorConfig.setMaxSplitEntryQueueSize(10); + pulsarConnectorConfig.setMaxSplitMessageQueueSize(100); + PulsarDispatchingRowDecoderFactory dispatchingRowDecoderFactory = + new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager()); + PulsarMetadata pulsarMetadata = new PulsarMetadata(pulsarConnectorId, pulsarConnectorConfig, dispatchingRowDecoderFactory); + return pulsarMetadata; + } + public static PulsarConnectorId getPulsarConnectorId() { assertNotNull(pulsarConnectorId); return pulsarConnectorId; diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java index 6be688a16065a..513d96f41a5ff 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java @@ -18,20 +18,51 @@ */ package org.apache.pulsar.sql.presto; +import com.fasterxml.jackson.databind.ObjectMapper; import io.airlift.log.Logger; +import io.netty.buffer.ByteBuf; +import io.prestosql.spi.predicate.TupleDomain; import io.prestosql.spi.type.RowType; +import lombok.Data; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.ManagedLedgerFactory; +import org.apache.bookkeeper.mledger.ReadOnlyCursor; +import org.apache.bookkeeper.mledger.impl.EntryImpl; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.impl.ReadOnlyCursorImpl; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; +import org.apache.bookkeeper.stats.NullStatsProvider; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.schema.KeyValueSchema; +import org.apache.pulsar.common.api.proto.PulsarApi; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.schema.KeyValueEncodingType; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import org.testng.annotations.Test; +import java.util.Arrays; +import java.util.Base64; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import static java.util.concurrent.CompletableFuture.completedFuture; +import static org.apache.pulsar.common.protocol.Commands.serializeMetadataAndPayload; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; public class TestPulsarRecordCursor extends TestPulsarConnector { @@ -112,4 +143,278 @@ public void testTopics() throws Exception { } } + @Test(singleThreaded = true) + public void TestKeyValueStructSchema() throws Exception { + + TopicName topicName = TopicName.get("persistent", NAMESPACE_NAME_1, "topic-4"); + Long entriesNum = 5L; + + for (KeyValueEncodingType encodingType : + Arrays.asList(KeyValueEncodingType.INLINE, KeyValueEncodingType.SEPARATED)) { + + KeyValueSchema schema = (KeyValueSchema) Schema.KeyValue(Schema.JSON(Foo.class), Schema.AVRO(Boo.class), + encodingType); + + Foo foo = new Foo(); + foo.field1 = "field1-value"; + foo.field2 = 20; + Boo boo = new Boo(); + boo.field1 = "field1-value"; + boo.field2 = true; + boo.field3 = 10.2; + + KeyValue message = new KeyValue<>(foo, boo); + List ColumnHandles = getColumnColumnHandles(topicName, schema.getSchemaInfo(), PulsarColumnHandle.HandleKeyValueType.NONE, true); + PulsarRecordCursor pulsarRecordCursor = mockKeyValueSchemaPulsarRecordCursor(entriesNum, topicName, + schema, message, ColumnHandles); + + assertNotNull(pulsarRecordCursor); + Long count = 0L; + while (pulsarRecordCursor.advanceNextPosition()) { + List columnsSeen = new LinkedList<>(); + for (int i = 0; i < ColumnHandles.size(); i++) { + if (pulsarRecordCursor.isNull(i)) { + columnsSeen.add(ColumnHandles.get(i).getName()); + } else { + if (ColumnHandles.get(i).getName().equals("field1")) { + assertEquals(pulsarRecordCursor.getSlice(i).getBytes(), boo.field1.getBytes()); + columnsSeen.add(ColumnHandles.get(i).getName()); + } else if (ColumnHandles.get(i).getName().equals("field2")) { + assertEquals(pulsarRecordCursor.getBoolean(i), boo.field2.booleanValue()); + columnsSeen.add(ColumnHandles.get(i).getName()); + } else if (ColumnHandles.get(i).getName().equals("field3")) { + assertEquals((Double) pulsarRecordCursor.getDouble(i), (Double) boo.field3); + columnsSeen.add(ColumnHandles.get(i).getName()); + } else if (ColumnHandles.get(i).getName().equals(PulsarColumnMetadata.KEY_SCHEMA_COLUMN_PREFIX + + "field1")) { + assertEquals(pulsarRecordCursor.getSlice(i).getBytes(), foo.field1.getBytes()); + columnsSeen.add(ColumnHandles.get(i).getName()); + } else if (ColumnHandles.get(i).getName().equals(PulsarColumnMetadata.KEY_SCHEMA_COLUMN_PREFIX + + "field2")) { + assertEquals(pulsarRecordCursor.getLong(i), Long.valueOf(foo.field2).longValue()); + columnsSeen.add(ColumnHandles.get(i).getName()); + } else { + if (PulsarInternalColumn.getInternalFieldsMap().containsKey(ColumnHandles.get(i).getName())) { + columnsSeen.add(ColumnHandles.get(i).getName()); + } + } + } + } + assertEquals(columnsSeen.size(), ColumnHandles.size()); + count++; + } + assertEquals(count, entriesNum); + pulsarRecordCursor.close(); + } + } + + + @Test(singleThreaded = true) + public void TestKeyValuePrimitiveSchema() throws Exception { + + TopicName topicName = TopicName.get("persistent", NAMESPACE_NAME_1, "topic-4"); + Long entriesNum = 5L; + + for (KeyValueEncodingType encodingType : + Arrays.asList(KeyValueEncodingType.INLINE, KeyValueEncodingType.SEPARATED)) { + + KeyValueSchema schema = (KeyValueSchema) Schema.KeyValue(Schema.INT32, Schema.STRING, + encodingType); + + String value = "primitive_message_value"; + Integer key = 23; + KeyValue message = new KeyValue<>(key, value); + + List ColumnHandles = getColumnColumnHandles(topicName, schema.getSchemaInfo(), PulsarColumnHandle.HandleKeyValueType.NONE, true); + PulsarRecordCursor pulsarRecordCursor = mockKeyValueSchemaPulsarRecordCursor(entriesNum, topicName, + schema, message, ColumnHandles); + + assertNotNull(pulsarRecordCursor); + Long count = 0L; + while (pulsarRecordCursor.advanceNextPosition()) { + List columnsSeen = new LinkedList<>(); + for (int i = 0; i < ColumnHandles.size(); i++) { + if (pulsarRecordCursor.isNull(i)) { + columnsSeen.add(ColumnHandles.get(i).getName()); + } else { + if (ColumnHandles.get(i).getName().equals(PRIMITIVE_COLUMN_NAME)) { + assertEquals(pulsarRecordCursor.getSlice(i).getBytes(), value.getBytes()); + columnsSeen.add(ColumnHandles.get(i).getName()); + } else if (ColumnHandles.get(i).getName().equals(KEY_SCHEMA_COLUMN_PREFIX + + PRIMITIVE_COLUMN_NAME)) { + assertEquals((Long) pulsarRecordCursor.getLong(i), Long.valueOf(key)); + columnsSeen.add(ColumnHandles.get(i).getName()); + } else { + if (PulsarInternalColumn.getInternalFieldsMap().containsKey(ColumnHandles.get(i).getName())) { + columnsSeen.add(ColumnHandles.get(i).getName()); + } + } + } + } + assertEquals(columnsSeen.size(), ColumnHandles.size()); + count++; + } + assertEquals(count, entriesNum); + pulsarRecordCursor.close(); + } + } + + + /** + * mock a simple PulsarRecordCursor for KeyValueSchema test. + * @param entriesNum + * @param topicName + * @param schema + * @param message + * @param ColumnHandles + * @return + * @throws Exception + */ + private PulsarRecordCursor mockKeyValueSchemaPulsarRecordCursor(final Long entriesNum, final TopicName topicName, + final KeyValueSchema schema, KeyValue message, List ColumnHandles) throws Exception { + + ManagedLedgerFactory managedLedgerFactory = mock(ManagedLedgerFactory.class); + + when(managedLedgerFactory.openReadOnlyCursor(any(), any(), any())).then(new Answer() { + + private Map positions = new HashMap<>(); + + @Override + public ReadOnlyCursor answer(InvocationOnMock invocationOnMock) throws Throwable { + Object[] args = invocationOnMock.getArguments(); + String topic = (String) args[0]; + PositionImpl positionImpl = (PositionImpl) args[1]; + int position = positionImpl.getEntryId() == -1 ? 0 : (int) positionImpl.getEntryId(); + + positions.put(topic, position); + ReadOnlyCursorImpl readOnlyCursor = mock(ReadOnlyCursorImpl.class); + doReturn(entriesNum).when(readOnlyCursor).getNumberOfEntries(); + + doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + Object[] args = invocation.getArguments(); + Integer skipEntries = (Integer) args[0]; + positions.put(topic, positions.get(topic) + skipEntries); + return null; + } + }).when(readOnlyCursor).skipEntries(anyInt()); + + when(readOnlyCursor.getReadPosition()).thenAnswer(new Answer() { + @Override + public PositionImpl answer(InvocationOnMock invocationOnMock) throws Throwable { + return PositionImpl.get(0, positions.get(topic)); + } + }); + + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocationOnMock) throws Throwable { + Object[] args = invocationOnMock.getArguments(); + Integer readEntries = (Integer) args[0]; + AsyncCallbacks.ReadEntriesCallback callback = (AsyncCallbacks.ReadEntriesCallback) args[1]; + Object ctx = args[2]; + + new Thread(new Runnable() { + @Override + public void run() { + List entries = new LinkedList<>(); + for (int i = 0; i < readEntries; i++) { + + PulsarApi.MessageMetadata.Builder messageBuilder = + PulsarApi.MessageMetadata.newBuilder() + .setProducerName("test-producer").setSequenceId(positions.get(topic)) + .setPublishTime(System.currentTimeMillis()); + if (KeyValueEncodingType.SEPARATED.equals(schema.getKeyValueEncodingType())) { + messageBuilder + .setPartitionKey(Base64.getEncoder().encodeToString(schema + .getKeySchema().encode(message.getKey()))) + .setPartitionKeyB64Encoded(true); + } + PulsarApi.MessageMetadata messageMetadata = + messageBuilder + .build(); + + ByteBuf dataPayload = io.netty.buffer.Unpooled + .copiedBuffer(schema.encode(message)); + + ByteBuf byteBuf = serializeMetadataAndPayload( + Commands.ChecksumType.Crc32c, messageMetadata, dataPayload); + + entries.add(EntryImpl.create(0, positions.get(topic), byteBuf)); + positions.put(topic, positions.get(topic) + 1); + } + + callback.readEntriesComplete(entries, ctx); + } + }).start(); + + return null; + } + }).when(readOnlyCursor).asyncReadEntries(anyInt(), any(), any()); + + when(readOnlyCursor.hasMoreEntries()).thenAnswer(new Answer() { + @Override + public Boolean answer(InvocationOnMock invocationOnMock) throws Throwable { + return positions.get(topic) < entriesNum; + } + }); + + when(readOnlyCursor.getNumberOfEntries(any())).then(new Answer() { + @Override + public Long answer(InvocationOnMock invocationOnMock) throws Throwable { + Object[] args = invocationOnMock.getArguments(); + com.google.common.collect.Range range + = (com.google.common.collect.Range) args[0]; + return (range.upperEndpoint().getEntryId() + 1) - range.lowerEndpoint().getEntryId(); + } + }); + + when(readOnlyCursor.getCurrentLedgerInfo()).thenReturn(MLDataFormats.ManagedLedgerInfo.LedgerInfo.newBuilder().setLedgerId(0).build()); + + return readOnlyCursor; + } + }); + + ObjectMapper objectMapper = new ObjectMapper(); + + PulsarSplit split = new PulsarSplit(0, pulsarConnectorId.toString(), + topicName.getNamespace(), topicName.getLocalName(), topicName.getLocalName(), + entriesNum, + new String(schema.getSchemaInfo().getSchema()), + schema.getSchemaInfo().getType(), + 0, entriesNum, + 0, 0, TupleDomain.all(), + objectMapper.writeValueAsString( + schema.getSchemaInfo().getProperties()), null); + + PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor( + ColumnHandles, split, + pulsarConnectorConfig, managedLedgerFactory, new ManagedLedgerConfig(), + new PulsarConnectorMetricsTracker(new NullStatsProvider()), dispatchingRowDecoderFactory)); + + PulsarSqlSchemaInfoProvider pulsarSqlSchemaInfoProvider = mock(PulsarSqlSchemaInfoProvider.class); + when(pulsarSqlSchemaInfoProvider.getSchemaByVersion(any())).thenReturn(completedFuture(schema.getSchemaInfo())); + pulsarRecordCursor.setPulsarSqlSchemaInfoProvider(pulsarSqlSchemaInfoProvider); + + return pulsarRecordCursor; + } + + + final static String KEY_SCHEMA_COLUMN_PREFIX = "__key."; + final static String PRIMITIVE_COLUMN_NAME = "__value__"; + + @Data + static class Foo { + private String field1; + private Integer field2; + } + + @Data + static class Boo { + private String field1; + private Boolean field2; + private Double field3; + } + } diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java index 44e6453baf598..cd4fcaf0d1e33 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/AbstractDecoderTester.java @@ -22,6 +22,7 @@ import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; import io.prestosql.spi.block.Block; +import io.prestosql.spi.connector.ColumnMetadata; import io.prestosql.spi.connector.ConnectorContext; import io.prestosql.spi.type.Type; import io.prestosql.testing.TestingConnectorContext; @@ -29,30 +30,44 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.sql.presto.PulsarColumnHandle; +import org.apache.pulsar.sql.presto.PulsarColumnMetadata; +import org.apache.pulsar.sql.presto.PulsarConnectorConfig; +import org.apache.pulsar.sql.presto.PulsarConnectorId; import org.apache.pulsar.sql.presto.PulsarDispatchingRowDecoderFactory; +import org.apache.pulsar.sql.presto.PulsarMetadata; import org.apache.pulsar.sql.presto.PulsarRowDecoder; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import static org.mockito.Mockito.spy; import static org.testng.Assert.assertNotNull; /** - * Abstract superclass for TestXXDecoder (e.g. TestAvroDecoder 态TestJsonDecoder) + * Abstract superclass for TestXXDecoder (e.g. TestAvroDecoder 态TestJsonDecoder). */ public abstract class AbstractDecoderTester { protected PulsarDispatchingRowDecoderFactory decoderFactory; + protected PulsarConnectorId pulsarConnectorId = new PulsarConnectorId("test-connector"); protected SchemaInfo schemaInfo; protected TopicName topicName; protected List pulsarColumnHandle; protected PulsarRowDecoder pulsarRowDecoder; protected DecoderTestUtil decoderTestUtil; + protected PulsarConnectorConfig pulsarConnectorConfig; + protected PulsarMetadata pulsarMetadata; protected void init() { ConnectorContext prestoConnectorContext = new TestingConnectorContext(); - decoderFactory = new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager()); - topicName = TopicName.get("persistent", NamespaceName.get("tenant-1", "ns-1"), "topic-1"); + this.decoderFactory = new PulsarDispatchingRowDecoderFactory(prestoConnectorContext.getTypeManager()); + this.pulsarConnectorConfig = spy(new PulsarConnectorConfig()); + this.pulsarConnectorConfig.setMaxEntryReadBatchSize(1); + this.pulsarConnectorConfig.setMaxSplitEntryQueueSize(10); + this.pulsarConnectorConfig.setMaxSplitMessageQueueSize(100); + this.pulsarMetadata = new PulsarMetadata(pulsarConnectorId, this.pulsarConnectorConfig, decoderFactory); + this.topicName = TopicName.get("persistent", NamespaceName.get("tenant-1", "ns-1"), "topic-1"); } protected void checkArrayValues(Block block, Type type, Object value) { @@ -93,4 +108,26 @@ protected Block getBlock(Map decodedRow return provider.getBlock(); } + protected List getColumnColumnHandles(TopicName topicName, SchemaInfo schemaInfo, + PulsarColumnHandle.HandleKeyValueType handleKeyValueType, boolean includeInternalColumn, PulsarDispatchingRowDecoderFactory dispatchingRowDecoderFactory) { + List columnHandles = new ArrayList<>(); + List columnMetadata = pulsarMetadata.getPulsarColumns(topicName, schemaInfo, + includeInternalColumn, handleKeyValueType); + + columnMetadata.forEach(column -> { + PulsarColumnMetadata pulsarColumnMetadata = (PulsarColumnMetadata) column; + columnHandles.add(new PulsarColumnHandle( + pulsarConnectorId.toString(), + pulsarColumnMetadata.getNameWithCase(), + pulsarColumnMetadata.getType(), + pulsarColumnMetadata.isHidden(), + pulsarColumnMetadata.isInternal(), + pulsarColumnMetadata.getDecoderExtraInfo().getMapping(), + pulsarColumnMetadata.getDecoderExtraInfo().getDataFormat(), pulsarColumnMetadata.getDecoderExtraInfo().getFormatHint(), + pulsarColumnMetadata.getHandleKeyValueType())); + + }); + return columnHandles; + } + } diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java index 4b95e881cd809..b4db0483f8daa 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.sql.presto.decoder; +import lombok.Data; + import java.util.List; import java.util.Map; @@ -68,4 +70,22 @@ public static class CompositeRow { public NestedRow nestedRow; public Map> structedField; } + + /** + * POJO for cyclic detect. + */ + @Data + static public class CyclicFoo { + private String field1; + private Integer field2; + private CyclicBoo boo; + } + + @Data + static public class CyclicBoo { + private String field1; + private Boolean field2; + private CyclicFoo foo; + } + } diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java index d0fdd59578eb3..4c3c4a634472c 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestUtil.java @@ -26,7 +26,6 @@ import io.prestosql.spi.type.MapType; import io.prestosql.spi.type.RowType; import io.prestosql.spi.type.Type; - import java.util.Map; import static io.prestosql.testing.TestingConnectorSession.SESSION; @@ -49,7 +48,6 @@ protected DecoderTestUtil() { public abstract void checkPrimitiveValue(Object actual, Object expected); - public void checkField(Block actualBlock, Type type, int position, Object expectedValue) { assertNotNull(type, "Type is null"); assertNotNull(actualBlock, "actualBlock is null"); @@ -67,7 +65,6 @@ public void checkField(Block actualBlock, Type type, int position, Object expect } } - public boolean isIntegralType(Object value) { return value instanceof Long || value instanceof Integer @@ -79,7 +76,6 @@ public boolean isRealType(Object value) { return value instanceof Float || value instanceof Double; } - public Object getObjectValue(Type type, Block block, int position) { if (block.isNull(position)) { return null; @@ -87,7 +83,6 @@ public Object getObjectValue(Type type, Block block, int position) { return type.getObjectValue(SESSION, block, position); } - public void checkValue(Map decodedRow, DecoderColumnHandle handle, Slice value) { FieldValueProvider provider = decodedRow.get(handle); assertNotNull(provider); diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java index 9e3a38e8eb6d3..39295a91517eb 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java @@ -22,6 +22,7 @@ import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; +import io.prestosql.spi.PrestoException; import io.prestosql.spi.type.*; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord; @@ -45,9 +46,10 @@ import static io.prestosql.spi.type.RealType.REAL; import static io.prestosql.spi.type.VarcharType.VARCHAR; import static java.lang.Float.floatToIntBits; -import static org.apache.pulsar.sql.presto.TestPulsarConnector.getColumnColumnHandles; import static org.apache.pulsar.sql.presto.TestPulsarConnector.getPulsarConnectorId; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; public class TestAvroDecoder extends AbstractDecoderTester { @@ -270,4 +272,19 @@ public void testCompositeType() { checkRowValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); } + + @Test(singleThreaded = true) + public void testCyclicDefinitionDetect() { + AvroSchema cyclicSchema = AvroSchema.of(DecoderTestMessage.CyclicFoo.class); + PrestoException exception = expectThrows(PrestoException.class, + () -> { + decoderFactory.extractColumnMetadata(topicName, cyclicSchema.getSchemaInfo(), + PulsarColumnHandle.HandleKeyValueType.NONE); + }); + + assertEquals("Topic " + + topicName.toString() + " schema may contains cyclic definitions.", exception.getMessage()); + + } + } diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java index ace6ca56e887b..9f56e3006c9b0 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java @@ -22,6 +22,7 @@ import io.netty.buffer.ByteBuf; import io.prestosql.decoder.DecoderColumnHandle; import io.prestosql.decoder.FieldValueProvider; +import io.prestosql.spi.PrestoException; import io.prestosql.spi.type.*; import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; @@ -45,9 +46,10 @@ import static io.prestosql.spi.type.RealType.REAL; import static io.prestosql.spi.type.VarcharType.VARCHAR; import static java.lang.Float.floatToIntBits; -import static org.apache.pulsar.sql.presto.TestPulsarConnector.getColumnColumnHandles; import static org.apache.pulsar.sql.presto.TestPulsarConnector.getPulsarConnectorId; +import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; public class TestJsonDecoder extends AbstractDecoderTester { @@ -265,4 +267,18 @@ public void testCompositeType() { checkRowValues(getBlock(decodedRow, columnHandle), columnHandle.getType(), fieldValue); } + @Test(singleThreaded = true) + public void testCyclicDefinitionDetect() { + JSONSchema cyclicSchema = JSONSchema.of(DecoderTestMessage.CyclicFoo.class); + PrestoException exception = expectThrows(PrestoException.class, + () -> { + decoderFactory.extractColumnMetadata(topicName, cyclicSchema.getSchemaInfo(), + PulsarColumnHandle.HandleKeyValueType.NONE); + }); + + assertEquals("Topic " + + topicName.toString() + " schema may contains cyclic definitions.", exception.getMessage()); + + } + } diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/PrimitiveDecoderTestUtil.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/PrimitiveDecoderTestUtil.java new file mode 100644 index 0000000000000..53813d5707d58 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/PrimitiveDecoderTestUtil.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder.primitive; + +import io.prestosql.spi.block.Block; +import io.prestosql.spi.type.Type; +import org.apache.pulsar.sql.presto.decoder.DecoderTestUtil; + +/** + * TestUtil for PrimitiveDecoder. + * CheckXXXValues() is mock method. Because Primitive is single hierarchy, so CheckXXXValues are never actually + * invoked. + */ +public class PrimitiveDecoderTestUtil extends DecoderTestUtil { + + public PrimitiveDecoderTestUtil() { + super(); + } + + @Override + public void checkArrayValues(Block block, Type type, Object value) { + + } + + @Override + public void checkMapValues(Block block, Type type, Object value) { + + } + + @Override + public void checkRowValues(Block block, Type type, Object value) { + + } + + @Override + public void checkPrimitiveValue(Object actual, Object expected) { + + } + +} diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/TestPrimitiveDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/TestPrimitiveDecoder.java new file mode 100644 index 0000000000000..f4810ba70d816 --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/primitive/TestPrimitiveDecoder.java @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto.decoder.primitive; + +import io.airlift.slice.Slices; +import io.prestosql.decoder.DecoderColumnHandle; +import io.prestosql.decoder.FieldValueProvider; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; +import org.apache.pulsar.sql.presto.PulsarColumnHandle; +import org.apache.pulsar.sql.presto.PulsarRowDecoder; +import org.apache.pulsar.sql.presto.decoder.AbstractDecoderTester; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.sql.Time; +import java.sql.Timestamp; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import static io.prestosql.spi.type.DateType.DATE; +import static io.prestosql.spi.type.SmallintType.SMALLINT; +import static io.prestosql.spi.type.TimeType.TIME; +import static io.prestosql.spi.type.TimestampType.TIMESTAMP; +import static io.prestosql.spi.type.TinyintType.TINYINT; +import static io.prestosql.spi.type.VarbinaryType.VARBINARY; +import static io.prestosql.spi.type.VarcharType.VARCHAR; +import static io.prestosql.spi.type.BigintType.BIGINT; +import static io.prestosql.spi.type.BooleanType.BOOLEAN; +import static io.prestosql.spi.type.DoubleType.DOUBLE; +import static io.prestosql.spi.type.IntegerType.INTEGER; +import static io.prestosql.spi.type.RealType.REAL; +import static org.apache.pulsar.sql.presto.TestPulsarConnector.getPulsarConnectorId; + +public class TestPrimitiveDecoder extends AbstractDecoderTester { + + public static final String PRIMITIVE_COLUMN_NAME = "__value__"; + + @BeforeMethod + public void init() { + decoderTestUtil = new PrimitiveDecoderTestUtil(); + super.init(); + } + + @Test(singleThreaded = true) + public void testPrimitiveType() { + + byte int8Value = 1; + SchemaInfo schemaInfoInt8 = SchemaInfo.builder().type(SchemaType.INT8).build(); + Schema schemaInt8 = Schema.getSchema(schemaInfoInt8); + List pulsarColumnHandleInt8 = getColumnColumnHandles(topicName, schemaInfoInt8, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); + PulsarRowDecoder pulsarRowDecoderInt8 = decoderFactory.createRowDecoder(topicName, schemaInfoInt8, + new HashSet<>(pulsarColumnHandleInt8)); + Map decodedRowInt8 = + pulsarRowDecoderInt8.decodeRow(io.netty.buffer.Unpooled + .copiedBuffer(schemaInt8.encode(int8Value))).get(); + checkValue(decodedRowInt8, new PulsarColumnHandle(getPulsarConnectorId().toString(), + PRIMITIVE_COLUMN_NAME, TINYINT, false, false, PRIMITIVE_COLUMN_NAME, null, null, PulsarColumnHandle.HandleKeyValueType.NONE), int8Value); + + short int16Value = 2; + SchemaInfo schemaInfoInt16 = SchemaInfo.builder().type(SchemaType.INT16).build(); + Schema schemaInt16 = Schema.getSchema(schemaInfoInt16); + List pulsarColumnHandleInt16 = getColumnColumnHandles(topicName, schemaInfoInt16, PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); + PulsarRowDecoder pulsarRowDecoderInt16 = decoderFactory.createRowDecoder(topicName, schemaInfoInt16, + new HashSet<>(pulsarColumnHandleInt16)); + Map decodedRowInt16 = + pulsarRowDecoderInt16.decodeRow(io.netty.buffer.Unpooled + .copiedBuffer(schemaInt16.encode(int16Value))).get(); + checkValue(decodedRowInt16, new PulsarColumnHandle(getPulsarConnectorId().toString(), + PRIMITIVE_COLUMN_NAME, SMALLINT, false, false, PRIMITIVE_COLUMN_NAME, null, null, PulsarColumnHandle.HandleKeyValueType.NONE), int16Value); + + int int32Value = 2; + SchemaInfo schemaInfoInt32 = SchemaInfo.builder().type(SchemaType.INT32).build(); + Schema schemaInt32 = Schema.getSchema(schemaInfoInt32); + List pulsarColumnHandleInt32 = getColumnColumnHandles(topicName, schemaInfoInt32, + PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); + PulsarRowDecoder pulsarRowDecoderInt32 = decoderFactory.createRowDecoder(topicName, schemaInfoInt32, + new HashSet<>(pulsarColumnHandleInt32)); + Map decodedRowInt32 = + pulsarRowDecoderInt32.decodeRow(io.netty.buffer.Unpooled + .copiedBuffer(schemaInt32.encode(int32Value))).get(); + checkValue(decodedRowInt32, new PulsarColumnHandle(getPulsarConnectorId().toString(), + PRIMITIVE_COLUMN_NAME, INTEGER, false, false, PRIMITIVE_COLUMN_NAME, null, null, PulsarColumnHandle.HandleKeyValueType.NONE), int32Value); + + long int64Value = 2; + SchemaInfo schemaInfoInt64 = SchemaInfo.builder().type(SchemaType.INT64).build(); + Schema schemaInt64 = Schema.getSchema(schemaInfoInt64); + List pulsarColumnHandleInt64 = getColumnColumnHandles(topicName, schemaInfoInt64, + PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); + PulsarRowDecoder pulsarRowDecoderInt64 = decoderFactory.createRowDecoder(topicName, schemaInfoInt64, + new HashSet<>(pulsarColumnHandleInt64)); + Map decodedRowInt64 = + pulsarRowDecoderInt64.decodeRow(io.netty.buffer.Unpooled + .copiedBuffer(schemaInt64.encode(int64Value))).get(); + checkValue(decodedRowInt64, new PulsarColumnHandle(getPulsarConnectorId().toString(), + PRIMITIVE_COLUMN_NAME, BIGINT, false, false, PRIMITIVE_COLUMN_NAME, null, null, + PulsarColumnHandle.HandleKeyValueType.NONE), int64Value); + + String stringValue = "test"; + SchemaInfo schemaInfoString = SchemaInfo.builder().type(SchemaType.STRING).build(); + Schema schemaString = Schema.getSchema(schemaInfoString); + List pulsarColumnHandleString = getColumnColumnHandles(topicName, schemaInfoString, + PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); + PulsarRowDecoder pulsarRowDecoderString = decoderFactory.createRowDecoder(topicName, schemaInfoString, + new HashSet<>(pulsarColumnHandleString)); + Map decodedRowString = + pulsarRowDecoderString.decodeRow(io.netty.buffer.Unpooled + .copiedBuffer(schemaString.encode(stringValue))).get(); + checkValue(decodedRowString, new PulsarColumnHandle(getPulsarConnectorId().toString(), + PRIMITIVE_COLUMN_NAME, VARCHAR, false, false, PRIMITIVE_COLUMN_NAME, null, null, + PulsarColumnHandle.HandleKeyValueType.NONE), stringValue); + + float floatValue = 0.2f; + SchemaInfo schemaInfoFloat = SchemaInfo.builder().type(SchemaType.FLOAT).build(); + Schema schemaFloat = Schema.getSchema(schemaInfoFloat); + List pulsarColumnHandleFloat = getColumnColumnHandles(topicName, schemaInfoFloat, + PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); + PulsarRowDecoder pulsarRowDecoderFloat = decoderFactory.createRowDecoder(topicName, schemaInfoFloat, + new HashSet<>(pulsarColumnHandleFloat)); + Map decodedRowFloat = + pulsarRowDecoderFloat.decodeRow(io.netty.buffer.Unpooled + .copiedBuffer(schemaFloat.encode(floatValue))).get(); + checkValue(decodedRowFloat, new PulsarColumnHandle(getPulsarConnectorId().toString(), + PRIMITIVE_COLUMN_NAME, REAL, false, false, PRIMITIVE_COLUMN_NAME, null, null, + PulsarColumnHandle.HandleKeyValueType.NONE), Long.valueOf(Float.floatToIntBits(floatValue))); + + double doubleValue = 0.22d; + SchemaInfo schemaInfoDouble = SchemaInfo.builder().type(SchemaType.DOUBLE).build(); + Schema schemaDouble = Schema.getSchema(schemaInfoDouble); + List pulsarColumnHandleDouble = getColumnColumnHandles(topicName, schemaInfoDouble, + PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); + PulsarRowDecoder pulsarRowDecoderDouble = decoderFactory.createRowDecoder(topicName, schemaInfoDouble, + new HashSet<>(pulsarColumnHandleDouble)); + Map decodedRowDouble = + pulsarRowDecoderDouble.decodeRow(io.netty.buffer.Unpooled + .copiedBuffer(schemaDouble.encode(doubleValue))).get(); + checkValue(decodedRowDouble, new PulsarColumnHandle(getPulsarConnectorId().toString(), + PRIMITIVE_COLUMN_NAME, DOUBLE, false, false, PRIMITIVE_COLUMN_NAME, null, null, + PulsarColumnHandle.HandleKeyValueType.NONE), doubleValue); + + boolean booleanValue = true; + SchemaInfo schemaInfoBoolean = SchemaInfo.builder().type(SchemaType.BOOLEAN).build(); + Schema schemaBoolean = Schema.getSchema(schemaInfoBoolean); + List pulsarColumnHandleBoolean = getColumnColumnHandles(topicName, schemaInfoBoolean, + PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); + PulsarRowDecoder pulsarRowDecoderBoolean = decoderFactory.createRowDecoder(topicName, schemaInfoBoolean, + new HashSet<>(pulsarColumnHandleBoolean)); + Map decodedRowBoolean = + pulsarRowDecoderBoolean.decodeRow(io.netty.buffer.Unpooled + .copiedBuffer(schemaBoolean.encode(booleanValue))).get(); + checkValue(decodedRowBoolean, new PulsarColumnHandle(getPulsarConnectorId().toString(), + PRIMITIVE_COLUMN_NAME, BOOLEAN, false, false, PRIMITIVE_COLUMN_NAME, null, null, + PulsarColumnHandle.HandleKeyValueType.NONE), booleanValue); + + byte[] bytesValue = new byte[1]; + bytesValue[0] = 1; + SchemaInfo schemaInfoBytes = SchemaInfo.builder().type(SchemaType.BYTES).build(); + Schema schemaBytes = Schema.getSchema(schemaInfoBytes); + List pulsarColumnHandleBytes = getColumnColumnHandles(topicName, schemaInfoBytes, + PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); + PulsarRowDecoder pulsarRowDecoderBytes = decoderFactory.createRowDecoder(topicName, schemaInfoBytes, + new HashSet<>(pulsarColumnHandleBytes)); + Map decodedRowBytes = + pulsarRowDecoderBytes.decodeRow(io.netty.buffer.Unpooled + .copiedBuffer(schemaBytes.encode(bytesValue))).get(); + checkValue(decodedRowBytes, new PulsarColumnHandle(getPulsarConnectorId().toString(), + PRIMITIVE_COLUMN_NAME, VARBINARY, false, false, PRIMITIVE_COLUMN_NAME, null, null, + PulsarColumnHandle.HandleKeyValueType.NONE), Slices.wrappedBuffer(bytesValue)); + + Date dateValue = new Date(System.currentTimeMillis()); + SchemaInfo schemaInfoDate = SchemaInfo.builder().type(SchemaType.DATE).build(); + Schema schemaDate = Schema.getSchema(schemaInfoDate); + List pulsarColumnHandleDate = getColumnColumnHandles(topicName, schemaInfoDate, + PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); + PulsarRowDecoder pulsarRowDecoderDate = decoderFactory.createRowDecoder(topicName, schemaInfoDate, + new HashSet<>(pulsarColumnHandleDate)); + Map decodedRowDate = + pulsarRowDecoderDate.decodeRow(io.netty.buffer.Unpooled + .copiedBuffer(schemaDate.encode(dateValue))).get(); + checkValue(decodedRowDate, new PulsarColumnHandle(getPulsarConnectorId().toString(), + PRIMITIVE_COLUMN_NAME, DATE, false, false, PRIMITIVE_COLUMN_NAME, null, null, + PulsarColumnHandle.HandleKeyValueType.NONE), dateValue.getTime()); + + Time timeValue = new Time(System.currentTimeMillis()); + SchemaInfo schemaInfoTime = SchemaInfo.builder().type(SchemaType.TIME).build(); + Schema schemaTime = Schema.getSchema(schemaInfoTime); + List pulsarColumnHandleTime = getColumnColumnHandles(topicName, schemaInfoTime, + PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); + PulsarRowDecoder pulsarRowDecoderTime = decoderFactory.createRowDecoder(topicName, schemaInfoTime, + new HashSet<>(pulsarColumnHandleTime)); + Map decodedRowTime = + pulsarRowDecoderTime.decodeRow(io.netty.buffer.Unpooled + .copiedBuffer(schemaTime.encode(timeValue))).get(); + checkValue(decodedRowTime, new PulsarColumnHandle(getPulsarConnectorId().toString(), + PRIMITIVE_COLUMN_NAME, TIME, false, false, PRIMITIVE_COLUMN_NAME, null, null, + PulsarColumnHandle.HandleKeyValueType.NONE), timeValue.getTime()); + + Timestamp timestampValue = new Timestamp(System.currentTimeMillis()); + SchemaInfo schemaInfoTimestamp = SchemaInfo.builder().type(SchemaType.TIMESTAMP).build(); + Schema schemaTimestamp = Schema.getSchema(schemaInfoTimestamp); + List pulsarColumnHandleTimestamp = getColumnColumnHandles(topicName, schemaInfoTimestamp, + PulsarColumnHandle.HandleKeyValueType.NONE, false, decoderFactory); + PulsarRowDecoder pulsarRowDecoderTimestamp = decoderFactory.createRowDecoder(topicName, schemaInfoTimestamp, + new HashSet<>(pulsarColumnHandleTimestamp)); + Map decodedRowTimestamp = + pulsarRowDecoderTimestamp.decodeRow(io.netty.buffer.Unpooled + .copiedBuffer(schemaTimestamp.encode(timestampValue))).get(); + checkValue(decodedRowTimestamp, new PulsarColumnHandle(getPulsarConnectorId().toString(), + PRIMITIVE_COLUMN_NAME, TIMESTAMP, false, false, PRIMITIVE_COLUMN_NAME, null, null, + PulsarColumnHandle.HandleKeyValueType.NONE), timestampValue.getTime()); + + } + +} From c54c1d00fe35cc67e0356d0bddf6fc7cb89b19d3 Mon Sep 17 00:00:00 2001 From: wangguowei Date: Tue, 19 Jan 2021 10:39:29 +0800 Subject: [PATCH 18/19] merge master --- .../sql/presto/TestPulsarConnector.java | 3 +-- .../sql/presto/TestPulsarRecordCursor.java | 22 +++++++++---------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java index 681fff98a7005..dd48f48927eb8 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java @@ -37,8 +37,7 @@ import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.client.impl.schema.JSONSchema; - import org.apache.pulsar.common.api.proto.MessageMetadata; -import org.apache.pulsar.common.api.proto.PulsarApi; +import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java index 513d96f41a5ff..5a401246ebd5b 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java @@ -36,7 +36,7 @@ import org.apache.bookkeeper.stats.NullStatsProvider; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.schema.KeyValueSchema; -import org.apache.pulsar.common.api.proto.PulsarApi; +import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.schema.KeyValue; @@ -45,8 +45,8 @@ import org.mockito.stubbing.Answer; import org.testng.annotations.Test; +import java.nio.charset.Charset; import java.util.Arrays; -import java.util.Base64; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -208,7 +208,6 @@ public void TestKeyValueStructSchema() throws Exception { } } - @Test(singleThreaded = true) public void TestKeyValuePrimitiveSchema() throws Exception { @@ -321,19 +320,18 @@ public void run() { List entries = new LinkedList<>(); for (int i = 0; i < readEntries; i++) { - PulsarApi.MessageMetadata.Builder messageBuilder = - PulsarApi.MessageMetadata.newBuilder() + MessageMetadata messageMetadata = + new MessageMetadata() .setProducerName("test-producer").setSequenceId(positions.get(topic)) .setPublishTime(System.currentTimeMillis()); + if (KeyValueEncodingType.SEPARATED.equals(schema.getKeyValueEncodingType())) { - messageBuilder - .setPartitionKey(Base64.getEncoder().encodeToString(schema - .getKeySchema().encode(message.getKey()))) - .setPartitionKeyB64Encoded(true); + messageMetadata + .setPartitionKey(new String(schema + .getKeySchema().encode(message.getKey()), Charset.forName( + "UTF-8"))) + .setPartitionKeyB64Encoded(false); } - PulsarApi.MessageMetadata messageMetadata = - messageBuilder - .build(); ByteBuf dataPayload = io.netty.buffer.Unpooled .copiedBuffer(schema.encode(message)); From 73dadc6b99d32ef958e584c791eb910e24cd90d6 Mon Sep 17 00:00:00 2001 From: wangguowei Date: Sun, 31 Jan 2021 15:46:26 +0800 Subject: [PATCH 19/19] merge master --- .../org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java index 5a401246ebd5b..aadf0dcc9f89d 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarRecordCursor.java @@ -349,7 +349,7 @@ public void run() { return null; } - }).when(readOnlyCursor).asyncReadEntries(anyInt(), any(), any()); + }).when(readOnlyCursor).asyncReadEntries(anyInt(), any(), any(), any()); when(readOnlyCursor.hasMoreEntries()).thenAnswer(new Answer() { @Override