From 5ab49983cd0dfbbb417d00c99db75b1fdbb2f574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Tue, 10 Sep 2019 10:18:33 +0800 Subject: [PATCH 01/24] Change import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; --- conf/broker.conf | 4 ++ .../pulsar/broker/ServiceConfiguration.java | 13 +++++ .../broker/admin/v2/SchemasResource.java | 2 +- .../pulsar/broker/service/AbstractTopic.java | 2 +- .../nonpersistent/NonPersistentTopic.java | 2 +- .../service/persistent/PersistentTopic.java | 2 +- .../AvroSchemaBasedCompatibilityCheck.java | 21 ++++++-- .../schema/DefaultSchemaRegistryService.java | 2 + .../schema/JsonSchemaCompatibilityCheck.java | 3 +- .../KeyValueSchemaCompatibilityCheck.java | 11 +++++ .../schema/SchemaCompatibilityCheck.java | 48 +++++++++++++++---- .../broker/service/schema/SchemaRegistry.java | 2 + .../schema/SchemaRegistryServiceImpl.java | 36 ++++++++++++-- ...egistryServiceWithSchemaDataValidator.java | 2 +- .../admin/AdminApiSchemaAutoUpdateTest.java | 2 +- .../BaseAvroSchemaCompatibilityTest.java | 1 + .../JsonSchemaCompatibilityCheckTest.java | 1 + .../KeyValueSchemaCompatibilityCheckTest.java | 1 + .../service/schema/SchemaServiceTest.java | 1 + ...tryServiceWithSchemaDataValidatorTest.java | 2 +- .../api/SimpleTypedProducerConsumerTest.java | 2 +- .../pulsar/common/policies/data/Policies.java | 3 ++ .../data}/SchemaCompatibilityStrategy.java | 10 ++-- 23 files changed, 146 insertions(+), 27 deletions(-) rename {pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema => pulsar-common/src/main/java/org/apache/pulsar/common/policies/data}/SchemaCompatibilityStrategy.java (95%) diff --git a/conf/broker.conf b/conf/broker.conf index b491336ca534c..b0ecc645e34dc 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -725,6 +725,10 @@ schemaRegistryStorageClassName=org.apache.pulsar.broker.service.schema.Bookkeepe # if you enable this setting, it will cause non-java clients failed to produce. isSchemaValidationEnforced=false +schemaCompatibilityStrategy = FULL + +isAllowAutoUpdateSchema = true + ### --- Ledger Offloading --- ### # The directory for all the offloader implementations diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 832f02759c681..d45bde6d6da00 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1193,6 +1193,19 @@ public class ServiceConfiguration implements PulsarConfiguration { doc = "The schema storage implementation used by this broker" ) private String schemaRegistryStorageClassName = "org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorageFactory"; + + @FieldContext( + category = CATEGORY_SCHEMA, + doc = "The schema storage implementation used by this broker" + ) + private String schemaCompatibilityStrategy = "FULL"; + + @FieldContext( + category = CATEGORY_SCHEMA, + doc = "The schema storage implementation used by this broker" + ) + private boolean isAllowSchemaAutoUpdate = true; + @FieldContext( category = CATEGORY_SCHEMA, doc = "The list compatibility checkers to be used in schema registry" diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java index b91e577a0d38e..245098d86b802 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java @@ -55,7 +55,7 @@ import org.apache.pulsar.broker.admin.AdminResource; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.broker.service.schema.LongSchemaVersion; -import org.apache.pulsar.broker.service.schema.SchemaCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.broker.service.schema.SchemaRegistry.SchemaAndMetadata; import org.apache.pulsar.broker.service.schema.exceptions.InvalidSchemaDataException; import org.apache.pulsar.broker.web.RestException; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 2c65f57edbd4d..5ec45f1af2d36 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -21,7 +21,7 @@ import com.google.common.base.MoreObjects; import org.apache.bookkeeper.mledger.util.StatsBuckets; import org.apache.pulsar.broker.admin.AdminResource; -import org.apache.pulsar.broker.service.schema.SchemaCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.Policies; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 60a8beb4c29a4..7d1328d9a393c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -60,7 +60,7 @@ import org.apache.pulsar.broker.service.StreamingStats; import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; -import org.apache.pulsar.broker.service.schema.SchemaCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.broker.stats.ClusterReplicationMetrics; import org.apache.pulsar.broker.stats.NamespaceStats; import org.apache.pulsar.client.api.MessageId; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 761e3011b7d9c..bae6c5612f973 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -80,7 +80,7 @@ import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type; -import org.apache.pulsar.broker.service.schema.SchemaCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.broker.stats.ClusterReplicationMetrics; import org.apache.pulsar.broker.stats.NamespaceStats; import org.apache.pulsar.broker.stats.ReplicationMetrics; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java index 8f215cac21ccc..e5a69b8168219 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java @@ -29,6 +29,7 @@ import org.apache.avro.SchemaValidator; import org.apache.avro.SchemaValidatorBuilder; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.protocol.schema.SchemaData; import lombok.extern.slf4j.Slf4j; @@ -39,12 +40,12 @@ abstract class AvroSchemaBasedCompatibilityCheck implements SchemaCompatibilityCheck { @Override - public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { + public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException { checkCompatible(Collections.singletonList(from), to, strategy); } @Override - public void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { + public void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException { LinkedList fromList = new LinkedList<>(); try { for (SchemaData schemaData : from) { @@ -53,7 +54,8 @@ public void checkCompatible(Iterable from, SchemaData to, SchemaComp } Schema.Parser parser = new Schema.Parser(); Schema toSchema = parser.parse(new String(to.getData(), UTF_8)); - SchemaValidator schemaValidator = createSchemaValidator(strategy); + SchemaValidator schemaValidator = isConsumer ? + createConsumerSchemaValidator(strategy) : createSchemaValidator(strategy); schemaValidator.validate(toSchema, fromList); } catch (SchemaParseException e) { log.error("Error during schema parsing: {}", e.getMessage(), e); @@ -64,7 +66,7 @@ public void checkCompatible(Iterable from, SchemaData to, SchemaComp } } - static SchemaValidator createSchemaValidator(SchemaCompatibilityStrategy compatibilityStrategy) throws IncompatibleSchemaException { + static SchemaValidator createSchemaValidator(SchemaCompatibilityStrategy compatibilityStrategy) { final SchemaValidatorBuilder validatorBuilder = new SchemaValidatorBuilder(); switch (compatibilityStrategy) { case BACKWARD_TRANSITIVE: @@ -86,6 +88,17 @@ static SchemaValidator createSchemaValidator(SchemaCompatibilityStrategy compati } } + static SchemaValidator createConsumerSchemaValidator(SchemaCompatibilityStrategy compatibilityStrategy) { + if (compatibilityStrategy == SchemaCompatibilityStrategy.BACKWARD || + compatibilityStrategy == SchemaCompatibilityStrategy.FORWARD || + compatibilityStrategy == SchemaCompatibilityStrategy.FORWARD_TRANSITIVE || + compatibilityStrategy == SchemaCompatibilityStrategy.FULL) { + return createSchemaValidator(SchemaCompatibilityStrategy.BACKWARD); + } else { + return createSchemaValidator(compatibilityStrategy); + } + } + static SchemaValidator createLatestOrAllValidator(SchemaValidatorBuilder validatorBuilder, boolean onlyLatest) { return onlyLatest ? validatorBuilder.validateLatest() : validatorBuilder.validateAll(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/DefaultSchemaRegistryService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/DefaultSchemaRegistryService.java index c4f4d94e3f6f2..9d0945e9dc7b8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/DefaultSchemaRegistryService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/DefaultSchemaRegistryService.java @@ -23,6 +23,8 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; + +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.protocol.schema.SchemaVersion; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java index beb78d5301c85..cda5cc04cfe5e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java @@ -26,6 +26,7 @@ import org.apache.avro.Schema; import org.apache.avro.SchemaParseException; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaType; @@ -41,7 +42,7 @@ public SchemaType getSchemaType() { } @Override - public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { + public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException { if (isAvroSchema(from)) { if (isAvroSchema(to)) { // if both producer and broker have the schema in avro format diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheck.java index 006b63cbed64f..96e38dec38764 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheck.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheck.java @@ -20,6 +20,7 @@ import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.client.impl.schema.KeyValueSchemaInfo; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaInfo; @@ -95,4 +96,14 @@ public void checkCompatible(Iterable from, SchemaData to, SchemaComp keyCheck.checkCompatible(fromKeyList, toKeyValue.getKey(), strategy); valueCheck.checkCompatible(fromValueList, toKeyValue.getValue(), strategy); } + + @Override + public void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException { + throw new IncompatibleSchemaException("KeyValueSchemaCompatibilityCheck don't support this method"); + } + + @Override + public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException { + throw new IncompatibleSchemaException("KeyValueSchemaCompatibilityCheck don't support this method"); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaCompatibilityCheck.java index 800c2afc392ad..ea4e95692f7fe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaCompatibilityCheck.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaCompatibilityCheck.java @@ -19,11 +19,10 @@ package org.apache.pulsar.broker.service.schema; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaType; -import java.util.Collections; - public interface SchemaCompatibilityCheck { SchemaType getSchemaType(); @@ -34,16 +33,37 @@ public interface SchemaCompatibilityCheck { * @param to the future schema i.e. the schema sent by the producer * @param strategy the strategy to use when comparing schemas */ - void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException; + default void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { + checkCompatible(from, to, strategy, false); + } + + /** + * + * @param from the current schemas i.e. schemas that the broker has + * @param to the future schema i.e. the schema sent by the producer + * @param strategy the strategy to use when comparing schemas + */ + default void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { + checkCompatible(from, to, strategy, false); + } /** * * @param from the current schemas i.e. schemas that the broker has * @param to the future schema i.e. the schema sent by the producer * @param strategy the strategy to use when comparing schemas - * @return whether the schemas are compatible + * @param isConsumer the flag to judge whether it is consumer + */ + void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException; + + /** + * + * @param from the current schema i.e. schema that the broker has + * @param to the future schema i.e. the schema sent by the producer + * @param strategy the strategy to use when comparing schemas + * @param isConsumer the flag to judge whether it is consumer */ - void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException; + void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException; default boolean isCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy) { try { @@ -72,13 +92,25 @@ public SchemaType getSchemaType() { @Override public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { - if (strategy == SchemaCompatibilityStrategy.ALWAYS_INCOMPATIBLE) { - throw new IncompatibleSchemaException("Schema compatibility strategy is ALWAYS_INCOMPATIBLE"); - } + defaultCompatibilityCheck(strategy); } @Override public void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { + defaultCompatibilityCheck(strategy); + } + + @Override + public void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException { + defaultCompatibilityCheck(strategy); + } + + @Override + public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException { + defaultCompatibilityCheck(strategy); + } + + private void defaultCompatibilityCheck(SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { if (strategy == SchemaCompatibilityStrategy.ALWAYS_INCOMPATIBLE) { throw new IncompatibleSchemaException("Schema compatibility strategy is ALWAYS_INCOMPATIBLE"); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistry.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistry.java index b5f93767770ba..6d4cb4161ce6b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistry.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistry.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Objects; import java.util.concurrent.CompletableFuture; + +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.protocol.schema.SchemaVersion; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java index 26f030950bd4c..21a15ed6a13ba 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java @@ -20,9 +20,9 @@ import static java.util.Objects.isNull; import static java.util.concurrent.CompletableFuture.completedFuture; -import static org.apache.pulsar.broker.service.schema.SchemaCompatibilityStrategy.BACKWARD_TRANSITIVE; -import static org.apache.pulsar.broker.service.schema.SchemaCompatibilityStrategy.FORWARD_TRANSITIVE; -import static org.apache.pulsar.broker.service.schema.SchemaCompatibilityStrategy.FULL_TRANSITIVE; +import static org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy.BACKWARD_TRANSITIVE; +import static org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy.FORWARD_TRANSITIVE; +import static org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy.FULL_TRANSITIVE; import static org.apache.pulsar.broker.service.schema.SchemaRegistryServiceImpl.Functions.toMap; import static org.apache.pulsar.broker.service.schema.SchemaRegistryServiceImpl.Functions.toPairs; @@ -46,6 +46,7 @@ import org.apache.bookkeeper.common.concurrent.FutureUtils; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.broker.service.schema.proto.SchemaRegistryFormat; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.protocol.schema.SchemaVersion; @@ -245,6 +246,35 @@ public CompletableFuture findSchemaVersion(String schemaId, SchemaData sch }); } + public CompletableFuture checkConsumerCompatibility(String schemaId, SchemaData schemaData, + SchemaCompatibilityStrategy strategy) { + return getSchema(schemaId).thenCompose(existingSchema -> { + if (existingSchema != null && !existingSchema.schema.isDeleted()) { + CompletableFuture result = new CompletableFuture<>(); + try { + compatibilityChecks.getOrDefault(schemaData.getType(), SchemaCompatibilityCheck.DEFAULT) + .checkCompatible(existingSchema.schema, schemaData, strategy, true); + result.complete(null); + } catch (IncompatibleSchemaException e) { + result.completeExceptionally(e); + } + return result; + } else { + return FutureUtils.exception(new IncompatibleSchemaException("Do not have existing schema.")); + } + }); + } + + public CompletableFuture checkProducerCompatibility(String schemaId, SchemaData schemaData, + SchemaCompatibilityStrategy strategy) { + if (isTransitiveStrategy(strategy)) { + + } + + return null; + + } + private CompletableFuture checkCompatibilityWithLatest(String schemaId, SchemaData schema, SchemaCompatibilityStrategy strategy) { return getSchema(schemaId).thenCompose(existingSchema -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java index 9d23285680373..90f7bbe0ee1ea 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java @@ -20,7 +20,7 @@ import java.util.List; import java.util.concurrent.CompletableFuture; -import org.apache.pulsar.broker.service.schema.SchemaCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.broker.service.schema.exceptions.InvalidSchemaDataException; import org.apache.pulsar.common.protocol.schema.SchemaData; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaAutoUpdateTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaAutoUpdateTest.java index 5af5c7fced71a..ab1bd09db25fe 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaAutoUpdateTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaAutoUpdateTest.java @@ -28,7 +28,7 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.Topic; -import org.apache.pulsar.broker.service.schema.SchemaCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BaseAvroSchemaCompatibilityTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BaseAvroSchemaCompatibilityTest.java index bea4e91e6ed63..70881d69449d9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BaseAvroSchemaCompatibilityTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BaseAvroSchemaCompatibilityTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service.schema; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaType; import org.testng.Assert; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java index f234b18ee2235..4aa6dac4495de 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java @@ -35,6 +35,7 @@ import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.JSONSchema; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheckTest.java index 0755781e04323..d5912240b378e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheckTest.java @@ -27,6 +27,7 @@ import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.client.impl.schema.StringSchema; import org.apache.pulsar.client.impl.schema.KeyValueSchema; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaType; import org.testng.Assert; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java index 0c2a174f82825..21128b7f4005f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java @@ -39,6 +39,7 @@ import com.google.common.hash.Hashing; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.schema.SchemaRegistry.SchemaAndMetadata; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.protocol.schema.SchemaVersion; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidatorTest.java index 885a9e980b61c..bc05957b085b0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidatorTest.java @@ -30,7 +30,7 @@ import static org.testng.Assert.fail; import java.util.concurrent.CompletableFuture; -import org.apache.pulsar.broker.service.schema.SchemaCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.broker.service.schema.SchemaRegistry.SchemaAndMetadata; import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.broker.service.schema.exceptions.InvalidSchemaDataException; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java index 163185025a398..2dd5141a25f83 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java @@ -36,7 +36,7 @@ import lombok.Cleanup; -import org.apache.pulsar.broker.service.schema.SchemaCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.broker.service.schema.SchemaRegistry; import org.apache.pulsar.broker.service.schema.exceptions.InvalidSchemaDataException; import org.apache.pulsar.client.api.schema.GenericRecord; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java index 5f23be77fb6bf..e72c25ed887d7 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java @@ -80,9 +80,12 @@ public class Policies { public Long offload_deletion_lag_ms = null; @SuppressWarnings("checkstyle:MemberName") + @Deprecated public SchemaAutoUpdateCompatibilityStrategy schema_auto_update_compatibility_strategy = SchemaAutoUpdateCompatibilityStrategy.Full; + public SchemaCompatibilityStrategy schemaCompatibilityStrategy = SchemaCompatibilityStrategy.UNDEFINED; + @SuppressWarnings("checkstyle:MemberName") public boolean schema_validation_enforced = false; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaCompatibilityStrategy.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/SchemaCompatibilityStrategy.java similarity index 95% rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaCompatibilityStrategy.java rename to pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/SchemaCompatibilityStrategy.java index b15568d6f20c3..52953d685c4b4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaCompatibilityStrategy.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/SchemaCompatibilityStrategy.java @@ -16,11 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.broker.service.schema; - -import org.apache.pulsar.common.policies.data.SchemaAutoUpdateCompatibilityStrategy; +package org.apache.pulsar.common.policies.data; public enum SchemaCompatibilityStrategy { + + /** + * Undefined + */ + UNDEFINED, + /** * Always incompatible */ From 96990c32b0608715cef58240f858b945cb599bd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Tue, 17 Sep 2019 10:27:26 +0800 Subject: [PATCH 02/24] Schema compatibility check change --- conf/broker.conf | 4 - .../broker/admin/impl/NamespacesBase.java | 38 +++++++- .../pulsar/broker/admin/v2/Namespaces.java | 54 +++++++++++ .../broker/admin/v2/SchemasResource.java | 58 +++++++----- .../pulsar/broker/service/AbstractTopic.java | 16 +++- .../pulsar/broker/service/ServerCnx.java | 17 +--- .../apache/pulsar/broker/service/Topic.java | 4 +- .../nonpersistent/NonPersistentTopic.java | 16 ++-- .../service/persistent/PersistentTopic.java | 14 +-- .../AvroSchemaBasedCompatibilityCheck.java | 2 +- .../schema/DefaultSchemaRegistryService.java | 9 +- .../broker/service/schema/SchemaRegistry.java | 5 +- .../schema/SchemaRegistryServiceImpl.java | 94 +++++++++---------- ...egistryServiceWithSchemaDataValidator.java | 9 +- .../service/schema/SchemaServiceTest.java | 6 +- ...tryServiceWithSchemaDataValidatorTest.java | 12 +-- .../api/SimpleTypedProducerConsumerTest.java | 12 ++- .../pulsar/client/admin/Namespaces.java | 77 ++++++++++++++- .../client/admin/internal/NamespacesImpl.java | 47 ++++++++++ .../pulsar/admin/cli/CmdNamespaces.java | 78 +++++++++++++++ .../pulsar/common/policies/data/Policies.java | 18 +++- .../data/SchemaCompatibilityStrategy.java | 17 ++-- 22 files changed, 465 insertions(+), 142 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index b0ecc645e34dc..b491336ca534c 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -725,10 +725,6 @@ schemaRegistryStorageClassName=org.apache.pulsar.broker.service.schema.Bookkeepe # if you enable this setting, it will cause non-java clients failed to produce. isSchemaValidationEnforced=false -schemaCompatibilityStrategy = FULL - -isAllowAutoUpdateSchema = true - ### --- Ledger Offloading --- ### # The directory for all the offloader implementations diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 5d6d8fc6daffd..0f35f9f1e900d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -24,8 +24,6 @@ import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.pulsar.broker.cache.ConfigurationCacheService.POLICIES; import static org.apache.pulsar.broker.cache.LocalZooKeeperCacheService.LOCAL_POLICIES_ROOT; -import static org.apache.pulsar.broker.web.PulsarWebResource.joinPath; -import static org.apache.pulsar.common.naming.NamespaceBundleFactory.getBundlesData; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -82,6 +80,7 @@ import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.SchemaAutoUpdateCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.SubscriptionAuthMode; import org.apache.pulsar.common.util.FutureUtil; @@ -1985,11 +1984,18 @@ protected void internalSetOffloadDeletionLag(Long newDeletionLagMs) { } } + @Deprecated protected SchemaAutoUpdateCompatibilityStrategy internalGetSchemaAutoUpdateCompatibilityStrategy() { validateAdminAccessForTenant(namespaceName.getTenant()); return getNamespacePolicies(namespaceName).schema_auto_update_compatibility_strategy; } + protected SchemaCompatibilityStrategy internalGetSchemaCompatibilityStrategy() { + validateAdminAccessForTenant(namespaceName.getTenant()); + return getNamespacePolicies(namespaceName).schema_compatibility_strategy; + } + + @Deprecated protected void internalSetSchemaAutoUpdateCompatibilityStrategy(SchemaAutoUpdateCompatibilityStrategy strategy) { validateSuperUserAccess(); validatePoliciesReadOnlyAccess(); @@ -2001,6 +2007,17 @@ protected void internalSetSchemaAutoUpdateCompatibilityStrategy(SchemaAutoUpdate "schemaAutoUpdateCompatibilityStrategy"); } + protected void internalSetSchemaCompatibilityStrategy(SchemaCompatibilityStrategy strategy) { + validateSuperUserAccess(); + validatePoliciesReadOnlyAccess(); + + mutatePolicy((policies) -> { + policies.schema_compatibility_strategy = strategy; + return policies; + }, (policies) -> policies.schema_compatibility_strategy, + "schemaCompatibilityStrategy"); + } + protected boolean internalGetSchemaValidationEnforced() { validateSuperUserAccess(); validateAdminAccessForTenant(namespaceName.getTenant()); @@ -2018,6 +2035,23 @@ protected void internalSetSchemaValidationEnforced(boolean schemaValidationEnfor "schemaValidationEnforced"); } + protected boolean internalGetIsAllowAutoUpdateSchema() { + validateSuperUserAccess(); + validateAdminAccessForTenant(namespaceName.getTenant()); + return getNamespacePolicies(namespaceName).is_allow_auto_update_Schema; + } + + protected void internalSetIsAllowAutoUpdateSchema(boolean isAllowAutoUpdateSchema) { + validateSuperUserAccess(); + validatePoliciesReadOnlyAccess(); + + mutatePolicy((policies) -> { + policies.is_allow_auto_update_Schema = isAllowAutoUpdateSchema; + return policies; + }, (policies) -> policies.is_allow_auto_update_Schema, + "isAllowAutoUpdateSchema"); + } + private void mutatePolicy(Function policyTransformation, Function getter, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 435892eefb262..6575bda306292 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -49,6 +49,7 @@ import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.SubscriptionAuthMode; import org.apache.pulsar.common.policies.data.SchemaAutoUpdateCompatibilityStrategy; @@ -940,6 +941,59 @@ public void setSchemaAutoUpdateCompatibilityStrategy(@PathParam("tenant") String internalSetSchemaAutoUpdateCompatibilityStrategy(strategy); } + @GET + @Path("/{tenant}/{cluster}/{namespace}/schemaCompatibilityStrategy") + @ApiOperation(value = "The strategy of the namespace schema compatibility ") + @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Namespace doesn't exist"), + @ApiResponse(code = 409, message = "Concurrent modification") }) + public SchemaCompatibilityStrategy getSchemaCompatibilityStrategy( + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace) { + validateNamespaceName(tenant, namespace); + return internalGetSchemaCompatibilityStrategy(); + } + + @PUT + @Path("/{tenant}/{cluster}/{namespace}/schemaAutoUpdateCompatibilityStrategy") + @ApiOperation(value = "Update the strategy used to check the compatibility of new schema") + @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Namespace doesn't exist"), + @ApiResponse(code = 409, message = "Concurrent modification") }) + public void setSchemaCompatibilityStrategy(@PathParam("tenant") String tenant, + @PathParam("namespace") String namespace, + SchemaCompatibilityStrategy strategy) { + validateNamespaceName(tenant, namespace); + internalSetSchemaCompatibilityStrategy(strategy); + } + + @GET + @Path("/{tenant}/{cluster}/{namespace}/isAllowAutoUpdateSchema") + @ApiOperation(value = "The flag of whether allow auto update schema") + @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Namespace doesn't exist"), + @ApiResponse(code = 409, message = "Concurrent modification") }) + public boolean getIsAllowAutoUpdateSchema( + @PathParam("tenant") String tenant, + @PathParam("namespace") String namespace) { + validateNamespaceName(tenant, namespace); + return internalGetIsAllowAutoUpdateSchema(); + } + + @POST + @Path("/{tenant}/{cluster}/{namespace}/isAllowAutoUpdateSchema") + @ApiOperation(value = "Update flag of whether allow auto update schema") + @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Namespace doesn't exist"), + @ApiResponse(code = 409, message = "Concurrent modification") }) + public void setIsAllowAutoUpdateSchema(@PathParam("tenant") String tenant, + @PathParam("namespace") String namespace, + boolean isAllowAutoUpdateSchema) { + validateNamespaceName(tenant, namespace); + internalSetIsAllowAutoUpdateSchema(isAllowAutoUpdateSchema); + } + + @GET @Path("/{tenant}/{namespace}/schemaValidationEnforced") @ApiOperation(value = "Get schema validation enforced flag for namespace.", diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java index 245098d86b802..3761a86d8ea3b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java @@ -55,6 +55,7 @@ import org.apache.pulsar.broker.admin.AdminResource; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.broker.service.schema.LongSchemaVersion; +import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.broker.service.schema.SchemaRegistry.SchemaAndMetadata; import org.apache.pulsar.broker.service.schema.exceptions.InvalidSchemaDataException; @@ -314,18 +315,21 @@ public void postSchema( NamespaceName namespaceName = NamespaceName.get(tenant, namespace); getNamespacePoliciesAsync(namespaceName).thenAccept(policies -> { - SchemaCompatibilityStrategy schemaCompatibilityStrategy = SchemaCompatibilityStrategy - .fromAutoUpdatePolicy(policies.schema_auto_update_compatibility_strategy); - byte[] data; - if (SchemaType.KEY_VALUE.name().equals(payload.getType())) { - data = DefaultImplementation - .convertKeyValueDataStringToSchemaInfoSchema(payload.getSchema().getBytes(Charsets.UTF_8)); - } else { - data = payload.getSchema().getBytes(Charsets.UTF_8); - } - pulsar().getSchemaRegistryService().putSchemaIfAbsent( - buildSchemaId(tenant, namespace, topic), - SchemaData.builder() + SchemaCompatibilityStrategy schemaCompatibilityStrategy = policies.schema_compatibility_strategy; + if (schemaCompatibilityStrategy == SchemaCompatibilityStrategy.UNDEFINED) { + schemaCompatibilityStrategy = SchemaCompatibilityStrategy + .fromAutoUpdatePolicy(policies.schema_auto_update_compatibility_strategy); + } + byte[] data; + if (SchemaType.KEY_VALUE.name().equals(payload.getType())) { + data = DefaultImplementation + .convertKeyValueDataStringToSchemaInfoSchema(payload.getSchema().getBytes(Charsets.UTF_8)); + } else { + data = payload.getSchema().getBytes(Charsets.UTF_8); + } + pulsar().getSchemaRegistryService().putSchemaIfAbsent( + buildSchemaId(tenant, namespace, topic), + SchemaData.builder() .data(data) .isDeleted(false) .timestamp(clock.millis()) @@ -333,15 +337,16 @@ public void postSchema( .user(defaultIfEmpty(clientAppId(), "")) .props(payload.getProperties()) .build(), - schemaCompatibilityStrategy - ).thenAccept(version -> - response.resume( - Response.accepted().entity( - PostSchemaResponse.builder() - .version(version) - .build() - ).build() - ) + schemaCompatibilityStrategy, + true + ).thenAccept(version -> + response.resume( + Response.accepted().entity( + PostSchemaResponse.builder() + .version(version) + .build() + ).build() + ) ).exceptionally(error -> { if (error.getCause() instanceof IncompatibleSchemaException) { response.resume(Response.status(Response.Status.CONFLICT.getStatusCode(), @@ -407,10 +412,15 @@ public void testCompatibility( validateDestinationAndAdminOperation(tenant, namespace, topic, authoritative); String schemaId = buildSchemaId(tenant, namespace, topic); + Policies policies = getNamespacePolicies(NamespaceName.get(tenant, namespace)); - SchemaCompatibilityStrategy schemaCompatibilityStrategy = SchemaCompatibilityStrategy - .fromAutoUpdatePolicy(getNamespacePolicies(NamespaceName.get(tenant, namespace)) - .schema_auto_update_compatibility_strategy); + SchemaCompatibilityStrategy schemaCompatibilityStrategy; + if (policies.schema_compatibility_strategy == SchemaCompatibilityStrategy.UNDEFINED) { + schemaCompatibilityStrategy = SchemaCompatibilityStrategy + .fromAutoUpdatePolicy(policies.schema_auto_update_compatibility_strategy); + } else { + schemaCompatibilityStrategy = policies.schema_compatibility_strategy; + } pulsar().getSchemaRegistryService().isCompatible(schemaId, SchemaData.builder() .data(payload.getSchema().getBytes(Charsets.UTF_8)) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 5ec45f1af2d36..721fec6b4f414 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -71,6 +71,7 @@ public abstract class AbstractTopic implements Topic { protected volatile boolean isEncryptionRequired = false; protected volatile SchemaCompatibilityStrategy schemaCompatibilityStrategy = SchemaCompatibilityStrategy.FULL; + protected volatile boolean isAllowAutoUpdateSchema = true; // schema validation enforced flag protected volatile boolean schemaValidationEnforced = false; @@ -169,7 +170,7 @@ public CompletableFuture addSchema(SchemaData schema) { String id = TopicName.get(base).getSchemaName(); return brokerService.pulsar() .getSchemaRegistryService() - .putSchemaIfAbsent(id, schema, schemaCompatibilityStrategy); + .putSchemaIfAbsent(id, schema, schemaCompatibilityStrategy, isAllowAutoUpdateSchema); } @Override @@ -188,16 +189,25 @@ public CompletableFuture deleteSchema() { } @Override - public CompletableFuture isSchemaCompatible(SchemaData schema) { + public CompletableFuture isSchemaCompatible(SchemaData schema) { String base = TopicName.get(getName()).getPartitionedTopicName(); String id = TopicName.get(base).getSchemaName(); return brokerService.pulsar() .getSchemaRegistryService() - .isCompatible(id, schema, schemaCompatibilityStrategy); + .checkConsumerCompatibility(id, schema, schemaCompatibilityStrategy); } @Override public void recordAddLatency(long latencyUSec) { addEntryLatencyStatsUsec.addValue(latencyUSec); } + + protected void setSchemaCompatibilityStrategy (Policies policies) { + if (policies.schema_compatibility_strategy == SchemaCompatibilityStrategy.UNDEFINED) { + schemaCompatibilityStrategy = SchemaCompatibilityStrategy.fromAutoUpdatePolicy( + policies.schema_auto_update_compatibility_strategy); + } else { + schemaCompatibilityStrategy = policies.schema_compatibility_strategy; + } + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index d91bd1a7d55fe..f34c5b61557bb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -680,19 +680,10 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { .thenCompose(topic -> { if (schema != null) { return topic.addSchemaIfIdleOrCheckCompatible(schema) - .thenCompose(isCompatible -> { - if (isCompatible) { - return topic.subscribe(ServerCnx.this, subscriptionName, consumerId, - subType, priorityLevel, consumerName, isDurable, - startMessageId, metadata, - readCompacted, initialPosition, isReplicated); - } else { - return FutureUtil.failedFuture( - new IncompatibleSchemaException( - "Trying to subscribe with incompatible schema" - )); - } - }); + .thenCompose(isCompatible -> topic.subscribe(ServerCnx.this, subscriptionName, consumerId, + subType, priorityLevel, consumerName, isDurable, + startMessageId, metadata, + readCompacted, initialPosition, isReplicated)); } else { return topic.subscribe(ServerCnx.this, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java index 028292b5d1e71..519f9b28c9281 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java @@ -165,14 +165,14 @@ void updateRates(NamespaceStats nsStats, NamespaceBundleStats currentBundleStats /** * Check if schema is compatible with current topic schema. */ - CompletableFuture isSchemaCompatible(SchemaData schema); + CompletableFuture isSchemaCompatible(SchemaData schema); /** * If the topic is idle (no producers, no entries, no subscribers and no existing schema), * add the passed schema to the topic. Otherwise, check that the passed schema is compatible * with what the topic already has. */ - CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema); + CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema); CompletableFuture deleteForcefully(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 7d1328d9a393c..69125495bacff 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -144,8 +144,9 @@ public NonPersistentTopic(String topic, BrokerService brokerService) { .get(AdminResource.path(POLICIES, TopicName.get(topic).getNamespace())) .orElseThrow(() -> new KeeperException.NoNodeException()); isEncryptionRequired = policies.encryption_required; - schemaCompatibilityStrategy = SchemaCompatibilityStrategy.fromAutoUpdatePolicy( - policies.schema_auto_update_compatibility_strategy); + isAllowAutoUpdateSchema = policies.is_allow_auto_update_Schema; + setSchemaCompatibilityStrategy(policies); + schemaValidationEnforced = policies.schema_validation_enforced; } catch (Exception e) { @@ -870,8 +871,8 @@ public CompletableFuture onPoliciesUpdate(Policies data) { log.debug("[{}] isEncryptionRequired changes: {} -> {}", topic, isEncryptionRequired, data.encryption_required); } isEncryptionRequired = data.encryption_required; - schemaCompatibilityStrategy = SchemaCompatibilityStrategy.fromAutoUpdatePolicy( - data.schema_auto_update_compatibility_strategy); + setSchemaCompatibilityStrategy(data); + isAllowAutoUpdateSchema = data.is_allow_auto_update_Schema; schemaValidationEnforced = data.schema_validation_enforced; producers.forEach(producer -> { @@ -922,13 +923,14 @@ public Position getLastMessageId() { @Override - public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema) { + public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema) { return hasSchema() - .thenCompose((hasSchema) -> { + .thenCompose((hasSchema) -> { if (hasSchema || isActive() || ENTRIES_ADDED_COUNTER_UPDATER.get(this) != 0) { return isSchemaCompatible(schema); } else { - return addSchema(schema).thenApply((ignore) -> true); + addSchema(schema); + return CompletableFuture.completedFuture(null); } }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index bae6c5612f973..bf13527b7b6fe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -226,8 +226,8 @@ public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerS .orElseThrow(() -> new KeeperException.NoNodeException()); isEncryptionRequired = policies.encryption_required; - schemaCompatibilityStrategy = SchemaCompatibilityStrategy.fromAutoUpdatePolicy( - policies.schema_auto_update_compatibility_strategy); + setSchemaCompatibilityStrategy(policies); + isAllowAutoUpdateSchema = policies.is_allow_auto_update_Schema; schemaValidationEnforced = policies.schema_validation_enforced; } catch (Exception e) { @@ -1586,12 +1586,11 @@ public CompletableFuture onPoliciesUpdate(Policies data) { } isEncryptionRequired = data.encryption_required; - schemaCompatibilityStrategy = SchemaCompatibilityStrategy.fromAutoUpdatePolicy( - data.schema_auto_update_compatibility_strategy); + setSchemaCompatibilityStrategy(data); + isAllowAutoUpdateSchema = data.is_allow_auto_update_Schema; schemaValidationEnforced = data.schema_validation_enforced; - initializeDispatchRateLimiterIfNeeded(Optional.ofNullable(data)); producers.forEach(producer -> { @@ -1844,13 +1843,14 @@ public synchronized OffloadProcessStatus offloadStatus() { private static final Logger log = LoggerFactory.getLogger(PersistentTopic.class); @Override - public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema) { + public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema) { return hasSchema() .thenCompose((hasSchema) -> { if (hasSchema || isActive() || ledger.getTotalSize() != 0) { return isSchemaCompatible(schema); } else { - return addSchema(schema).thenApply((ignore) -> true); + addSchema(schema); + return CompletableFuture.completedFuture(null); } }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java index e5a69b8168219..679c8968aeda1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java @@ -41,7 +41,7 @@ abstract class AvroSchemaBasedCompatibilityCheck implements SchemaCompatibilityC @Override public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException { - checkCompatible(Collections.singletonList(from), to, strategy); + checkCompatible(Collections.singletonList(from), to, strategy, isConsumer); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/DefaultSchemaRegistryService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/DefaultSchemaRegistryService.java index 9d0945e9dc7b8..1e1155e593e23 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/DefaultSchemaRegistryService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/DefaultSchemaRegistryService.java @@ -44,6 +44,11 @@ public CompletableFuture>> getAllSchem return completedFuture(Collections.emptyList()); } + @Override + public CompletableFuture putSchemaIfAbsent(String schemaId, SchemaData schema, SchemaCompatibilityStrategy strategy, boolean isAllowAutoUpdateSchema) { + return completedFuture(null); + } + @Override public CompletableFuture> trimDeletedSchemaAndGetList(String schemaId) { return completedFuture(Collections.emptyList()); @@ -54,10 +59,8 @@ public CompletableFuture findSchemaVersion(String schemaId, SchemaData sch return completedFuture(NO_SCHEMA_VERSION); } - @Override - public CompletableFuture putSchemaIfAbsent(String schemaId, SchemaData schema, - SchemaCompatibilityStrategy strategy) { + public CompletableFuture checkConsumerCompatibility(String schemaId, SchemaData schemaData, SchemaCompatibilityStrategy strategy) { return completedFuture(null); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistry.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistry.java index 6d4cb4161ce6b..90cd92cb7b806 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistry.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistry.java @@ -37,7 +37,7 @@ public interface SchemaRegistry extends AutoCloseable { CompletableFuture>> getAllSchemas(String schemaId); CompletableFuture putSchemaIfAbsent(String schemaId, SchemaData schema, - SchemaCompatibilityStrategy strategy); + SchemaCompatibilityStrategy strategy, boolean isAllowAutoUpdateSchema); CompletableFuture deleteSchema(String schemaId, String user); @@ -51,6 +51,9 @@ CompletableFuture checkCompatible(String schemaId, SchemaData schema, CompletableFuture findSchemaVersion(String schemaId, SchemaData schemaData); + CompletableFuture checkConsumerCompatibility(String schemaId, SchemaData schemaData, + SchemaCompatibilityStrategy strategy); + SchemaVersion versionFromBytes(byte[] version); class SchemaAndMetadata { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java index 21a15ed6a13ba..c3b2845f91729 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java @@ -110,51 +110,60 @@ public CompletableFuture>> getAllSchem @Override @NotNull public CompletableFuture putSchemaIfAbsent(String schemaId, SchemaData schema, - SchemaCompatibilityStrategy strategy) { - return getSchema(schemaId, SchemaVersion.Latest) - .thenCompose( - (existingSchema) -> - { + SchemaCompatibilityStrategy strategy, boolean isAllowAutoUpdateSchema) { + return trimDeletedSchemaAndGetList(schemaId).thenCompose(schemaAndMetadataList -> { + CompletableFuture completableFuture = new CompletableFuture<>(); + SchemaVersion schemaVersion = null; + for (SchemaAndMetadata schemaAndMetadata : schemaAndMetadataList) { + if (Arrays.equals(hashFunction.hashBytes(schemaAndMetadata.schema.getData()).asBytes(), + hashFunction.hashBytes(schema.getData()).asBytes())) { + schemaVersion = schemaAndMetadata.version; + } + } + if (schemaVersion != null) { + completableFuture.complete(schemaVersion); + return completableFuture; + } else { + if (isAllowAutoUpdateSchema == true) { CompletableFuture maxDeleteVersionFuture; - if (existingSchema == null) { - maxDeleteVersionFuture = completedFuture(NO_DELETED_VERSION); - } else if (existingSchema.schema.isDeleted()) { - maxDeleteVersionFuture = completedFuture(((LongSchemaVersion)schemaStorage - .versionFromBytes(existingSchema.version.bytes())).getVersion()); - } else { + if (schemaAndMetadataList.size() == 0) { if (isTransitiveStrategy(strategy)) { maxDeleteVersionFuture = checkCompatibilityWithAll(schemaId, schema, strategy); - } else { maxDeleteVersionFuture = new CompletableFuture<>(); - trimDeletedSchemaAndGetList(schemaId).thenAccept(schemaAndMetadataList -> { - checkCompatibilityWithLatest(schemaId, schema, strategy).whenComplete((v, ex) -> { - if (ex == null) { - Long maxDeleteVersion = ((LongSchemaVersion)schemaStorage - .versionFromBytes(schemaAndMetadataList.get(0).version.bytes())).getVersion() - 1L; - maxDeleteVersionFuture.complete(maxDeleteVersion); - } else { - maxDeleteVersionFuture.completeExceptionally(ex); - } - }); + checkCompatibilityWithLatest(schemaId, schema, strategy).whenComplete((v, ex) -> { + if (ex == null) { + Long maxDeleteVersion = ((LongSchemaVersion) schemaStorage + .versionFromBytes(schemaAndMetadataList.get(0).version.bytes())).getVersion() - 1L; + maxDeleteVersionFuture.complete(maxDeleteVersion); + } else { + maxDeleteVersionFuture.completeExceptionally(ex); + } }); } + } else { + maxDeleteVersionFuture = new CompletableFuture<>(); + maxDeleteVersionFuture.complete(NO_DELETED_VERSION); } - return maxDeleteVersionFuture; + return maxDeleteVersionFuture.thenCompose(maxDeleteVersion -> { + byte[] context = hashFunction.hashBytes(schema.getData()).asBytes(); + SchemaRegistryFormat.SchemaInfo info = SchemaRegistryFormat.SchemaInfo.newBuilder() + .setType(Functions.convertFromDomainType(schema.getType())) + .setSchema(ByteString.copyFrom(schema.getData())) + .setSchemaId(schemaId) + .setUser(schema.getUser()) + .setDeleted(false) + .setTimestamp(clock.millis()) + .addAllProps(toPairs(schema.getProps())) + .build(); + return schemaStorage.put(schemaId, info.toByteArray(), context, maxDeleteVersion); + }); + + } else { + return FutureUtils.exception(new IncompatibleSchemaException("Do allow auto update schema.")); } - ).thenCompose(maxDeleteVersion -> { - byte[] context = hashFunction.hashBytes(schema.getData()).asBytes(); - SchemaRegistryFormat.SchemaInfo info = SchemaRegistryFormat.SchemaInfo.newBuilder() - .setType(Functions.convertFromDomainType(schema.getType())) - .setSchema(ByteString.copyFrom(schema.getData())) - .setSchemaId(schemaId) - .setUser(schema.getUser()) - .setDeleted(false) - .setTimestamp(clock.millis()) - .addAllProps(toPairs(schema.getProps())) - .build(); - return schemaStorage.put(schemaId, info.toByteArray(), context, maxDeleteVersion); - }); + } + }); } @Override @@ -246,6 +255,7 @@ public CompletableFuture findSchemaVersion(String schemaId, SchemaData sch }); } + @Override public CompletableFuture checkConsumerCompatibility(String schemaId, SchemaData schemaData, SchemaCompatibilityStrategy strategy) { return getSchema(schemaId).thenCompose(existingSchema -> { @@ -260,21 +270,11 @@ public CompletableFuture checkConsumerCompatibility(String schemaId, Schem } return result; } else { - return FutureUtils.exception(new IncompatibleSchemaException("Do not have existing schema.")); + return FutureUtils.exception(new IncompatibleSchemaException("Trying to subscribe with incompatible schema")); } }); } - public CompletableFuture checkProducerCompatibility(String schemaId, SchemaData schemaData, - SchemaCompatibilityStrategy strategy) { - if (isTransitiveStrategy(strategy)) { - - } - - return null; - - } - private CompletableFuture checkCompatibilityWithLatest(String schemaId, SchemaData schema, SchemaCompatibilityStrategy strategy) { return getSchema(schemaId).thenCompose(existingSchema -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java index 90f7bbe0ee1ea..28b47fdd79684 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java @@ -72,16 +72,21 @@ public CompletableFuture findSchemaVersion(String schemaId, SchemaData sch return this.service.findSchemaVersion(schemaId, schemaData); } + @Override + public CompletableFuture checkConsumerCompatibility(String schemaId, SchemaData schemaData, SchemaCompatibilityStrategy strategy) { + return this.checkConsumerCompatibility(schemaId, schemaData, strategy); + } + @Override public CompletableFuture putSchemaIfAbsent(String schemaId, SchemaData schema, - SchemaCompatibilityStrategy strategy) { + SchemaCompatibilityStrategy strategy, boolean isAllowAutoUpdateSchema) { try { SchemaDataValidator.validateSchemaData(schema); } catch (InvalidSchemaDataException e) { return FutureUtil.failedFuture(e); } - return service.putSchemaIfAbsent(schemaId, schema, strategy); + return service.putSchemaIfAbsent(schemaId, schema, strategy, isAllowAutoUpdateSchema); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java index 21128b7f4005f..14fe70c0b48db 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java @@ -258,11 +258,11 @@ public void dontReAddExistingSchemaAtRoot() throws Exception { public void trimDeletedSchemaAndGetListTest() throws Exception { List list = new ArrayList<>(); CompletableFuture put = schemaRegistryService.putSchemaIfAbsent( - schemaId1, schema1, SchemaCompatibilityStrategy.FULL); + schemaId1, schema1, SchemaCompatibilityStrategy.FULL, true); SchemaVersion newVersion = put.get(); list.add(new SchemaAndMetadata(schemaId1, schema1, newVersion)); put = schemaRegistryService.putSchemaIfAbsent( - schemaId1, schema2, SchemaCompatibilityStrategy.FULL); + schemaId1, schema2, SchemaCompatibilityStrategy.FULL, true); newVersion = put.get(); list.add(new SchemaAndMetadata(schemaId1, schema2, newVersion)); List list1 = schemaRegistryService.trimDeletedSchemaAndGetList(schemaId1).get(); @@ -326,7 +326,7 @@ private void putSchema(String schemaId, SchemaData schema, SchemaVersion expecte private void putSchema(String schemaId, SchemaData schema, SchemaVersion expectedVersion, SchemaCompatibilityStrategy strategy) throws ExecutionException, InterruptedException { CompletableFuture put = schemaRegistryService.putSchemaIfAbsent( - schemaId, schema, strategy); + schemaId, schema, strategy, true); SchemaVersion newVersion = put.get(); assertEquals(expectedVersion, newVersion); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidatorTest.java index bc05957b085b0..93f3afe72ccd6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidatorTest.java @@ -128,15 +128,15 @@ public void testPutSchemaIfAbsentWithGoodSchemaData() { String schemaId = "test-schema-id"; SchemaCompatibilityStrategy strategy = SchemaCompatibilityStrategy.FULL; CompletableFuture future = new CompletableFuture<>(); - when(underlyingService.putSchemaIfAbsent(eq(schemaId), any(SchemaData.class), eq(strategy))) + when(underlyingService.putSchemaIfAbsent(eq(schemaId), any(SchemaData.class), eq(strategy), eq(true))) .thenReturn(future); SchemaData schemaData = SchemaData.builder() .type(SchemaType.BOOLEAN) .data(new byte[0]) .build(); - assertSame(future, service.putSchemaIfAbsent(schemaId, schemaData, strategy)); + assertSame(future, service.putSchemaIfAbsent(schemaId, schemaData, strategy, true)); verify(underlyingService, times(1)) - .putSchemaIfAbsent(eq(schemaId), same(schemaData), eq(strategy)); + .putSchemaIfAbsent(eq(schemaId), same(schemaData), eq(strategy), eq(true)); } @Test @@ -144,20 +144,20 @@ public void testPutSchemaIfAbsentWithBadSchemaData() { String schemaId = "test-schema-id"; SchemaCompatibilityStrategy strategy = SchemaCompatibilityStrategy.FULL; CompletableFuture future = new CompletableFuture<>(); - when(underlyingService.putSchemaIfAbsent(eq(schemaId), any(SchemaData.class), eq(strategy))) + when(underlyingService.putSchemaIfAbsent(eq(schemaId), any(SchemaData.class), eq(strategy), eq(true))) .thenReturn(future); SchemaData schemaData = SchemaData.builder() .type(SchemaType.BOOLEAN) .data(new byte[10]) .build(); try { - service.putSchemaIfAbsent(schemaId, schemaData, strategy).get(); + service.putSchemaIfAbsent(schemaId, schemaData, strategy, true).get(); fail("Should fail putSchemaIfAbsent"); } catch (Exception e) { assertTrue(e.getCause() instanceof InvalidSchemaDataException); } verify(underlyingService, times(0)) - .putSchemaIfAbsent(eq(schemaId), same(schemaData), eq(strategy)); + .putSchemaIfAbsent(eq(schemaId), same(schemaData), eq(strategy), eq(true)); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java index 2dd5141a25f83..16a588d3337d8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java @@ -131,7 +131,8 @@ public void testJsonProducerAndConsumerWithPrestoredSchema() throws Exception { .data(jsonSchema.getSchemaInfo().getSchema()) .props(Collections.emptyMap()) .build(), - SchemaCompatibilityStrategy.FULL + SchemaCompatibilityStrategy.FULL, + true ).get(); Consumer consumer = pulsarClient @@ -174,7 +175,8 @@ public void testWrongCorruptedSchema() throws Exception { .data(randomSchemaBytes) .props(Collections.emptyMap()) .build(), - SchemaCompatibilityStrategy.FULL + SchemaCompatibilityStrategy.FULL, + true ).get(); fail("Should fail to add corrupted schema data"); } catch (Exception e) { @@ -250,7 +252,8 @@ public void testProtobufConsumerWithWrongPrestoredSchema() throws Exception { .data(schema.getSchemaInfo().getSchema()) .props(Collections.emptyMap()) .build(), - SchemaCompatibilityStrategy.FULL + SchemaCompatibilityStrategy.FULL, + true ).get(); Consumer consumer = pulsarClient @@ -335,7 +338,8 @@ public void testAvroConsumerWithWrongPrestoredSchema() throws Exception { .data(randomSchemaBytes) .props(Collections.emptyMap()) .build(), - SchemaCompatibilityStrategy.FULL + SchemaCompatibilityStrategy.FULL, + true ).get(); Consumer consumer = pulsarClient diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java index c69a4a078f857..c9dd0a4d89899 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java @@ -24,6 +24,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; import org.apache.pulsar.client.admin.PulsarAdminException.ConflictException; import org.apache.pulsar.client.admin.PulsarAdminException.NotAuthorizedException; import org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException; @@ -37,9 +40,15 @@ import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.SchemaAutoUpdateCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.SubscriptionAuthMode; +import javax.ws.rs.GET; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; + /** * Admin interface for namespaces management */ @@ -1450,6 +1459,7 @@ CompletableFuture clearNamespaceBundleBacklogForSubscriptionAsync(String n * @throws PulsarAdminException * Unexpected error */ + @Deprecated SchemaAutoUpdateCompatibilityStrategy getSchemaAutoUpdateCompatibilityStrategy(String namespace) throws PulsarAdminException; @@ -1469,6 +1479,7 @@ SchemaAutoUpdateCompatibilityStrategy getSchemaAutoUpdateCompatibilityStrategy(S * @throws PulsarAdminException * Unexpected error */ + @Deprecated void setSchemaAutoUpdateCompatibilityStrategy(String namespace, SchemaAutoUpdateCompatibilityStrategy strategy) throws PulsarAdminException; @@ -1483,7 +1494,6 @@ void setSchemaAutoUpdateCompatibilityStrategy(String namespace, * @throws PulsarAdminException * Unexpected error */ - boolean getSchemaValidationEnforced(String namespace) throws PulsarAdminException; /** @@ -1501,7 +1511,70 @@ boolean getSchemaValidationEnforced(String namespace) * @throws PulsarAdminException * Unexpected error */ - void setSchemaValidationEnforced(String namespace, boolean schemaValidationEnforced) throws PulsarAdminException; + + /** + * Get the strategy used to check the a new schema provided by a producer is compatible with the current schema + * before it is installed. + * + * @param namespace The namespace in whose policy we are interested + * @return the strategy used to check compatibility + * @throws NotAuthorizedException + * Don't have admin permission + * @throws NotFoundException + * Namespace does not exist + * @throws PulsarAdminException + * Unexpected error + */ + SchemaCompatibilityStrategy getSchemaCompatibilityStrategy(String namespace) + throws PulsarAdminException;; + + /** + * Set the strategy used to check the a new schema provided by a producer is compatible with the current schema + * before it is installed. + * + * @param namespace The namespace in whose policy should be set + * @param strategy The schema compatibility strategy + * @throws NotAuthorizedException + * Don't have admin permission + * @throws NotFoundException + * Namespace does not exist + * @throws PulsarAdminException + * Unexpected error + */ + void setSchemaCompatibilityStrategy(String namespace, + SchemaCompatibilityStrategy strategy) + throws PulsarAdminException; + + /** + * Get whether allow auto update schema + * + * @param namespace pulsar namespace name + * @return the schema validation enforced flag + * @throws NotAuthorizedException + * Don't have admin permission + * @throws NotFoundException + * Tenant or Namespace does not exist + * @throws PulsarAdminException + * Unexpected error + */ + boolean getIsAllowAutoUpdateSchema(String namespace) + throws PulsarAdminException; + + /** + * The flag is when producer bring a new schema and the schema pass compatibility check + * whether allow schema auto registered + * + * @param namespace pulsar namespace name + * @param isAllowAutoUpdateSchema flag to enable or disable auto update schema + * @throws NotAuthorizedException + * Don't have admin permission + * @throws NotFoundException + * Tenant or Namespace does not exist + * @throws PulsarAdminException + * Unexpected error + */ + void setIsAllowAutoUpdateSchema(String namespace, boolean isAllowAutoUpdateSchema) + throws PulsarAdminException; } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java index a5470a43042b4..83e830165efb5 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java @@ -47,6 +47,7 @@ import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.SchemaAutoUpdateCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.SubscriptionAuthMode; @@ -934,6 +935,52 @@ public void setSchemaValidationEnforced(String namespace, boolean schemaValidati } } + @Override + public SchemaCompatibilityStrategy getSchemaCompatibilityStrategy(String namespace) throws PulsarAdminException { + try { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "schemaCompatibilityStrategy"); + return request(path).get(SchemaCompatibilityStrategy.class); + } catch (Exception e) { + throw getApiException(e); + } + } + + @Override + public void setSchemaCompatibilityStrategy(String namespace, SchemaCompatibilityStrategy strategy) throws PulsarAdminException { + try { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "schemaCompatibilityStrategy"); + request(path).put(Entity.entity(strategy, MediaType.APPLICATION_JSON), + ErrorData.class); + } catch (Exception e) { + throw getApiException(e); + } + + } + + @Override + public boolean getIsAllowAutoUpdateSchema(String namespace) throws PulsarAdminException { + try { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "isAllowAutoUpdateSchema"); + return request(path).get(Boolean.class); + } catch (Exception e) { + throw getApiException(e); + } + } + + @Override + public void setIsAllowAutoUpdateSchema(String namespace, boolean isAllowAutoUpdateSchema) throws PulsarAdminException { + try { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "isAllowAutoUpdateSchema"); + request(path).post(Entity.entity(isAllowAutoUpdateSchema, MediaType.APPLICATION_JSON), ErrorData.class); + } catch (Exception e) { + throw getApiException(e); + } + } + private WebTarget namespacePath(NamespaceName namespace, String... parts) { final WebTarget base = namespace.isV2() ? adminV2Namespaces : adminNamespaces; WebTarget namespacePath = base.path(namespace.toString()); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 5a996e418b262..47a53fb66a81c 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -44,6 +44,7 @@ import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.SchemaAutoUpdateCompatibilityStrategy; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.SubscriptionAuthMode; import org.apache.pulsar.common.util.RelativeTimeUtil; @@ -1096,6 +1097,77 @@ void run() throws PulsarAdminException { } } + @Parameters(commandDescription = "Get the schema compatibility strategy for a namespace") + private class GetSchemaCompatibilityStrategy extends CliCommand { + @Parameter(description = "tenant/namespace", required = true) + private java.util.List params; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(params); + System.out.println(admin.namespaces().getSchemaCompatibilityStrategy(namespace) + .toString().toUpperCase()); + } + } + + @Parameters(commandDescription = "Set the schema compatibility strategy for a namespace") + private class SetSchemaCompatibilityStrategy extends CliCommand { + @Parameter(description = "tenant/namespace", required = true) + private java.util.List params; + + @Parameter(names = { "--compatibility", "-c" }, + description = "Compatibility level required for new schemas created via a Producer. " + + "Possible values (FULL, BACKWARD, FORWARD, " + + "UNDEFINED, BACKWARD_TRANSITIVE, " + + "FORWARD_TRANSITIVE, FULL_TRANSITIVE, " + + "ALWAYS_INCOMPATIBLE," + + "ALWAYS_COMPATIBLE).") + private String strategyParam = null; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(params); + + String strategyStr = strategyParam != null ? strategyParam.toUpperCase() : ""; + admin.namespaces().setSchemaCompatibilityStrategy(namespace, SchemaCompatibilityStrategy.valueOf(strategyStr)); + } + } + + @Parameters(commandDescription = "Get the namespace whether allow auto update schema") + private class GetIsAllowAutoUpdateSchema extends CliCommand { + @Parameter(description = "tenant/namespace", required = true) + private java.util.List params; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(params); + + System.out.println(admin.namespaces().getIsAllowAutoUpdateSchema(namespace)); + } + } + + @Parameters(commandDescription = "Set the namespace whether allow auto update schema") + private class SetIsAllowAutoUpdateSchema extends CliCommand { + @Parameter(description = "tenant/namespace", required = true) + private java.util.List params; + + @Parameter(names = { "--enable", "-e" }, description = "Enable schema validation enforced") + private boolean enable = false; + + @Parameter(names = { "--disable", "-d" }, description = "Disable schema validation enforced") + private boolean disable = false; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(params); + + if (enable == disable) { + throw new ParameterException("Need to specify either --enable or --disable"); + } + admin.namespaces().setIsAllowAutoUpdateSchema(namespace, enable); + } + } + @Parameters(commandDescription = "Get the schema validation enforced") private class GetSchemaValidationEnforced extends CliCommand { @Parameter(description = "tenant/namespace", required = true) @@ -1219,6 +1291,12 @@ public CmdNamespaces(PulsarAdmin admin) { jcommander.addCommand("get-schema-autoupdate-strategy", new GetSchemaAutoUpdateStrategy()); jcommander.addCommand("set-schema-autoupdate-strategy", new SetSchemaAutoUpdateStrategy()); + jcommander.addCommand("get-schema-compatibility-strategy", new GetSchemaCompatibilityStrategy()); + jcommander.addCommand("set-schema-compatibility-strategy", new GetSchemaCompatibilityStrategy()); + + jcommander.addCommand("get-is-allow-auto-update-schema", new GetIsAllowAutoUpdateSchema()); + jcommander.addCommand("set-is-allow-auto-update-schema", new SetIsAllowAutoUpdateSchema()); + jcommander.addCommand("get-schema-validation-enforce", new GetSchemaValidationEnforced()); jcommander.addCommand("set-schema-validation-enforce", new SetSchemaValidationEnforced()); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java index e72c25ed887d7..9bc038d35e835 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java @@ -84,7 +84,11 @@ public class Policies { public SchemaAutoUpdateCompatibilityStrategy schema_auto_update_compatibility_strategy = SchemaAutoUpdateCompatibilityStrategy.Full; - public SchemaCompatibilityStrategy schemaCompatibilityStrategy = SchemaCompatibilityStrategy.UNDEFINED; + @SuppressWarnings("checkstyle:MemberName") + public SchemaCompatibilityStrategy schema_compatibility_strategy = SchemaCompatibilityStrategy.UNDEFINED; + + @SuppressWarnings("checkstyle:MemberName") + public boolean is_allow_auto_update_Schema = true; @SuppressWarnings("checkstyle:MemberName") public boolean schema_validation_enforced = false; @@ -103,7 +107,9 @@ public int hashCode() { compaction_threshold, offload_threshold, offload_deletion_lag_ms, schema_auto_update_compatibility_strategy, - schema_validation_enforced); + schema_validation_enforced, + schema_compatibility_strategy, + is_allow_auto_update_Schema); } @Override @@ -133,7 +139,9 @@ public boolean equals(Object obj) { && offload_threshold == other.offload_threshold && offload_deletion_lag_ms == other.offload_deletion_lag_ms && schema_auto_update_compatibility_strategy == other.schema_auto_update_compatibility_strategy - && schema_validation_enforced == other.schema_validation_enforced; + && schema_validation_enforced == other.schema_validation_enforced + && schema_compatibility_strategy == other.schema_compatibility_strategy + && is_allow_auto_update_Schema == other.is_allow_auto_update_Schema; } return false; @@ -178,6 +186,8 @@ public String toString() { .add("offload_threshold", offload_threshold) .add("offload_deletion_lag_ms", offload_deletion_lag_ms) .add("schema_auto_update_compatibility_strategy", schema_auto_update_compatibility_strategy) - .add("schema_validation_enforced", schema_validation_enforced).toString(); + .add("schema_validation_enforced", schema_validation_enforced) + .add("schema_compatibility_Strategy", schema_compatibility_strategy) + .add("is_allow_auto_update_Schema", is_allow_auto_update_Schema).toString(); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/SchemaCompatibilityStrategy.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/SchemaCompatibilityStrategy.java index 52953d685c4b4..9a4f74c437b14 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/SchemaCompatibilityStrategy.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/SchemaCompatibilityStrategy.java @@ -18,35 +18,38 @@ */ package org.apache.pulsar.common.policies.data; +/** + * Pulsar Schema compatibility strategy. + */ public enum SchemaCompatibilityStrategy { /** - * Undefined + * Undefined. */ UNDEFINED, /** - * Always incompatible + * Always incompatible. */ ALWAYS_INCOMPATIBLE, /** - * Always compatible + * Always compatible. */ ALWAYS_COMPATIBLE, /** - * Messages written by an old schema can be read by a new schema + * Messages written by an old schema can be read by a new schema. */ BACKWARD, /** - * Messages written by a new schema can be read by an old schema + * Messages written by a new schema can be read by an old schema. */ FORWARD, /** - * Equivalent to both FORWARD and BACKWARD + * Equivalent to both FORWARD and BACKWARD. */ FULL, @@ -63,7 +66,7 @@ public enum SchemaCompatibilityStrategy { FORWARD_TRANSITIVE, /** - * Equivalent to both FORWARD_TRANSITIVE and BACKWARD_TRANSITIVE + * Equivalent to both FORWARD_TRANSITIVE and BACKWARD_TRANSITIVE. */ FULL_TRANSITIVE; From 3e4aa71f621a5f8ff9f1367e351c98ab44ce5073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 19 Sep 2019 16:46:18 +0800 Subject: [PATCH 03/24] Add the test --- .../pulsar/broker/admin/v2/Namespaces.java | 8 +- .../pulsar/broker/service/ServerCnx.java | 1 - .../AvroSchemaBasedCompatibilityCheck.java | 22 +- .../schema/BookkeeperSchemaStorage.java | 61 +-- .../schema/JsonSchemaCompatibilityCheck.java | 2 +- .../KeyValueSchemaCompatibilityCheck.java | 10 - .../schema/SchemaCompatibilityCheck.java | 45 +-- .../schema/SchemaRegistryServiceImpl.java | 102 +++-- .../broker/service/schema/SchemaStorage.java | 2 +- ...egistryServiceWithSchemaDataValidator.java | 2 +- .../service/schema/SchemaServiceTest.java | 11 + .../client/impl/schema/reader/AvroReader.java | 4 +- .../schema/SchemaCompatibilityCheckTest.java | 366 ++++++++++++++++++ .../tests/integration/schema/Schemas.java | 49 +++ 14 files changed, 505 insertions(+), 180 deletions(-) create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 6575bda306292..cdb59667eead5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -942,7 +942,7 @@ public void setSchemaAutoUpdateCompatibilityStrategy(@PathParam("tenant") String } @GET - @Path("/{tenant}/{cluster}/{namespace}/schemaCompatibilityStrategy") + @Path("/{tenant}/{namespace}/schemaCompatibilityStrategy") @ApiOperation(value = "The strategy of the namespace schema compatibility ") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @@ -955,7 +955,7 @@ public SchemaCompatibilityStrategy getSchemaCompatibilityStrategy( } @PUT - @Path("/{tenant}/{cluster}/{namespace}/schemaAutoUpdateCompatibilityStrategy") + @Path("/{tenant}/{namespace}/schemaCompatibilityStrategy") @ApiOperation(value = "Update the strategy used to check the compatibility of new schema") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @@ -968,7 +968,7 @@ public void setSchemaCompatibilityStrategy(@PathParam("tenant") String tenant, } @GET - @Path("/{tenant}/{cluster}/{namespace}/isAllowAutoUpdateSchema") + @Path("/{tenant}/{namespace}/isAllowAutoUpdateSchema") @ApiOperation(value = "The flag of whether allow auto update schema") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), @@ -981,7 +981,7 @@ public boolean getIsAllowAutoUpdateSchema( } @POST - @Path("/{tenant}/{cluster}/{namespace}/isAllowAutoUpdateSchema") + @Path("/{tenant}/{namespace}/isAllowAutoUpdateSchema") @ApiOperation(value = "Update flag of whether allow auto update schema") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace doesn't exist"), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index f34c5b61557bb..2d7e226ff49fb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -103,7 +103,6 @@ import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.protocol.schema.SchemaInfoUtil; import org.apache.pulsar.common.protocol.schema.SchemaVersion; -import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; import org.apache.pulsar.shaded.com.google.protobuf.v241.GeneratedMessageLite; import org.slf4j.Logger; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java index 679c8968aeda1..5e2cf22edae70 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java @@ -40,12 +40,12 @@ abstract class AvroSchemaBasedCompatibilityCheck implements SchemaCompatibilityCheck { @Override - public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException { - checkCompatible(Collections.singletonList(from), to, strategy, isConsumer); + public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { + checkCompatible(Collections.singletonList(from), to, strategy); } @Override - public void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException { + public void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { LinkedList fromList = new LinkedList<>(); try { for (SchemaData schemaData : from) { @@ -54,8 +54,7 @@ public void checkCompatible(Iterable from, SchemaData to, SchemaComp } Schema.Parser parser = new Schema.Parser(); Schema toSchema = parser.parse(new String(to.getData(), UTF_8)); - SchemaValidator schemaValidator = isConsumer ? - createConsumerSchemaValidator(strategy) : createSchemaValidator(strategy); + SchemaValidator schemaValidator = createSchemaValidator(strategy); schemaValidator.validate(toSchema, fromList); } catch (SchemaParseException e) { log.error("Error during schema parsing: {}", e.getMessage(), e); @@ -66,7 +65,7 @@ public void checkCompatible(Iterable from, SchemaData to, SchemaComp } } - static SchemaValidator createSchemaValidator(SchemaCompatibilityStrategy compatibilityStrategy) { + static SchemaValidator createSchemaValidator(SchemaCompatibilityStrategy compatibilityStrategy) throws IncompatibleSchemaException { final SchemaValidatorBuilder validatorBuilder = new SchemaValidatorBuilder(); switch (compatibilityStrategy) { case BACKWARD_TRANSITIVE: @@ -88,17 +87,6 @@ static SchemaValidator createSchemaValidator(SchemaCompatibilityStrategy compati } } - static SchemaValidator createConsumerSchemaValidator(SchemaCompatibilityStrategy compatibilityStrategy) { - if (compatibilityStrategy == SchemaCompatibilityStrategy.BACKWARD || - compatibilityStrategy == SchemaCompatibilityStrategy.FORWARD || - compatibilityStrategy == SchemaCompatibilityStrategy.FORWARD_TRANSITIVE || - compatibilityStrategy == SchemaCompatibilityStrategy.FULL) { - return createSchemaValidator(SchemaCompatibilityStrategy.BACKWARD); - } else { - return createSchemaValidator(compatibilityStrategy); - } - } - static SchemaValidator createLatestOrAllValidator(SchemaValidatorBuilder validatorBuilder, boolean onlyLatest) { return onlyLatest ? validatorBuilder.validateLatest() : validatorBuilder.validateAll(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java index 78d3e8a238088..f5cc422fe6fa7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java @@ -25,7 +25,6 @@ import static java.util.Objects.nonNull; import static java.util.concurrent.CompletableFuture.completedFuture; import static org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage.Functions.newSchemaEntry; -import static org.apache.pulsar.broker.service.schema.SchemaRegistryServiceImpl.NO_DELETED_VERSION; import com.google.common.annotations.VisibleForTesting; @@ -108,8 +107,8 @@ public void start() throws IOException { } @Override - public CompletableFuture put(String key, byte[] value, byte[] hash, long maxDeletedVersion) { - return putSchemaIfAbsent(key, value, hash, maxDeletedVersion).thenApply(LongSchemaVersion::new); + public CompletableFuture put(String key, byte[] value, byte[] hash) { + return putSchemaIfAbsent(key, value, hash).thenApply(LongSchemaVersion::new); } @Override @@ -172,6 +171,7 @@ private CompletableFuture getSchema(String schemaId) { } SchemaStorageFormat.SchemaLocator schemaLocator = locator.get().locator; + return readSchemaEntry(schemaLocator.getInfo().getPosition()) .thenApply(entry -> new StoredSchema(entry.getSchemaData().toByteArray(), new LongSchemaVersion(schemaLocator.getInfo().getVersion()))); @@ -254,14 +254,14 @@ private CompletableFuture putSchema(String schemaId, byte[] data, byte[] h // No schema was defined yet CompletableFuture future = new CompletableFuture<>(); createNewSchema(schemaId, data, hash) - .thenAccept(version -> future.complete(version)) + .thenAccept(future::complete) .exceptionally(ex -> { if (ex.getCause() instanceof NodeExistsException) { // There was a race condition on the schema creation. Since it has now been created, // retry the whole operation so that we have a chance to recover without bubbling error // back to producer/consumer putSchema(schemaId, data, hash) - .thenAccept(version -> future.complete(version)) + .thenAccept(future::complete) .exceptionally(ex2 -> { future.completeExceptionally(ex2); return null; @@ -280,7 +280,7 @@ private CompletableFuture putSchema(String schemaId, byte[] data, byte[] h } @NotNull - private CompletableFuture putSchemaIfAbsent(String schemaId, byte[] data, byte[] hash, long maxDeletedVersion) { + private CompletableFuture putSchemaIfAbsent(String schemaId, byte[] data, byte[] hash) { return getSchemaLocator(getSchemaPath(schemaId)).thenCompose(optLocatorEntry -> { if (optLocatorEntry.isPresent()) { @@ -294,27 +294,21 @@ private CompletableFuture putSchemaIfAbsent(String schemaId, byte[] data, if (log.isDebugEnabled()) { log.debug("[{}] findSchemaEntryByHash - hash={}", schemaId, hash); } - - return findSchemaEntryByHash(locator.getIndexList(), hash, maxDeletedVersion).thenCompose(version -> { - if (isNull(version)) { - return addNewSchemaEntryToStore(schemaId, locator.getIndexList(), data).thenCompose( - position -> updateSchemaLocator(schemaId, optLocatorEntry.get(), position, hash)); - } else { - return completedFuture(version); - } - }); + return readSchemaEntry(locator.getIndexList().get(0).getPosition()) + .thenCompose(schemaEntry -> addNewSchemaEntryToStore(schemaId, locator.getIndexList(), data).thenCompose( + position -> updateSchemaLocator(schemaId, optLocatorEntry.get(), position, hash))); } else { // No schema was defined yet CompletableFuture future = new CompletableFuture<>(); createNewSchema(schemaId, data, hash) - .thenAccept(version -> future.complete(version)) + .thenAccept(future::complete) .exceptionally(ex -> { if (ex.getCause() instanceof NodeExistsException) { // There was a race condition on the schema creation. Since it has now been created, // retry the whole operation so that we have a chance to recover without bubbling error // back to producer/consumer - putSchemaIfAbsent(schemaId, data, hash, NO_DELETED_VERSION) - .thenAccept(version -> future.complete(version)) + putSchemaIfAbsent(schemaId, data, hash) + .thenAccept(future::complete) .exceptionally(ex2 -> { future.completeExceptionally(ex2); return null; @@ -323,7 +317,6 @@ private CompletableFuture putSchemaIfAbsent(String schemaId, byte[] data, // For other errors, just fail the operation future.completeExceptionally(ex); } - return null; }); @@ -439,36 +432,6 @@ private CompletableFuture findSchemaEntryByVers return completedFuture(null); } - @NotNull - private CompletableFuture findSchemaEntryByHash( - List index, - byte[] hash, - long maxDeletedVersion - ) { - - if (index.isEmpty()) { - return completedFuture(null); - } - - for (SchemaStorageFormat.IndexEntry entry : index) { - if (Arrays.equals(entry.getHash().toByteArray(), hash)) { - if (entry.getVersion() > maxDeletedVersion) { - return completedFuture(entry.getVersion()); - } else { - return completedFuture(null); - } - } - } - - if (index.get(0).getPosition().getLedgerId() == -1) { - return completedFuture(null); - } else { - return readSchemaEntry(index.get(0).getPosition()) - .thenCompose(entry -> findSchemaEntryByHash(entry.getIndexList(), hash, maxDeletedVersion)); - } - - } - @NotNull private CompletableFuture readSchemaEntry( SchemaStorageFormat.PositionInfo position diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java index cda5cc04cfe5e..8f4f1398446b2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java @@ -42,7 +42,7 @@ public SchemaType getSchemaType() { } @Override - public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException { + public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { if (isAvroSchema(from)) { if (isAvroSchema(to)) { // if both producer and broker have the schema in avro format diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheck.java index 96e38dec38764..8d0487c316076 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheck.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/KeyValueSchemaCompatibilityCheck.java @@ -96,14 +96,4 @@ public void checkCompatible(Iterable from, SchemaData to, SchemaComp keyCheck.checkCompatible(fromKeyList, toKeyValue.getKey(), strategy); valueCheck.checkCompatible(fromValueList, toKeyValue.getValue(), strategy); } - - @Override - public void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException { - throw new IncompatibleSchemaException("KeyValueSchemaCompatibilityCheck don't support this method"); - } - - @Override - public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException { - throw new IncompatibleSchemaException("KeyValueSchemaCompatibilityCheck don't support this method"); - } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaCompatibilityCheck.java index ea4e95692f7fe..9b6ffccf04000 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaCompatibilityCheck.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaCompatibilityCheck.java @@ -33,37 +33,16 @@ public interface SchemaCompatibilityCheck { * @param to the future schema i.e. the schema sent by the producer * @param strategy the strategy to use when comparing schemas */ - default void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { - checkCompatible(from, to, strategy, false); - } - - /** - * - * @param from the current schemas i.e. schemas that the broker has - * @param to the future schema i.e. the schema sent by the producer - * @param strategy the strategy to use when comparing schemas - */ - default void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { - checkCompatible(from, to, strategy, false); - } + void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException; /** * * @param from the current schemas i.e. schemas that the broker has * @param to the future schema i.e. the schema sent by the producer * @param strategy the strategy to use when comparing schemas - * @param isConsumer the flag to judge whether it is consumer + * @return whether the schemas are compatible */ - void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException; - - /** - * - * @param from the current schema i.e. schema that the broker has - * @param to the future schema i.e. the schema sent by the producer - * @param strategy the strategy to use when comparing schemas - * @param isConsumer the flag to judge whether it is consumer - */ - void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException; + void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException; default boolean isCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy) { try { @@ -92,25 +71,13 @@ public SchemaType getSchemaType() { @Override public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { - defaultCompatibilityCheck(strategy); + if (strategy == SchemaCompatibilityStrategy.ALWAYS_INCOMPATIBLE) { + throw new IncompatibleSchemaException("Schema compatibility strategy is ALWAYS_INCOMPATIBLE"); + } } @Override public void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { - defaultCompatibilityCheck(strategy); - } - - @Override - public void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException { - defaultCompatibilityCheck(strategy); - } - - @Override - public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityStrategy strategy, boolean isConsumer) throws IncompatibleSchemaException { - defaultCompatibilityCheck(strategy); - } - - private void defaultCompatibilityCheck(SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { if (strategy == SchemaCompatibilityStrategy.ALWAYS_INCOMPATIBLE) { throw new IncompatibleSchemaException("Schema compatibility strategy is ALWAYS_INCOMPATIBLE"); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java index c3b2845f91729..b6fd62e96cabe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java @@ -43,6 +43,7 @@ import java.util.stream.Collectors; import javax.validation.constraints.NotNull; +import org.apache.avro.SchemaValidator; import org.apache.bookkeeper.common.concurrent.FutureUtils; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.broker.service.schema.proto.SchemaRegistryFormat; @@ -50,13 +51,13 @@ import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.protocol.schema.SchemaVersion; +import org.apache.pulsar.common.util.FutureUtil; public class SchemaRegistryServiceImpl implements SchemaRegistryService { private static HashFunction hashFunction = Hashing.sha256(); private final Map compatibilityChecks; private final SchemaStorage schemaStorage; private final Clock clock; - protected static final long NO_DELETED_VERSION = -1L; @VisibleForTesting SchemaRegistryServiceImpl(SchemaStorage schemaStorage, Map compatibilityChecks, Clock clock) { @@ -112,56 +113,37 @@ public CompletableFuture>> getAllSchem public CompletableFuture putSchemaIfAbsent(String schemaId, SchemaData schema, SchemaCompatibilityStrategy strategy, boolean isAllowAutoUpdateSchema) { return trimDeletedSchemaAndGetList(schemaId).thenCompose(schemaAndMetadataList -> { - CompletableFuture completableFuture = new CompletableFuture<>(); - SchemaVersion schemaVersion = null; + final CompletableFuture completableFuture = new CompletableFuture<>(); + SchemaVersion schemaVersion; for (SchemaAndMetadata schemaAndMetadata : schemaAndMetadataList) { if (Arrays.equals(hashFunction.hashBytes(schemaAndMetadata.schema.getData()).asBytes(), hashFunction.hashBytes(schema.getData()).asBytes())) { schemaVersion = schemaAndMetadata.version; + completableFuture.complete(schemaVersion); + return completableFuture; } } - if (schemaVersion != null) { - completableFuture.complete(schemaVersion); - return completableFuture; - } else { - if (isAllowAutoUpdateSchema == true) { - CompletableFuture maxDeleteVersionFuture; - if (schemaAndMetadataList.size() == 0) { - if (isTransitiveStrategy(strategy)) { - maxDeleteVersionFuture = checkCompatibilityWithAll(schemaId, schema, strategy); - } else { - maxDeleteVersionFuture = new CompletableFuture<>(); - checkCompatibilityWithLatest(schemaId, schema, strategy).whenComplete((v, ex) -> { - if (ex == null) { - Long maxDeleteVersion = ((LongSchemaVersion) schemaStorage - .versionFromBytes(schemaAndMetadataList.get(0).version.bytes())).getVersion() - 1L; - maxDeleteVersionFuture.complete(maxDeleteVersion); - } else { - maxDeleteVersionFuture.completeExceptionally(ex); - } - }); - } + if (isAllowAutoUpdateSchema) { + if (schemaAndMetadataList.size() != 0) { + if (isTransitiveStrategy(strategy)) { + checkCompatibilityWithAll(schemaId, schema, strategy); } else { - maxDeleteVersionFuture = new CompletableFuture<>(); - maxDeleteVersionFuture.complete(NO_DELETED_VERSION); + checkCompatibilityWithLatest(schemaId, schema, strategy); } - return maxDeleteVersionFuture.thenCompose(maxDeleteVersion -> { - byte[] context = hashFunction.hashBytes(schema.getData()).asBytes(); - SchemaRegistryFormat.SchemaInfo info = SchemaRegistryFormat.SchemaInfo.newBuilder() - .setType(Functions.convertFromDomainType(schema.getType())) - .setSchema(ByteString.copyFrom(schema.getData())) - .setSchemaId(schemaId) - .setUser(schema.getUser()) - .setDeleted(false) - .setTimestamp(clock.millis()) - .addAllProps(toPairs(schema.getProps())) - .build(); - return schemaStorage.put(schemaId, info.toByteArray(), context, maxDeleteVersion); - }); - - } else { - return FutureUtils.exception(new IncompatibleSchemaException("Do allow auto update schema.")); } + byte[] context = hashFunction.hashBytes(schema.getData()).asBytes(); + SchemaRegistryFormat.SchemaInfo info = SchemaRegistryFormat.SchemaInfo.newBuilder() + .setType(Functions.convertFromDomainType(schema.getType())) + .setSchema(ByteString.copyFrom(schema.getData())) + .setSchemaId(schemaId) + .setUser(schema.getUser()) + .setDeleted(false) + .setTimestamp(clock.millis()) + .addAllProps(toPairs(schema.getProps())) + .build(); + return schemaStorage.put(schemaId, info.toByteArray(), context); + } else { + return FutureUtils.exception(new IncompatibleSchemaException("Do allow auto update schema.")); } }); } @@ -171,9 +153,7 @@ public CompletableFuture putSchemaIfAbsent(String schemaId, Schem public CompletableFuture deleteSchema(String schemaId, String user) { byte[] deletedEntry = deleted(schemaId, user).toByteArray(); return trimDeletedSchemaAndGetList(schemaId).thenCompose(schemaAndMetadataList -> - schemaStorage.put(schemaId, deletedEntry, new byte[]{}, ((LongSchemaVersion)schemaStorage - .versionFromBytes(schemaAndMetadataList.get(0).version.bytes())).getVersion() - 1L)); - + schemaStorage.put(schemaId, deletedEntry, new byte[]{})); } @Override @@ -261,16 +241,31 @@ public CompletableFuture checkConsumerCompatibility(String schemaId, Schem return getSchema(schemaId).thenCompose(existingSchema -> { if (existingSchema != null && !existingSchema.schema.isDeleted()) { CompletableFuture result = new CompletableFuture<>(); - try { - compatibilityChecks.getOrDefault(schemaData.getType(), SchemaCompatibilityCheck.DEFAULT) - .checkCompatible(existingSchema.schema, schemaData, strategy, true); + if (strategy == SchemaCompatibilityStrategy.BACKWARD || + strategy == SchemaCompatibilityStrategy.FORWARD || + strategy == SchemaCompatibilityStrategy.FORWARD_TRANSITIVE || + strategy == SchemaCompatibilityStrategy.FULL) { + checkCompatibilityWithLatest(schemaId, schemaData, SchemaCompatibilityStrategy.BACKWARD).whenComplete((v, ex) -> { + if (ex != null) { + result.completeExceptionally(ex); + } + }); + } else { + checkCompatibilityWithAll(schemaId, schemaData, strategy).whenComplete((v, ex) -> { + if (ex != null) { + result.completeExceptionally(ex); + } + }); + } + if (!result.isCompletedExceptionally()) { result.complete(null); - } catch (IncompatibleSchemaException e) { - result.completeExceptionally(e); } return result; } else { - return FutureUtils.exception(new IncompatibleSchemaException("Trying to subscribe with incompatible schema")); + return FutureUtil.failedFuture( + new IncompatibleSchemaException( + "Trying to subscribe with incompatible schema" + )); } }); } @@ -293,17 +288,16 @@ private CompletableFuture checkCompatibilityWithLatest(String schemaId, Sc }); } - private CompletableFuture checkCompatibilityWithAll(String schemaId, SchemaData schema, + private CompletableFuture checkCompatibilityWithAll(String schemaId, SchemaData schema, SchemaCompatibilityStrategy strategy) { return trimDeletedSchemaAndGetList(schemaId).thenCompose(schemaAndMetadataList -> { - CompletableFuture result = new CompletableFuture<>(); + CompletableFuture result = new CompletableFuture<>(); try { compatibilityChecks.getOrDefault(schema.getType(), SchemaCompatibilityCheck.DEFAULT).checkCompatible(schemaAndMetadataList .stream() .map(schemaAndMetadata -> schemaAndMetadata.schema) .collect(Collectors.toList()), schema, strategy); - result.complete(((LongSchemaVersion)schemaStorage.versionFromBytes(schemaAndMetadataList.get(0).version.bytes())).getVersion()); } catch (IncompatibleSchemaException e) { result.completeExceptionally(e); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaStorage.java index e59fa33231afa..f1336667bd5a1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaStorage.java @@ -24,7 +24,7 @@ public interface SchemaStorage { - CompletableFuture put(String key, byte[] value, byte[] hash, long maxDeletedVersion); + CompletableFuture put(String key, byte[] value, byte[] hash); CompletableFuture get(String key, SchemaVersion version); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java index 28b47fdd79684..8a00ea8dcda94 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java @@ -74,7 +74,7 @@ public CompletableFuture findSchemaVersion(String schemaId, SchemaData sch @Override public CompletableFuture checkConsumerCompatibility(String schemaId, SchemaData schemaData, SchemaCompatibilityStrategy strategy) { - return this.checkConsumerCompatibility(schemaId, schemaData, strategy); + return this.service.checkConsumerCompatibility(schemaId, schemaData, strategy); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java index 14fe70c0b48db..706a333557191 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java @@ -253,6 +253,17 @@ public void dontReAddExistingSchemaAtRoot() throws Exception { putSchema(schemaId1, schema1, version(0)); } + @Test + public void testSchemaAutoUpdate() throws Exception { + putSchema(schemaId1, schema1, version(0)); + try { + schemaRegistryService.putSchemaIfAbsent(schemaId1, schema2, SchemaCompatibilityStrategy.FULL, false).get(); + } catch (Exception e) { + assertEquals("Do allow auto update schema.", e.getCause().getMessage()); + } + + } + @Test public void trimDeletedSchemaAndGetListTest() throws Exception { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/reader/AvroReader.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/reader/AvroReader.java index c502df5c66f79..f8d0211279a61 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/reader/AvroReader.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/reader/AvroReader.java @@ -25,8 +25,6 @@ import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.api.schema.SchemaReader; -import java.io.IOException; - public class AvroReader implements SchemaReader { private ReflectDatumReader reader; @@ -50,7 +48,7 @@ public T read(byte[] bytes) { decoders.set(decoder); } return reader.read(null, DecoderFactory.get().binaryDecoder(bytes, decoder)); - } catch (IOException e) { + } catch (Exception e) { throw new SchemaSerializationException(e); } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java new file mode 100644 index 0000000000000..922de18ab6d73 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java @@ -0,0 +1,366 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.schema; + +import com.google.common.collect.Sets; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SchemaSerializationException; +import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; +import org.apache.pulsar.tests.integration.suites.PulsarTestSuite; + +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import static org.apache.pulsar.common.naming.TopicName.PUBLIC_TENANT; +import static org.junit.Assert.assertEquals; + +@Slf4j +public class SchemaCompatibilityCheckTest extends PulsarTestSuite { + + private PulsarClient client; + private PulsarAdmin admin; + + @DataProvider(name = "CanReadLastSchemaCompatibilityStrategy") + public Object[] canReadLastSchemaCompatibilityStrategy() { + return new Object[] { + SchemaCompatibilityStrategy.BACKWARD, + SchemaCompatibilityStrategy.FORWARD_TRANSITIVE, + SchemaCompatibilityStrategy.FORWARD, + SchemaCompatibilityStrategy.FULL + }; + } + + @DataProvider(name = "ReadAllCheckSchemaCompatibilityStrategy") + public Object[] readAllCheckSchemaCompatibilityStrategy() { + return new Object[] { + SchemaCompatibilityStrategy.BACKWARD_TRANSITIVE, + SchemaCompatibilityStrategy.FULL_TRANSITIVE + }; + } + + @DataProvider(name = "AllCheckSchemaCompatibilityStrategy") + public Object[] allCheckSchemaCompatibilityStrategy() { + return new Object[] { + SchemaCompatibilityStrategy.BACKWARD, + SchemaCompatibilityStrategy.FORWARD_TRANSITIVE, + SchemaCompatibilityStrategy.FORWARD, + SchemaCompatibilityStrategy.FULL, + SchemaCompatibilityStrategy.BACKWARD_TRANSITIVE, + SchemaCompatibilityStrategy.FULL_TRANSITIVE + }; + } + + @BeforeMethod + public void setup() throws Exception { + this.client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + this.admin = PulsarAdmin.builder() + .serviceHttpUrl(pulsarCluster.getHttpServiceUrl()) + .build(); + } + + @Test(dataProvider = "CanReadLastSchemaCompatibilityStrategy") + public void testConsumerCompatibilityCheckCanReadLastTest(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception { + final String tenant = PUBLIC_TENANT; + final String topic = "test-consumer-compatibility"; + + String namespace = "test-namespace-" + randomName(16); + String fqtn = TopicName.get( + TopicDomain.persistent.value(), + tenant, + namespace, + topic + ).toString(); + + NamespaceName namespaceName = NamespaceName.get(tenant, namespace); + + admin.namespaces().createNamespace( + tenant + "/" + namespace, + Sets.newHashSet(pulsarCluster.getClusterName()) + ); + + admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); + admin.schemas().createSchema(topic, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); + admin.schemas().createSchema(topic, Schema.AVRO(SchemaDefinition.builder() + .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo()); + + Consumer consumerThree = client.newConsumer(Schema.AVRO( + SchemaDefinition.builder().withAlwaysAllowNull + (false).withSupportSchemaVersioning(true). + withPojo(Schemas.PersonThree.class).build())) + .subscriptionName("test") + .topic(fqtn) + .subscribe(); + + Producer producerOne = client + .newProducer(Schema.AVRO(Schemas.PersonOne.class)) + .topic(fqtn) + .create(); + + + Schemas.PersonOne personOne = new Schemas.PersonOne(); + personOne.id = 1; + + producerOne.send(personOne); + Message message = null; + + try { + message = consumerThree.receive(); + message.getValue(); + } catch (Exception e) { + Assert.assertTrue(e instanceof SchemaSerializationException); + consumerThree.acknowledge(message); + } + + Producer producerTwo = client + .newProducer(Schema.AVRO(SchemaDefinition.builder().withAlwaysAllowNull + (false).withSupportSchemaVersioning(true). + withPojo(Schemas.PersonTwo.class).build())) + .topic(fqtn) + .create(); + + Schemas.PersonTwo personTwo = new Schemas.PersonTwo(); + personTwo.id = 1; + personTwo.name = "Jerry"; + producerTwo.send(personTwo); + + message = consumerThree.receive(); + Schemas.PersonThree personThree = message.getValue(); + consumerThree.acknowledge(message); + + assertEquals(personThree.id, 1); + assertEquals(personThree.name, "Jerry"); + + consumerThree.close(); + producerOne.close(); + producerTwo.close(); + } + + @Test(dataProvider = "ReadAllCheckSchemaCompatibilityStrategy") + public void testConsumerCompatibilityReadAllCheckTest(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception { + final String tenant = PUBLIC_TENANT; + final String topic = "test-consumer-compatibility"; + String namespace = "test-namespace-" + randomName(16); + String fqtn = TopicName.get( + TopicDomain.persistent.value(), + tenant, + namespace, + topic + ).toString(); + + NamespaceName namespaceName = NamespaceName.get(tenant, namespace); + + admin.namespaces().createNamespace( + tenant + "/" + namespace, + Sets.newHashSet(pulsarCluster.getClusterName()) + ); + + admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); + admin.schemas().createSchema(topic, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); + admin.schemas().createSchema(topic, Schema.AVRO(SchemaDefinition.builder() + .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo()); + Consumer consumerThree = null; + try { + consumerThree = client.newConsumer(Schema.AVRO( + SchemaDefinition.builder().withAlwaysAllowNull + (false).withSupportSchemaVersioning(true). + withPojo(Schemas.PersonThree.class).build())) + .subscriptionName("test") + .topic(fqtn) + .subscribe(); + + } catch (Exception e) { + consumerThree.close(); + Assert.assertTrue(e.getMessage().contains("Unable to read schema")); + } + } + + @Test(dataProvider = "AllCheckSchemaCompatibilityStrategy") + public void testIsAutoUpdateSchema(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception { + final String tenant = PUBLIC_TENANT; + final String topic = "test-consumer-compatibility"; + + String namespace = "test-namespace-" + randomName(16); + String fqtn = TopicName.get( + TopicDomain.persistent.value(), + tenant, + namespace, + topic + ).toString(); + + NamespaceName namespaceName = NamespaceName.get(tenant, namespace); + + admin.namespaces().createNamespace( + tenant + "/" + namespace, + Sets.newHashSet(pulsarCluster.getClusterName()) + ); + + admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); + admin.schemas().createSchema(topic, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); + + admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false); + ProducerBuilder producerThreeBuilder = client + .newProducer(Schema.AVRO(SchemaDefinition.builder().withAlwaysAllowNull + (false).withSupportSchemaVersioning(true). + withPojo(Schemas.PersonTwo.class).build())) + .topic(fqtn); + try { + producerThreeBuilder.create(); + } catch (Exception e) { + Assert.assertTrue(e.getCause().getMessage().contains("Do allow auto update schema.")); + } + + admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), true); + + Producer producer = producerThreeBuilder.create(); + Consumer consumerTwo = client.newConsumer(Schema.AVRO( + SchemaDefinition.builder().withAlwaysAllowNull + (false).withSupportSchemaVersioning(true). + withPojo(Schemas.PersonTwo.class).build())) + .subscriptionName("test") + .topic(fqtn) + .subscribe(); + producer.send(new Schemas.PersonTwo(2, "Lucy")); + Message message = consumerTwo.receive(); + + Schemas.PersonTwo personTwo = consumerTwo.receive().getValue(); + consumerTwo.acknowledge(message); + + assertEquals(personTwo.id, 1); + assertEquals(personTwo.name, "Lucy"); + + consumerTwo.close(); + producer.close(); + } + + @Test(dataProvider = "AllCheckSchemaCompatibilityStrategy") + public void testProducerSendWithOldSchemaAndConsumerCanRead(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception { + final String tenant = PUBLIC_TENANT; + final String topic = "test-consumer-compatibility"; + String namespace = "test-namespace-" + randomName(16); + String fqtn = TopicName.get( + TopicDomain.persistent.value(), + tenant, + namespace, + topic + ).toString(); + + NamespaceName namespaceName = NamespaceName.get(tenant, namespace); + + admin.namespaces().createNamespace( + tenant + "/" + namespace, + Sets.newHashSet(pulsarCluster.getClusterName()) + ); + + admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); + admin.schemas().createSchema(topic, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); + admin.schemas().createSchema(topic, Schema.AVRO(SchemaDefinition.builder() + .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo()); + + Producer producerOne = client + .newProducer(Schema.AVRO(Schemas.PersonOne.class)) + .topic(fqtn) + .create(); + + + Schemas.PersonOne personOne = new Schemas.PersonOne(); + personOne.id = 10; + + Consumer consumerTwo = client.newConsumer(Schema.AVRO( + SchemaDefinition.builder().withAlwaysAllowNull + (false).withSupportSchemaVersioning(true). + withPojo(Schemas.PersonOne.class).build())) + .subscriptionName("test") + .topic(fqtn) + .subscribe(); + + producerOne.send(personOne); + + Message message = consumerTwo.receive(); + + personOne = message.getValue(); + + assertEquals(10, personOne.id); + + consumerTwo.close(); + producerOne.close(); + + } + + @Test(dataProvider = "CanReadLastSchemaCompatibilityStrategy") + public void testConsumerWithNotCompatibilitySchema(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception { + final String tenant = PUBLIC_TENANT; + final String topic = "test-consumer-compatibility"; + + String namespace = "test-namespace-" + randomName(16); + String fqtn = TopicName.get( + TopicDomain.persistent.value(), + tenant, + namespace, + topic + ).toString(); + + NamespaceName namespaceName = NamespaceName.get(tenant, namespace); + + admin.namespaces().createNamespace( + tenant + "/" + namespace, + Sets.newHashSet(pulsarCluster.getClusterName()) + ); + + admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); + admin.schemas().createSchema(topic, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); + admin.schemas().createSchema(topic, Schema.AVRO(SchemaDefinition.builder() + .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo()); + + Consumer consumerFour = null; + try { + consumerFour = client.newConsumer(Schema.AVRO( + SchemaDefinition.builder().withAlwaysAllowNull + (false).withSupportSchemaVersioning(true). + withPojo(Schemas.PersonFour.class).build())) + .subscriptionName("test") + .topic(fqtn) + .subscribe(); + } catch (Exception e) { + consumerFour.close(); + Assert.assertTrue(e.getMessage().contains("Unable to read schema")); + } + + } + + + + +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/Schemas.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/Schemas.java index babb3c7858bc9..390371c523375 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/Schemas.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/Schemas.java @@ -39,7 +39,9 @@ import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; + import org.apache.avro.reflect.AvroDefault; + import org.joda.time.DateTime; import org.joda.time.LocalDate; import org.joda.time.LocalTime; @@ -130,4 +132,51 @@ public static class AvroLogicalType{ private Schemas() {} + @Data + @Getter + @Setter + @ToString + @EqualsAndHashCode + public static class PersonOne{ + int id; + } + + @Data + @Getter + @Setter + @ToString + @EqualsAndHashCode + @AllArgsConstructor + @NoArgsConstructor + public static class PersonTwo{ + int id; + + @AvroDefault("\"Tom\"") + String name; + } + + @Data + @Getter + @Setter + @ToString + @EqualsAndHashCode + public static class PersonThree{ + int id; + + String name; + } + + @Data + @Getter + @Setter + @ToString + @EqualsAndHashCode + public static class PersonFour{ + int id; + + String name; + + int age; + } + } From e6fbadd0637796a68a47409b42d04e9bcea2ff1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 19 Sep 2019 17:18:25 +0800 Subject: [PATCH 04/24] modify the cmdNamespaces --- .../main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 47a53fb66a81c..e8c1f8a0c381b 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -1292,7 +1292,7 @@ public CmdNamespaces(PulsarAdmin admin) { jcommander.addCommand("set-schema-autoupdate-strategy", new SetSchemaAutoUpdateStrategy()); jcommander.addCommand("get-schema-compatibility-strategy", new GetSchemaCompatibilityStrategy()); - jcommander.addCommand("set-schema-compatibility-strategy", new GetSchemaCompatibilityStrategy()); + jcommander.addCommand("set-schema-compatibility-strategy", new SetSchemaCompatibilityStrategy()); jcommander.addCommand("get-is-allow-auto-update-schema", new GetIsAllowAutoUpdateSchema()); jcommander.addCommand("set-is-allow-auto-update-schema", new SetIsAllowAutoUpdateSchema()); From b00cfb666b00009b6cbf326f7069e71bb192149c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 19 Sep 2019 17:54:13 +0800 Subject: [PATCH 05/24] Modify the variable name --- .../org/apache/pulsar/common/policies/data/Policies.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java index 9bc038d35e835..80daae5a3efc4 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java @@ -88,7 +88,7 @@ public class Policies { public SchemaCompatibilityStrategy schema_compatibility_strategy = SchemaCompatibilityStrategy.UNDEFINED; @SuppressWarnings("checkstyle:MemberName") - public boolean is_allow_auto_update_Schema = true; + public boolean is_allow_auto_update_schema = true; @SuppressWarnings("checkstyle:MemberName") public boolean schema_validation_enforced = false; @@ -109,7 +109,7 @@ public int hashCode() { schema_auto_update_compatibility_strategy, schema_validation_enforced, schema_compatibility_strategy, - is_allow_auto_update_Schema); + is_allow_auto_update_schema); } @Override @@ -141,7 +141,7 @@ public boolean equals(Object obj) { && schema_auto_update_compatibility_strategy == other.schema_auto_update_compatibility_strategy && schema_validation_enforced == other.schema_validation_enforced && schema_compatibility_strategy == other.schema_compatibility_strategy - && is_allow_auto_update_Schema == other.is_allow_auto_update_Schema; + && is_allow_auto_update_schema == other.is_allow_auto_update_schema; } return false; @@ -188,6 +188,6 @@ public String toString() { .add("schema_auto_update_compatibility_strategy", schema_auto_update_compatibility_strategy) .add("schema_validation_enforced", schema_validation_enforced) .add("schema_compatibility_Strategy", schema_compatibility_strategy) - .add("is_allow_auto_update_Schema", is_allow_auto_update_Schema).toString(); + .add("is_allow_auto_update_Schema", is_allow_auto_update_schema).toString(); } } From 35ca6516d3cf6cb5161e4ed3c854aadc7bb0b092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 19 Sep 2019 18:03:33 +0800 Subject: [PATCH 06/24] Delete the configuration --- .../pulsar/broker/ServiceConfiguration.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 823399deb50bf..ffea06de40ed5 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -29,10 +29,8 @@ import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; - import lombok.Getter; import lombok.Setter; - import org.apache.bookkeeper.client.api.DigestType; import org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider; import org.apache.pulsar.common.protocol.Commands; @@ -1222,19 +1220,6 @@ public class ServiceConfiguration implements PulsarConfiguration { doc = "The schema storage implementation used by this broker" ) private String schemaRegistryStorageClassName = "org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorageFactory"; - - @FieldContext( - category = CATEGORY_SCHEMA, - doc = "The schema storage implementation used by this broker" - ) - private String schemaCompatibilityStrategy = "FULL"; - - @FieldContext( - category = CATEGORY_SCHEMA, - doc = "The schema storage implementation used by this broker" - ) - private boolean isAllowSchemaAutoUpdate = true; - @FieldContext( category = CATEGORY_SCHEMA, doc = "The list compatibility checkers to be used in schema registry" From 92e51ac31408ec1c93ab2735ffe9a2ffd409bed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 19 Sep 2019 18:58:06 +0800 Subject: [PATCH 07/24] Modify the variable name --- .../broker/admin/impl/NamespacesBase.java | 6 +-- .../nonpersistent/NonPersistentTopic.java | 4 +- .../service/persistent/PersistentTopic.java | 11 +++-- .../schema/BookkeeperSchemaStorage.java | 42 ++----------------- .../schema/SchemaRegistryServiceImpl.java | 32 ++++++++------ .../schema/SchemaCompatibilityCheckTest.java | 4 -- 6 files changed, 36 insertions(+), 63 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 1f660fa41d98d..4822cf62a3af0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -2039,7 +2039,7 @@ protected void internalSetSchemaValidationEnforced(boolean schemaValidationEnfor protected boolean internalGetIsAllowAutoUpdateSchema() { validateSuperUserAccess(); validateAdminAccessForTenant(namespaceName.getTenant()); - return getNamespacePolicies(namespaceName).is_allow_auto_update_Schema; + return getNamespacePolicies(namespaceName).is_allow_auto_update_schema; } protected void internalSetIsAllowAutoUpdateSchema(boolean isAllowAutoUpdateSchema) { @@ -2047,9 +2047,9 @@ protected void internalSetIsAllowAutoUpdateSchema(boolean isAllowAutoUpdateSchem validatePoliciesReadOnlyAccess(); mutatePolicy((policies) -> { - policies.is_allow_auto_update_Schema = isAllowAutoUpdateSchema; + policies.is_allow_auto_update_schema = isAllowAutoUpdateSchema; return policies; - }, (policies) -> policies.is_allow_auto_update_Schema, + }, (policies) -> policies.is_allow_auto_update_schema, "isAllowAutoUpdateSchema"); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index f6242bcda0c2b..49a6020d98f86 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -144,7 +144,7 @@ public NonPersistentTopic(String topic, BrokerService brokerService) { .get(AdminResource.path(POLICIES, TopicName.get(topic).getNamespace())) .orElseThrow(() -> new KeeperException.NoNodeException()); isEncryptionRequired = policies.encryption_required; - isAllowAutoUpdateSchema = policies.is_allow_auto_update_Schema; + isAllowAutoUpdateSchema = policies.is_allow_auto_update_schema; setSchemaCompatibilityStrategy(policies); schemaValidationEnforced = policies.schema_validation_enforced; @@ -872,7 +872,7 @@ public CompletableFuture onPoliciesUpdate(Policies data) { } isEncryptionRequired = data.encryption_required; setSchemaCompatibilityStrategy(data); - isAllowAutoUpdateSchema = data.is_allow_auto_update_Schema; + isAllowAutoUpdateSchema = data.is_allow_auto_update_schema; schemaValidationEnforced = data.schema_validation_enforced; producers.forEach(producer -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 6227d4f67e4fb..1d03ca8918806 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -19,11 +19,14 @@ package org.apache.pulsar.broker.service.persistent; import com.carrotsearch.hppc.ObjectObjectHashMap; + import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; + import io.netty.buffer.ByteBuf; import io.netty.util.concurrent.FastThreadLocal; + import java.util.Collections; import java.util.List; import java.util.Map; @@ -35,6 +38,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLongFieldUpdater; + import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback; @@ -80,7 +84,6 @@ import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type; -import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.broker.stats.ClusterReplicationMetrics; import org.apache.pulsar.broker.stats.NamespaceStats; import org.apache.pulsar.broker.stats.ReplicationMetrics; @@ -117,8 +120,10 @@ import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; + import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.pulsar.broker.cache.ConfigurationCacheService.POLICIES; @@ -227,7 +232,7 @@ public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerS isEncryptionRequired = policies.encryption_required; setSchemaCompatibilityStrategy(policies); - isAllowAutoUpdateSchema = policies.is_allow_auto_update_Schema; + isAllowAutoUpdateSchema = policies.is_allow_auto_update_schema; schemaValidationEnforced = policies.schema_validation_enforced; } catch (Exception e) { @@ -1596,7 +1601,7 @@ public CompletableFuture onPoliciesUpdate(Policies data) { isEncryptionRequired = data.encryption_required; setSchemaCompatibilityStrategy(data); - isAllowAutoUpdateSchema = data.is_allow_auto_update_Schema; + isAllowAutoUpdateSchema = data.is_allow_auto_update_schema; schemaValidationEnforced = data.schema_validation_enforced; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java index f5cc422fe6fa7..ae361db739ce9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java @@ -108,7 +108,7 @@ public void start() throws IOException { @Override public CompletableFuture put(String key, byte[] value, byte[] hash) { - return putSchemaIfAbsent(key, value, hash).thenApply(LongSchemaVersion::new); + return putSchema(key, value, hash).thenApply(LongSchemaVersion::new); } @Override @@ -245,42 +245,6 @@ private CompletableFuture getSchema(String schemaId, long version) @NotNull private CompletableFuture putSchema(String schemaId, byte[] data, byte[] hash) { - return getSchemaLocator(getSchemaPath(schemaId)).thenCompose(optLocatorEntry -> { - if (optLocatorEntry.isPresent()) { - // Schema locator was already present - return addNewSchemaEntryToStore(schemaId, optLocatorEntry.get().locator.getIndexList(), data) - .thenCompose(position -> updateSchemaLocator(schemaId, optLocatorEntry.get(), position, hash)); - } else { - // No schema was defined yet - CompletableFuture future = new CompletableFuture<>(); - createNewSchema(schemaId, data, hash) - .thenAccept(future::complete) - .exceptionally(ex -> { - if (ex.getCause() instanceof NodeExistsException) { - // There was a race condition on the schema creation. Since it has now been created, - // retry the whole operation so that we have a chance to recover without bubbling error - // back to producer/consumer - putSchema(schemaId, data, hash) - .thenAccept(future::complete) - .exceptionally(ex2 -> { - future.completeExceptionally(ex2); - return null; - }); - } else { - // For other errors, just fail the operation - future.completeExceptionally(ex); - } - - return null; - }); - - return future; - } - }); - } - - @NotNull - private CompletableFuture putSchemaIfAbsent(String schemaId, byte[] data, byte[] hash) { return getSchemaLocator(getSchemaPath(schemaId)).thenCompose(optLocatorEntry -> { if (optLocatorEntry.isPresent()) { @@ -294,6 +258,8 @@ private CompletableFuture putSchemaIfAbsent(String schemaId, byte[] data, if (log.isDebugEnabled()) { log.debug("[{}] findSchemaEntryByHash - hash={}", schemaId, hash); } + + //don't check the schema whether already exist return readSchemaEntry(locator.getIndexList().get(0).getPosition()) .thenCompose(schemaEntry -> addNewSchemaEntryToStore(schemaId, locator.getIndexList(), data).thenCompose( position -> updateSchemaLocator(schemaId, optLocatorEntry.get(), position, hash))); @@ -307,7 +273,7 @@ private CompletableFuture putSchemaIfAbsent(String schemaId, byte[] data, // There was a race condition on the schema creation. Since it has now been created, // retry the whole operation so that we have a chance to recover without bubbling error // back to producer/consumer - putSchemaIfAbsent(schemaId, data, hash) + putSchema(schemaId, data, hash) .thenAccept(future::complete) .exceptionally(ex2 -> { future.completeExceptionally(ex2); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java index b6fd62e96cabe..bf0be2a8d24a2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java @@ -124,24 +124,30 @@ public CompletableFuture putSchemaIfAbsent(String schemaId, Schem } } if (isAllowAutoUpdateSchema) { + CompletableFuture isCompatibility = new CompletableFuture<>(); if (schemaAndMetadataList.size() != 0) { if (isTransitiveStrategy(strategy)) { - checkCompatibilityWithAll(schemaId, schema, strategy); + isCompatibility = checkCompatibilityWithAll(schemaId, schema, strategy); } else { - checkCompatibilityWithLatest(schemaId, schema, strategy); + isCompatibility = checkCompatibilityWithLatest(schemaId, schema, strategy); } + } else { + isCompatibility.complete(null); } - byte[] context = hashFunction.hashBytes(schema.getData()).asBytes(); - SchemaRegistryFormat.SchemaInfo info = SchemaRegistryFormat.SchemaInfo.newBuilder() - .setType(Functions.convertFromDomainType(schema.getType())) - .setSchema(ByteString.copyFrom(schema.getData())) - .setSchemaId(schemaId) - .setUser(schema.getUser()) - .setDeleted(false) - .setTimestamp(clock.millis()) - .addAllProps(toPairs(schema.getProps())) - .build(); - return schemaStorage.put(schemaId, info.toByteArray(), context); + return isCompatibility.thenCompose((v) -> { + byte[] context = hashFunction.hashBytes(schema.getData()).asBytes(); + SchemaRegistryFormat.SchemaInfo info = SchemaRegistryFormat.SchemaInfo.newBuilder() + .setType(Functions.convertFromDomainType(schema.getType())) + .setSchema(ByteString.copyFrom(schema.getData())) + .setSchemaId(schemaId) + .setUser(schema.getUser()) + .setDeleted(false) + .setTimestamp(clock.millis()) + .addAllProps(toPairs(schema.getProps())) + .build(); + return schemaStorage.put(schemaId, info.toByteArray(), context); + + }); } else { return FutureUtils.exception(new IncompatibleSchemaException("Do allow auto update schema.")); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java index 922de18ab6d73..b5fa6aaef46d8 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java @@ -359,8 +359,4 @@ public void testConsumerWithNotCompatibilitySchema(SchemaCompatibilityStrategy s } } - - - - } From 0573f9aaeba6b4c5aa6944d6ad1140c455047e55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Fri, 20 Sep 2019 11:33:20 +0800 Subject: [PATCH 08/24] modify the method --- .../broker/service/nonpersistent/NonPersistentTopic.java | 4 ++-- .../pulsar/broker/service/persistent/PersistentTopic.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 49a6020d98f86..82ee3815c5827 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -929,8 +929,8 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem if (hasSchema || isActive() || ENTRIES_ADDED_COUNTER_UPDATER.get(this) != 0) { return isSchemaCompatible(schema); } else { - addSchema(schema); - return CompletableFuture.completedFuture(null); + return addSchema(schema).thenCompose(schemaVersion-> + CompletableFuture.completedFuture(null)); } }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 1d03ca8918806..26ac69389efb1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1863,8 +1863,8 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem if (hasSchema || isActive() || ledger.getTotalSize() != 0) { return isSchemaCompatible(schema); } else { - addSchema(schema); - return CompletableFuture.completedFuture(null); + return addSchema(schema).thenCompose(schemaVersion -> + CompletableFuture.completedFuture(null)); } }); } From 5e967906f49ae4e328915158fcbbca614cfef89b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Fri, 20 Sep 2019 14:43:57 +0800 Subject: [PATCH 09/24] Fix some tests --- .../schema/SchemaRegistryServiceImpl.java | 17 ++--------------- .../api/SimpleTypedProducerConsumerTest.java | 3 +-- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java index 18f1e6120cb81..91d168648d147 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java @@ -245,27 +245,14 @@ public CompletableFuture checkConsumerCompatibility(String schemaId, Schem SchemaCompatibilityStrategy strategy) { return getSchema(schemaId).thenCompose(existingSchema -> { if (existingSchema != null && !existingSchema.schema.isDeleted()) { - CompletableFuture result = new CompletableFuture<>(); if (strategy == SchemaCompatibilityStrategy.BACKWARD || strategy == SchemaCompatibilityStrategy.FORWARD || strategy == SchemaCompatibilityStrategy.FORWARD_TRANSITIVE || strategy == SchemaCompatibilityStrategy.FULL) { - checkCompatibilityWithLatest(schemaId, schemaData, SchemaCompatibilityStrategy.BACKWARD).whenComplete((v, ex) -> { - if (ex != null) { - result.completeExceptionally(ex); - } - }); + return checkCompatibilityWithLatest(schemaId, schemaData, SchemaCompatibilityStrategy.BACKWARD); } else { - checkCompatibilityWithAll(schemaId, schemaData, strategy).whenComplete((v, ex) -> { - if (ex != null) { - result.completeExceptionally(ex); - } - }); + return checkCompatibilityWithAll(schemaId, schemaData, strategy); } - if (!result.isCompletedExceptionally()) { - result.complete(null); - } - return result; } else { return FutureUtil.failedFuture( new IncompatibleSchemaException( diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java index 16a588d3337d8..2d0f40d2cec2e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java @@ -341,10 +341,9 @@ public void testAvroConsumerWithWrongPrestoredSchema() throws Exception { SchemaCompatibilityStrategy.FULL, true ).get(); - Consumer consumer = pulsarClient .newConsumer(AvroSchema.of(SchemaDefinition.builder(). - withPojo(AvroEncodedPojo.class).build())) + withPojo(AvroEncodedPojo.class).withAlwaysAllowNull(false).build())) .topic("persistent://my-property/use/my-ns/my-topic1") .subscriptionName("my-subscriber-name") .subscribe(); From 54e5c0a6ebfdf459dd4c05f4aa30ed99ed3d0a9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Fri, 20 Sep 2019 19:06:03 +0800 Subject: [PATCH 10/24] add the integration test --- .../pulsar/client/admin/Namespaces.java | 8 ----- .../tests/integration/schema/SchemaTest.java | 21 ++------------ .../test/resources/pulsar-schema-suite.xml | 28 ++++++++++++++++++ .../src/test/resources/pulsar-schema.xml | 29 +++++++++++++++++++ .../integration/src/test/resources/pulsar.xml | 1 + 5 files changed, 60 insertions(+), 27 deletions(-) create mode 100644 tests/integration/src/test/resources/pulsar-schema-suite.xml create mode 100644 tests/integration/src/test/resources/pulsar-schema.xml diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java index c9dd0a4d89899..20b59cb353f7e 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java @@ -24,9 +24,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import io.swagger.annotations.ApiOperation; -import io.swagger.annotations.ApiResponse; -import io.swagger.annotations.ApiResponses; import org.apache.pulsar.client.admin.PulsarAdminException.ConflictException; import org.apache.pulsar.client.admin.PulsarAdminException.NotAuthorizedException; import org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException; @@ -44,11 +41,6 @@ import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.policies.data.SubscriptionAuthMode; -import javax.ws.rs.GET; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; - /** * Admin interface for namespaces management */ diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java index 942748a0de08b..89e9291af0965 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java @@ -268,7 +268,6 @@ public void testPrimitiveSchemaTypeCompatibilityCheck() { List schemas = new ArrayList<>(); schemas.add(Schema.STRING); - schemas.add(Schema.BYTES); schemas.add(Schema.INT8); schemas.add(Schema.INT16); schemas.add(Schema.INT32); @@ -279,35 +278,19 @@ public void testPrimitiveSchemaTypeCompatibilityCheck() { schemas.add(Schema.DATE); schemas.add(Schema.TIME); schemas.add(Schema.TIMESTAMP); - schemas.add(null); - - schemas.stream().forEach(schemaProducer -> { - schemas.stream().forEach(schemaConsumer -> { + schemas.forEach(schemaProducer -> { + schemas.forEach(schemaConsumer -> { try { String topicName = schemaProducer.getSchemaInfo().getName() + schemaConsumer.getSchemaInfo().getName(); - if (schemaProducer == null) { - client.newProducer() - .topic(topicName) - .create().close(); - } else { client.newProducer(schemaProducer) .topic(topicName) .create().close(); - } - if (schemaConsumer == null) { - client.newConsumer() - .topic(topicName) - .subscriptionName("test") - .subscribe().close(); - } else { client.newConsumer(schemaConsumer) .topic(topicName) .subscriptionName("test") .subscribe().close(); - } - assertEquals(schemaProducer.getSchemaInfo().getType(), schemaConsumer.getSchemaInfo().getType()); diff --git a/tests/integration/src/test/resources/pulsar-schema-suite.xml b/tests/integration/src/test/resources/pulsar-schema-suite.xml new file mode 100644 index 0000000000000..83792268aecc7 --- /dev/null +++ b/tests/integration/src/test/resources/pulsar-schema-suite.xml @@ -0,0 +1,28 @@ + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-schema.xml b/tests/integration/src/test/resources/pulsar-schema.xml new file mode 100644 index 0000000000000..02fcff2dd2e7c --- /dev/null +++ b/tests/integration/src/test/resources/pulsar-schema.xml @@ -0,0 +1,29 @@ + + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar.xml b/tests/integration/src/test/resources/pulsar.xml index 79bf744360130..9c761bac49f9f 100644 --- a/tests/integration/src/test/resources/pulsar.xml +++ b/tests/integration/src/test/resources/pulsar.xml @@ -29,5 +29,6 @@ + From 65e1abb9383237f16542242ce9aac48037343969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Mon, 23 Sep 2019 11:33:23 +0800 Subject: [PATCH 11/24] Modify the test --- .../schema/SchemaCompatibilityCheckTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java index b5fa6aaef46d8..f78128d663b45 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java @@ -112,8 +112,8 @@ public void testConsumerCompatibilityCheckCanReadLastTest(SchemaCompatibilityStr ); admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); - admin.schemas().createSchema(topic, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); - admin.schemas().createSchema(topic, Schema.AVRO(SchemaDefinition.builder() + admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); + admin.schemas().createSchema(fqtn, Schema.AVRO(SchemaDefinition.builder() .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo()); Consumer consumerThree = client.newConsumer(Schema.AVRO( @@ -188,8 +188,8 @@ public void testConsumerCompatibilityReadAllCheckTest(SchemaCompatibilityStrateg ); admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); - admin.schemas().createSchema(topic, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); - admin.schemas().createSchema(topic, Schema.AVRO(SchemaDefinition.builder() + admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); + admin.schemas().createSchema(fqtn, Schema.AVRO(SchemaDefinition.builder() .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo()); Consumer consumerThree = null; try { @@ -228,7 +228,7 @@ public void testIsAutoUpdateSchema(SchemaCompatibilityStrategy schemaCompatibili ); admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); - admin.schemas().createSchema(topic, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); + admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false); ProducerBuilder producerThreeBuilder = client @@ -239,7 +239,7 @@ public void testIsAutoUpdateSchema(SchemaCompatibilityStrategy schemaCompatibili try { producerThreeBuilder.create(); } catch (Exception e) { - Assert.assertTrue(e.getCause().getMessage().contains("Do allow auto update schema.")); + Assert.assertTrue(e.getMessage().contains("Do allow auto update schema.")); } admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), true); @@ -285,8 +285,8 @@ public void testProducerSendWithOldSchemaAndConsumerCanRead(SchemaCompatibilityS ); admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); - admin.schemas().createSchema(topic, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); - admin.schemas().createSchema(topic, Schema.AVRO(SchemaDefinition.builder() + admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); + admin.schemas().createSchema(fqtn, Schema.AVRO(SchemaDefinition.builder() .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo()); Producer producerOne = client @@ -340,8 +340,8 @@ public void testConsumerWithNotCompatibilitySchema(SchemaCompatibilityStrategy s ); admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); - admin.schemas().createSchema(topic, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); - admin.schemas().createSchema(topic, Schema.AVRO(SchemaDefinition.builder() + admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); + admin.schemas().createSchema(fqtn, Schema.AVRO(SchemaDefinition.builder() .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo()); Consumer consumerFour = null; From 2c0fe6c2ae53e856b008bc767bd5fa0b720d71db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 25 Sep 2019 10:52:11 +0800 Subject: [PATCH 12/24] modity the Timeout time --- .../java/org/apache/pulsar/proxy/server/AdminProxyHandler.java | 2 +- .../tests/integration/schema/SchemaCompatibilityCheckTest.java | 2 +- .../org/apache/pulsar/tests/integration/schema/Schemas.java | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java index fdf687dfc9ea0..ae10bb2530de7 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java @@ -104,7 +104,7 @@ protected HttpClient createHttpClient() throws ServletException { value = config.getInitParameter("idleTimeout"); if (value == null) - value = "30000"; + value = "120000"; client.setIdleTimeout(Long.parseLong(value)); value = config.getInitParameter("requestBufferSize"); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java index f78128d663b45..ea52aa99fe3fa 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java @@ -239,7 +239,7 @@ public void testIsAutoUpdateSchema(SchemaCompatibilityStrategy schemaCompatibili try { producerThreeBuilder.create(); } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("Do allow auto update schema.")); + Assert.assertTrue(e.getMessage().contains("Don't allow auto update schema.")); } admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), true); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/Schemas.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/Schemas.java index 390371c523375..8158e107c6b33 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/Schemas.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/Schemas.java @@ -137,6 +137,8 @@ private Schemas() {} @Setter @ToString @EqualsAndHashCode + @AllArgsConstructor + @NoArgsConstructor public static class PersonOne{ int id; } From 5b8500d61fcfdb72f236c89ac7392544de838ad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 25 Sep 2019 11:00:22 +0800 Subject: [PATCH 13/24] Modify the test's sequence --- tests/integration/src/test/resources/pulsar.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/test/resources/pulsar.xml b/tests/integration/src/test/resources/pulsar.xml index 9c761bac49f9f..c811fdc9b51da 100644 --- a/tests/integration/src/test/resources/pulsar.xml +++ b/tests/integration/src/test/resources/pulsar.xml @@ -24,11 +24,11 @@ + - From 02caf3f0aa44e8b130f70b5f8a8787d421d0e0bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 25 Sep 2019 16:58:21 +0800 Subject: [PATCH 14/24] modify the exception message --- .../pulsar/broker/service/schema/SchemaRegistryServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java index 91d168648d147..b239856770ac1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java @@ -148,7 +148,7 @@ public CompletableFuture putSchemaIfAbsent(String schemaId, Schem }); } else { - return FutureUtils.exception(new IncompatibleSchemaException("Do allow auto update schema.")); + return FutureUtils.exception(new IncompatibleSchemaException("Don't allow auto update schema.")); } }); } From 6f5f845e5c763ca18eede8adbb40d02d10674faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 25 Sep 2019 19:41:42 +0800 Subject: [PATCH 15/24] Add the test --- .../SchemaCompatibilityCheckTest.java | 93 ++++++++++--------- .../pulsar/schema/compatibility/Schemas.java | 67 +++++++++++++ .../src/test/resources/pulsar-schema.xml | 1 - 3 files changed, 114 insertions(+), 47 deletions(-) rename {tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema => pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility}/SchemaCompatibilityCheckTest.java (86%) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/Schemas.java diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java similarity index 86% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java index ea52aa99fe3fa..b5405c2f865bb 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java @@ -16,40 +16,58 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.schema; +package org.apache.pulsar.schema.compatibility; import com.google.common.collect.Sets; - import lombok.extern.slf4j.Slf4j; - -import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; -import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; -import org.apache.pulsar.tests.integration.suites.PulsarTestSuite; - +import org.apache.pulsar.common.policies.data.TenantInfo; import org.testng.Assert; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import java.util.Collections; + +import static org.apache.pulsar.client.admin.internal.SchemaCompatibilityCheckTest.randomName; import static org.apache.pulsar.common.naming.TopicName.PUBLIC_TENANT; import static org.junit.Assert.assertEquals; + @Slf4j -public class SchemaCompatibilityCheckTest extends PulsarTestSuite { +public class SchemaCompatibilityCheckTest extends MockedPulsarServiceBaseTest { + private final static String CLUSTER_NAME = "test"; - private PulsarClient client; - private PulsarAdmin admin; + @BeforeMethod + @Override + public void setup() throws Exception { + super.internalSetup(); + + // Setup namespaces + admin.clusters().createCluster(CLUSTER_NAME, new ClusterData("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT)); + TenantInfo tenantInfo = new TenantInfo(); + tenantInfo.setAllowedClusters(Collections.singleton(CLUSTER_NAME)); + admin.tenants().createTenant(PUBLIC_TENANT, tenantInfo); + } + + @AfterMethod + @Override + public void cleanup() throws Exception { + super.internalCleanup(); + } @DataProvider(name = "CanReadLastSchemaCompatibilityStrategy") public Object[] canReadLastSchemaCompatibilityStrategy() { @@ -81,16 +99,6 @@ public Object[] allCheckSchemaCompatibilityStrategy() { }; } - @BeforeMethod - public void setup() throws Exception { - this.client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - this.admin = PulsarAdmin.builder() - .serviceHttpUrl(pulsarCluster.getHttpServiceUrl()) - .build(); - } - @Test(dataProvider = "CanReadLastSchemaCompatibilityStrategy") public void testConsumerCompatibilityCheckCanReadLastTest(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception { final String tenant = PUBLIC_TENANT; @@ -108,7 +116,7 @@ public void testConsumerCompatibilityCheckCanReadLastTest(SchemaCompatibilityStr admin.namespaces().createNamespace( tenant + "/" + namespace, - Sets.newHashSet(pulsarCluster.getClusterName()) + Sets.newHashSet(CLUSTER_NAME) ); admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); @@ -116,7 +124,7 @@ public void testConsumerCompatibilityCheckCanReadLastTest(SchemaCompatibilityStr admin.schemas().createSchema(fqtn, Schema.AVRO(SchemaDefinition.builder() .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo()); - Consumer consumerThree = client.newConsumer(Schema.AVRO( + Consumer consumerThree = pulsarClient.newConsumer(Schema.AVRO( SchemaDefinition.builder().withAlwaysAllowNull (false).withSupportSchemaVersioning(true). withPojo(Schemas.PersonThree.class).build())) @@ -124,7 +132,7 @@ public void testConsumerCompatibilityCheckCanReadLastTest(SchemaCompatibilityStr .topic(fqtn) .subscribe(); - Producer producerOne = client + Producer producerOne = pulsarClient .newProducer(Schema.AVRO(Schemas.PersonOne.class)) .topic(fqtn) .create(); @@ -144,7 +152,7 @@ public void testConsumerCompatibilityCheckCanReadLastTest(SchemaCompatibilityStr consumerThree.acknowledge(message); } - Producer producerTwo = client + Producer producerTwo = pulsarClient .newProducer(Schema.AVRO(SchemaDefinition.builder().withAlwaysAllowNull (false).withSupportSchemaVersioning(true). withPojo(Schemas.PersonTwo.class).build())) @@ -184,16 +192,15 @@ public void testConsumerCompatibilityReadAllCheckTest(SchemaCompatibilityStrateg admin.namespaces().createNamespace( tenant + "/" + namespace, - Sets.newHashSet(pulsarCluster.getClusterName()) + Sets.newHashSet(CLUSTER_NAME) ); admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); admin.schemas().createSchema(fqtn, Schema.AVRO(SchemaDefinition.builder() .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo()); - Consumer consumerThree = null; try { - consumerThree = client.newConsumer(Schema.AVRO( + pulsarClient.newConsumer(Schema.AVRO( SchemaDefinition.builder().withAlwaysAllowNull (false).withSupportSchemaVersioning(true). withPojo(Schemas.PersonThree.class).build())) @@ -202,7 +209,6 @@ public void testConsumerCompatibilityReadAllCheckTest(SchemaCompatibilityStrateg .subscribe(); } catch (Exception e) { - consumerThree.close(); Assert.assertTrue(e.getMessage().contains("Unable to read schema")); } } @@ -224,14 +230,14 @@ public void testIsAutoUpdateSchema(SchemaCompatibilityStrategy schemaCompatibili admin.namespaces().createNamespace( tenant + "/" + namespace, - Sets.newHashSet(pulsarCluster.getClusterName()) + Sets.newHashSet(CLUSTER_NAME) ); admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo()); admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false); - ProducerBuilder producerThreeBuilder = client + ProducerBuilder producerThreeBuilder = pulsarClient .newProducer(Schema.AVRO(SchemaDefinition.builder().withAlwaysAllowNull (false).withSupportSchemaVersioning(true). withPojo(Schemas.PersonTwo.class).build())) @@ -245,7 +251,7 @@ public void testIsAutoUpdateSchema(SchemaCompatibilityStrategy schemaCompatibili admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), true); Producer producer = producerThreeBuilder.create(); - Consumer consumerTwo = client.newConsumer(Schema.AVRO( + Consumer consumerTwo = pulsarClient.newConsumer(Schema.AVRO( SchemaDefinition.builder().withAlwaysAllowNull (false).withSupportSchemaVersioning(true). withPojo(Schemas.PersonTwo.class).build())) @@ -255,10 +261,10 @@ public void testIsAutoUpdateSchema(SchemaCompatibilityStrategy schemaCompatibili producer.send(new Schemas.PersonTwo(2, "Lucy")); Message message = consumerTwo.receive(); - Schemas.PersonTwo personTwo = consumerTwo.receive().getValue(); + Schemas.PersonTwo personTwo = message.getValue(); consumerTwo.acknowledge(message); - assertEquals(personTwo.id, 1); + assertEquals(personTwo.id, 2); assertEquals(personTwo.name, "Lucy"); consumerTwo.close(); @@ -281,7 +287,7 @@ public void testProducerSendWithOldSchemaAndConsumerCanRead(SchemaCompatibilityS admin.namespaces().createNamespace( tenant + "/" + namespace, - Sets.newHashSet(pulsarCluster.getClusterName()) + Sets.newHashSet(CLUSTER_NAME) ); admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); @@ -289,16 +295,15 @@ public void testProducerSendWithOldSchemaAndConsumerCanRead(SchemaCompatibilityS admin.schemas().createSchema(fqtn, Schema.AVRO(SchemaDefinition.builder() .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo()); - Producer producerOne = client + Producer producerOne = pulsarClient .newProducer(Schema.AVRO(Schemas.PersonOne.class)) .topic(fqtn) .create(); - Schemas.PersonOne personOne = new Schemas.PersonOne(); - personOne.id = 10; + Schemas.PersonOne personOne = new Schemas.PersonOne(10); - Consumer consumerTwo = client.newConsumer(Schema.AVRO( + Consumer consumerOne = pulsarClient.newConsumer(Schema.AVRO( SchemaDefinition.builder().withAlwaysAllowNull (false).withSupportSchemaVersioning(true). withPojo(Schemas.PersonOne.class).build())) @@ -307,14 +312,12 @@ public void testProducerSendWithOldSchemaAndConsumerCanRead(SchemaCompatibilityS .subscribe(); producerOne.send(personOne); - - Message message = consumerTwo.receive(); - + Message message = consumerOne.receive(); personOne = message.getValue(); assertEquals(10, personOne.id); - consumerTwo.close(); + consumerOne.close(); producerOne.close(); } @@ -336,7 +339,7 @@ public void testConsumerWithNotCompatibilitySchema(SchemaCompatibilityStrategy s admin.namespaces().createNamespace( tenant + "/" + namespace, - Sets.newHashSet(pulsarCluster.getClusterName()) + Sets.newHashSet(CLUSTER_NAME) ); admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy); @@ -344,9 +347,8 @@ public void testConsumerWithNotCompatibilitySchema(SchemaCompatibilityStrategy s admin.schemas().createSchema(fqtn, Schema.AVRO(SchemaDefinition.builder() .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo()); - Consumer consumerFour = null; try { - consumerFour = client.newConsumer(Schema.AVRO( + pulsarClient.newConsumer(Schema.AVRO( SchemaDefinition.builder().withAlwaysAllowNull (false).withSupportSchemaVersioning(true). withPojo(Schemas.PersonFour.class).build())) @@ -354,7 +356,6 @@ public void testConsumerWithNotCompatibilitySchema(SchemaCompatibilityStrategy s .topic(fqtn) .subscribe(); } catch (Exception e) { - consumerFour.close(); Assert.assertTrue(e.getMessage().contains("Unable to read schema")); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/Schemas.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/Schemas.java new file mode 100644 index 0000000000000..7c228eadb3f77 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/Schemas.java @@ -0,0 +1,67 @@ + +package org.apache.pulsar.schema.compatibility; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.apache.avro.reflect.AvroDefault; + +public class Schemas { + + @Data + @Getter + @Setter + @ToString + @EqualsAndHashCode + @NoArgsConstructor + @AllArgsConstructor + public static class PersonOne{ + int id; + } + + @Data + @Getter + @Setter + @ToString + @EqualsAndHashCode + @AllArgsConstructor + @NoArgsConstructor + public static class PersonTwo{ + int id; + + @AvroDefault("\"Tom\"") + String name; + } + + @Data + @Getter + @Setter + @ToString + @EqualsAndHashCode + @AllArgsConstructor + @NoArgsConstructor + public static class PersonThree{ + int id; + + String name; + } + + @Data + @Getter + @Setter + @ToString + @EqualsAndHashCode + @AllArgsConstructor + @NoArgsConstructor + public static class PersonFour{ + int id; + + String name; + + int age; + } +} diff --git a/tests/integration/src/test/resources/pulsar-schema.xml b/tests/integration/src/test/resources/pulsar-schema.xml index 02fcff2dd2e7c..7b3b21ef0dea6 100644 --- a/tests/integration/src/test/resources/pulsar-schema.xml +++ b/tests/integration/src/test/resources/pulsar-schema.xml @@ -22,7 +22,6 @@ - From a7aecfdbabeefec159bd5c22d0caaa20680cf441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 25 Sep 2019 21:07:34 +0800 Subject: [PATCH 16/24] modify the httpLookupService --- .../org/apache/pulsar/client/impl/HttpLookupService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpLookupService.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpLookupService.java index e5334dfd1b6a0..6a0746c8af327 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpLookupService.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpLookupService.java @@ -24,7 +24,6 @@ import java.net.InetSocketAddress; import java.net.URI; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Base64; import java.util.List; @@ -47,6 +46,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static com.yahoo.sketches.Util.bytesToLong; + class HttpLookupService implements LookupService { private final HttpClient httpClient; @@ -151,7 +152,7 @@ public CompletableFuture> getSchema(TopicName topicName, by if (version != null) { path = String.format("admin/v2/schemas/%s/schema/%s", schemaName, - new String(version, StandardCharsets.UTF_8)); + bytesToLong(version)); } httpClient.get(path, GetSchemaResponse.class).thenAccept(response -> { future.complete(Optional.of(SchemaInfoUtil.newSchemaInfo(schemaName, response))); From 03694e31405403191a615ddd94558d754a7ec984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 26 Sep 2019 10:27:55 +0800 Subject: [PATCH 17/24] Add the license --- .../pulsar/schema/compatibility/Schemas.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/Schemas.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/Schemas.java index 7c228eadb3f77..6b63a25e5448a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/Schemas.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/Schemas.java @@ -1,4 +1,21 @@ - +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.pulsar.schema.compatibility; import lombok.AllArgsConstructor; From 542ae9e0108e894741a7cf793b83c6e0e97688f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 26 Sep 2019 10:44:19 +0800 Subject: [PATCH 18/24] add the method --- .../compatibility/SchemaCompatibilityCheckTest.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java index b5405c2f865bb..f091d9a890f52 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java @@ -41,8 +41,8 @@ import org.testng.annotations.Test; import java.util.Collections; +import java.util.concurrent.ThreadLocalRandom; -import static org.apache.pulsar.client.admin.internal.SchemaCompatibilityCheckTest.randomName; import static org.apache.pulsar.common.naming.TopicName.PUBLIC_TENANT; import static org.junit.Assert.assertEquals; @@ -360,4 +360,11 @@ public void testConsumerWithNotCompatibilitySchema(SchemaCompatibilityStrategy s } } + public static String randomName(int numChars) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < numChars; i++) { + sb.append((char) (ThreadLocalRandom.current().nextInt(26) + 'a')); + } + return sb.toString(); + } } From 1d123cf336b134f3095558669ece18b3f3067475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 26 Sep 2019 13:15:22 +0800 Subject: [PATCH 19/24] Modify decode message throw exception --- .../pulsar/broker/service/schema/SchemaServiceTest.java | 2 +- .../org/apache/pulsar/client/impl/schema/StructSchema.java | 7 ++++++- .../org/apache/pulsar/proxy/server/AdminProxyHandler.java | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java index 706a333557191..94fef1648519e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java @@ -259,7 +259,7 @@ public void testSchemaAutoUpdate() throws Exception { try { schemaRegistryService.putSchemaIfAbsent(schemaId1, schema2, SchemaCompatibilityStrategy.FULL, false).get(); } catch (Exception e) { - assertEquals("Do allow auto update schema.", e.getCause().getMessage()); + assertEquals("Don't allow auto update schema.", e.getCause().getMessage()); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java index b38df410fa571..ede229cb74e63 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java @@ -28,11 +28,13 @@ import com.google.common.cache.LoadingCache; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufInputStream; +import org.apache.avro.AvroTypeException; import org.apache.avro.Schema.Parser; import org.apache.avro.reflect.ReflectData; import org.apache.commons.codec.binary.Hex; import org.apache.commons.lang3.SerializationException; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.api.schema.SchemaInfoProvider; import org.apache.pulsar.client.api.schema.SchemaReader; @@ -94,7 +96,10 @@ public T decode(byte[] bytes) { public T decode(byte[] bytes, byte[] schemaVersion) { try { return readerCache.get(BytesSchemaVersion.of(schemaVersion)).read(bytes); - } catch (ExecutionException e) { + } catch (ExecutionException | AvroTypeException e) { + if (e instanceof AvroTypeException) { + throw new SchemaSerializationException(e); + } LOG.error("Can't get generic schema for topic {} schema version {}", schemaInfoProvider.getTopicName(), Hex.encodeHexString(schemaVersion), e); throw new RuntimeException("Can't get generic schema for topic " + schemaInfoProvider.getTopicName()); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java index ae10bb2530de7..fdf687dfc9ea0 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java @@ -104,7 +104,7 @@ protected HttpClient createHttpClient() throws ServletException { value = config.getInitParameter("idleTimeout"); if (value == null) - value = "120000"; + value = "30000"; client.setIdleTimeout(Long.parseLong(value)); value = config.getInitParameter("requestBufferSize"); From a045acdd69ee5d89ee295c09401374fdcafec235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Mon, 7 Oct 2019 23:03:25 +0800 Subject: [PATCH 20/24] Add the import class --- .../main/java/org/apache/pulsar/broker/service/ServerCnx.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index f4d03c6369ed9..abdf92a4a39b2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -104,6 +104,7 @@ import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.protocol.schema.SchemaInfoUtil; import org.apache.pulsar.common.protocol.schema.SchemaVersion; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; import org.apache.pulsar.shaded.com.google.protobuf.v241.GeneratedMessageLite; import org.slf4j.Logger; From ce044fe2f0964769acc3cfb33eaf0ec82cb8198b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 10 Oct 2019 19:46:10 +0800 Subject: [PATCH 21/24] fix some comment --- .../broker/admin/v2/SchemasResource.java | 17 +++-- .../pulsar/broker/service/AbstractTopic.java | 14 ++-- .../pulsar/broker/service/ServerCnx.java | 2 +- .../apache/pulsar/broker/service/Topic.java | 2 +- .../nonpersistent/NonPersistentTopic.java | 2 +- .../service/persistent/PersistentTopic.java | 13 +--- .../AvroSchemaBasedCompatibilityCheck.java | 2 + .../broker/service/schema/SchemaRegistry.java | 2 +- .../schema/SchemaRegistryServiceImpl.java | 64 +++++++++---------- ...egistryServiceWithSchemaDataValidator.java | 4 +- .../service/schema/SchemaServiceTest.java | 18 +----- ...tryServiceWithSchemaDataValidatorTest.java | 12 ++-- .../api/SimpleTypedProducerConsumerTest.java | 12 ++-- 13 files changed, 70 insertions(+), 94 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java index 3761a86d8ea3b..868c7ec90ab60 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/SchemasResource.java @@ -330,15 +330,14 @@ public void postSchema( pulsar().getSchemaRegistryService().putSchemaIfAbsent( buildSchemaId(tenant, namespace, topic), SchemaData.builder() - .data(data) - .isDeleted(false) - .timestamp(clock.millis()) - .type(SchemaType.valueOf(payload.getType())) - .user(defaultIfEmpty(clientAppId(), "")) - .props(payload.getProperties()) - .build(), - schemaCompatibilityStrategy, - true + .data(data) + .isDeleted(false) + .timestamp(clock.millis()) + .type(SchemaType.valueOf(payload.getType())) + .user(defaultIfEmpty(clientAppId(), "")) + .props(payload.getProperties()) + .build(), + schemaCompatibilityStrategy ).thenAccept(version -> response.resume( Response.accepted().entity( diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 6308930f01feb..c85ec7ebf6f0a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -21,6 +21,7 @@ import com.google.common.base.MoreObjects; import org.apache.bookkeeper.mledger.util.StatsBuckets; import org.apache.pulsar.broker.admin.AdminResource; +import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; @@ -28,6 +29,7 @@ import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.protocol.schema.SchemaVersion; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -170,9 +172,13 @@ public CompletableFuture addSchema(SchemaData schema) { String base = TopicName.get(getName()).getPartitionedTopicName(); String id = TopicName.get(base).getSchemaName(); - return brokerService.pulsar() - .getSchemaRegistryService() - .putSchemaIfAbsent(id, schema, schemaCompatibilityStrategy, isAllowAutoUpdateSchema); + if (isAllowAutoUpdateSchema) { + return brokerService.pulsar() + .getSchemaRegistryService() + .putSchemaIfAbsent(id, schema, schemaCompatibilityStrategy); + } else { + return FutureUtil.failedFuture(new IncompatibleSchemaException("Don't allow auto update schema.")); + } } @Override @@ -191,7 +197,7 @@ public CompletableFuture deleteSchema() { } @Override - public CompletableFuture isSchemaCompatible(SchemaData schema) { + public CompletableFuture checkSchemaCompatibleForConsumer(SchemaData schema) { String base = TopicName.get(getName()).getPartitionedTopicName(); String id = TopicName.get(base).getSchemaName(); return brokerService.pulsar() diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 97d704ca41227..d903da2dd3e98 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -693,7 +693,7 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { if (schema != null) { return topic.addSchemaIfIdleOrCheckCompatible(schema) - .thenCompose(isCompatible -> topic.subscribe(ServerCnx.this, subscriptionName, consumerId, + .thenCompose(v -> topic.subscribe(ServerCnx.this, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, startMessageId, metadata, readCompacted, initialPosition, isReplicated)); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java index 09db235e148ee..71ee57993316d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java @@ -164,7 +164,7 @@ void updateRates(NamespaceStats nsStats, NamespaceBundleStats currentBundleStats /** * Check if schema is compatible with current topic schema. */ - CompletableFuture isSchemaCompatible(SchemaData schema); + CompletableFuture checkSchemaCompatibleForConsumer(SchemaData schema); /** * If the topic is idle (no producers, no entries, no subscribers and no existing schema), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 82ee3815c5827..bedd8258cd2d6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -927,7 +927,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem return hasSchema() .thenCompose((hasSchema) -> { if (hasSchema || isActive() || ENTRIES_ADDED_COUNTER_UPDATER.get(this) != 0) { - return isSchemaCompatible(schema); + return checkSchemaCompatibleForConsumer(schema); } else { return addSchema(schema).thenCompose(schemaVersion-> CompletableFuture.completedFuture(null)); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 1b2ed8c0d32c2..f22cd056cf45a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -122,18 +122,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CancellationException; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.function.BiFunction; import static com.google.common.base.Preconditions.checkArgument; @@ -1935,7 +1924,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem return hasSchema() .thenCompose((hasSchema) -> { if (hasSchema || isActive() || ledger.getTotalSize() != 0) { - return isSchemaCompatible(schema); + return checkSchemaCompatibleForConsumer(schema); } else { return addSchema(schema).thenCompose(schemaVersion -> CompletableFuture.completedFuture(null)); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java index 5e2cf22edae70..9b8d4a10d6a06 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service.schema; +import static com.google.common.base.Preconditions.checkArgument; import static java.nio.charset.StandardCharsets.UTF_8; import java.util.Collections; @@ -47,6 +48,7 @@ public void checkCompatible(SchemaData from, SchemaData to, SchemaCompatibilityS @Override public void checkCompatible(Iterable from, SchemaData to, SchemaCompatibilityStrategy strategy) throws IncompatibleSchemaException { LinkedList fromList = new LinkedList<>(); + checkArgument(from != null, "check compatibility list is null"); try { for (SchemaData schemaData : from) { Schema.Parser parser = new Schema.Parser(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistry.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistry.java index 90cd92cb7b806..5b041648952b1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistry.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistry.java @@ -37,7 +37,7 @@ public interface SchemaRegistry extends AutoCloseable { CompletableFuture>> getAllSchemas(String schemaId); CompletableFuture putSchemaIfAbsent(String schemaId, SchemaData schema, - SchemaCompatibilityStrategy strategy, boolean isAllowAutoUpdateSchema); + SchemaCompatibilityStrategy strategy); CompletableFuture deleteSchema(String schemaId, String user); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java index b239856770ac1..4ad1cc12427b2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java @@ -50,7 +50,6 @@ import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.protocol.schema.SchemaVersion; -import org.apache.pulsar.common.util.FutureUtil; public class SchemaRegistryServiceImpl implements SchemaRegistryService { private static HashFunction hashFunction = Hashing.sha256(); @@ -110,7 +109,7 @@ public CompletableFuture>> getAllSchem @Override @NotNull public CompletableFuture putSchemaIfAbsent(String schemaId, SchemaData schema, - SchemaCompatibilityStrategy strategy, boolean isAllowAutoUpdateSchema) { + SchemaCompatibilityStrategy strategy) { return trimDeletedSchemaAndGetList(schemaId).thenCompose(schemaAndMetadataList -> { final CompletableFuture completableFuture = new CompletableFuture<>(); SchemaVersion schemaVersion; @@ -122,34 +121,31 @@ public CompletableFuture putSchemaIfAbsent(String schemaId, Schem return completableFuture; } } - if (isAllowAutoUpdateSchema) { - CompletableFuture isCompatibility = new CompletableFuture<>(); - if (schemaAndMetadataList.size() != 0) { - if (isTransitiveStrategy(strategy)) { - isCompatibility = checkCompatibilityWithAll(schema, strategy, schemaAndMetadataList); - } else { - isCompatibility = checkCompatibilityWithLatest(schemaId, schema, strategy); - } + CompletableFuture checkCompatibilityFurture = new CompletableFuture<>(); + if (schemaAndMetadataList.size() != 0) { + if (isTransitiveStrategy(strategy)) { + checkCompatibilityFurture = checkCompatibilityWithAll(schema, strategy, schemaAndMetadataList); } else { - isCompatibility.complete(null); + checkCompatibilityFurture = checkCompatibilityWithLatest(schemaId, schema, strategy); } - return isCompatibility.thenCompose((v) -> { - byte[] context = hashFunction.hashBytes(schema.getData()).asBytes(); - SchemaRegistryFormat.SchemaInfo info = SchemaRegistryFormat.SchemaInfo.newBuilder() - .setType(Functions.convertFromDomainType(schema.getType())) - .setSchema(ByteString.copyFrom(schema.getData())) - .setSchemaId(schemaId) - .setUser(schema.getUser()) - .setDeleted(false) - .setTimestamp(clock.millis()) - .addAllProps(toPairs(schema.getProps())) - .build(); - return schemaStorage.put(schemaId, info.toByteArray(), context); - - }); } else { - return FutureUtils.exception(new IncompatibleSchemaException("Don't allow auto update schema.")); + checkCompatibilityFurture.complete(null); } + return checkCompatibilityFurture.thenCompose(v -> { + byte[] context = hashFunction.hashBytes(schema.getData()).asBytes(); + SchemaRegistryFormat.SchemaInfo info = SchemaRegistryFormat.SchemaInfo.newBuilder() + .setType(Functions.convertFromDomainType(schema.getType())) + .setSchema(ByteString.copyFrom(schema.getData())) + .setSchemaId(schemaId) + .setUser(schema.getUser()) + .setDeleted(false) + .setTimestamp(clock.millis()) + .addAllProps(toPairs(schema.getProps())) + .build(); + return schemaStorage.put(schemaId, info.toByteArray(), context); + + }); + }); } @@ -157,8 +153,7 @@ public CompletableFuture putSchemaIfAbsent(String schemaId, Schem @NotNull public CompletableFuture deleteSchema(String schemaId, String user) { byte[] deletedEntry = deleted(schemaId, user).toByteArray(); - return trimDeletedSchemaAndGetList(schemaId).thenCompose(schemaAndMetadataList -> - schemaStorage.put(schemaId, deletedEntry, new byte[]{})); + return schemaStorage.put(schemaId, deletedEntry, new byte[]{}); } @Override @@ -254,10 +249,7 @@ public CompletableFuture checkConsumerCompatibility(String schemaId, Schem return checkCompatibilityWithAll(schemaId, schemaData, strategy); } } else { - return FutureUtil.failedFuture( - new IncompatibleSchemaException( - "Trying to subscribe with incompatible schema" - )); + return CompletableFuture.completedFuture(null); } }); } @@ -297,8 +289,12 @@ private CompletableFuture checkCompatibilityWithAll(SchemaData schema, .map(schemaAndMetadata -> schemaAndMetadata.schema) .collect(Collectors.toList()), schema, strategy); result.complete(null); - } catch (IncompatibleSchemaException e) { - result.completeExceptionally(e); + } catch (Exception e) { + if (e instanceof IncompatibleSchemaException) { + result.completeExceptionally(e); + } else { + result.completeExceptionally(new IncompatibleSchemaException(e)); + } } return result; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java index 8a00ea8dcda94..e589d06e09e48 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidator.java @@ -80,13 +80,13 @@ public CompletableFuture checkConsumerCompatibility(String schemaId, Schem @Override public CompletableFuture putSchemaIfAbsent(String schemaId, SchemaData schema, - SchemaCompatibilityStrategy strategy, boolean isAllowAutoUpdateSchema) { + SchemaCompatibilityStrategy strategy) { try { SchemaDataValidator.validateSchemaData(schema); } catch (InvalidSchemaDataException e) { return FutureUtil.failedFuture(e); } - return service.putSchemaIfAbsent(schemaId, schema, strategy, isAllowAutoUpdateSchema); + return service.putSchemaIfAbsent(schemaId, schema, strategy); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java index 94fef1648519e..e18fa839dec7b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java @@ -253,27 +253,15 @@ public void dontReAddExistingSchemaAtRoot() throws Exception { putSchema(schemaId1, schema1, version(0)); } - @Test - public void testSchemaAutoUpdate() throws Exception { - putSchema(schemaId1, schema1, version(0)); - try { - schemaRegistryService.putSchemaIfAbsent(schemaId1, schema2, SchemaCompatibilityStrategy.FULL, false).get(); - } catch (Exception e) { - assertEquals("Don't allow auto update schema.", e.getCause().getMessage()); - } - - } - - @Test public void trimDeletedSchemaAndGetListTest() throws Exception { List list = new ArrayList<>(); CompletableFuture put = schemaRegistryService.putSchemaIfAbsent( - schemaId1, schema1, SchemaCompatibilityStrategy.FULL, true); + schemaId1, schema1, SchemaCompatibilityStrategy.FULL); SchemaVersion newVersion = put.get(); list.add(new SchemaAndMetadata(schemaId1, schema1, newVersion)); put = schemaRegistryService.putSchemaIfAbsent( - schemaId1, schema2, SchemaCompatibilityStrategy.FULL, true); + schemaId1, schema2, SchemaCompatibilityStrategy.FULL); newVersion = put.get(); list.add(new SchemaAndMetadata(schemaId1, schema2, newVersion)); List list1 = schemaRegistryService.trimDeletedSchemaAndGetList(schemaId1).get(); @@ -337,7 +325,7 @@ private void putSchema(String schemaId, SchemaData schema, SchemaVersion expecte private void putSchema(String schemaId, SchemaData schema, SchemaVersion expectedVersion, SchemaCompatibilityStrategy strategy) throws ExecutionException, InterruptedException { CompletableFuture put = schemaRegistryService.putSchemaIfAbsent( - schemaId, schema, strategy, true); + schemaId, schema, strategy); SchemaVersion newVersion = put.get(); assertEquals(expectedVersion, newVersion); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidatorTest.java index 93f3afe72ccd6..bc05957b085b0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/validator/SchemaRegistryServiceWithSchemaDataValidatorTest.java @@ -128,15 +128,15 @@ public void testPutSchemaIfAbsentWithGoodSchemaData() { String schemaId = "test-schema-id"; SchemaCompatibilityStrategy strategy = SchemaCompatibilityStrategy.FULL; CompletableFuture future = new CompletableFuture<>(); - when(underlyingService.putSchemaIfAbsent(eq(schemaId), any(SchemaData.class), eq(strategy), eq(true))) + when(underlyingService.putSchemaIfAbsent(eq(schemaId), any(SchemaData.class), eq(strategy))) .thenReturn(future); SchemaData schemaData = SchemaData.builder() .type(SchemaType.BOOLEAN) .data(new byte[0]) .build(); - assertSame(future, service.putSchemaIfAbsent(schemaId, schemaData, strategy, true)); + assertSame(future, service.putSchemaIfAbsent(schemaId, schemaData, strategy)); verify(underlyingService, times(1)) - .putSchemaIfAbsent(eq(schemaId), same(schemaData), eq(strategy), eq(true)); + .putSchemaIfAbsent(eq(schemaId), same(schemaData), eq(strategy)); } @Test @@ -144,20 +144,20 @@ public void testPutSchemaIfAbsentWithBadSchemaData() { String schemaId = "test-schema-id"; SchemaCompatibilityStrategy strategy = SchemaCompatibilityStrategy.FULL; CompletableFuture future = new CompletableFuture<>(); - when(underlyingService.putSchemaIfAbsent(eq(schemaId), any(SchemaData.class), eq(strategy), eq(true))) + when(underlyingService.putSchemaIfAbsent(eq(schemaId), any(SchemaData.class), eq(strategy))) .thenReturn(future); SchemaData schemaData = SchemaData.builder() .type(SchemaType.BOOLEAN) .data(new byte[10]) .build(); try { - service.putSchemaIfAbsent(schemaId, schemaData, strategy, true).get(); + service.putSchemaIfAbsent(schemaId, schemaData, strategy).get(); fail("Should fail putSchemaIfAbsent"); } catch (Exception e) { assertTrue(e.getCause() instanceof InvalidSchemaDataException); } verify(underlyingService, times(0)) - .putSchemaIfAbsent(eq(schemaId), same(schemaData), eq(strategy), eq(true)); + .putSchemaIfAbsent(eq(schemaId), same(schemaData), eq(strategy)); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java index 2d0f40d2cec2e..112bbe7cbe25a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java @@ -131,8 +131,7 @@ public void testJsonProducerAndConsumerWithPrestoredSchema() throws Exception { .data(jsonSchema.getSchemaInfo().getSchema()) .props(Collections.emptyMap()) .build(), - SchemaCompatibilityStrategy.FULL, - true + SchemaCompatibilityStrategy.FULL ).get(); Consumer consumer = pulsarClient @@ -175,8 +174,7 @@ public void testWrongCorruptedSchema() throws Exception { .data(randomSchemaBytes) .props(Collections.emptyMap()) .build(), - SchemaCompatibilityStrategy.FULL, - true + SchemaCompatibilityStrategy.FULL ).get(); fail("Should fail to add corrupted schema data"); } catch (Exception e) { @@ -252,8 +250,7 @@ public void testProtobufConsumerWithWrongPrestoredSchema() throws Exception { .data(schema.getSchemaInfo().getSchema()) .props(Collections.emptyMap()) .build(), - SchemaCompatibilityStrategy.FULL, - true + SchemaCompatibilityStrategy.FULL ).get(); Consumer consumer = pulsarClient @@ -338,8 +335,7 @@ public void testAvroConsumerWithWrongPrestoredSchema() throws Exception { .data(randomSchemaBytes) .props(Collections.emptyMap()) .build(), - SchemaCompatibilityStrategy.FULL, - true + SchemaCompatibilityStrategy.FULL ).get(); Consumer consumer = pulsarClient .newConsumer(AvroSchema.of(SchemaDefinition.builder(). From f7f50970d621090ea1c66fefefdc3762f77c465e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Fri, 11 Oct 2019 10:23:17 +0800 Subject: [PATCH 22/24] Modify the implments method --- .../broker/service/schema/DefaultSchemaRegistryService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/DefaultSchemaRegistryService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/DefaultSchemaRegistryService.java index 1e1155e593e23..8bbda1119990d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/DefaultSchemaRegistryService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/DefaultSchemaRegistryService.java @@ -45,7 +45,7 @@ public CompletableFuture>> getAllSchem } @Override - public CompletableFuture putSchemaIfAbsent(String schemaId, SchemaData schema, SchemaCompatibilityStrategy strategy, boolean isAllowAutoUpdateSchema) { + public CompletableFuture putSchemaIfAbsent(String schemaId, SchemaData schema, SchemaCompatibilityStrategy strategy) { return completedFuture(null); } From 78694d28cadc5a9e3c30c8c07651a779dc187de5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Fri, 11 Oct 2019 14:50:35 +0800 Subject: [PATCH 23/24] add the judgement for consumer schema compatibility check --- .../broker/service/schema/SchemaRegistryServiceImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java index 4ad1cc12427b2..4b0acbc9cc6f2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java @@ -50,6 +50,7 @@ import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.protocol.schema.SchemaVersion; +import org.apache.pulsar.common.util.FutureUtil; public class SchemaRegistryServiceImpl implements SchemaRegistryService { private static HashFunction hashFunction = Hashing.sha256(); @@ -249,7 +250,7 @@ public CompletableFuture checkConsumerCompatibility(String schemaId, Schem return checkCompatibilityWithAll(schemaId, schemaData, strategy); } } else { - return CompletableFuture.completedFuture(null); + return FutureUtil.failedFuture(new IncompatibleSchemaException("Topic does not have schema to check")); } }); } From 711d9ad9b7eba09041879f9c244118c43e434a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 31 Oct 2019 14:56:03 +0800 Subject: [PATCH 24/24] add the import --- .../broker/service/schema/SchemaRegistryServiceImpl.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java index a40eb890f88a8..d4e395695d35e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/SchemaRegistryServiceImpl.java @@ -27,10 +27,14 @@ import static org.apache.pulsar.broker.service.schema.SchemaRegistryServiceImpl.Functions.toPairs; import com.google.common.annotations.VisibleForTesting; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; + import java.time.Clock; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -50,6 +54,7 @@ import org.apache.pulsar.common.util.FutureUtil; public class SchemaRegistryServiceImpl implements SchemaRegistryService { + private static HashFunction hashFunction = Hashing.sha256(); private final Map compatibilityChecks; private final SchemaStorage schemaStorage; private final Clock clock;