From 96c1299d8a431e5e47aaee9de314443785cbcd2d Mon Sep 17 00:00:00 2001 From: rdhabalia Date: Wed, 7 Sep 2016 15:12:15 -0700 Subject: [PATCH] Support Topic lookup using Pulsar binary protocol --- .../pulsar/broker/admin/AdminResource.java | 1 + .../pulsar/broker/admin/PersistentTopics.java | 92 +- .../broker/lookup/DestinationLookup.java | 130 +- .../pulsar/broker/lookup/LookupResult.java | 28 +- .../broker/namespace/NamespaceService.java | 91 +- .../yahoo/pulsar/broker/service/Consumer.java | 1 - .../pulsar/broker/service/ServerCnx.java | 52 +- .../pulsar/broker/web/PulsarWebResource.java | 254 +- .../pulsar/broker/admin/NamespacesTest.java | 62 +- .../auth/MockedPulsarServiceBaseTest.java | 16 +- .../SimpleLoadManagerImplTest.java | 3 + .../http/HttpDestinationLookupv2Test.java | 7 + .../client/api/BrokerServiceLookupTest.java | 551 +++ .../pulsar/client/api/MockBrokerService.java | 2 +- .../api/SimpleProducerConsumerTest.java | 6 +- .../client/impl/BinaryProtoLookupService.java | 215 ++ .../yahoo/pulsar/client/impl/ClientCnx.java | 98 +- .../pulsar/client/impl/ConnectionPool.java | 10 +- .../pulsar/client/impl/HttpLookupService.java | 81 + .../pulsar/client/impl/LookupService.java | 72 +- .../impl/PartitionMetadataLookupService.java | 35 - .../pulsar/client/impl/PulsarClientImpl.java | 34 +- .../com/yahoo/pulsar/common/api/Commands.java | 98 + .../pulsar/common/api/PulsarDecoder.java | 44 + .../pulsar/common/api/PulsarHandler.java | 6 +- .../pulsar/common/api/proto/PulsarApi.java | 2991 +++++++++++++++++ .../pulsar/common/lookup/data/LookupData.java | 18 +- .../common/policies/data/ClusterData.java | 54 +- .../protobuf/ByteBufCodedOutputStream.java | 11 + pulsar-common/src/main/proto/PulsarApi.proto | 56 + .../common/lookup/data/LookupDataTest.java | 7 +- .../service/BrokerDiscoveryProvider.java | 83 + .../discovery/service/DiscoveryService.java | 184 + .../service/PulsarServerException.java | 35 + .../discovery/service/ServerConnection.java | 107 + .../service/ServiceChannelInitializer.java | 64 + .../server/DiscoveryServiceStarter.java | 11 + .../service/server/ServerManager.java | 4 + .../service/server/ServiceConfig.java | 31 + .../service/web/DiscoveryServiceServlet.java | 2 +- .../service/web/ZookeeperCacheLoader.java | 2 +- .../service/BaseDiscoveryTestSetup.java | 86 + .../service/DiscoveryServiceTest.java | 208 ++ .../server/DiscoveryServiceWebTest.java | 58 + .../service/web/DiscoveryServiceWebTest.java | 16 +- .../src/test/resources/certificate/client.crt | 20 + .../src/test/resources/certificate/client.key | 28 + .../service/BaseDiscoveryTestSetup.java | 86 + .../service/DiscoveryServiceTest.java | 208 ++ .../server/DiscoveryServiceWebTest.java | 58 + .../service/web/BaseZKStarterTest.java | 68 + .../service/web/DiscoveryServiceWebTest.java | 317 ++ .../service/web/ZookeeperCacheLoaderTest.java | 101 + .../pulsar/zookeeper/ZooKeeperCache.java | 11 + 54 files changed, 6601 insertions(+), 313 deletions(-) create mode 100644 pulsar-broker/src/test/java/com/yahoo/pulsar/client/api/BrokerServiceLookupTest.java create mode 100644 pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/BinaryProtoLookupService.java create mode 100644 pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/HttpLookupService.java delete mode 100644 pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/PartitionMetadataLookupService.java create mode 100644 pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/BrokerDiscoveryProvider.java create mode 100644 pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/DiscoveryService.java create mode 100644 pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/PulsarServerException.java create mode 100644 pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/ServerConnection.java create mode 100644 pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/ServiceChannelInitializer.java create mode 100644 pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/BaseDiscoveryTestSetup.java create mode 100644 pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/DiscoveryServiceTest.java create mode 100644 pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/server/DiscoveryServiceWebTest.java create mode 100644 pulsar-discovery-service/src/test/resources/certificate/client.crt create mode 100644 pulsar-discovery-service/src/test/resources/certificate/client.key create mode 100644 pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/BaseDiscoveryTestSetup.java create mode 100644 pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/DiscoveryServiceTest.java create mode 100644 pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/server/DiscoveryServiceWebTest.java create mode 100644 pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/web/BaseZKStarterTest.java create mode 100644 pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/web/DiscoveryServiceWebTest.java create mode 100644 pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/web/ZookeeperCacheLoaderTest.java diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/AdminResource.java index e7a982d2cf3da..ecf6923c03d1c 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/AdminResource.java @@ -39,6 +39,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; +import com.yahoo.pulsar.broker.PulsarService; import com.yahoo.pulsar.broker.cache.LocalZooKeeperCacheService; import com.yahoo.pulsar.broker.web.PulsarWebResource; import com.yahoo.pulsar.broker.web.RestException; diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/PersistentTopics.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/PersistentTopics.java index 6572fb49803ef..7d93a4abf8449 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/PersistentTopics.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/PersistentTopics.java @@ -93,6 +93,8 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; +import com.yahoo.pulsar.broker.PulsarService; +import com.yahoo.pulsar.client.api.PulsarClientException; /** */ @@ -356,21 +358,7 @@ public PartitionedTopicMetadata getPartitionedTopicMetadata(@PathParam("property String path = path(PARTITIONED_TOPIC_PATH_ZNODE, property, cluster, namespace, domain(), dn.getEncodedLocalName()); - PartitionedTopicMetadata partitionMetadata; - try { - // gets the number of partitions from the zk cache - partitionMetadata = globalZkCache().getData(path, new Deserializer() { - - @Override - public PartitionedTopicMetadata deserialize(String key, byte[] content) throws Exception { - return jsonMapper().readValue(content, PartitionedTopicMetadata.class); - } - }).orElse( - // if the partitioned topic is not found in zk, then the topic is not partitioned - new PartitionedTopicMetadata()); - } catch (Exception e) { - throw new RestException(e); - } + PartitionedTopicMetadata partitionMetadata = fetchPartitionedTopicMetadata(pulsar(), path); if (log.isDebugEnabled()) { log.debug("[{}] Total number of partitions for topic {} is {}", clientAppId(), dn, @@ -993,7 +981,79 @@ public PersistentOfflineTopicStats getBacklog(@PathParam("property") String prop return offlineTopicStats; } - /** + public static CompletableFuture getPartitionedTopicMetadata(PulsarService pulsar, + String clientAppId, DestinationName dn) { + CompletableFuture metadataFuture = new CompletableFuture<>(); + try { + // (1) authorize client + try { + checkAuthorization(pulsar, dn, clientAppId); + } catch (RestException e) { + try { + validateAdminAccessOnProperty(pulsar, clientAppId, dn.getProperty()); + } catch (RestException authException) { + log.warn("Failed to authorize {} on cluster {}", clientAppId, dn.toString()); + throw new PulsarClientException(String.format("Authorization failed %s on cluster %s with error %s", + clientAppId, dn.toString(), authException.getMessage())); + } + } + String path = path(PARTITIONED_TOPIC_PATH_ZNODE, dn.getProperty(), dn.getCluster(), + dn.getNamespacePortion(), "persistent", dn.getEncodedLocalName()); + fetchPartitionedTopicMetadataAsync(pulsar, path).thenAccept(metadata -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Total number of partitions for topic {} is {}", clientAppId, dn, + metadata.partitions); + } + metadataFuture.complete(metadata); + }).exceptionally(ex -> { + metadataFuture.completeExceptionally(ex); + return null; + }); + } catch (Exception ex) { + metadataFuture.completeExceptionally(ex); + } + return metadataFuture; + } + + private static PartitionedTopicMetadata fetchPartitionedTopicMetadata(PulsarService pulsar, String path) { + try { + return fetchPartitionedTopicMetadataAsync(pulsar, path).get(); + } catch (Exception e) { + if (e.getCause() instanceof RestException) { + throw (RestException) e; + } + throw new RestException(e); + } + } + + private static CompletableFuture fetchPartitionedTopicMetadataAsync(PulsarService pulsar, + String path) { + CompletableFuture metadataFuture = new CompletableFuture<>(); + try { + // gets the number of partitions from the zk cache + pulsar.getGlobalZkCache().getDataAsync(path, new Deserializer() { + @Override + public PartitionedTopicMetadata deserialize(String key, byte[] content) throws Exception { + return jsonMapper().readValue(content, PartitionedTopicMetadata.class); + } + }).thenAccept(metadata -> { + // if the partitioned topic is not found in zk, then the topic is not partitioned + if (metadata.isPresent()) { + metadataFuture.complete(metadata.get()); + } else { + metadataFuture.complete(new PartitionedTopicMetadata()); + } + }).exceptionally(ex -> { + metadataFuture.complete(new PartitionedTopicMetadata()); + return null; + }); + } catch (Exception e) { + metadataFuture.completeExceptionally(e); + } + return metadataFuture; + } + + /** * Get the Topic object reference from the Pulsar broker */ private PersistentTopic getTopicReference(DestinationName dn) { diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/lookup/DestinationLookup.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/lookup/DestinationLookup.java index 81f801a92c8eb..ca3c77a1bef77 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/lookup/DestinationLookup.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/lookup/DestinationLookup.java @@ -30,6 +30,7 @@ import javax.ws.rs.container.Suspended; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +38,16 @@ import com.yahoo.pulsar.broker.web.NoSwaggerDocumentation; import com.yahoo.pulsar.broker.web.PulsarWebResource; import com.yahoo.pulsar.common.naming.DestinationName; +import com.yahoo.pulsar.broker.PulsarService; +import com.yahoo.pulsar.broker.web.RestException; +import static com.yahoo.pulsar.common.api.Commands.newLookupResponse; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType; +import com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError; +import com.yahoo.pulsar.common.lookup.data.LookupData; +import com.yahoo.pulsar.common.policies.data.ClusterData; +import static com.google.common.base.Preconditions.checkNotNull; + +import io.netty.buffer.ByteBuf; @Path("/v2/destination/") @NoSwaggerDocumentation @@ -55,7 +66,7 @@ public void lookupDestinationAsync(@PathParam("property") String property, @Path try { validateClusterOwnership(topic.getCluster()); checkConnect(topic); - validateReplicationSettingsOnNamespace(topic.getNamespaceObject()); + validateReplicationSettingsOnNamespace(pulsar(), topic.getNamespaceObject()); } catch (Throwable t) { // Validation checks failed log.error("Validation check failed: {}", t.getMessage()); @@ -74,23 +85,24 @@ public void lookupDestinationAsync(@PathParam("property") String property, @Path } // We have found either a broker that owns the topic, or a broker to which we should redirect the client to - if (result.isHttpRedirect()) { + if (result.isRedirect()) { boolean newAuthoritative = this.isLeaderBroker(); URI redirect; try { - redirect = new URI(String.format("%s%s%s?authoritative=%s", result.getHttpRedirectAddress(), - "/lookup/v2/destination/", topic.getLookupName(), newAuthoritative)); + String redirectUrl = isRequestHttps() ? result.getLookupData().getHttpUrlTls() + : result.getLookupData().getHttpUrl(); + redirect = new URI(String.format("%s%s%s?authoritative=%s", redirectUrl, "/lookup/v2/destination/", + topic.getLookupName(), newAuthoritative)); } catch (URISyntaxException e) { log.error("Error in preparing redirect url for {}: {}", topic, e.getMessage(), e); asyncResponse.resume(e); return; } - if (log.isDebugEnabled()) { log.debug("Redirect lookup for topic {} to {}", topic, redirect); } - asyncResponse.resume(new WebApplicationException(Response.temporaryRedirect(redirect).build())); + } else { // Found broker owning the topic if (log.isDebugEnabled()) { @@ -103,7 +115,113 @@ public void lookupDestinationAsync(@PathParam("property") String property, @Path asyncResponse.resume(exception); return null; }); + } + + /** + * + * Lookup broker-service address for a given namespace-bundle which contains given topic. + * + * a. Returns broker-address if namespace-bundle is already owned by any broker + * b. If current-broker receives lookup-request and if it's not a leader + * then current broker redirects request to leader by returning leader-service address. + * c. If current-broker is leader then it finds out least-loaded broker to own namespace bundle and + * redirects request by returning least-loaded broker. + * d. If current-broker receives request to own the namespace-bundle then it owns a bundle and returns + * success(connect) response to client. + * + * @param pulsarService + * @param fqdn + * @param authoritative + * @param clientAppId + * @param requestId + * @return + */ + public static CompletableFuture lookupDestinationAsync(PulsarService pulsarService, DestinationName fqdn, boolean authoritative, + String clientAppId, long requestId) { + + final CompletableFuture validationFuture = new CompletableFuture<>(); + final CompletableFuture lookupfuture = new CompletableFuture<>(); + final String cluster = fqdn.getCluster(); + + // (1) validate cluster + getClusterDataIfDifferentCluster(pulsarService, cluster, clientAppId).thenAccept(differentClusterData -> { + + if (differentClusterData != null) { + if (log.isDebugEnabled()) { + log.debug("[{}] Redirecting the lookup call to {}/{} cluster={}", clientAppId, + differentClusterData.getBrokerServiceUrl(), differentClusterData.getBrokerServiceUrlTls(), cluster); + } + validationFuture.complete(newLookupResponse(differentClusterData.getBrokerServiceUrl(), + differentClusterData.getBrokerServiceUrlTls(), true, LookupType.Redirect, requestId)); + } else { + // (2) authorize client + try { + checkAuthorization(pulsarService, fqdn, clientAppId); + } catch (Exception e) { + log.warn("Failed to authorized {} on cluster {}", clientAppId, fqdn.toString()); + validationFuture + .complete(newLookupResponse(ServerError.AuthorizationError, e.getMessage(), requestId)); + return; + } + // (3) validate global namespace + validateReplicationSettingsOnNamespaceAsync(pulsarService, fqdn.getNamespaceObject()) + .thenAccept(success -> { + // (4) all validation passed: initiate lookup + validationFuture.complete(null); + }).exceptionally(ex -> { + validationFuture + .complete(newLookupResponse(ServerError.MetadataError, ex.getMessage(), requestId)); + return null; + }); + } + }).exceptionally(ex -> { + validationFuture.completeExceptionally(ex); + return null; + }); + + // Initiate lookup once validation completes + validationFuture.thenAccept(validaitonFailureResponse -> { + if (validaitonFailureResponse != null) { + lookupfuture.complete(validaitonFailureResponse); + } else { + pulsarService.getNamespaceService().getBrokerServiceUrlAsync(fqdn, authoritative) + .thenAccept(lookupResult -> { + + if (log.isDebugEnabled()) { + log.debug("[{}] Lookup result {}", fqdn.toString(), lookupResult); + } + + LookupData lookupData = lookupResult.getLookupData(); + if (lookupResult.isRedirect()) { + boolean newAuthoritative = isLeaderBroker(pulsarService); + lookupfuture.complete( + newLookupResponse(lookupData.getBrokerUrl(), lookupData.getBrokerUrlTls(), + newAuthoritative, LookupType.Redirect, requestId)); + } else { + lookupfuture.complete( + newLookupResponse(lookupData.getBrokerUrl(), lookupData.getBrokerUrlTls(), + true /* authoritative */, LookupType.Connect, requestId)); + } + }).exceptionally(e -> { + log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, fqdn.toString(), + e.getMessage(), e); + lookupfuture.complete( + newLookupResponse(ServerError.ServiceNotReady, e.getMessage(), requestId)); + return null; + }); + } + + }).exceptionally(ex -> { + log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, fqdn.toString(), ex.getMessage(), + ex); + lookupfuture.complete(newLookupResponse(ServerError.ServiceNotReady, ex.getMessage(), requestId)); + return null; + }); + + return lookupfuture; + } + private static final Logger log = LoggerFactory.getLogger(DestinationLookup.class); } diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/lookup/LookupResult.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/lookup/LookupResult.java index 421af84afb029..2f96147cb07c9 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/lookup/LookupResult.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/lookup/LookupResult.java @@ -30,41 +30,41 @@ */ public class LookupResult { enum Type { - BrokerUrl, HttpRedirectUrl + BrokerUrl, RedirectUrl } private final Type type; private final LookupData lookupData; - private final URI httpRedirectAddress; public LookupResult(NamespaceEphemeralData namespaceEphemeralData) { this.type = Type.BrokerUrl; this.lookupData = new LookupData(namespaceEphemeralData.getNativeUrl(), - namespaceEphemeralData.getNativeUrlTls(), namespaceEphemeralData.getHttpUrl()); - this.httpRedirectAddress = null; + namespaceEphemeralData.getNativeUrlTls(), namespaceEphemeralData.getHttpUrl(), + namespaceEphemeralData.getHttpUrlTls()); } - public LookupResult(URI httpRedirectAddress) { - this.type = Type.HttpRedirectUrl; - this.lookupData = null; - this.httpRedirectAddress = httpRedirectAddress; + public LookupResult(String httpUrl, String httpUrlTls, String brokerServiceUrl, String brokerServiceUrlTls) { + this.type = Type.RedirectUrl; // type = reidrect => as current broker is + // not owner and prepares LookupResult + // with other broker's urls + this.lookupData = new LookupData(brokerServiceUrl, brokerServiceUrlTls, httpUrl, httpUrlTls); } public boolean isBrokerUrl() { return type == Type.BrokerUrl; } - public boolean isHttpRedirect() { - return type == Type.HttpRedirectUrl; + public boolean isRedirect() { + return type == Type.RedirectUrl; } public LookupData getLookupData() { - checkArgument(isBrokerUrl()); return lookupData; } - public URI getHttpRedirectAddress() { - checkArgument(isHttpRedirect()); - return httpRedirectAddress; + @Override + public String toString() { + return "LookupResult [type=" + type + ", lookupData=" + lookupData + "]"; } + } diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/namespace/NamespaceService.java index 63c7c78a93632..658b304671a85 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/namespace/NamespaceService.java @@ -22,11 +22,8 @@ import static com.yahoo.pulsar.common.naming.NamespaceBundleFactory.getBundlesData; import static java.lang.String.format; -import java.net.MalformedURLException; import java.net.URI; -import java.net.URISyntaxException; import java.net.URL; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -37,6 +34,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.zookeeper.AsyncCallback.StatCallback; import org.apache.zookeeper.KeeperException; @@ -51,6 +49,7 @@ import com.yahoo.pulsar.broker.ServiceConfiguration; import com.yahoo.pulsar.broker.admin.AdminResource; import com.yahoo.pulsar.broker.loadbalance.LoadManager; +import com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport; import com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl; import com.yahoo.pulsar.broker.lookup.LookupResult; import com.yahoo.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; @@ -70,6 +69,8 @@ import com.yahoo.pulsar.common.policies.impl.NamespaceIsolationPolicies; import com.yahoo.pulsar.common.util.Codec; import com.yahoo.pulsar.common.util.ObjectMapperFactory; +import com.yahoo.pulsar.zookeeper.ZooKeeperCache.Deserializer; +import static com.yahoo.pulsar.broker.admin.AdminResource.jsonMapper; /** * The NamespaceService provides resource ownership lookup as well as resource ownership claiming services @@ -147,56 +148,51 @@ private NamespaceBundle getFullBundle(NamespaceName fqnn) throws Exception { return bundleFactory.getFullBundle(fqnn); } - public URL getWebServiceUrl(ServiceUnitId suName, boolean authoritative, boolean readOnly) throws Exception { + private static final Deserializer loadReportDeserializer = new Deserializer() { + @Override + public LoadReport deserialize(String key, byte[] content) throws Exception { + return jsonMapper().readValue(content, LoadReport.class); + } + }; + + public URL getWebServiceUrl(ServiceUnitId suName, boolean authoritative, boolean isRequestHttps, boolean readOnly) + throws Exception { if (suName instanceof DestinationName) { DestinationName name = (DestinationName) suName; if (LOG.isDebugEnabled()) { LOG.debug("Getting web service URL of destination: {} - auth: {}", name, authoritative); } - - return this.internalGetWebServiceUrl(getBundle(name), authoritative, readOnly).get(); + return this.internalGetWebServiceUrl(getBundle(name), authoritative, isRequestHttps, readOnly).get(); } if (suName instanceof NamespaceName) { - return this.internalGetWebServiceUrl(getFullBundle((NamespaceName) suName), authoritative, readOnly).get(); + return this.internalGetWebServiceUrl(getFullBundle((NamespaceName) suName), authoritative, isRequestHttps, readOnly).get(); } if (suName instanceof NamespaceBundle) { - return this.internalGetWebServiceUrl((NamespaceBundle) suName, authoritative, readOnly).get(); + return this.internalGetWebServiceUrl((NamespaceBundle) suName, authoritative, isRequestHttps, readOnly).get(); } throw new IllegalArgumentException("Unrecognized class of NamespaceBundle: " + suName.getClass().getName()); } private CompletableFuture internalGetWebServiceUrl(NamespaceBundle bundle, boolean authoritative, - boolean readOnly) { + boolean isRequestHttps, boolean readOnly) { return findBrokerServiceUrl(bundle, authoritative, readOnly).thenApply(lookupResult -> { - if (lookupResult == null) { - return null; - } - - try { - if (lookupResult.isBrokerUrl()) { - // Somebody already owns the service unit + if (lookupResult != null) { + try { LookupData lookupData = lookupResult.getLookupData(); - if (lookupData.getHttpUrl() != null) { - // If the broker uses the new format, we know the correct address - return new URL(lookupData.getHttpUrl()); - } else { - // Fallback to use same port as current broker - URI brokerAddress = new URI(lookupData.getBrokerUrl()); - String host = brokerAddress.getHost(); - int port = config.getWebServicePort(); - return new URL(String.format("http://%s:%s", host, port)); - } - } else { - // We have the HTTP address to redirect to - return lookupResult.getHttpRedirectAddress().toURL(); + final String redirectUrl = isRequestHttps ? lookupData.getHttpUrlTls() : lookupData.getHttpUrl(); + return new URL(redirectUrl); + + } catch (Exception e) { + // just log the exception, nothing else to do + LOG.warn("internalGetWebServiceUrl [{}]", e.getMessage(), e); } - } catch (MalformedURLException | URISyntaxException e) { - throw new RuntimeException(e); + } + return null; }); } @@ -391,7 +387,12 @@ private void searchForCandidateBroker(NamespaceBundle bundle, CompletableFuture< } // Now setting the redirect url - lookupFuture.complete(new LookupResult(new URI(candidateBroker))); + createLookupResult(candidateBroker).thenAccept(lookupResult -> lookupFuture.complete(lookupResult)) + .exceptionally(ex -> { + lookupFuture.completeExceptionally(ex); + return null; + }); + } } catch (Exception e) { LOG.warn("Error in trying to acquire namespace bundle ownership for {}: {}", bundle, e.getMessage(), e); @@ -399,6 +400,32 @@ private void searchForCandidateBroker(NamespaceBundle bundle, CompletableFuture< } } + private CompletableFuture createLookupResult(String candidateBroker) throws Exception { + + CompletableFuture lookupFuture = new CompletableFuture<>(); + try { + checkArgument(StringUtils.isNotBlank(candidateBroker), "Lookup broker can't be null " + candidateBroker); + URI uri = new URI(candidateBroker); + String path = String.format("%s/%s:%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, uri.getHost(), + uri.getPort()); + pulsar.getLocalZkCache().getDataAsync(path, loadReportDeserializer).thenAccept(reportData -> { + if (reportData.isPresent()) { + LoadReport report = reportData.get(); + lookupFuture.complete(new LookupResult(report.getWebServiceUrl(), report.getWebServiceUrlTls(), + report.getPulsarServiceUrl(), report.getPulsarServieUrlTls())); + } else { + lookupFuture.completeExceptionally(new KeeperException.NoNodeException(path)); + } + }).exceptionally(ex -> { + lookupFuture.completeExceptionally(ex); + return null; + }); + } catch (Exception e) { + lookupFuture.completeExceptionally(e); + } + return lookupFuture; + } + private boolean isBrokerActive(String candidateBroker) throws KeeperException, InterruptedException { Set activeNativeBrokers = pulsar.getLocalZkCache() .getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT); diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/Consumer.java index a6aaad0b0edf1..60c5777d4fcb6 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/Consumer.java @@ -448,7 +448,6 @@ public ConcurrentOpenHashMap getPendingAcks() { private static final Logger log = LoggerFactory.getLogger(Consumer.class); public void redeliverUnacknowledgedMessages() { - // cleanup unackedMessage bucket and redeliver those unack-msgs again unackedMessages.set(0); blockedConsumerOnUnackedMsgs = false; diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/ServerCnx.java index 5c737c4ed6993..9c18e5a7d3fba 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/ServerCnx.java @@ -16,6 +16,10 @@ package com.yahoo.pulsar.broker.service; import static com.google.common.base.Preconditions.checkArgument; +import static com.yahoo.pulsar.broker.admin.PersistentTopics.getPartitionedTopicMetadata; +import static com.yahoo.pulsar.broker.lookup.DestinationLookup.lookupDestinationAsync; +import static com.yahoo.pulsar.common.api.Commands.newLookupResponse; +import static com.yahoo.pulsar.common.api.proto.PulsarApi.ProtocolVersion.v5; import java.net.SocketAddress; import java.util.concurrent.CompletableFuture; @@ -29,6 +33,7 @@ import com.yahoo.pulsar.broker.authentication.AuthenticationDataCommand; import com.yahoo.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; +import com.yahoo.pulsar.client.api.PulsarClientException; import com.yahoo.pulsar.common.api.Commands; import com.yahoo.pulsar.common.api.PulsarHandler; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandAck; @@ -36,6 +41,8 @@ import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandCloseProducer; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandConnect; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandFlow; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandProducer; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandRedeliverUnacknowledgedMessages; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandSend; @@ -43,7 +50,6 @@ import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandUnsubscribe; import com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata; -import static com.yahoo.pulsar.common.api.proto.PulsarApi.ProtocolVersion.v5; import com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError; import com.yahoo.pulsar.common.naming.DestinationName; import com.yahoo.pulsar.common.policies.data.BacklogQuota; @@ -138,7 +144,51 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E // //// // // Incoming commands handling // //// + + + @Override + protected void handleLookup(CommandLookupTopic lookup) { + if (log.isDebugEnabled()) { + log.debug("Received Lookup from {}", remoteAddress); + } + lookupDestinationAsync(getBrokerService().pulsar(), DestinationName.get(lookup.getTopic()), + lookup.getAuthoritative(), getRole(), lookup.getRequestId()).thenAccept(lookupResponse -> { + ctx.writeAndFlush(lookupResponse); + }).exceptionally(ex -> { + // it should never happen + log.warn("[{}] lookup failed with error {}", remoteAddress, ex.getMessage(), ex); + ctx.writeAndFlush( + newLookupResponse(ServerError.ServiceNotReady, ex.getMessage(), lookup.getRequestId())); + return null; + }); + + } + @Override + protected void handlePartitionMetadataRequest(CommandPartitionedTopicMetadata partitionMetadata) { + if (log.isDebugEnabled()) { + log.debug("Received PartitionMetadataLookup from {}", remoteAddress); + } + getPartitionedTopicMetadata(getBrokerService().pulsar(), getRole(), + DestinationName.get(partitionMetadata.getTopic())).thenAccept(metadata -> { + int partitions = metadata.partitions; + ctx.writeAndFlush( + Commands.newPartitionMetadataResponse(partitions, partitionMetadata.getRequestId())); + }).exceptionally(ex -> { + if (ex instanceof PulsarClientException) { + log.warn("Failed to authorize {} at [{}] on topic {} : {}", getRole(), remoteAddress, + partitionMetadata.getTopic(), ex.getMessage()); + ctx.writeAndFlush(Commands.newPartitionMetadataResponse(ServerError.AuthorizationError, + ex.getMessage(), partitionMetadata.getRequestId())); + } else { + log.warn("Failed to get Partitioned Metadata [{}] {}: {}", remoteAddress, ex.getMessage()); + ctx.writeAndFlush(Commands.newPartitionMetadataResponse(ServerError.ServiceNotReady, + ex.getMessage(), partitionMetadata.getRequestId())); + } + return null; + }); + } + @Override protected void handleConnect(CommandConnect connect) { checkArgument(state == State.Start); diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/web/PulsarWebResource.java index c36c6cf69e84f..d520945b1210f 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/web/PulsarWebResource.java @@ -16,10 +16,13 @@ package com.yahoo.pulsar.broker.web; import static com.google.common.base.Preconditions.checkArgument; +import static com.yahoo.pulsar.common.api.Commands.newLookupResponse; import java.net.URI; import java.net.URL; import java.util.Iterator; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; @@ -43,6 +46,7 @@ import com.yahoo.pulsar.broker.admin.AdminResource; import com.yahoo.pulsar.broker.admin.Namespaces; import com.yahoo.pulsar.broker.namespace.NamespaceService; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType; import com.yahoo.pulsar.common.naming.DestinationName; import com.yahoo.pulsar.common.naming.NamespaceBundle; import com.yahoo.pulsar.common.naming.NamespaceBundles; @@ -113,9 +117,13 @@ public static String splitPath(String source, int slice) { public String clientAppId() { return (String) httpRequest.getAttribute(AuthenticationFilter.AuthenticatedRoleAttributeName); } + + public boolean isRequestHttps() { + return "https".equalsIgnoreCase(httpRequest.getScheme()); + } - public boolean isClientAuthenticated() { - return clientAppId() != null; + public static boolean isClientAuthenticated(String appId) { + return appId != null; } /** @@ -127,9 +135,10 @@ public boolean isClientAuthenticated() { protected void validateSuperUserAccess() { if (config().isAuthenticationEnabled()) { String appId = clientAppId(); - log.debug("[{}] Check super user access: Authenticated: {} -- Role: {}", uri.getRequestUri(), - isClientAuthenticated(), appId); - + if(log.isDebugEnabled()) { + log.debug("[{}] Check super user access: Authenticated: {} -- Role: {}", uri.getRequestUri(), + isClientAuthenticated(appId), appId); + } if (!config().getSuperUserRoles().contains(appId)) { throw new RestException(Status.UNAUTHORIZED, "This operation requires super-user access"); } @@ -145,35 +154,41 @@ protected void validateSuperUserAccess() { * if not authorized */ protected void validateAdminAccessOnProperty(String property) { - if (config().isAuthenticationEnabled() && config().isAuthorizationEnabled()) { - String appId = clientAppId(); + validateAdminAccessOnProperty(pulsar(), clientAppId(), property); + } + + protected static void validateAdminAccessOnProperty(PulsarService pulsar, String clientAppId, String property) { + if (pulsar.getConfiguration().isAuthenticationEnabled() && pulsar.getConfiguration().isAuthorizationEnabled()) { log.debug("check admin access on property: {} - Authenticated: {} -- role: {}", property, - isClientAuthenticated(), appId); + (isClientAuthenticated(clientAppId)), clientAppId); - if (!isClientAuthenticated()) { + if (!isClientAuthenticated(clientAppId)) { throw new RestException(Status.FORBIDDEN, "Need to authenticate to perform the request"); } - if (config().getSuperUserRoles().contains(appId)) { + if (pulsar.getConfiguration().getSuperUserRoles().contains(clientAppId)) { // Super-user has access to configure all the policies - log.debug("granting access to super-user {} on property {}", appId, property); + log.debug("granting access to super-user {} on property {}", clientAppId, property); } else { PropertyAdmin propertyAdmin; try { - propertyAdmin = pulsar().getConfigurationCache().propertiesCache().get(path("policies", property)) + propertyAdmin = pulsar.getConfigurationCache().propertiesCache().get(path("policies", property)) .orElseThrow(() -> new RestException(Status.UNAUTHORIZED, "Property does not exist")); + } catch (KeeperException.NoNodeException e) { + log.warn("Failed to get property admin data for non existing property {}", property); + throw new RestException(Status.UNAUTHORIZED, "Property does not exist"); } catch (Exception e) { log.error("Failed to get property admin data for property"); throw new RestException(e); } - if (!propertyAdmin.getAdminRoles().contains(appId)) { + if (!propertyAdmin.getAdminRoles().contains(clientAppId)) { throw new RestException(Status.UNAUTHORIZED, "Don't have permission to administrate resources on this property"); } - log.debug("Successfully authorized {} on property {}", appId, property); + log.debug("Successfully authorized {} on property {}", clientAppId, property); } } } @@ -207,45 +222,83 @@ protected void validateClusterForProperty(String property, String cluster) { * In case the redirect happens */ protected void validateClusterOwnership(String cluster) throws WebApplicationException { - // If the cluster name is "global", don't validate the cluster ownership. - // The validation will be done by checking the namespace configuration - if (cluster.equals(Namespaces.GLOBAL_CLUSTER)) { - return; - } - - if (!pulsar().getConfiguration().isAuthorizationEnabled()) { - // Without authorization, any cluster name should be valid and accepted by the broker - return; - } - try { - if (!pulsar().getConfigurationCache().clustersListCache().get().contains(cluster)) { - log.warn("[{}] Cluster does not exist: requested={}, registered={}", clientAppId(), cluster, - pulsar().getConfigurationCache().clustersListCache().get()); - throw new RestException(Status.NOT_FOUND, "Cluster does not exist: cluster=" + cluster); - } - - if (!config().getClusterName().equals(cluster)) { - // redirect to the cluster requested - ClusterData clusterData = pulsar().getConfigurationCache().clustersCache() - .get(path("clusters", cluster)).orElseThrow(() -> new RestException(Status.NOT_FOUND, - "Cluster does not exist: cluster=" + cluster)); + ClusterData differentClusterData = getClusterDataIfDifferentCluster(pulsar(), cluster, clientAppId()).get(); + if (differentClusterData != null) { URL webUrl; - if (config().isTlsEnabled() && !clusterData.getServiceUrlTls().isEmpty()) { - webUrl = new URL(clusterData.getServiceUrlTls()); + if (pulsar.getConfiguration().isTlsEnabled() && !differentClusterData.getServiceUrlTls().isEmpty()) { + webUrl = new URL(differentClusterData.getServiceUrlTls()); } else { - webUrl = new URL(clusterData.getServiceUrl()); + webUrl = new URL(differentClusterData.getServiceUrl()); } URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(webUrl.getHost()).port(webUrl.getPort()) .build(); - log.debug("[{}] Redirecting the rest call to {}: cluster={}", clientAppId(), redirect, cluster); + if (log.isDebugEnabled()) { + log.debug("[{}] Redirecting the rest call to {}: cluster={}", clientAppId(), redirect, cluster); + + } + // redirect to the cluster requested throw new WebApplicationException(Response.temporaryRedirect(redirect).build()); } } catch (WebApplicationException wae) { throw wae; } catch (Exception e) { - throw new RestException(e); + if (e.getCause() instanceof WebApplicationException) { + throw (WebApplicationException) e.getCause(); + } + throw new RestException(Status.SERVICE_UNAVAILABLE, String + .format("Failed to validate Cluster configuration : cluster=%s emsg=%s", cluster, e.getMessage())); } + + } + + protected static CompletableFuture getClusterDataIfDifferentCluster(PulsarService pulsar, + String cluster, String clientAppId) { + + CompletableFuture clusterDataFuture = new CompletableFuture<>(); + + if (!isValidCluster(pulsar, cluster)) { + try { + if (!pulsar.getConfiguration().getClusterName().equals(cluster)) { + // redirect to the cluster requested + pulsar.getConfigurationCache().clustersCache().getAsync(path("clusters", cluster)) + .thenAccept(clusterDataResult -> { + if (clusterDataResult.isPresent()) { + clusterDataFuture.complete(clusterDataResult.get()); + } else { + log.warn("[{}] Cluster does not exist: requested={}", clientAppId, cluster); + clusterDataFuture.completeExceptionally(new RestException(Status.NOT_FOUND, + "Cluster does not exist: cluster=" + cluster)); + } + }).exceptionally(ex -> { + clusterDataFuture.completeExceptionally(ex); + return null; + }); + } else { + clusterDataFuture.complete(null); + } + } catch (Exception e) { + clusterDataFuture.completeExceptionally(e); + } + } else { + clusterDataFuture.complete(null); + } + return clusterDataFuture; + } + + protected static boolean isValidCluster(PulsarService pulsarSevice, String cluster) {// If the cluster name is + // "global", don't validate the + // cluster ownership. + // The validation will be done by checking the namespace configuration + if (cluster.equals(Namespaces.GLOBAL_CLUSTER)) { + return true; + } + + if (!pulsarSevice.getConfiguration().isAuthorizationEnabled()) { + // Without authorization, any cluster name should be valid and accepted by the broker + return true; + } + return false; } /** @@ -342,7 +395,7 @@ public void validateBundleOwnership(NamespaceBundle bundle, boolean authoritativ // - If authoritative is false and this broker is not leader, forward to leader // - If authoritative is false and this broker is leader, determine owner and forward w/ authoritative=true // - If authoritative is true, own the namespace and continue - URL webUrl = nsService.getWebServiceUrl(bundle, authoritative, readOnly); + URL webUrl = nsService.getWebServiceUrl(bundle, authoritative, isRequestHttps(), readOnly); // Ensure we get a url if (webUrl == null) { log.warn("Unable to get web service url"); @@ -394,7 +447,7 @@ protected void validateDestinationOwnership(DestinationName fqdn, boolean author try { // per function name, this is trying to acquire the whole namespace ownership - URL webUrl = nsService.getWebServiceUrl(fqdn, authoritative, false); + URL webUrl = nsService.getWebServiceUrl(fqdn, authoritative, isRequestHttps(), false); // Ensure we get a url if (webUrl == null) { log.info("Unable to get web service url"); @@ -402,7 +455,7 @@ protected void validateDestinationOwnership(DestinationName fqdn, boolean author } if (!nsService.isServiceUnitOwned(fqdn)) { - boolean newAuthoritative = this.isLeaderBroker(); + boolean newAuthoritative = this.isLeaderBroker(pulsar()); // Replace the host and port of the current request and redirect URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(webUrl.getHost()).port(webUrl.getPort()) .replaceQueryParam("authoritative", newAuthoritative).build(); @@ -427,7 +480,7 @@ protected void validateDestinationOwnership(DestinationName fqdn, boolean author protected void validateReplicationSettingsOnNamespace(String property, String cluster, String namespace) { NamespaceName namespaceName = new NamespaceName(property, cluster, namespace); - validateReplicationSettingsOnNamespace(namespaceName); + validateReplicationSettingsOnNamespace(pulsar(), namespaceName); } /** @@ -435,66 +488,91 @@ protected void validateReplicationSettingsOnNamespace(String property, String cl * namespace 2. If local cluster belonging to this namespace is replicated 3. If replication is enabled for this * namespace * + * @param pulsarService * @param namespace * @throws Exception */ + protected static void validateReplicationSettingsOnNamespace(PulsarService pulsarService, NamespaceName namespace) { + try { + validateReplicationSettingsOnNamespaceAsync(pulsarService, namespace).get(); + } catch (Exception e) { + if(e.getCause() instanceof WebApplicationException) { + throw (WebApplicationException) e.getCause(); + } + throw new RestException(Status.SERVICE_UNAVAILABLE, String.format( + "Failed to validate global cluster configuration : ns=%s emsg=%s", namespace, e.getMessage())); + } + } + + protected static CompletableFuture validateReplicationSettingsOnNamespaceAsync(PulsarService pulsarService, + NamespaceName namespace) { - protected void validateReplicationSettingsOnNamespace(NamespaceName namespace) { - if (namespace.isGlobal()) { - String localCluster = pulsar().getConfiguration().getClusterName(); - Policies policies; - try { - String path = AdminResource.path("policies", namespace.getProperty(), namespace.getCluster(), - namespace.getLocalName()); - policies = pulsar().getConfigurationCache().policiesCache().get(path) - .orElseThrow(() -> new KeeperException.NoNodeException(path)); - - if (policies.replication_clusters.isEmpty()) { - String msg = String.format( - "Global namespace does not have any clusters configured : local_cluster=%s ns=%s", - localCluster, namespace.toString()); + CompletableFuture validationFuture = new CompletableFuture<>(); - log.warn(msg); + if (namespace.isGlobal()) { + String localCluster = pulsarService.getConfiguration().getClusterName(); + + String path = AdminResource.path("policies", namespace.getProperty(), namespace.getCluster(), + namespace.getLocalName()); + + pulsarService.getConfigurationCache().policiesCache().getAsync(path).thenAccept(policiesResult -> { + + if (policiesResult.isPresent()) { + Policies policies = policiesResult.get(); + if (policies.replication_clusters.isEmpty()) { + String msg = String.format( + "Global namespace does not have any clusters configured : local_cluster=%s ns=%s", + localCluster, namespace.toString()); + log.warn(msg); + validationFuture.completeExceptionally(new RestException(Status.PRECONDITION_FAILED, msg)); + } else if (!policies.replication_clusters.contains(localCluster)) { + String msg = String.format( + "Global namespace missing local cluster name in replication list : local_cluster=%s ns=%s repl_clusters=%s", + localCluster, namespace.toString(), policies.replication_clusters); + + log.warn(msg); + // TODO: when we have a fail-over policy defined, we should find the next cluster in the + // replication + // clusters to re-direct the request to + validationFuture.completeExceptionally(new RestException(Status.PRECONDITION_FAILED, msg)); + } else { + validationFuture.complete(null); + } - throw new RestException(Status.PRECONDITION_FAILED, msg); + } else { + String msg = String.format("Policies not found for %s namespace", namespace.toString()); + log.error(msg); + validationFuture.completeExceptionally(new RestException(Status.NOT_FOUND, msg)); } - if (!policies.replication_clusters.contains(localCluster)) { - String msg = String.format( - "Global namespace missing local cluster name in replication list : local_cluster=%s ns=%s repl_clusters=%s", - localCluster, namespace.toString(), policies.replication_clusters); - - log.warn(msg); - - // TODO: when we have a fail-over policy defined, we should find the next cluster in the replication - // clusters to re-direct the request to - throw new RestException(Status.PRECONDITION_FAILED, msg); - } - } catch (RestException re) { - // pass-through the exception - throw re; - } catch (Exception e) { + }).exceptionally(ex -> { String msg = String.format( "Failed to validate global cluster configuration : cluster=%s ns=%s emsg=%s", localCluster, - namespace, e.getMessage()); - + namespace, ex.getMessage()); log.error(msg); + validationFuture.completeExceptionally(new RestException(ex)); + return null; + }); - throw new RestException(e); - } + } else { + validationFuture.complete(null); } + + return validationFuture; } protected void checkConnect(DestinationName destination) { - if (!pulsar().getConfiguration().isAuthorizationEnabled()) { + checkAuthorization(pulsar(), destination, clientAppId()); + } + + protected static void checkAuthorization(PulsarService pulsarService, DestinationName destination, String role) { + if (!pulsarService.getConfiguration().isAuthorizationEnabled()) { // No enforcing of authorization policies return; } - - String role = clientAppId(); try { // get zk policy manager - if (!pulsar().getBrokerService().getAuthorizationManager().canLookup(destination, role)) { + if (!pulsarService.getBrokerService().getAuthorizationManager().canLookup(destination, role)) { log.warn("[{}] Role {} is not allowed to lookup topic", destination, role); throw new RestException(Status.UNAUTHORIZED, "Don't have permission to connect to this namespace"); } @@ -515,10 +593,14 @@ public void setPulsar(PulsarService pulsar) { } protected boolean isLeaderBroker() { + return isLeaderBroker(pulsar()); + } + + protected static boolean isLeaderBroker(PulsarService pulsar) { - String leaderAddress = pulsar().getLeaderElectionService().getCurrentLeader().getServiceUrl(); + String leaderAddress = pulsar.getLeaderElectionService().getCurrentLeader().getServiceUrl(); - String myAddress = pulsar().getWebServiceAddress(); + String myAddress = pulsar.getWebServiceAddress(); return myAddress.equals(leaderAddress); // If i am the leader, my decisions are } diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/NamespacesTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/NamespacesTest.java index 29573f964478c..54f50ffa9077a 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/NamespacesTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/NamespacesTest.java @@ -96,7 +96,7 @@ public NamespacesTest() { } @BeforeClass - public void init() throws Exception { + public void initNamespace() throws Exception { testLocalNamespaces = Lists.newArrayList(); testGlobalNamespaces = Lists.newArrayList(); @@ -123,6 +123,7 @@ public void setup() throws Exception { doReturn(mockZookKeeper).when(namespaces).localZk(); doReturn(pulsar.getConfigurationCache().propertiesCache()).when(namespaces).propertiesCache(); doReturn(pulsar.getConfigurationCache().policiesCache()).when(namespaces).policiesCache(); + doReturn(false).when(namespaces).isRequestHttps(); doReturn("test").when(namespaces).clientAppId(); doReturn(Sets.newTreeSet(Lists.newArrayList("use", "usw", "usc", "global"))).when(namespaces).clusters(); doNothing().when(namespaces).validateAdminAccessOnProperty("my-property"); @@ -557,7 +558,7 @@ public void _dont_implement_Matcher___instead_extend_BaseMatcher_() { } - }), Mockito.anyBoolean(), Mockito.anyBoolean()); + }), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean()); admin.namespaces().setNamespaceReplicationClusters(testGlobalNamespaces.get(0).toString(), Lists.newArrayList("usw")); @@ -596,7 +597,7 @@ public void testDeleteNamespaces() throws Exception { // setup ownership to localhost URL localWebServiceUrl = new URL(pulsar.getWebServiceAddress()); - doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(testNs, false, false); + doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(testNs, false, false, false); doReturn(true).when(nsSvc).isServiceUnitOwned(testNs); try { namespaces.deleteNamespace(testNs.getProperty(), testNs.getCluster(), testNs.getLocalName(), false); @@ -608,13 +609,13 @@ public void testDeleteNamespaces() throws Exception { testNs = this.testGlobalNamespaces.get(0); // setup ownership to localhost - doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(testNs, false, false); + doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(testNs, false, false, false); doReturn(true).when(nsSvc).isServiceUnitOwned(testNs); namespaces.deleteNamespace(testNs.getProperty(), testNs.getCluster(), testNs.getLocalName(), false); testNs = this.testLocalNamespaces.get(0); // setup ownership to localhost - doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(testNs, false, false); + doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(testNs, false, false, false); doReturn(true).when(nsSvc).isServiceUnitOwned(testNs); namespaces.deleteNamespace(testNs.getProperty(), testNs.getCluster(), testNs.getLocalName(), false); List nsList = Lists.newArrayList(this.testLocalNamespaces.get(1).toString(), @@ -635,7 +636,7 @@ public void testDeleteNamespaces() throws Exception { // ensure refreshed destination list in the cache pulsar.getLocalZkCacheService().managedLedgerListCache().clearTree(); // setup ownership to localhost - doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(testNs, false, false); + doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(testNs, false, false, false); doReturn(true).when(nsSvc).isServiceUnitOwned(testNs); namespaces.deleteNamespace(testNs.getProperty(), testNs.getCluster(), testNs.getLocalName(), false); } @@ -670,7 +671,7 @@ public boolean matches(Object item) { public void _dont_implement_Matcher___instead_extend_BaseMatcher_() { } - }), Mockito.anyBoolean(), Mockito.anyBoolean()); + }), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean()); doReturn(false).when(nsSvc).isServiceUnitOwned(Mockito.argThat(new Matcher() { @Override @@ -691,25 +692,26 @@ public void _dont_implement_Matcher___instead_extend_BaseMatcher_() { } })); - doReturn(Optional.of(new NamespaceEphemeralData())).when(nsSvc).getOwner(Mockito.argThat(new Matcher() { + doReturn(Optional.of(new NamespaceEphemeralData())).when(nsSvc) + .getOwner(Mockito.argThat(new Matcher() { - @Override - public void describeTo(Description description) { - } + @Override + public void describeTo(Description description) { + } - @Override - public boolean matches(Object item) { - if (item instanceof NamespaceBundle) { - NamespaceBundle bundle = (NamespaceBundle) item; - return bundle.getNamespaceObject().equals(testNs); - } - return false; - } + @Override + public boolean matches(Object item) { + if (item instanceof NamespaceBundle) { + NamespaceBundle bundle = (NamespaceBundle) item; + return bundle.getNamespaceObject().equals(testNs); + } + return false; + } - @Override - public void _dont_implement_Matcher___instead_extend_BaseMatcher_() { - } - })); + @Override + public void _dont_implement_Matcher___instead_extend_BaseMatcher_() { + } + })); doThrow(new PulsarAdminException.PreconditionFailedException( new ClientErrorException(Status.PRECONDITION_FAILED))).when(namespacesAdmin) @@ -732,7 +734,7 @@ public void _dont_implement_Matcher___instead_extend_BaseMatcher_() { NamespaceBundles nsBundles = nsSvc.getNamespaceBundleFactory().getBundles(testNs, bundleData); // make one bundle owned - doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(nsBundles.getBundles().get(0), false, false); + doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(nsBundles.getBundles().get(0), false, true, false); doReturn(true).when(nsSvc).isServiceUnitOwned(nsBundles.getBundles().get(0)); doNothing().when(namespacesAdmin).deleteNamespaceBundle( testProperty + "/" + testLocalCluster + "/" + bundledNsLocal, "0x00000000_0x80000000"); @@ -754,7 +756,7 @@ public void _dont_implement_Matcher___instead_extend_BaseMatcher_() { // ensure all three bundles are owned by the local broker for (NamespaceBundle bundle : nsBundles.getBundles()) { - doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(bundle, false, false); + doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(bundle, false, true, false); doReturn(true).when(nsSvc).isServiceUnitOwned(bundle); } doNothing().when(namespacesAdmin).deleteNamespaceBundle(Mockito.anyString(), Mockito.anyString()); @@ -787,7 +789,7 @@ public void _dont_implement_Matcher___instead_extend_BaseMatcher_() { } - }), Mockito.anyBoolean(), Mockito.anyBoolean()); + }), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean()); doReturn(true).when(nsSvc).isServiceUnitOwned(Mockito.argThat(new Matcher() { @Override @@ -889,7 +891,7 @@ public void _dont_implement_Matcher___instead_extend_BaseMatcher_() { } - }), Mockito.anyBoolean(), Mockito.anyBoolean()); + }), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean()); doReturn(true).when(nsSvc).isServiceUnitOwned(Mockito.argThat(new Matcher() { @Override @@ -918,7 +920,7 @@ public void _dont_implement_Matcher___instead_extend_BaseMatcher_() { NamespaceBundles nsBundles = nsSvc.getNamespaceBundleFactory().getBundles(testNs, bundleData); NamespaceBundle testBundle = nsBundles.getBundles().get(0); // make one bundle owned - doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(testBundle, false, false); + doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(testBundle, false, true, false); doReturn(true).when(nsSvc).isServiceUnitOwned(testBundle); doNothing().when(nsSvc).unloadNamespaceBundle(testBundle); namespaces.unloadNamespaceBundle(testProperty, testLocalCluster, bundledNsLocal, "0x00000000_0x80000000", @@ -1034,6 +1036,8 @@ public void testValidateDestinationOwnership() throws Exception { PersistentTopics topics = spy(new PersistentTopics()); topics.setServletContext(new MockServletContext()); topics.setPulsar(pulsar); + doReturn(false).when(topics).isRequestHttps(); + doReturn("test").when(topics).clientAppId(); mockWebUrl(localWebServiceUrl, testNs); try { @@ -1077,7 +1081,7 @@ public boolean matches(Object item) { @Override public void _dont_implement_Matcher___instead_extend_BaseMatcher_() { } - }), Mockito.anyBoolean(), Mockito.anyBoolean()); + }), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean()); doReturn(true).when(nsSvc).isServiceUnitOwned(Mockito.argThat(new Matcher() { @Override public void describeTo(Description description) { diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index af7bdc01a6a40..87fa2621f9a7c 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -19,6 +19,7 @@ import static org.mockito.Mockito.spy; import java.io.IOException; +import java.net.URI; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -67,6 +68,7 @@ public abstract class MockedPulsarServiceBaseTest { protected MockZooKeeper mockZookKeeper; protected NonClosableMockBookKeeper mockBookKeeper; + protected boolean isTcpLookup = false; private SameThreadOrderedSafeExecutor sameThreadOrderedSafeExecutor; @@ -84,17 +86,25 @@ protected final void internalSetup() throws Exception { init(); com.yahoo.pulsar.client.api.ClientConfiguration clientConf = new com.yahoo.pulsar.client.api.ClientConfiguration(); clientConf.setStatsInterval(0, TimeUnit.SECONDS); - pulsarClient = PulsarClient.create(brokerUrl.toString(), clientConf); + String lookupUrl = brokerUrl.toString(); + if (isTcpLookup) { + lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString(); + } + pulsarClient = PulsarClient.create(lookupUrl, clientConf); } protected final void internalSetupForStatsTest() throws Exception { init(); com.yahoo.pulsar.client.api.ClientConfiguration clientConf = new com.yahoo.pulsar.client.api.ClientConfiguration(); clientConf.setStatsInterval(1, TimeUnit.SECONDS); - pulsarClient = PulsarClient.create(brokerUrl.toString(), clientConf); + String lookupUrl = brokerUrl.toString(); + if (isTcpLookup) { + lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString(); + } + pulsarClient = PulsarClient.create(lookupUrl, clientConf); } - private final void init() throws Exception { + protected final void init() throws Exception { mockZookKeeper = createMockZooKeeper(); mockBookKeeper = new NonClosableMockBookKeeper(new ClientConfiguration(), mockZookKeeper); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java index f0d7144e97f3e..68b644cd6ab8c 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java @@ -66,6 +66,8 @@ import com.yahoo.pulsar.client.admin.BrokerStats; import com.yahoo.pulsar.client.admin.PulsarAdmin; import com.yahoo.pulsar.client.api.Authentication; +import com.yahoo.pulsar.client.api.ClientConfiguration; +import com.yahoo.pulsar.client.api.PulsarClient; import com.yahoo.pulsar.common.naming.NamespaceName; import com.yahoo.pulsar.common.policies.data.AutoFailoverPolicyData; import com.yahoo.pulsar.common.policies.data.AutoFailoverPolicyType; @@ -485,4 +487,5 @@ public void testUsage() { usage.reset(); assertNotEquals(usage.getBandwidthIn().usage, usageLimit); } + } diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/lookup/http/HttpDestinationLookupv2Test.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/lookup/http/HttpDestinationLookupv2Test.java index 4d485d5977df8..0b10a86f5bbbf 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/lookup/http/HttpDestinationLookupv2Test.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/lookup/http/HttpDestinationLookupv2Test.java @@ -26,6 +26,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.CompletableFuture; import javax.ws.rs.WebApplicationException; import javax.ws.rs.container.AsyncResponse; @@ -33,6 +34,7 @@ import javax.ws.rs.core.UriInfo; import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -98,6 +100,9 @@ public void setUp() throws Exception { doReturn(Optional.of(useData)).when(clustersCache).get(AdminResource.path("clusters", "use")); doReturn(Optional.of(uscData)).when(clustersCache).get(AdminResource.path("clusters", "usc")); doReturn(Optional.of(uswData)).when(clustersCache).get(AdminResource.path("clusters", "usw")); + doReturn(CompletableFuture.completedFuture(Optional.of(useData))).when(clustersCache).getAsync(AdminResource.path("clusters", "use")); + doReturn(CompletableFuture.completedFuture(Optional.of(uscData))).when(clustersCache).getAsync(AdminResource.path("clusters", "usc")); + doReturn(CompletableFuture.completedFuture(Optional.of(uswData))).when(clustersCache).getAsync(AdminResource.path("clusters", "usw")); doReturn(clusters).when(clustersListCache).get(); doReturn(ns).when(pulsar).getNamespaceService(); BrokerService brokerService = mock(BrokerService.class); @@ -109,6 +114,7 @@ public void setUp() throws Exception { public void crossColoLookup() throws Exception { DestinationLookup destLookup = spy(new DestinationLookup()); + doReturn(false).when(destLookup).isRequestHttps(); destLookup.setPulsar(pulsar); doReturn("null").when(destLookup).clientAppId(); Field uriField = PulsarWebResource.class.getDeclaredField("uri"); @@ -145,6 +151,7 @@ public void testValidateReplicationSettingsOnNamespace() throws Exception { .get(AdminResource.path("policies", property, cluster, ns2)); DestinationLookup destLookup = spy(new DestinationLookup()); + doReturn(false).when(destLookup).isRequestHttps(); destLookup.setPulsar(pulsar); doReturn("null").when(destLookup).clientAppId(); Field uriField = PulsarWebResource.class.getDeclaredField("uri"); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/client/api/BrokerServiceLookupTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/client/api/BrokerServiceLookupTest.java new file mode 100644 index 0000000000000..fb76e10ecd3a5 --- /dev/null +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/client/api/BrokerServiceLookupTest.java @@ -0,0 +1,551 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.client.api; + +import static org.apache.bookkeeper.test.PortManager.nextFreePort; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; + +import java.io.InputStream; +import java.lang.reflect.Field; +import java.net.URI; +import java.net.URL; +import java.net.URLConnection; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.SecureRandom; +import java.security.cert.Certificate; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; + +import org.apache.bookkeeper.test.PortManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.yahoo.pulsar.broker.PulsarService; +import com.yahoo.pulsar.broker.ServiceConfiguration; +import com.yahoo.pulsar.broker.loadbalance.LoadManager; +import com.yahoo.pulsar.broker.loadbalance.impl.SimpleResourceUnit; +import com.yahoo.pulsar.broker.namespace.NamespaceService; +import com.yahoo.pulsar.client.api.ProducerConfiguration.MessageRoutingMode; +import com.yahoo.pulsar.client.impl.auth.AuthenticationTls; +import com.yahoo.pulsar.common.naming.DestinationName; +import com.yahoo.pulsar.common.naming.ServiceUnitId; +import com.yahoo.pulsar.common.policies.data.ClusterData; +import com.yahoo.pulsar.common.policies.data.PropertyAdmin; +import com.yahoo.pulsar.common.util.SecurityUtility; +import com.yahoo.pulsar.discovery.service.DiscoveryService; +import com.yahoo.pulsar.discovery.service.server.ServiceConfig; + +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; + +public class BrokerServiceLookupTest extends ProducerConsumerBase { + private static final Logger log = LoggerFactory.getLogger(BrokerServiceLookupTest.class); + + @BeforeMethod + @Override + protected void setup() throws Exception { + super.init(); + com.yahoo.pulsar.client.api.ClientConfiguration clientConf = new com.yahoo.pulsar.client.api.ClientConfiguration(); + clientConf.setStatsInterval(0, TimeUnit.SECONDS); + URI brokerServiceUrl = new URI("pulsar://localhost:" + BROKER_PORT); + pulsarClient = PulsarClient.create(brokerServiceUrl.toString(), clientConf); + super.producerBaseSetup(); + } + + @AfterMethod + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + + /** + * UsecaseL Multiple Broker => Lookup Redirection test + * + * 1. Broker1 is a leader + * 2. Lookup request reaches to Broker2 which redirects to leader (Broker1) with authoritative = false + * 3. Leader (Broker1) finds out least loaded broker as Broker2 and redirects request to Broker2 with authoritative = true + * 4. Broker2 receives final request to own a bundle with authoritative = true and client connects to Broker2 + * + * @throws Exception + */ + @Test + public void testMultipleBrokerLookup() throws Exception { + log.info("-- Starting {} test --", methodName); + + /**** start broker-2 ****/ + ServiceConfiguration conf2 = new ServiceConfiguration(); + conf2.setBrokerServicePort(PortManager.nextFreePort()); + conf2.setBrokerServicePortTls(PortManager.nextFreePort()); + conf2.setWebServicePort(PortManager.nextFreePort()); + conf2.setWebServicePortTls(PortManager.nextFreePort()); + conf2.setAdvertisedAddress("localhost"); + conf2.setClusterName(conf.getClusterName()); + PulsarService pulsar2 = startBroker(conf2); + pulsar.getLoadManager().writeLoadReportOnZookeeper(); + pulsar2.getLoadManager().writeLoadReportOnZookeeper(); + + + LoadManager loadManager1 = spy(pulsar.getLoadManager()); + LoadManager loadManager2 = spy(pulsar2.getLoadManager()); + Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager"); + loadManagerField.setAccessible(true); + + // mock: redirect request to leader [2] + doReturn(true).when(loadManager2).isCentralized(); + loadManagerField.set(pulsar2.getNamespaceService(), loadManager2); + + // mock: return Broker2 as a Least-loaded broker when leader receies request [3] + doReturn(true).when(loadManager1).isCentralized(); + SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getWebServiceAddress(), null); + doReturn(resourceUnit).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class)); + loadManagerField.set(pulsar.getNamespaceService(), loadManager1); + + /**** started broker-2 ****/ + + URI brokerServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort()); + PulsarClient pulsarClient2 = PulsarClient.create(brokerServiceUrl.toString(), new ClientConfiguration()); + + // load namespace-bundle by calling Broker2 + Consumer consumer = pulsarClient2.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", + new ConsumerConfiguration()); + Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", new ProducerConfiguration()); + + for (int i = 0; i < 10; i++) { + String message = "my-message-" + i; + producer.send(message.getBytes()); + } + + Message msg = null; + Set messageSet = Sets.newHashSet(); + for (int i = 0; i < 10; i++) { + msg = consumer.receive(5, TimeUnit.SECONDS); + String receivedMessage = new String(msg.getData()); + log.debug("Received message: [{}]", receivedMessage); + String expectedMessage = "my-message-" + i; + testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage); + } + // Acknowledge the consumption of all messages at once + consumer.acknowledgeCumulative(msg); + consumer.close(); + producer.close(); + + pulsarClient2.close(); + pulsar2.close(); + loadManager1 = null; + loadManager2 = null; + + } + + /** + * Usecase: Redirection due to different cluster + * 1. Broker1 runs on cluster: "use" and Broker2 runs on cluster: "use2" + * 2. Broker1 receives "use2" cluster request => Broker1 reads "/clusters" from global-zookkeeper and + * redirects request to Broker2 whch serves "use2" + * 3. Broker2 receives redirect request and own namespace bundle + * + * @throws Exception + */ + @Test + public void testMultipleBrokerDifferentClusterLookup() throws Exception { + log.info("-- Starting {} test --", methodName); + + /**** start broker-2 ****/ + final String newCluster = "use2"; + final String property = "my-property2"; + ServiceConfiguration conf2 = new ServiceConfiguration(); + conf2.setBrokerServicePort(PortManager.nextFreePort()); + conf2.setBrokerServicePortTls(PortManager.nextFreePort()); + conf2.setWebServicePort(PortManager.nextFreePort()); + conf2.setWebServicePortTls(PortManager.nextFreePort()); + conf2.setAdvertisedAddress("localhost"); + conf2.setClusterName(newCluster); // Broker2 serves newCluster + String broker2ServiceUrl = "pulsar://localhost:" + conf2.getBrokerServicePort(); + + admin.clusters().createCluster(newCluster, new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT, null, broker2ServiceUrl, null)); + admin.properties().createProperty(property, + new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet(newCluster))); + admin.namespaces().createNamespace(property + "/" + newCluster + "/my-ns"); + + + PulsarService pulsar2 = startBroker(conf2); + pulsar.getLoadManager().writeLoadReportOnZookeeper(); + pulsar2.getLoadManager().writeLoadReportOnZookeeper(); + + URI brokerServiceUrl = new URI(broker2ServiceUrl); + PulsarClient pulsarClient2 = PulsarClient.create(brokerServiceUrl.toString(), new ClientConfiguration()); + + // enable authorization: so, broker can validate cluster and redirect if finds different cluster + pulsar.getConfiguration().setAuthorizationEnabled(true); + + LoadManager loadManager2 = spy(pulsar2.getLoadManager()); + Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager"); + loadManagerField.setAccessible(true); + + // mock: return Broker2 as a Least-loaded broker when leader receies request + doReturn(true).when(loadManager2).isCentralized(); + SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getWebServiceAddress(), null); + doReturn(resourceUnit).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class)); + loadManagerField.set(pulsar.getNamespaceService(), loadManager2); + /**** started broker-2 ****/ + + // load namespace-bundle by calling Broker2 + Consumer consumer = pulsarClient.subscribe("persistent://my-property2/use2/my-ns/my-topic1", "my-subscriber-name", + new ConsumerConfiguration()); + Producer producer = pulsarClient2.createProducer("persistent://my-property2/use2/my-ns/my-topic1", new ProducerConfiguration()); + + for (int i = 0; i < 10; i++) { + String message = "my-message-" + i; + producer.send(message.getBytes()); + } + + Message msg = null; + Set messageSet = Sets.newHashSet(); + for (int i = 0; i < 10; i++) { + msg = consumer.receive(5, TimeUnit.SECONDS); + String receivedMessage = new String(msg.getData()); + log.debug("Received message: [{}]", receivedMessage); + String expectedMessage = "my-message-" + i; + testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage); + } + // Acknowledge the consumption of all messages at once + consumer.acknowledgeCumulative(msg); + consumer.close(); + producer.close(); + + // disable authorization + pulsar.getConfiguration().setAuthorizationEnabled(false); + pulsarClient2.close(); + pulsar2.close(); + loadManager2 = null; + + } + + /** + * Create #PartitionedTopic and let it served by multiple brokers which requries + * a. tcp partitioned-metadata-lookup + * b. multiple topic-lookup + * c. partitioned producer-consumer + * + * @throws Exception + */ + @Test + public void testPartitionTopicLookup() throws Exception { + log.info("-- Starting {} test --", methodName); + + int numPartitions = 8; + DestinationName dn = DestinationName.get("persistent://my-property/use/my-ns/my-partitionedtopic1"); + + ConsumerConfiguration conf = new ConsumerConfiguration(); + conf.setSubscriptionType(SubscriptionType.Exclusive); + + admin.persistentTopics().createPartitionedTopic(dn.toString(), numPartitions); + + /**** start broker-2 ****/ + ServiceConfiguration conf2 = new ServiceConfiguration(); + conf2.setBrokerServicePort(PortManager.nextFreePort()); + conf2.setBrokerServicePortTls(PortManager.nextFreePort()); + conf2.setWebServicePort(PortManager.nextFreePort()); + conf2.setWebServicePortTls(PortManager.nextFreePort()); + conf2.setAdvertisedAddress("localhost"); + conf2.setClusterName(pulsar.getConfiguration().getClusterName()); + PulsarService pulsar2 = startBroker(conf2); + pulsar.getLoadManager().writeLoadReportOnZookeeper(); + pulsar2.getLoadManager().writeLoadReportOnZookeeper(); + + + LoadManager loadManager1 = spy(pulsar.getLoadManager()); + LoadManager loadManager2 = spy(pulsar2.getLoadManager()); + Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager"); + loadManagerField.setAccessible(true); + + // mock: return Broker2 as a Least-loaded broker when leader receies request + doReturn(true).when(loadManager1).isCentralized(); + loadManagerField.set(pulsar.getNamespaceService(), loadManager1); + + // mock: redirect request to leader + doReturn(true).when(loadManager2).isCentralized(); + loadManagerField.set(pulsar2.getNamespaceService(), loadManager2); + /**** broker-2 started ****/ + + ProducerConfiguration producerConf = new ProducerConfiguration(); + producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition); + Producer producer = pulsarClient.createProducer(dn.toString(), producerConf); + + Consumer consumer = pulsarClient.subscribe(dn.toString(), "my-partitioned-subscriber", conf); + + for (int i = 0; i < 20; i++) { + String message = "my-message-" + i; + producer.send(message.getBytes()); + } + + Message msg = null; + Set messageSet = Sets.newHashSet(); + for (int i = 0; i < 20; i++) { + msg = consumer.receive(5, TimeUnit.SECONDS); + Assert.assertNotNull(msg, "Message should not be null"); + consumer.acknowledge(msg); + String receivedMessage = new String(msg.getData()); + log.debug("Received message: [{}]", receivedMessage); + Assert.assertTrue(messageSet.add(receivedMessage), "Message " + receivedMessage + " already received"); + } + + producer.close(); + consumer.unsubscribe(); + consumer.close(); + admin.persistentTopics().deletePartitionedTopic(dn.toString()); + + pulsar2.close(); + loadManager2 = null; + + log.info("-- Exiting {} test --", methodName); + } + + /** + * 1. Start broker1 and broker2 with tls enable + * 2. Hit HTTPS lookup url at broker2 which redirects to HTTPS broker1 + * + * @throws Exception + */ + @Test + public void testWebserviceServiceTls() throws Exception { + log.info("-- Starting {} test --", methodName); + final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt"; + final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key"; + final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt"; + final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key"; + + /**** start broker-2 ****/ + ServiceConfiguration conf2 = new ServiceConfiguration(); + conf2.setBrokerServicePort(PortManager.nextFreePort()); + conf2.setBrokerServicePortTls(PortManager.nextFreePort()); + conf2.setWebServicePort(PortManager.nextFreePort()); + conf2.setWebServicePortTls(PortManager.nextFreePort()); + conf2.setAdvertisedAddress("localhost"); + conf2.setTlsAllowInsecureConnection(true); + conf2.setTlsEnabled(true); + conf2.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); + conf2.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); + conf2.setClusterName(conf.getClusterName()); + PulsarService pulsar2 = startBroker(conf2); + + // restart broker1 with tls enabled + conf.setTlsAllowInsecureConnection(true); + conf.setTlsEnabled(true); + conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); + stopBroker(); + startBroker(); + pulsar.getLoadManager().writeLoadReportOnZookeeper(); + pulsar2.getLoadManager().writeLoadReportOnZookeeper(); + + LoadManager loadManager1 = spy(pulsar.getLoadManager()); + LoadManager loadManager2 = spy(pulsar2.getLoadManager()); + Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager"); + loadManagerField.setAccessible(true); + + // mock: redirect request to leader [2] + doReturn(true).when(loadManager2).isCentralized(); + loadManagerField.set(pulsar2.getNamespaceService(), loadManager2); + + // mock: return Broker2 as a Least-loaded broker when leader receies + // request [3] + doReturn(true).when(loadManager1).isCentralized(); + SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getWebServiceAddress(), null); + doReturn(resourceUnit).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class)); + loadManagerField.set(pulsar.getNamespaceService(), loadManager1); + + /**** started broker-2 ****/ + + URI brokerServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort()); + PulsarClient pulsarClient2 = PulsarClient.create(brokerServiceUrl.toString(), new ClientConfiguration()); + + final String lookupResourceUrl = "/lookup/v2/destination/persistent/my-property/use/my-ns/my-topic1"; + + // set client cert_key file + KeyManager[] keyManagers = null; + Certificate[] tlsCert = SecurityUtility.loadCertificatesFromPemFile(TLS_CLIENT_CERT_FILE_PATH); + PrivateKey tlsKey = SecurityUtility.loadPrivateKeyFromPemFile(TLS_CLIENT_KEY_FILE_PATH); + KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType()); + ks.load(null, null); + ks.setKeyEntry("private", tlsKey, "".toCharArray(), tlsCert); + KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + kmf.init(ks, "".toCharArray()); + keyManagers = kmf.getKeyManagers(); + TrustManager[] trustManagers = InsecureTrustManagerFactory.INSTANCE.getTrustManagers(); + SSLContext sslCtx = SSLContext.getInstance("TLS"); + sslCtx.init(keyManagers, trustManagers, new SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory()); + + + // hit broker2 url + URLConnection con = new URL(pulsar2.getWebServiceAddressTls() + lookupResourceUrl).openConnection(); + log.info("orignal url: {}", con.getURL()); + con.connect(); + log.info("connected url: {} ", con.getURL()); + // assert connect-url: broker2-https + Assert.assertEquals(con.getURL().getPort(), conf2.getWebServicePortTls()); + InputStream is = con.getInputStream(); + // assert redirect-url: broker1-https only + log.info("redirected url: {}", con.getURL()); + Assert.assertEquals(con.getURL().getPort(), conf.getWebServicePortTls()); + is.close(); + + pulsarClient2.close(); + pulsar2.close(); + loadManager1 = null; + loadManager2 = null; + + } + + /** + * Discovery-Service lookup over binary-protocol + * 1. Start discovery service + * 2. start broker + * 3. Create Producer/Consumer: by calling Discovery service for partitionedMetadata and topic lookup + * + * @throws Exception + */ + @Test + public void testDiscoveryLookup() throws Exception { + + // (1) start discovery service + ServiceConfig config = new ServiceConfig(); + config.setServicePort(nextFreePort()); + config.setBindOnLocalhost(true); + DiscoveryService discoveryService = spy(new DiscoveryService(config)); + doReturn(mockZooKeeperClientFactory).when(discoveryService).getZooKeeperClientFactory(); + discoveryService.start(); + + // (2) lookup using discovery service + final String discoverySvcUrl = discoveryService.getServiceUrl(); + ClientConfiguration clientConfig = new ClientConfiguration(); + PulsarClient pulsarClient2 = PulsarClient.create(discoverySvcUrl, clientConfig); + Consumer consumer = pulsarClient2.subscribe("persistent://my-property2/use2/my-ns/my-topic1", "my-subscriber-name", + new ConsumerConfiguration()); + Producer producer = pulsarClient2.createProducer("persistent://my-property2/use2/my-ns/my-topic1", new ProducerConfiguration()); + + for (int i = 0; i < 10; i++) { + String message = "my-message-" + i; + producer.send(message.getBytes()); + } + + Message msg = null; + Set messageSet = Sets.newHashSet(); + for (int i = 0; i < 10; i++) { + msg = consumer.receive(5, TimeUnit.SECONDS); + String receivedMessage = new String(msg.getData()); + log.debug("Received message: [{}]", receivedMessage); + String expectedMessage = "my-message-" + i; + testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage); + } + // Acknowledge the consumption of all messages at once + consumer.acknowledgeCumulative(msg); + consumer.close(); + producer.close(); + + } + + /** + * Verify discovery-service binary-proto lookup using tls + * + * @throws Exception + */ + @Test + public void testDiscoveryLookupTls() throws Exception { + + final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt"; + final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key"; + final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt"; + final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key"; + + // (1) restart broker1 with tls enabled + conf.setTlsAllowInsecureConnection(true); + conf.setTlsEnabled(true); + conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); + stopBroker(); + startBroker(); + + // (2) start discovery service + ServiceConfig config = new ServiceConfig(); + config.setServicePort(nextFreePort()); + config.setServicePortTls(nextFreePort()); + config.setTlsEnabled(true); + config.setBindOnLocalhost(true); + config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); + config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); + DiscoveryService discoveryService = spy(new DiscoveryService(config)); + doReturn(mockZooKeeperClientFactory).when(discoveryService).getZooKeeperClientFactory(); + discoveryService.start(); + + // (3) lookup using discovery service + final String discoverySvcUrl = discoveryService.getServiceUrlTls(); + ClientConfiguration clientConfig = new ClientConfiguration(); + + Map authParams = new HashMap<>(); + authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); + authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); + Authentication auth = new AuthenticationTls(); + auth.configure(authParams); + clientConfig.setAuthentication(auth); + clientConfig.setUseTls(true); + clientConfig.setTlsAllowInsecureConnection(true); + + + PulsarClient pulsarClient2 = PulsarClient.create(discoverySvcUrl, clientConfig); + Consumer consumer = pulsarClient2.subscribe("persistent://my-property2/use2/my-ns/my-topic1", "my-subscriber-name", + new ConsumerConfiguration()); + Producer producer = pulsarClient2.createProducer("persistent://my-property2/use2/my-ns/my-topic1", new ProducerConfiguration()); + + for (int i = 0; i < 10; i++) { + String message = "my-message-" + i; + producer.send(message.getBytes()); + } + + Message msg = null; + Set messageSet = Sets.newHashSet(); + for (int i = 0; i < 10; i++) { + msg = consumer.receive(5, TimeUnit.SECONDS); + String receivedMessage = new String(msg.getData()); + log.debug("Received message: [{}]", receivedMessage); + String expectedMessage = "my-message-" + i; + testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage); + } + // Acknowledge the consumption of all messages at once + consumer.acknowledgeCumulative(msg); + consumer.close(); + producer.close(); + + } + +} diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/client/api/MockBrokerService.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/client/api/MockBrokerService.java index 79a11f19c1b5a..b1077b2dfaee1 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/client/api/MockBrokerService.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/client/api/MockBrokerService.java @@ -69,7 +69,7 @@ private class genericResponseHandler extends AbstractHandler { private final String lookupURI = "/lookup/v2/destination/persistent"; private final String partitionMetadataURI = "/admin/persistent"; private final LookupData lookupData = new LookupData("pulsar://127.0.0.1:" + brokerServicePort, - "pulsar://127.0.0.1:" + brokerServicePortTls, "http://127.0.0.1:" + webServicePort); + "pulsar://127.0.0.1:" + brokerServicePortTls, "http://127.0.0.1:" + webServicePort, null); private final PartitionedTopicMetadata singlePartitionedTopicMetadata = new PartitionedTopicMetadata(1); private final PartitionedTopicMetadata multiPartitionedTopicMetadata = new PartitionedTopicMetadata(4); private final PartitionedTopicMetadata nonPartitionedTopicMetadata = new PartitionedTopicMetadata(); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/client/api/SimpleProducerConsumerTest.java index 706a98cf677f8..2161e3e5619c7 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/client/api/SimpleProducerConsumerTest.java @@ -1238,7 +1238,7 @@ public void testMutlipleSharedConsumerBlockingWithUnAckedMessages() throws Excep pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages); } } - + @Test public void testUnackBlockRedeliverMessages() throws Exception { log.info("-- Starting {} test --", methodName); @@ -1246,8 +1246,8 @@ public void testUnackBlockRedeliverMessages() throws Exception { int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer(); int totalReceiveMsg = 0; try { - final int unAckedMessagesBufferSize = 10; - final int receiverQueueSize = 20; + final int unAckedMessagesBufferSize = 20; + final int receiverQueueSize = 10; final int totalProducedMsgs = 100; pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize); diff --git a/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/BinaryProtoLookupService.java b/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/BinaryProtoLookupService.java new file mode 100644 index 0000000000000..9029eeeb61bfd --- /dev/null +++ b/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/BinaryProtoLookupService.java @@ -0,0 +1,215 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.client.impl; + +import static com.yahoo.pulsar.client.impl.PulsarClientImpl.requestIdGenerator; +import static java.lang.String.format; + +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.concurrent.CompletableFuture; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.yahoo.pulsar.client.api.PulsarClientException; +import com.yahoo.pulsar.common.api.Commands; +import com.yahoo.pulsar.common.naming.DestinationName; +import com.yahoo.pulsar.common.partition.PartitionedTopicMetadata; + +import io.netty.buffer.ByteBuf; + +class BinaryProtoLookupService implements LookupService { + + private final ConnectionPool cnxPool; + protected final InetSocketAddress serviceAddress; + private final boolean useTls; + + public BinaryProtoLookupService(ConnectionPool cnxPool, String serviceUrl, boolean useTls) + throws PulsarClientException { + this.cnxPool = cnxPool; + this.useTls = useTls; + URI uri; + try { + uri = new URI(serviceUrl); + this.serviceAddress = new InetSocketAddress(uri.getHost(), uri.getPort()); + } catch (Exception e) { + log.error("Invalid service-url {} provided {}", serviceUrl, e.getMessage(), e); + throw new PulsarClientException.InvalidServiceURL(e); + } + } + + /** + * Calls broker binaryProto-lookup api to find broker-service address which can serve a given topic. + * + * @param destination: topic-name + * @return broker-socket-address that serves given topic + */ + public CompletableFuture getBroker(DestinationName destination) { + return findBroker(serviceAddress, false, destination); + } + + /** + * calls broker binaryProto-lookup api to get metadata of partitioned-topic. + * + */ + public CompletableFuture getPartitionedTopicMetadata(DestinationName destination) { + return getPartitionedTopicMetadata(serviceAddress, destination); + } + + + private CompletableFuture findBroker(InetSocketAddress socketAddress, boolean authoritative, + DestinationName destination) { + CompletableFuture addressFuture = new CompletableFuture(); + + cnxPool.getConnection(socketAddress).thenAccept(clientCnx -> { + long requestId = requestIdGenerator.getAndIncrement(); + ByteBuf request = Commands.newLookup(destination.toString(), authoritative, requestId); + clientCnx.newLookup(request, requestId).thenAccept(lookupDataResult -> { + URI uri = null; + try { + // (1) build response broker-address + if (useTls) { + uri = new URI(lookupDataResult.brokerUrlTls); + } else { + String serviceUrl = lookupDataResult.brokerUrl; + uri = new URI(serviceUrl); + } + InetSocketAddress responseBrokerAddress = new InetSocketAddress(uri.getHost(), uri.getPort()); + + // (2) redirect to given address if response is: redirect + if (lookupDataResult.redirect) { + findBroker(responseBrokerAddress, lookupDataResult.authoritative, destination) + .thenAccept(brokerAddress -> { + addressFuture.complete(brokerAddress); + }).exceptionally((lookupException) -> { + // lookup failed + log.warn("[{}] lookup failed : {}", destination.toString(), + lookupException.getMessage(), lookupException); + addressFuture.completeExceptionally(lookupException); + return null; + }); + } else { + // (3) received correct broker to connect + addressFuture.complete(responseBrokerAddress); + } + + } catch (Exception parseUrlException) { + // Failed to parse url + log.warn("[{}] invalid url {} : {}", destination.toString(), uri, parseUrlException.getMessage(), + parseUrlException); + addressFuture.completeExceptionally(parseUrlException); + } + }).exceptionally((sendException) -> { + // lookup failed + log.warn("[{}] failed to send lookup request : {}", destination.toString(), sendException.getMessage(), + sendException); + addressFuture.completeExceptionally(sendException); + return null; + }); + }).exceptionally(connectionException -> { + addressFuture.completeExceptionally(connectionException); + return null; + }); + return addressFuture; + } + + + private CompletableFuture getPartitionedTopicMetadata(InetSocketAddress socketAddress, + DestinationName destination) { + + CompletableFuture partitionFuture = new CompletableFuture(); + + cnxPool.getConnection(socketAddress).thenAccept(clientCnx -> { + long requestId = requestIdGenerator.getAndIncrement(); + ByteBuf request = Commands.newPartitionMetadataRequest(destination.toString(), requestId); + clientCnx.newLookup(request, requestId).thenAccept(lookupDataResult -> { + try { + URI uri = null; + // (1) if redirect request for different broker lookup + if (lookupDataResult.redirect) { + + if (useTls) { + uri = new URI(lookupDataResult.brokerUrlTls); + } else { + String serviceUrl = lookupDataResult.brokerUrl; + uri = new URI(serviceUrl); + } + InetSocketAddress responseBrokerAddress = new InetSocketAddress(uri.getHost(), uri.getPort()); + //(1.a) retry getPartitionedMetadata with different redirected broker + getPartitionedTopicMetadata(responseBrokerAddress, destination).thenAccept(metadata -> { + partitionFuture.complete(metadata); + }).exceptionally((lookupException) -> { + // lookup failed + log.warn("[{}] lookup failed : {}", destination.toString(), lookupException.getMessage(), + lookupException); + partitionFuture.completeExceptionally(lookupException); + return null; + }); + } else { + // (2) received result partitions + partitionFuture.complete(new PartitionedTopicMetadata(lookupDataResult.partitions)); + } + } catch (Exception e) { + partitionFuture.completeExceptionally(new PulsarClientException.LookupException( + format("Failed to parse partition-response redirect=%s , partitions with %s", + lookupDataResult.redirect, lookupDataResult.partitions, e.getMessage()))); + } + }).exceptionally((e) -> { + log.warn("[{}] failed to get Partitioned metadata : {}", destination.toString(), e.getMessage(), e); + partitionFuture.completeExceptionally(e); + return null; + }); + }).exceptionally(connectionException -> { + partitionFuture.completeExceptionally(connectionException); + return null; + }); + + return partitionFuture; + } + + public String getServiceUrl() { + return serviceAddress.toString(); + } + + static class LookupDataResult { + + private String brokerUrl; + private String brokerUrlTls; + private int partitions; + private boolean authoritative; + private boolean redirect; + + public LookupDataResult(String brokerUrl, String brokerUrlTls, boolean redirect, boolean authoritative) { + super(); + this.brokerUrl = brokerUrl; + this.brokerUrlTls = brokerUrlTls; + this.authoritative = authoritative; + this.redirect = redirect; + } + + public LookupDataResult(int partitions, String brokerUrl, String brokerUrlTls, boolean redirect) { + super(); + this.brokerUrl = brokerUrl; + this.brokerUrlTls = brokerUrlTls; + this.partitions = partitions; + this.redirect = redirect; + } + + } + + private static final Logger log = LoggerFactory.getLogger(BinaryProtoLookupService.class); +} diff --git a/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/ClientCnx.java index c7f6e921bd50b..12d94faaf61d8 100644 --- a/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/ClientCnx.java @@ -26,6 +26,7 @@ import com.yahoo.pulsar.client.api.Authentication; import com.yahoo.pulsar.client.api.PulsarClientException; +import com.yahoo.pulsar.client.impl.BinaryProtoLookupService.LookupDataResult; import com.yahoo.pulsar.common.api.Commands; import com.yahoo.pulsar.common.api.PulsarHandler; import com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError; @@ -33,11 +34,15 @@ import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandCloseProducer; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandConnected; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandError; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandMessage; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandProducerSuccess; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandSendError; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandSendReceipt; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandSuccess; +import com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError; import com.yahoo.pulsar.common.util.collections.ConcurrentLongHashMap; import io.netty.buffer.ByteBuf; @@ -51,6 +56,7 @@ public class ClientCnx extends PulsarHandler { private State state; private final ConcurrentLongHashMap> pendingRequests = new ConcurrentLongHashMap<>(16, 1); + private final ConcurrentLongHashMap> pendingLookupRequests = new ConcurrentLongHashMap<>(16, 1); private final ConcurrentLongHashMap producers = new ConcurrentLongHashMap<>(16, 1); private final ConcurrentLongHashMap consumers = new ConcurrentLongHashMap<>(16, 1); @@ -191,6 +197,62 @@ protected void handleProducerSuccess(CommandProducerSuccess success) { log.warn("{} Received unknown request id from server: {}", ctx.channel(), success.getRequestId()); } } + + @Override + protected void handleLookupResponse(CommandLookupTopicResponse lookupResult) { + log.info("Received Broker lookup response: {}", lookupResult.getResponse()); + + long requestId = lookupResult.getRequestId(); + CompletableFuture requestFuture = pendingLookupRequests.remove(requestId); + + if (requestFuture != null) { + // Complete future with exception if : Result.response=fail/null + if (lookupResult.getResponse() == null || LookupType.Failed.equals(lookupResult.getResponse())) { + if (lookupResult.hasError()) { + requestFuture.completeExceptionally( + getPulsarClientException(lookupResult.getError(), lookupResult.getMessage())); + } else { + requestFuture + .completeExceptionally(new PulsarClientException.LookupException("Empty lookup response")); + } + } else { + // return LookupDataResult when Result.response = connect/redirect + boolean redirect = LookupType.Redirect.equals(lookupResult.getResponse()); + requestFuture.complete(new LookupDataResult(lookupResult.getBrokerServiceUrl(), + lookupResult.getBrokerServiceUrlTls(), redirect, lookupResult.getAuthoritative())); + } + } else { + log.warn("{} Received unknown request id from server: {}", ctx.channel(), lookupResult.getRequestId()); + } + } + + @Override + protected void handlePartitionResponse(CommandPartitionedTopicMetadataResponse lookupResult) { + log.info("Received Broker Partition response: {}", lookupResult.getPartitions()); + + long requestId = lookupResult.getRequestId(); + CompletableFuture requestFuture = pendingLookupRequests.remove(requestId); + + if (requestFuture != null) { + // Complete future with exception if : Result.response=fail/null + if (lookupResult.getResponse() == null || LookupType.Failed.equals(lookupResult.getResponse())) { + if (lookupResult.hasError()) { + requestFuture.completeExceptionally( + getPulsarClientException(lookupResult.getError(), lookupResult.getMessage())); + } else { + requestFuture + .completeExceptionally(new PulsarClientException.LookupException("Empty lookup response")); + } + } else { + // return LookupDataResult when Result.response = success/redirect + boolean redirect = LookupType.Redirect.equals(lookupResult.getResponse()); + requestFuture.complete(new LookupDataResult(lookupResult.getPartitions(), + lookupResult.getBrokerServiceUrl(), lookupResult.getBrokerServiceUrlTls(), redirect)); + } + } else { + log.warn("{} Received unknown request id from server: {}", ctx.channel(), lookupResult.getRequestId()); + } + } @Override protected void handleSendError(CommandSendError sendError) { @@ -216,7 +278,7 @@ protected void handleError(CommandError error) { } CompletableFuture requestFuture = pendingRequests.remove(requestId); if (requestFuture != null) { - requestFuture.completeExceptionally(getPulsarClientException(error)); + requestFuture.completeExceptionally(getPulsarClientException(error.getError(), error.getMessage())); } else { log.warn("{} Received unknown request id from server: {}", ctx.channel(), error.getRequestId()); } @@ -251,6 +313,18 @@ protected boolean isHandshakeCompleted() { return state == State.Ready; } + CompletableFuture newLookup(ByteBuf request, long requestId) { + CompletableFuture future = new CompletableFuture<>(); + pendingLookupRequests.put(requestId, future); + ctx.writeAndFlush(request).addListener(writeFuture -> { + if (!writeFuture.isSuccess()) { + log.warn("{} Failed to send request to broker: {}", ctx.channel(), writeFuture.cause().getMessage()); + future.completeExceptionally(writeFuture.cause()); + } + }); + return future; + } + Promise newPromise() { return ctx.newPromise(); } @@ -299,27 +373,27 @@ void removeConsumer(final long consumerId) { consumers.remove(consumerId); } - private PulsarClientException getPulsarClientException(CommandError error) { - switch (error.getError()) { + private PulsarClientException getPulsarClientException(ServerError error, String errorMsg) { + switch (error) { case AuthenticationError: - return new PulsarClientException.AuthenticationException(error.getMessage()); + return new PulsarClientException.AuthenticationException(errorMsg); case AuthorizationError: - return new PulsarClientException.AuthorizationException(error.getMessage()); + return new PulsarClientException.AuthorizationException(errorMsg); case ConsumerBusy: - return new PulsarClientException.ConsumerBusyException(error.getMessage()); + return new PulsarClientException.ConsumerBusyException(errorMsg); case MetadataError: - return new PulsarClientException.BrokerMetadataException(error.getMessage()); + return new PulsarClientException.BrokerMetadataException(errorMsg); case PersistenceError: - return new PulsarClientException.BrokerPersistenceException(error.getMessage()); + return new PulsarClientException.BrokerPersistenceException(errorMsg); case ServiceNotReady: - return new PulsarClientException.LookupException(error.getMessage()); + return new PulsarClientException.LookupException(errorMsg); case ProducerBlockedQuotaExceededError: - return new PulsarClientException.ProducerBlockedQuotaExceededError(error.getMessage()); + return new PulsarClientException.ProducerBlockedQuotaExceededError(errorMsg); case ProducerBlockedQuotaExceededException: - return new PulsarClientException.ProducerBlockedQuotaExceededException(error.getMessage()); + return new PulsarClientException.ProducerBlockedQuotaExceededException(errorMsg); case UnknownError: default: - return new PulsarClientException(error.getMessage()); + return new PulsarClientException(errorMsg); } } diff --git a/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/ConnectionPool.java b/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/ConnectionPool.java index 796e4c5655d09..9a868f8d65879 100644 --- a/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/ConnectionPool.java +++ b/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/ConnectionPool.java @@ -115,7 +115,7 @@ public CompletableFuture getConnection(final InetSocketAddress addres return createConnection(address, -1); } - final int randomKey = Math.abs(random.nextInt()) % maxConnectionsPerHosts; + final int randomKey = signSafeMod(random.nextInt(), maxConnectionsPerHosts); return pool.computeIfAbsent(address, a -> new ConcurrentHashMap<>()) // .computeIfAbsent(randomKey, k -> createConnection(address, randomKey)); @@ -176,6 +176,14 @@ private void cleanupConnection(InetSocketAddress address, int connectionKey, map.remove(connectionKey, connectionFuture); } } + + public static int signSafeMod(long dividend, int divisor) { + int mod = (int) (dividend % (long) divisor); + if (mod < 0) { + mod += divisor; + } + return mod; + } private static final Logger log = LoggerFactory.getLogger(ConnectionPool.class); } diff --git a/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/HttpLookupService.java b/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/HttpLookupService.java new file mode 100644 index 0000000000000..5639c2ad5ee47 --- /dev/null +++ b/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/HttpLookupService.java @@ -0,0 +1,81 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.client.impl; + +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.concurrent.CompletableFuture; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.yahoo.pulsar.client.util.FutureUtil; +import com.yahoo.pulsar.common.lookup.data.LookupData; +import com.yahoo.pulsar.common.naming.DestinationName; +import com.yahoo.pulsar.common.partition.PartitionedTopicMetadata; + +class HttpLookupService implements LookupService { + + private final HttpClient httpClient; + private final boolean useTls; + private static final String BasePath = "lookup/v2/destination/"; + + public HttpLookupService(HttpClient httpClient, boolean useTls) { + this.httpClient = httpClient; + this.useTls = useTls; + } + + /** + * Calls http-lookup api to find broker-service address which can serve a given topic. + * + * @param destination: topic-name + * @return broker-socket-address that serves given topic + */ + @SuppressWarnings("deprecation") + public CompletableFuture getBroker(DestinationName destination) { + return httpClient.get(BasePath + destination.getLookupName(), LookupData.class).thenCompose(lookupData -> { + // Convert LookupData into as SocketAddress, handling exceptions + URI uri = null; + try { + if (useTls) { + uri = new URI(lookupData.getBrokerUrlTls()); + } else { + String serviceUrl = lookupData.getBrokerUrl(); + if (serviceUrl == null) { + serviceUrl = lookupData.getNativeUrl(); + } + uri = new URI(serviceUrl); + } + return CompletableFuture.completedFuture(new InetSocketAddress(uri.getHost(), uri.getPort())); + } catch (Exception e) { + // Failed to parse url + log.warn("[{}] Lookup Failed due to invalid url {}, {}", destination, uri, e.getMessage()); + return FutureUtil.failedFuture(e); + } + }); + } + + public CompletableFuture getPartitionedTopicMetadata(DestinationName destination) { + return httpClient.get(String.format("admin/%s/partitions", destination.getLookupName()), + PartitionedTopicMetadata.class); + } + + public String getServiceUrl() { + return httpClient.url.toString(); + } + + private static final Logger log = LoggerFactory.getLogger(HttpLookupService.class); +} diff --git a/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/LookupService.java b/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/LookupService.java index bdd4e0c695d33..00d07e5b9aa27 100644 --- a/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/LookupService.java +++ b/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/LookupService.java @@ -16,44 +16,46 @@ package com.yahoo.pulsar.client.impl; import java.net.InetSocketAddress; -import java.net.URI; import java.util.concurrent.CompletableFuture; -import com.yahoo.pulsar.client.util.FutureUtil; -import com.yahoo.pulsar.common.lookup.data.LookupData; import com.yahoo.pulsar.common.naming.DestinationName; +import com.yahoo.pulsar.common.partition.PartitionedTopicMetadata; -class LookupService { - - private final HttpClient httpClient; - private final boolean useTls; - private static final String BasePath = "lookup/v2/destination/"; - - public LookupService(HttpClient httpClient, boolean useTls) { - this.httpClient = httpClient; - this.useTls = useTls; - } +/** + * Provides lookup service to find broker which serves given topic. It helps to + * lookup + *
    + *
  • topic-lookup: lookup to find broker-address which serves given + * topic
    • + *
  • Partitioned-topic-Metadata-lookup: lookup to find + * PartitionedMetadata for a given topic
    • + *
+ * + */ +interface LookupService { - @SuppressWarnings("deprecation") - public CompletableFuture getBroker(DestinationName destination) { - return httpClient.get(BasePath + destination.getLookupName(), LookupData.class).thenCompose(lookupData -> { - // Convert LookupData into as SocketAddress, handling exceptions - try { - URI uri; - if (useTls) { - uri = new URI(lookupData.getBrokerUrlTls()); - } else { - String serviceUrl = lookupData.getBrokerUrl(); - if (serviceUrl == null) { - serviceUrl = lookupData.getNativeUrl(); - } - uri = new URI(serviceUrl); - } - return CompletableFuture.completedFuture(new InetSocketAddress(uri.getHost(), uri.getPort())); - } catch (Exception e) { - // Failed to parse url - return FutureUtil.failedFuture(e); - } - }); - } + /** + * Calls broker lookup-api to get broker {@link InetSocketAddress} which serves namespacebundle that + * contains given topic. + * + * @param destination: + * topic-name + * @return broker-socket-address that serves given topic + */ + public CompletableFuture getBroker(DestinationName topic); + + /** + * Returns {@link PartitionedTopicMetadata} for a given topic. + * + * @param destination : topic-name + * @return + */ + public CompletableFuture getPartitionedTopicMetadata(DestinationName destination); + + /** + * Returns broker-service lookup api url. + * + * @return + */ + public String getServiceUrl(); } diff --git a/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/PartitionMetadataLookupService.java b/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/PartitionMetadataLookupService.java deleted file mode 100644 index 3b864133e6bfb..0000000000000 --- a/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/PartitionMetadataLookupService.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2016 Yahoo Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.yahoo.pulsar.client.impl; - -import java.util.concurrent.CompletableFuture; - -import com.yahoo.pulsar.common.naming.DestinationName; -import com.yahoo.pulsar.common.partition.PartitionedTopicMetadata; - -public class PartitionMetadataLookupService { - - private final HttpClient httpClient; - - public PartitionMetadataLookupService(HttpClient httpClient) { - this.httpClient = httpClient; - } - - public CompletableFuture getPartitionedTopicMetadata(DestinationName destination) { - return httpClient.get(String.format("admin/%s/partitions", destination.getLookupName()), - PartitionedTopicMetadata.class); - } -}; \ No newline at end of file diff --git a/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/PulsarClientImpl.java index 79a11874636a9..9e3c4f5765ed7 100644 --- a/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/com/yahoo/pulsar/client/impl/PulsarClientImpl.java @@ -57,9 +57,8 @@ public class PulsarClientImpl implements PulsarClient { private static final Logger log = LoggerFactory.getLogger(PulsarClientImpl.class); private final ClientConfiguration conf; - private final HttpClient httpClient; + private HttpClient httpClient; private final LookupService lookup; - private final PartitionMetadataLookupService partition; private final ConnectionPool cnxPool; private final Timer timer; private final ExecutorProvider externalExecutorProvider; @@ -75,7 +74,7 @@ enum State { private final AtomicLong producerIdGenerator = new AtomicLong(); private final AtomicLong consumerIdGenerator = new AtomicLong(); - private final AtomicLong requestIdGenerator = new AtomicLong(); + protected static final AtomicLong requestIdGenerator = new AtomicLong(); public PulsarClientImpl(String serviceUrl, ClientConfiguration conf) throws PulsarClientException { this(serviceUrl, conf, getEventLoopGroup(conf), null); @@ -88,12 +87,14 @@ public PulsarClientImpl(String serviceUrl, ClientConfiguration conf, EventLoopGr } this.conf = conf; conf.getAuthentication().start(); - httpClient = new HttpClient(serviceUrl, conf.getAuthentication(), eventLoopGroup, - conf.isTlsAllowInsecureConnection(), conf.getTlsTrustCertsFilePath()); - lookup = new LookupService(httpClient, conf.isUseTls()); - partition = new PartitionMetadataLookupService(httpClient); cnxPool = new ConnectionPool(this, eventLoopGroup); - + if (serviceUrl.startsWith("http")) { + httpClient = new HttpClient(serviceUrl, conf.getAuthentication(), eventLoopGroup, + conf.isTlsAllowInsecureConnection(), conf.getTlsTrustCertsFilePath()); + lookup = new HttpLookupService(httpClient, conf.isUseTls()); + } else { + lookup = new BinaryProtoLookupService(cnxPool, serviceUrl, conf.isUseTls()); + } timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-timer"), 1, TimeUnit.MILLISECONDS); externalExecutorProvider = new ExecutorProvider(conf.getListenerThreads(), "pulsar-external-listener"); internalExecutorProvider = new ExecutorProvider(conf.getListenerThreads(), "pulsar-internal-listener"); @@ -287,7 +288,7 @@ public void close() throws PulsarClientException { @Override public CompletableFuture closeAsync() { - log.info("Client closing. URL: {}", this.httpClient.url.toString()); + log.info("Client closing. URL: {}", lookup.getServiceUrl()); if (!state.compareAndSet(State.Open, State.Closing)) { return FutureUtil.failedFuture(new PulsarClientException.AlreadyClosedException("Client already closed")); } @@ -327,7 +328,9 @@ public CompletableFuture closeAsync() { @Override public void shutdown() throws PulsarClientException { try { - httpClient.close(); + if (httpClient!= null) { + httpClient.close(); + } cnxPool.close(); timer.stop(); externalExecutorProvider.shutdownNow(); @@ -339,11 +342,10 @@ public void shutdown() throws PulsarClientException { } } - protected CompletableFuture getConnection(final String topic) { - DestinationName destinationName = DestinationName.get(topic); - - return lookup.getBroker(destinationName).thenCompose((brokerAddress) -> cnxPool.getConnection(brokerAddress)); - } + protected CompletableFuture getConnection(final String topic) { + DestinationName destinationName = DestinationName.get(topic); + return lookup.getBroker(destinationName).thenCompose((brokerAddress) -> cnxPool.getConnection(brokerAddress)); + } protected Timer timer() { return timer; @@ -375,7 +377,7 @@ private CompletableFuture getPartitionedTopicMetadata( try { DestinationName destinationName = DestinationName.get(topic); - metadataFuture = partition.getPartitionedTopicMetadata(destinationName); + metadataFuture = lookup.getPartitionedTopicMetadata(destinationName); } catch (IllegalArgumentException e) { return FutureUtil.failedFuture(e); } diff --git a/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/Commands.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/Commands.java index 201ad85ee564d..058538309c773 100644 --- a/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/Commands.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/Commands.java @@ -34,7 +34,12 @@ import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandConnected; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandError; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandFlow; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandMessage; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPing; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPong; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandProducer; @@ -342,6 +347,99 @@ public static ByteBuf newProducer(String topic, long producerId, long requestId, return res; } + public static ByteBuf newPartitionMetadataResponse(ServerError error, String errorMsg, long requestId) { + CommandPartitionedTopicMetadataResponse.Builder partitionMetadataResponseBuilder = CommandPartitionedTopicMetadataResponse + .newBuilder(); + partitionMetadataResponseBuilder.setRequestId(requestId); + partitionMetadataResponseBuilder.setError(error); + if (errorMsg != null) { + partitionMetadataResponseBuilder.setMessage(errorMsg); + } + + CommandPartitionedTopicMetadataResponse partitionMetadataResponse = partitionMetadataResponseBuilder.build(); + ByteBuf res = serializeWithSize(BaseCommand.newBuilder().setType(Type.PARTITIONED_METADATA_RESPONSE) + .setPartitionMetadataResponse(partitionMetadataResponse)); + partitionMetadataResponseBuilder.recycle(); + partitionMetadataResponse.recycle(); + return res; + } + + public static ByteBuf newPartitionMetadataRequest(String topic, long requestId) { + CommandPartitionedTopicMetadata.Builder partitionMetadataBuilder = CommandPartitionedTopicMetadata.newBuilder(); + partitionMetadataBuilder.setTopic(topic); + partitionMetadataBuilder.setRequestId(requestId); + + CommandPartitionedTopicMetadata partitionMetadata = partitionMetadataBuilder.build(); + ByteBuf res = serializeWithSize( + BaseCommand.newBuilder().setType(Type.PARTITIONED_METADATA).setPartitionMetadata(partitionMetadata)); + partitionMetadataBuilder.recycle(); + partitionMetadata.recycle(); + return res; + } + + public static ByteBuf newPartitionMetadataResponse(int partitions, long requestId) { + CommandPartitionedTopicMetadataResponse.Builder partitionMetadataResponseBuilder = CommandPartitionedTopicMetadataResponse + .newBuilder(); + partitionMetadataResponseBuilder.setPartitions(partitions); + partitionMetadataResponseBuilder.setRequestId(requestId); + + CommandPartitionedTopicMetadataResponse partitionMetadataResponse = partitionMetadataResponseBuilder.build(); + ByteBuf res = serializeWithSize(BaseCommand.newBuilder().setType(Type.PARTITIONED_METADATA_RESPONSE) + .setPartitionMetadataResponse(partitionMetadataResponse)); + partitionMetadataResponseBuilder.recycle(); + partitionMetadataResponse.recycle(); + return res; + } + + public static ByteBuf newLookup(String topic, boolean authoritative, long requestId) { + CommandLookupTopic.Builder lookupTopicBuilder = CommandLookupTopic.newBuilder(); + lookupTopicBuilder.setTopic(topic); + lookupTopicBuilder.setRequestId(requestId); + lookupTopicBuilder.setAuthoritative(authoritative); + + CommandLookupTopic lookupBroker = lookupTopicBuilder.build(); + ByteBuf res = serializeWithSize(BaseCommand.newBuilder().setType(Type.LOOKUP).setLookupTopic(lookupBroker)); + lookupTopicBuilder.recycle(); + lookupBroker.recycle(); + return res; + } + + + public static ByteBuf newLookupResponse(String brokerServiceUrl, String brokerServiceUrlTls, boolean authoritative, + LookupType response, long requestId) { + CommandLookupTopicResponse.Builder connectionBuilder = CommandLookupTopicResponse.newBuilder(); + connectionBuilder.setBrokerServiceUrl(brokerServiceUrl); + if (brokerServiceUrlTls != null) { + connectionBuilder.setBrokerServiceUrlTls(brokerServiceUrlTls); + } + connectionBuilder.setResponse(response); + connectionBuilder.setRequestId(requestId); + connectionBuilder.setAuthoritative(authoritative); + + CommandLookupTopicResponse connectionLookupResponse = connectionBuilder.build(); + ByteBuf res = serializeWithSize(BaseCommand.newBuilder().setType(Type.LOOKUP_RESPONSE) + .setLookupTopicResponse(connectionLookupResponse)); + connectionBuilder.recycle(); + connectionLookupResponse.recycle(); + return res; + } + + public static ByteBuf newLookupResponse(ServerError error, String errorMsg, long requestId) { + CommandLookupTopicResponse.Builder connectionBuilder = CommandLookupTopicResponse.newBuilder(); + connectionBuilder.setRequestId(requestId); + connectionBuilder.setError(error); + if (errorMsg != null) { + connectionBuilder.setMessage(errorMsg); + } + connectionBuilder.setResponse(LookupType.Failed); + + CommandLookupTopicResponse connectionBroker = connectionBuilder.build(); + ByteBuf res = serializeWithSize(BaseCommand.newBuilder().setType(Type.LOOKUP_RESPONSE).setLookupTopicResponse(connectionBroker)); + connectionBuilder.recycle(); + connectionBroker.recycle(); + return res; + } + public static ByteBuf newAck(long consumerId, long ledgerId, long entryId, AckType ackType, ValidationError validationError) { CommandAck.Builder ackBuilder = CommandAck.newBuilder(); diff --git a/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/PulsarDecoder.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/PulsarDecoder.java index d162002679e52..7cc0b331a5040 100644 --- a/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/PulsarDecoder.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/PulsarDecoder.java @@ -28,7 +28,11 @@ import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandConnected; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandError; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandFlow; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandMessage; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPing; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPong; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandProducer; @@ -79,6 +83,30 @@ final public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exce messageReceived(); switch (cmd.getType()) { + case PARTITIONED_METADATA: + checkArgument(cmd.hasPartitionMetadata()); + handlePartitionMetadataRequest(cmd.getPartitionMetadata()); + cmd.getPartitionMetadata().recycle(); + break; + + case PARTITIONED_METADATA_RESPONSE: + checkArgument(cmd.hasPartitionMetadataResponse()); + handlePartitionResponse(cmd.getPartitionMetadataResponse()); + cmd.getPartitionMetadataResponse().recycle(); + break; + + case LOOKUP: + checkArgument(cmd.hasLookupTopic()); + handleLookup(cmd.getLookupTopic()); + cmd.getLookupTopic().recycle(); + break; + + case LOOKUP_RESPONSE: + checkArgument(cmd.hasLookupTopicResponse()); + handleLookupResponse(cmd.getLookupTopicResponse()); + cmd.getLookupTopicResponse().recycle(); + break; + case ACK: checkArgument(cmd.hasAck()); handleAck(cmd.getAck()); @@ -212,6 +240,22 @@ final public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exce protected abstract void messageReceived(); + protected void handlePartitionMetadataRequest(CommandPartitionedTopicMetadata response) { + throw new UnsupportedOperationException(); + } + + protected void handlePartitionResponse(CommandPartitionedTopicMetadataResponse response) { + throw new UnsupportedOperationException(); + } + + protected void handleLookup(CommandLookupTopic lookup) { + throw new UnsupportedOperationException(); + } + + protected void handleLookupResponse(CommandLookupTopicResponse connection) { + throw new UnsupportedOperationException(); + } + protected void handleConnect(CommandConnect connect) { throw new UnsupportedOperationException(); } diff --git a/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/PulsarHandler.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/PulsarHandler.java index c501531a6b31b..09ff0ac0033f3 100644 --- a/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/PulsarHandler.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/PulsarHandler.java @@ -57,8 +57,10 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception { if (log.isDebugEnabled()) { log.debug("[{}] Scheduling keep-alive task every {} s", ctx.channel(), keepAliveIntervalSeconds); } - this.keepAliveTask = ctx.executor().scheduleAtFixedRate(this::handleKeepAliveTimeout, keepAliveIntervalSeconds, - keepAliveIntervalSeconds, TimeUnit.SECONDS); + if (keepAliveIntervalSeconds > 0) { + this.keepAliveTask = ctx.executor().scheduleAtFixedRate(this::handleKeepAliveTimeout, + keepAliveIntervalSeconds, keepAliveIntervalSeconds, TimeUnit.SECONDS); + } } @Override diff --git a/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/proto/PulsarApi.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/proto/PulsarApi.java index f2044caa8768f..a3bea01c5740f 100644 --- a/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/proto/PulsarApi.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/api/proto/PulsarApi.java @@ -4772,6 +4772,2601 @@ void setConsumerName(com.google.protobuf.ByteString value) { // @@protoc_insertion_point(class_scope:com.yahoo.pulsar.common.api.proto.CommandSubscribe) } + public interface CommandPartitionedTopicMetadataOrBuilder + extends com.google.protobuf.MessageLiteOrBuilder { + + // required string topic = 1; + boolean hasTopic(); + String getTopic(); + + // required uint64 request_id = 2; + boolean hasRequestId(); + long getRequestId(); + } + public static final class CommandPartitionedTopicMetadata extends + com.google.protobuf.GeneratedMessageLite + implements CommandPartitionedTopicMetadataOrBuilder, com.yahoo.pulsar.common.util.protobuf.ByteBufCodedOutputStream.ByteBufGeneratedMessage { + // Use CommandPartitionedTopicMetadata.newBuilder() to construct. + private io.netty.util.Recycler.Handle handle; + private CommandPartitionedTopicMetadata(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + } + + private static final io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected CommandPartitionedTopicMetadata newObject(Handle handle) { + return new CommandPartitionedTopicMetadata(handle); + } + }; + + public void recycle() { + this.initFields(); + this.memoizedIsInitialized = -1; + this.bitField0_ = 0; + this.memoizedSerializedSize = -1; + if (handle != null) { RECYCLER.recycle(this, handle); } + } + + private CommandPartitionedTopicMetadata(boolean noInit) {} + + private static final CommandPartitionedTopicMetadata defaultInstance; + public static CommandPartitionedTopicMetadata getDefaultInstance() { + return defaultInstance; + } + + public CommandPartitionedTopicMetadata getDefaultInstanceForType() { + return defaultInstance; + } + + private int bitField0_; + // required string topic = 1; + public static final int TOPIC_FIELD_NUMBER = 1; + private java.lang.Object topic_; + public boolean hasTopic() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getTopic() { + java.lang.Object ref = topic_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + topic_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getTopicBytes() { + java.lang.Object ref = topic_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + topic_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // required uint64 request_id = 2; + public static final int REQUEST_ID_FIELD_NUMBER = 2; + private long requestId_; + public boolean hasRequestId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getRequestId() { + return requestId_; + } + + private void initFields() { + topic_ = ""; + requestId_ = 0L; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasTopic()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasRequestId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + throw new RuntimeException("Cannot use CodedOutputStream"); + } + + public void writeTo(com.yahoo.pulsar.common.util.protobuf.ByteBufCodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getTopicBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeUInt64(2, requestId_); + } + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getTopicBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(2, requestId_); + } + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + public static final class Builder extends + com.google.protobuf.GeneratedMessageLite.Builder< + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata, Builder> + implements com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataOrBuilder, com.yahoo.pulsar.common.util.protobuf.ByteBufCodedInputStream.ByteBufMessageBuilder { + // Construct using com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata.newBuilder() + private final io.netty.util.Recycler.Handle handle; + private Builder(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + maybeForceBuilderInitialization(); + } + private final static io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected Builder newObject(io.netty.util.Recycler.Handle handle) { + return new Builder(handle); + } + }; + + public void recycle() { + clear(); + if (handle != null) {RECYCLER.recycle(this, handle);} + } + + private void maybeForceBuilderInitialization() { + } + private static Builder create() { + return RECYCLER.get(); + } + + public Builder clear() { + super.clear(); + topic_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + requestId_ = 0L; + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata getDefaultInstanceForType() { + return com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata.getDefaultInstance(); + } + + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata build() { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata buildPartial() { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata result = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata.RECYCLER.get(); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.topic_ = topic_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.requestId_ = requestId_; + result.bitField0_ = to_bitField0_; + return result; + } + + public Builder mergeFrom(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata other) { + if (other == com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata.getDefaultInstance()) return this; + if (other.hasTopic()) { + setTopic(other.getTopic()); + } + if (other.hasRequestId()) { + setRequestId(other.getRequestId()); + } + return this; + } + + public final boolean isInitialized() { + if (!hasTopic()) { + + return false; + } + if (!hasRequestId()) { + + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + throw new java.io.IOException("Merge from CodedInputStream is disabled"); + } + public Builder mergeFrom( + com.yahoo.pulsar.common.util.protobuf.ByteBufCodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + + return this; + default: { + if (!input.skipField(tag)) { + + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + topic_ = input.readBytes(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + requestId_ = input.readUInt64(); + break; + } + } + } + } + + private int bitField0_; + + // required string topic = 1; + private java.lang.Object topic_ = ""; + public boolean hasTopic() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getTopic() { + java.lang.Object ref = topic_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + topic_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setTopic(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + topic_ = value; + + return this; + } + public Builder clearTopic() { + bitField0_ = (bitField0_ & ~0x00000001); + topic_ = getDefaultInstance().getTopic(); + + return this; + } + void setTopic(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + topic_ = value; + + } + + // required uint64 request_id = 2; + private long requestId_ ; + public boolean hasRequestId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getRequestId() { + return requestId_; + } + public Builder setRequestId(long value) { + bitField0_ |= 0x00000002; + requestId_ = value; + + return this; + } + public Builder clearRequestId() { + bitField0_ = (bitField0_ & ~0x00000002); + requestId_ = 0L; + + return this; + } + + // @@protoc_insertion_point(builder_scope:com.yahoo.pulsar.common.api.proto.CommandPartitionedTopicMetadata) + } + + static { + defaultInstance = new CommandPartitionedTopicMetadata(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:com.yahoo.pulsar.common.api.proto.CommandPartitionedTopicMetadata) + } + + public interface CommandPartitionedTopicMetadataResponseOrBuilder + extends com.google.protobuf.MessageLiteOrBuilder { + + // optional uint32 partitions = 1; + boolean hasPartitions(); + int getPartitions(); + + // required uint64 request_id = 2; + boolean hasRequestId(); + long getRequestId(); + + // optional .com.yahoo.pulsar.common.api.proto.CommandPartitionedTopicMetadataResponse.LookupType response = 3; + boolean hasResponse(); + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.LookupType getResponse(); + + // optional string brokerServiceUrl = 4; + boolean hasBrokerServiceUrl(); + String getBrokerServiceUrl(); + + // optional string brokerServiceUrlTls = 5; + boolean hasBrokerServiceUrlTls(); + String getBrokerServiceUrlTls(); + + // optional .com.yahoo.pulsar.common.api.proto.ServerError error = 6; + boolean hasError(); + com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError getError(); + + // optional string message = 7; + boolean hasMessage(); + String getMessage(); + } + public static final class CommandPartitionedTopicMetadataResponse extends + com.google.protobuf.GeneratedMessageLite + implements CommandPartitionedTopicMetadataResponseOrBuilder, com.yahoo.pulsar.common.util.protobuf.ByteBufCodedOutputStream.ByteBufGeneratedMessage { + // Use CommandPartitionedTopicMetadataResponse.newBuilder() to construct. + private io.netty.util.Recycler.Handle handle; + private CommandPartitionedTopicMetadataResponse(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + } + + private static final io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected CommandPartitionedTopicMetadataResponse newObject(Handle handle) { + return new CommandPartitionedTopicMetadataResponse(handle); + } + }; + + public void recycle() { + this.initFields(); + this.memoizedIsInitialized = -1; + this.bitField0_ = 0; + this.memoizedSerializedSize = -1; + if (handle != null) { RECYCLER.recycle(this, handle); } + } + + private CommandPartitionedTopicMetadataResponse(boolean noInit) {} + + private static final CommandPartitionedTopicMetadataResponse defaultInstance; + public static CommandPartitionedTopicMetadataResponse getDefaultInstance() { + return defaultInstance; + } + + public CommandPartitionedTopicMetadataResponse getDefaultInstanceForType() { + return defaultInstance; + } + + public enum LookupType + implements com.google.protobuf.Internal.EnumLite { + Redirect(0, 0), + Success(1, 1), + Failed(2, 2), + ; + + public static final int Redirect_VALUE = 0; + public static final int Success_VALUE = 1; + public static final int Failed_VALUE = 2; + + + public final int getNumber() { return value; } + + public static LookupType valueOf(int value) { + switch (value) { + case 0: return Redirect; + case 1: return Success; + case 2: return Failed; + default: return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap + internalGetValueMap() { + return internalValueMap; + } + private static com.google.protobuf.Internal.EnumLiteMap + internalValueMap = + new com.google.protobuf.Internal.EnumLiteMap() { + public LookupType findValueByNumber(int number) { + return LookupType.valueOf(number); + } + }; + + private final int value; + + private LookupType(int index, int value) { + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:com.yahoo.pulsar.common.api.proto.CommandPartitionedTopicMetadataResponse.LookupType) + } + + private int bitField0_; + // optional uint32 partitions = 1; + public static final int PARTITIONS_FIELD_NUMBER = 1; + private int partitions_; + public boolean hasPartitions() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public int getPartitions() { + return partitions_; + } + + // required uint64 request_id = 2; + public static final int REQUEST_ID_FIELD_NUMBER = 2; + private long requestId_; + public boolean hasRequestId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getRequestId() { + return requestId_; + } + + // optional .com.yahoo.pulsar.common.api.proto.CommandPartitionedTopicMetadataResponse.LookupType response = 3; + public static final int RESPONSE_FIELD_NUMBER = 3; + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.LookupType response_; + public boolean hasResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.LookupType getResponse() { + return response_; + } + + // optional string brokerServiceUrl = 4; + public static final int BROKERSERVICEURL_FIELD_NUMBER = 4; + private java.lang.Object brokerServiceUrl_; + public boolean hasBrokerServiceUrl() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public String getBrokerServiceUrl() { + java.lang.Object ref = brokerServiceUrl_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + brokerServiceUrl_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getBrokerServiceUrlBytes() { + java.lang.Object ref = brokerServiceUrl_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + brokerServiceUrl_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional string brokerServiceUrlTls = 5; + public static final int BROKERSERVICEURLTLS_FIELD_NUMBER = 5; + private java.lang.Object brokerServiceUrlTls_; + public boolean hasBrokerServiceUrlTls() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public String getBrokerServiceUrlTls() { + java.lang.Object ref = brokerServiceUrlTls_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + brokerServiceUrlTls_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getBrokerServiceUrlTlsBytes() { + java.lang.Object ref = brokerServiceUrlTls_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + brokerServiceUrlTls_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional .com.yahoo.pulsar.common.api.proto.ServerError error = 6; + public static final int ERROR_FIELD_NUMBER = 6; + private com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError error_; + public boolean hasError() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError getError() { + return error_; + } + + // optional string message = 7; + public static final int MESSAGE_FIELD_NUMBER = 7; + private java.lang.Object message_; + public boolean hasMessage() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + public String getMessage() { + java.lang.Object ref = message_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + message_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getMessageBytes() { + java.lang.Object ref = message_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + message_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private void initFields() { + partitions_ = 0; + requestId_ = 0L; + response_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.LookupType.Redirect; + brokerServiceUrl_ = ""; + brokerServiceUrlTls_ = ""; + error_ = com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError.UnknownError; + message_ = ""; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasRequestId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + throw new RuntimeException("Cannot use CodedOutputStream"); + } + + public void writeTo(com.yahoo.pulsar.common.util.protobuf.ByteBufCodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeUInt32(1, partitions_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeUInt64(2, requestId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeEnum(3, response_.getNumber()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeBytes(4, getBrokerServiceUrlBytes()); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeBytes(5, getBrokerServiceUrlTlsBytes()); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + output.writeEnum(6, error_.getNumber()); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + output.writeBytes(7, getMessageBytes()); + } + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt32Size(1, partitions_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(2, requestId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(3, response_.getNumber()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(4, getBrokerServiceUrlBytes()); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(5, getBrokerServiceUrlTlsBytes()); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(6, error_.getNumber()); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(7, getMessageBytes()); + } + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + public static final class Builder extends + com.google.protobuf.GeneratedMessageLite.Builder< + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse, Builder> + implements com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponseOrBuilder, com.yahoo.pulsar.common.util.protobuf.ByteBufCodedInputStream.ByteBufMessageBuilder { + // Construct using com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.newBuilder() + private final io.netty.util.Recycler.Handle handle; + private Builder(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + maybeForceBuilderInitialization(); + } + private final static io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected Builder newObject(io.netty.util.Recycler.Handle handle) { + return new Builder(handle); + } + }; + + public void recycle() { + clear(); + if (handle != null) {RECYCLER.recycle(this, handle);} + } + + private void maybeForceBuilderInitialization() { + } + private static Builder create() { + return RECYCLER.get(); + } + + public Builder clear() { + super.clear(); + partitions_ = 0; + bitField0_ = (bitField0_ & ~0x00000001); + requestId_ = 0L; + bitField0_ = (bitField0_ & ~0x00000002); + response_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.LookupType.Redirect; + bitField0_ = (bitField0_ & ~0x00000004); + brokerServiceUrl_ = ""; + bitField0_ = (bitField0_ & ~0x00000008); + brokerServiceUrlTls_ = ""; + bitField0_ = (bitField0_ & ~0x00000010); + error_ = com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError.UnknownError; + bitField0_ = (bitField0_ & ~0x00000020); + message_ = ""; + bitField0_ = (bitField0_ & ~0x00000040); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse getDefaultInstanceForType() { + return com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.getDefaultInstance(); + } + + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse build() { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse buildPartial() { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse result = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.RECYCLER.get(); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.partitions_ = partitions_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.requestId_ = requestId_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.response_ = response_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.brokerServiceUrl_ = brokerServiceUrl_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.brokerServiceUrlTls_ = brokerServiceUrlTls_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000020; + } + result.error_ = error_; + if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000040; + } + result.message_ = message_; + result.bitField0_ = to_bitField0_; + return result; + } + + public Builder mergeFrom(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse other) { + if (other == com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.getDefaultInstance()) return this; + if (other.hasPartitions()) { + setPartitions(other.getPartitions()); + } + if (other.hasRequestId()) { + setRequestId(other.getRequestId()); + } + if (other.hasResponse()) { + setResponse(other.getResponse()); + } + if (other.hasBrokerServiceUrl()) { + setBrokerServiceUrl(other.getBrokerServiceUrl()); + } + if (other.hasBrokerServiceUrlTls()) { + setBrokerServiceUrlTls(other.getBrokerServiceUrlTls()); + } + if (other.hasError()) { + setError(other.getError()); + } + if (other.hasMessage()) { + setMessage(other.getMessage()); + } + return this; + } + + public final boolean isInitialized() { + if (!hasRequestId()) { + + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + throw new java.io.IOException("Merge from CodedInputStream is disabled"); + } + public Builder mergeFrom( + com.yahoo.pulsar.common.util.protobuf.ByteBufCodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + + return this; + default: { + if (!input.skipField(tag)) { + + return this; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + partitions_ = input.readUInt32(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + requestId_ = input.readUInt64(); + break; + } + case 24: { + int rawValue = input.readEnum(); + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.LookupType value = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.LookupType.valueOf(rawValue); + if (value != null) { + bitField0_ |= 0x00000004; + response_ = value; + } + break; + } + case 34: { + bitField0_ |= 0x00000008; + brokerServiceUrl_ = input.readBytes(); + break; + } + case 42: { + bitField0_ |= 0x00000010; + brokerServiceUrlTls_ = input.readBytes(); + break; + } + case 48: { + int rawValue = input.readEnum(); + com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError value = com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError.valueOf(rawValue); + if (value != null) { + bitField0_ |= 0x00000020; + error_ = value; + } + break; + } + case 58: { + bitField0_ |= 0x00000040; + message_ = input.readBytes(); + break; + } + } + } + } + + private int bitField0_; + + // optional uint32 partitions = 1; + private int partitions_ ; + public boolean hasPartitions() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public int getPartitions() { + return partitions_; + } + public Builder setPartitions(int value) { + bitField0_ |= 0x00000001; + partitions_ = value; + + return this; + } + public Builder clearPartitions() { + bitField0_ = (bitField0_ & ~0x00000001); + partitions_ = 0; + + return this; + } + + // required uint64 request_id = 2; + private long requestId_ ; + public boolean hasRequestId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getRequestId() { + return requestId_; + } + public Builder setRequestId(long value) { + bitField0_ |= 0x00000002; + requestId_ = value; + + return this; + } + public Builder clearRequestId() { + bitField0_ = (bitField0_ & ~0x00000002); + requestId_ = 0L; + + return this; + } + + // optional .com.yahoo.pulsar.common.api.proto.CommandPartitionedTopicMetadataResponse.LookupType response = 3; + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.LookupType response_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.LookupType.Redirect; + public boolean hasResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.LookupType getResponse() { + return response_; + } + public Builder setResponse(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.LookupType value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + response_ = value; + + return this; + } + public Builder clearResponse() { + bitField0_ = (bitField0_ & ~0x00000004); + response_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.LookupType.Redirect; + + return this; + } + + // optional string brokerServiceUrl = 4; + private java.lang.Object brokerServiceUrl_ = ""; + public boolean hasBrokerServiceUrl() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public String getBrokerServiceUrl() { + java.lang.Object ref = brokerServiceUrl_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + brokerServiceUrl_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setBrokerServiceUrl(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000008; + brokerServiceUrl_ = value; + + return this; + } + public Builder clearBrokerServiceUrl() { + bitField0_ = (bitField0_ & ~0x00000008); + brokerServiceUrl_ = getDefaultInstance().getBrokerServiceUrl(); + + return this; + } + void setBrokerServiceUrl(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000008; + brokerServiceUrl_ = value; + + } + + // optional string brokerServiceUrlTls = 5; + private java.lang.Object brokerServiceUrlTls_ = ""; + public boolean hasBrokerServiceUrlTls() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public String getBrokerServiceUrlTls() { + java.lang.Object ref = brokerServiceUrlTls_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + brokerServiceUrlTls_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setBrokerServiceUrlTls(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000010; + brokerServiceUrlTls_ = value; + + return this; + } + public Builder clearBrokerServiceUrlTls() { + bitField0_ = (bitField0_ & ~0x00000010); + brokerServiceUrlTls_ = getDefaultInstance().getBrokerServiceUrlTls(); + + return this; + } + void setBrokerServiceUrlTls(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000010; + brokerServiceUrlTls_ = value; + + } + + // optional .com.yahoo.pulsar.common.api.proto.ServerError error = 6; + private com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError error_ = com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError.UnknownError; + public boolean hasError() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError getError() { + return error_; + } + public Builder setError(com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000020; + error_ = value; + + return this; + } + public Builder clearError() { + bitField0_ = (bitField0_ & ~0x00000020); + error_ = com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError.UnknownError; + + return this; + } + + // optional string message = 7; + private java.lang.Object message_ = ""; + public boolean hasMessage() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + public String getMessage() { + java.lang.Object ref = message_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + message_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setMessage(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000040; + message_ = value; + + return this; + } + public Builder clearMessage() { + bitField0_ = (bitField0_ & ~0x00000040); + message_ = getDefaultInstance().getMessage(); + + return this; + } + void setMessage(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000040; + message_ = value; + + } + + // @@protoc_insertion_point(builder_scope:com.yahoo.pulsar.common.api.proto.CommandPartitionedTopicMetadataResponse) + } + + static { + defaultInstance = new CommandPartitionedTopicMetadataResponse(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:com.yahoo.pulsar.common.api.proto.CommandPartitionedTopicMetadataResponse) + } + + public interface CommandLookupTopicOrBuilder + extends com.google.protobuf.MessageLiteOrBuilder { + + // required string topic = 1; + boolean hasTopic(); + String getTopic(); + + // required uint64 request_id = 2; + boolean hasRequestId(); + long getRequestId(); + + // optional bool authoritative = 3 [default = false]; + boolean hasAuthoritative(); + boolean getAuthoritative(); + } + public static final class CommandLookupTopic extends + com.google.protobuf.GeneratedMessageLite + implements CommandLookupTopicOrBuilder, com.yahoo.pulsar.common.util.protobuf.ByteBufCodedOutputStream.ByteBufGeneratedMessage { + // Use CommandLookupTopic.newBuilder() to construct. + private io.netty.util.Recycler.Handle handle; + private CommandLookupTopic(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + } + + private static final io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected CommandLookupTopic newObject(Handle handle) { + return new CommandLookupTopic(handle); + } + }; + + public void recycle() { + this.initFields(); + this.memoizedIsInitialized = -1; + this.bitField0_ = 0; + this.memoizedSerializedSize = -1; + if (handle != null) { RECYCLER.recycle(this, handle); } + } + + private CommandLookupTopic(boolean noInit) {} + + private static final CommandLookupTopic defaultInstance; + public static CommandLookupTopic getDefaultInstance() { + return defaultInstance; + } + + public CommandLookupTopic getDefaultInstanceForType() { + return defaultInstance; + } + + private int bitField0_; + // required string topic = 1; + public static final int TOPIC_FIELD_NUMBER = 1; + private java.lang.Object topic_; + public boolean hasTopic() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getTopic() { + java.lang.Object ref = topic_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + topic_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getTopicBytes() { + java.lang.Object ref = topic_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + topic_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // required uint64 request_id = 2; + public static final int REQUEST_ID_FIELD_NUMBER = 2; + private long requestId_; + public boolean hasRequestId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getRequestId() { + return requestId_; + } + + // optional bool authoritative = 3 [default = false]; + public static final int AUTHORITATIVE_FIELD_NUMBER = 3; + private boolean authoritative_; + public boolean hasAuthoritative() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public boolean getAuthoritative() { + return authoritative_; + } + + private void initFields() { + topic_ = ""; + requestId_ = 0L; + authoritative_ = false; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasTopic()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasRequestId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + throw new RuntimeException("Cannot use CodedOutputStream"); + } + + public void writeTo(com.yahoo.pulsar.common.util.protobuf.ByteBufCodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getTopicBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeUInt64(2, requestId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeBool(3, authoritative_); + } + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getTopicBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(2, requestId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(3, authoritative_); + } + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + public static final class Builder extends + com.google.protobuf.GeneratedMessageLite.Builder< + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic, Builder> + implements com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicOrBuilder, com.yahoo.pulsar.common.util.protobuf.ByteBufCodedInputStream.ByteBufMessageBuilder { + // Construct using com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic.newBuilder() + private final io.netty.util.Recycler.Handle handle; + private Builder(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + maybeForceBuilderInitialization(); + } + private final static io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected Builder newObject(io.netty.util.Recycler.Handle handle) { + return new Builder(handle); + } + }; + + public void recycle() { + clear(); + if (handle != null) {RECYCLER.recycle(this, handle);} + } + + private void maybeForceBuilderInitialization() { + } + private static Builder create() { + return RECYCLER.get(); + } + + public Builder clear() { + super.clear(); + topic_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + requestId_ = 0L; + bitField0_ = (bitField0_ & ~0x00000002); + authoritative_ = false; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic getDefaultInstanceForType() { + return com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic.getDefaultInstance(); + } + + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic build() { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic buildPartial() { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic result = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic.RECYCLER.get(); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.topic_ = topic_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.requestId_ = requestId_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.authoritative_ = authoritative_; + result.bitField0_ = to_bitField0_; + return result; + } + + public Builder mergeFrom(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic other) { + if (other == com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic.getDefaultInstance()) return this; + if (other.hasTopic()) { + setTopic(other.getTopic()); + } + if (other.hasRequestId()) { + setRequestId(other.getRequestId()); + } + if (other.hasAuthoritative()) { + setAuthoritative(other.getAuthoritative()); + } + return this; + } + + public final boolean isInitialized() { + if (!hasTopic()) { + + return false; + } + if (!hasRequestId()) { + + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + throw new java.io.IOException("Merge from CodedInputStream is disabled"); + } + public Builder mergeFrom( + com.yahoo.pulsar.common.util.protobuf.ByteBufCodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + + return this; + default: { + if (!input.skipField(tag)) { + + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + topic_ = input.readBytes(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + requestId_ = input.readUInt64(); + break; + } + case 24: { + bitField0_ |= 0x00000004; + authoritative_ = input.readBool(); + break; + } + } + } + } + + private int bitField0_; + + // required string topic = 1; + private java.lang.Object topic_ = ""; + public boolean hasTopic() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getTopic() { + java.lang.Object ref = topic_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + topic_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setTopic(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + topic_ = value; + + return this; + } + public Builder clearTopic() { + bitField0_ = (bitField0_ & ~0x00000001); + topic_ = getDefaultInstance().getTopic(); + + return this; + } + void setTopic(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + topic_ = value; + + } + + // required uint64 request_id = 2; + private long requestId_ ; + public boolean hasRequestId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getRequestId() { + return requestId_; + } + public Builder setRequestId(long value) { + bitField0_ |= 0x00000002; + requestId_ = value; + + return this; + } + public Builder clearRequestId() { + bitField0_ = (bitField0_ & ~0x00000002); + requestId_ = 0L; + + return this; + } + + // optional bool authoritative = 3 [default = false]; + private boolean authoritative_ ; + public boolean hasAuthoritative() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public boolean getAuthoritative() { + return authoritative_; + } + public Builder setAuthoritative(boolean value) { + bitField0_ |= 0x00000004; + authoritative_ = value; + + return this; + } + public Builder clearAuthoritative() { + bitField0_ = (bitField0_ & ~0x00000004); + authoritative_ = false; + + return this; + } + + // @@protoc_insertion_point(builder_scope:com.yahoo.pulsar.common.api.proto.CommandLookupTopic) + } + + static { + defaultInstance = new CommandLookupTopic(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:com.yahoo.pulsar.common.api.proto.CommandLookupTopic) + } + + public interface CommandLookupTopicResponseOrBuilder + extends com.google.protobuf.MessageLiteOrBuilder { + + // optional string brokerServiceUrl = 1; + boolean hasBrokerServiceUrl(); + String getBrokerServiceUrl(); + + // optional string brokerServiceUrlTls = 2; + boolean hasBrokerServiceUrlTls(); + String getBrokerServiceUrlTls(); + + // optional .com.yahoo.pulsar.common.api.proto.CommandLookupTopicResponse.LookupType response = 3; + boolean hasResponse(); + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType getResponse(); + + // required uint64 request_id = 4; + boolean hasRequestId(); + long getRequestId(); + + // optional bool authoritative = 5 [default = false]; + boolean hasAuthoritative(); + boolean getAuthoritative(); + + // optional .com.yahoo.pulsar.common.api.proto.ServerError error = 6; + boolean hasError(); + com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError getError(); + + // optional string message = 7; + boolean hasMessage(); + String getMessage(); + } + public static final class CommandLookupTopicResponse extends + com.google.protobuf.GeneratedMessageLite + implements CommandLookupTopicResponseOrBuilder, com.yahoo.pulsar.common.util.protobuf.ByteBufCodedOutputStream.ByteBufGeneratedMessage { + // Use CommandLookupTopicResponse.newBuilder() to construct. + private io.netty.util.Recycler.Handle handle; + private CommandLookupTopicResponse(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + } + + private static final io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected CommandLookupTopicResponse newObject(Handle handle) { + return new CommandLookupTopicResponse(handle); + } + }; + + public void recycle() { + this.initFields(); + this.memoizedIsInitialized = -1; + this.bitField0_ = 0; + this.memoizedSerializedSize = -1; + if (handle != null) { RECYCLER.recycle(this, handle); } + } + + private CommandLookupTopicResponse(boolean noInit) {} + + private static final CommandLookupTopicResponse defaultInstance; + public static CommandLookupTopicResponse getDefaultInstance() { + return defaultInstance; + } + + public CommandLookupTopicResponse getDefaultInstanceForType() { + return defaultInstance; + } + + public enum LookupType + implements com.google.protobuf.Internal.EnumLite { + Redirect(0, 0), + Connect(1, 1), + Failed(2, 2), + ; + + public static final int Redirect_VALUE = 0; + public static final int Connect_VALUE = 1; + public static final int Failed_VALUE = 2; + + + public final int getNumber() { return value; } + + public static LookupType valueOf(int value) { + switch (value) { + case 0: return Redirect; + case 1: return Connect; + case 2: return Failed; + default: return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap + internalGetValueMap() { + return internalValueMap; + } + private static com.google.protobuf.Internal.EnumLiteMap + internalValueMap = + new com.google.protobuf.Internal.EnumLiteMap() { + public LookupType findValueByNumber(int number) { + return LookupType.valueOf(number); + } + }; + + private final int value; + + private LookupType(int index, int value) { + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:com.yahoo.pulsar.common.api.proto.CommandLookupTopicResponse.LookupType) + } + + private int bitField0_; + // optional string brokerServiceUrl = 1; + public static final int BROKERSERVICEURL_FIELD_NUMBER = 1; + private java.lang.Object brokerServiceUrl_; + public boolean hasBrokerServiceUrl() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getBrokerServiceUrl() { + java.lang.Object ref = brokerServiceUrl_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + brokerServiceUrl_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getBrokerServiceUrlBytes() { + java.lang.Object ref = brokerServiceUrl_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + brokerServiceUrl_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional string brokerServiceUrlTls = 2; + public static final int BROKERSERVICEURLTLS_FIELD_NUMBER = 2; + private java.lang.Object brokerServiceUrlTls_; + public boolean hasBrokerServiceUrlTls() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public String getBrokerServiceUrlTls() { + java.lang.Object ref = brokerServiceUrlTls_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + brokerServiceUrlTls_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getBrokerServiceUrlTlsBytes() { + java.lang.Object ref = brokerServiceUrlTls_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + brokerServiceUrlTls_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional .com.yahoo.pulsar.common.api.proto.CommandLookupTopicResponse.LookupType response = 3; + public static final int RESPONSE_FIELD_NUMBER = 3; + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType response_; + public boolean hasResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType getResponse() { + return response_; + } + + // required uint64 request_id = 4; + public static final int REQUEST_ID_FIELD_NUMBER = 4; + private long requestId_; + public boolean hasRequestId() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public long getRequestId() { + return requestId_; + } + + // optional bool authoritative = 5 [default = false]; + public static final int AUTHORITATIVE_FIELD_NUMBER = 5; + private boolean authoritative_; + public boolean hasAuthoritative() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public boolean getAuthoritative() { + return authoritative_; + } + + // optional .com.yahoo.pulsar.common.api.proto.ServerError error = 6; + public static final int ERROR_FIELD_NUMBER = 6; + private com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError error_; + public boolean hasError() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError getError() { + return error_; + } + + // optional string message = 7; + public static final int MESSAGE_FIELD_NUMBER = 7; + private java.lang.Object message_; + public boolean hasMessage() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + public String getMessage() { + java.lang.Object ref = message_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + if (com.google.protobuf.Internal.isValidUtf8(bs)) { + message_ = s; + } + return s; + } + } + private com.google.protobuf.ByteString getMessageBytes() { + java.lang.Object ref = message_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8((String) ref); + message_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private void initFields() { + brokerServiceUrl_ = ""; + brokerServiceUrlTls_ = ""; + response_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType.Redirect; + requestId_ = 0L; + authoritative_ = false; + error_ = com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError.UnknownError; + message_ = ""; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasRequestId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + throw new RuntimeException("Cannot use CodedOutputStream"); + } + + public void writeTo(com.yahoo.pulsar.common.util.protobuf.ByteBufCodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getBrokerServiceUrlBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBytes(2, getBrokerServiceUrlTlsBytes()); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeEnum(3, response_.getNumber()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeUInt64(4, requestId_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeBool(5, authoritative_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + output.writeEnum(6, error_.getNumber()); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + output.writeBytes(7, getMessageBytes()); + } + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getBrokerServiceUrlBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(2, getBrokerServiceUrlTlsBytes()); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(3, response_.getNumber()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(4, requestId_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(5, authoritative_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(6, error_.getNumber()); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(7, getMessageBytes()); + } + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + public static final class Builder extends + com.google.protobuf.GeneratedMessageLite.Builder< + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse, Builder> + implements com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponseOrBuilder, com.yahoo.pulsar.common.util.protobuf.ByteBufCodedInputStream.ByteBufMessageBuilder { + // Construct using com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.newBuilder() + private final io.netty.util.Recycler.Handle handle; + private Builder(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + maybeForceBuilderInitialization(); + } + private final static io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected Builder newObject(io.netty.util.Recycler.Handle handle) { + return new Builder(handle); + } + }; + + public void recycle() { + clear(); + if (handle != null) {RECYCLER.recycle(this, handle);} + } + + private void maybeForceBuilderInitialization() { + } + private static Builder create() { + return RECYCLER.get(); + } + + public Builder clear() { + super.clear(); + brokerServiceUrl_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + brokerServiceUrlTls_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + response_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType.Redirect; + bitField0_ = (bitField0_ & ~0x00000004); + requestId_ = 0L; + bitField0_ = (bitField0_ & ~0x00000008); + authoritative_ = false; + bitField0_ = (bitField0_ & ~0x00000010); + error_ = com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError.UnknownError; + bitField0_ = (bitField0_ & ~0x00000020); + message_ = ""; + bitField0_ = (bitField0_ & ~0x00000040); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse getDefaultInstanceForType() { + return com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.getDefaultInstance(); + } + + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse build() { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse buildParsed() + throws com.google.protobuf.InvalidProtocolBufferException { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse buildPartial() { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse result = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.RECYCLER.get(); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.brokerServiceUrl_ = brokerServiceUrl_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.brokerServiceUrlTls_ = brokerServiceUrlTls_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.response_ = response_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.requestId_ = requestId_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.authoritative_ = authoritative_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000020; + } + result.error_ = error_; + if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000040; + } + result.message_ = message_; + result.bitField0_ = to_bitField0_; + return result; + } + + public Builder mergeFrom(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse other) { + if (other == com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.getDefaultInstance()) return this; + if (other.hasBrokerServiceUrl()) { + setBrokerServiceUrl(other.getBrokerServiceUrl()); + } + if (other.hasBrokerServiceUrlTls()) { + setBrokerServiceUrlTls(other.getBrokerServiceUrlTls()); + } + if (other.hasResponse()) { + setResponse(other.getResponse()); + } + if (other.hasRequestId()) { + setRequestId(other.getRequestId()); + } + if (other.hasAuthoritative()) { + setAuthoritative(other.getAuthoritative()); + } + if (other.hasError()) { + setError(other.getError()); + } + if (other.hasMessage()) { + setMessage(other.getMessage()); + } + return this; + } + + public final boolean isInitialized() { + if (!hasRequestId()) { + + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + throw new java.io.IOException("Merge from CodedInputStream is disabled"); + } + public Builder mergeFrom( + com.yahoo.pulsar.common.util.protobuf.ByteBufCodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + + return this; + default: { + if (!input.skipField(tag)) { + + return this; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + brokerServiceUrl_ = input.readBytes(); + break; + } + case 18: { + bitField0_ |= 0x00000002; + brokerServiceUrlTls_ = input.readBytes(); + break; + } + case 24: { + int rawValue = input.readEnum(); + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType value = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType.valueOf(rawValue); + if (value != null) { + bitField0_ |= 0x00000004; + response_ = value; + } + break; + } + case 32: { + bitField0_ |= 0x00000008; + requestId_ = input.readUInt64(); + break; + } + case 40: { + bitField0_ |= 0x00000010; + authoritative_ = input.readBool(); + break; + } + case 48: { + int rawValue = input.readEnum(); + com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError value = com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError.valueOf(rawValue); + if (value != null) { + bitField0_ |= 0x00000020; + error_ = value; + } + break; + } + case 58: { + bitField0_ |= 0x00000040; + message_ = input.readBytes(); + break; + } + } + } + } + + private int bitField0_; + + // optional string brokerServiceUrl = 1; + private java.lang.Object brokerServiceUrl_ = ""; + public boolean hasBrokerServiceUrl() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public String getBrokerServiceUrl() { + java.lang.Object ref = brokerServiceUrl_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + brokerServiceUrl_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setBrokerServiceUrl(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + brokerServiceUrl_ = value; + + return this; + } + public Builder clearBrokerServiceUrl() { + bitField0_ = (bitField0_ & ~0x00000001); + brokerServiceUrl_ = getDefaultInstance().getBrokerServiceUrl(); + + return this; + } + void setBrokerServiceUrl(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000001; + brokerServiceUrl_ = value; + + } + + // optional string brokerServiceUrlTls = 2; + private java.lang.Object brokerServiceUrlTls_ = ""; + public boolean hasBrokerServiceUrlTls() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public String getBrokerServiceUrlTls() { + java.lang.Object ref = brokerServiceUrlTls_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + brokerServiceUrlTls_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setBrokerServiceUrlTls(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + brokerServiceUrlTls_ = value; + + return this; + } + public Builder clearBrokerServiceUrlTls() { + bitField0_ = (bitField0_ & ~0x00000002); + brokerServiceUrlTls_ = getDefaultInstance().getBrokerServiceUrlTls(); + + return this; + } + void setBrokerServiceUrlTls(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000002; + brokerServiceUrlTls_ = value; + + } + + // optional .com.yahoo.pulsar.common.api.proto.CommandLookupTopicResponse.LookupType response = 3; + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType response_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType.Redirect; + public boolean hasResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType getResponse() { + return response_; + } + public Builder setResponse(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + response_ = value; + + return this; + } + public Builder clearResponse() { + bitField0_ = (bitField0_ & ~0x00000004); + response_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType.Redirect; + + return this; + } + + // required uint64 request_id = 4; + private long requestId_ ; + public boolean hasRequestId() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public long getRequestId() { + return requestId_; + } + public Builder setRequestId(long value) { + bitField0_ |= 0x00000008; + requestId_ = value; + + return this; + } + public Builder clearRequestId() { + bitField0_ = (bitField0_ & ~0x00000008); + requestId_ = 0L; + + return this; + } + + // optional bool authoritative = 5 [default = false]; + private boolean authoritative_ ; + public boolean hasAuthoritative() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + public boolean getAuthoritative() { + return authoritative_; + } + public Builder setAuthoritative(boolean value) { + bitField0_ |= 0x00000010; + authoritative_ = value; + + return this; + } + public Builder clearAuthoritative() { + bitField0_ = (bitField0_ & ~0x00000010); + authoritative_ = false; + + return this; + } + + // optional .com.yahoo.pulsar.common.api.proto.ServerError error = 6; + private com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError error_ = com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError.UnknownError; + public boolean hasError() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError getError() { + return error_; + } + public Builder setError(com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000020; + error_ = value; + + return this; + } + public Builder clearError() { + bitField0_ = (bitField0_ & ~0x00000020); + error_ = com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError.UnknownError; + + return this; + } + + // optional string message = 7; + private java.lang.Object message_ = ""; + public boolean hasMessage() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + public String getMessage() { + java.lang.Object ref = message_; + if (!(ref instanceof String)) { + String s = ((com.google.protobuf.ByteString) ref).toStringUtf8(); + message_ = s; + return s; + } else { + return (String) ref; + } + } + public Builder setMessage(String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000040; + message_ = value; + + return this; + } + public Builder clearMessage() { + bitField0_ = (bitField0_ & ~0x00000040); + message_ = getDefaultInstance().getMessage(); + + return this; + } + void setMessage(com.google.protobuf.ByteString value) { + bitField0_ |= 0x00000040; + message_ = value; + + } + + // @@protoc_insertion_point(builder_scope:com.yahoo.pulsar.common.api.proto.CommandLookupTopicResponse) + } + + static { + defaultInstance = new CommandLookupTopicResponse(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:com.yahoo.pulsar.common.api.proto.CommandLookupTopicResponse) + } + public interface CommandProducerOrBuilder extends com.google.protobuf.MessageLiteOrBuilder { @@ -11862,6 +14457,22 @@ public interface BaseCommandOrBuilder // optional .com.yahoo.pulsar.common.api.proto.CommandRedeliverUnacknowledgedMessages redeliverUnacknowledgedMessages = 20; boolean hasRedeliverUnacknowledgedMessages(); com.yahoo.pulsar.common.api.proto.PulsarApi.CommandRedeliverUnacknowledgedMessages getRedeliverUnacknowledgedMessages(); + + // optional .com.yahoo.pulsar.common.api.proto.CommandPartitionedTopicMetadata partitionMetadata = 21; + boolean hasPartitionMetadata(); + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata getPartitionMetadata(); + + // optional .com.yahoo.pulsar.common.api.proto.CommandPartitionedTopicMetadataResponse partitionMetadataResponse = 22; + boolean hasPartitionMetadataResponse(); + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse getPartitionMetadataResponse(); + + // optional .com.yahoo.pulsar.common.api.proto.CommandLookupTopic lookupTopic = 23; + boolean hasLookupTopic(); + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic getLookupTopic(); + + // optional .com.yahoo.pulsar.common.api.proto.CommandLookupTopicResponse lookupTopicResponse = 24; + boolean hasLookupTopicResponse(); + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse getLookupTopicResponse(); } public static final class BaseCommand extends com.google.protobuf.GeneratedMessageLite @@ -11918,6 +14529,10 @@ public enum Type PING(16, 18), PONG(17, 19), REDELIVER_UNACKNOWLEDGED_MESSAGES(18, 20), + PARTITIONED_METADATA(19, 21), + PARTITIONED_METADATA_RESPONSE(20, 22), + LOOKUP(21, 23), + LOOKUP_RESPONSE(22, 24), ; public static final int CONNECT_VALUE = 2; @@ -11939,6 +14554,10 @@ public enum Type public static final int PING_VALUE = 18; public static final int PONG_VALUE = 19; public static final int REDELIVER_UNACKNOWLEDGED_MESSAGES_VALUE = 20; + public static final int PARTITIONED_METADATA_VALUE = 21; + public static final int PARTITIONED_METADATA_RESPONSE_VALUE = 22; + public static final int LOOKUP_VALUE = 23; + public static final int LOOKUP_RESPONSE_VALUE = 24; public final int getNumber() { return value; } @@ -11964,6 +14583,10 @@ public static Type valueOf(int value) { case 18: return PING; case 19: return PONG; case 20: return REDELIVER_UNACKNOWLEDGED_MESSAGES; + case 21: return PARTITIONED_METADATA; + case 22: return PARTITIONED_METADATA_RESPONSE; + case 23: return LOOKUP; + case 24: return LOOKUP_RESPONSE; default: return null; } } @@ -12190,6 +14813,46 @@ public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandRedeliverUnacknowledge return redeliverUnacknowledgedMessages_; } + // optional .com.yahoo.pulsar.common.api.proto.CommandPartitionedTopicMetadata partitionMetadata = 21; + public static final int PARTITIONMETADATA_FIELD_NUMBER = 21; + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata partitionMetadata_; + public boolean hasPartitionMetadata() { + return ((bitField0_ & 0x00100000) == 0x00100000); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata getPartitionMetadata() { + return partitionMetadata_; + } + + // optional .com.yahoo.pulsar.common.api.proto.CommandPartitionedTopicMetadataResponse partitionMetadataResponse = 22; + public static final int PARTITIONMETADATARESPONSE_FIELD_NUMBER = 22; + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse partitionMetadataResponse_; + public boolean hasPartitionMetadataResponse() { + return ((bitField0_ & 0x00200000) == 0x00200000); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse getPartitionMetadataResponse() { + return partitionMetadataResponse_; + } + + // optional .com.yahoo.pulsar.common.api.proto.CommandLookupTopic lookupTopic = 23; + public static final int LOOKUPTOPIC_FIELD_NUMBER = 23; + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic lookupTopic_; + public boolean hasLookupTopic() { + return ((bitField0_ & 0x00400000) == 0x00400000); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic getLookupTopic() { + return lookupTopic_; + } + + // optional .com.yahoo.pulsar.common.api.proto.CommandLookupTopicResponse lookupTopicResponse = 24; + public static final int LOOKUPTOPICRESPONSE_FIELD_NUMBER = 24; + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse lookupTopicResponse_; + public boolean hasLookupTopicResponse() { + return ((bitField0_ & 0x00800000) == 0x00800000); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse getLookupTopicResponse() { + return lookupTopicResponse_; + } + private void initFields() { type_ = com.yahoo.pulsar.common.api.proto.PulsarApi.BaseCommand.Type.CONNECT; connect_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandConnect.getDefaultInstance(); @@ -12211,6 +14874,10 @@ private void initFields() { ping_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPing.getDefaultInstance(); pong_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPong.getDefaultInstance(); redeliverUnacknowledgedMessages_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandRedeliverUnacknowledgedMessages.getDefaultInstance(); + partitionMetadata_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata.getDefaultInstance(); + partitionMetadataResponse_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.getDefaultInstance(); + lookupTopic_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic.getDefaultInstance(); + lookupTopicResponse_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.getDefaultInstance(); } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -12323,6 +14990,30 @@ public final boolean isInitialized() { return false; } } + if (hasPartitionMetadata()) { + if (!getPartitionMetadata().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + if (hasPartitionMetadataResponse()) { + if (!getPartitionMetadataResponse().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + if (hasLookupTopic()) { + if (!getLookupTopic().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + if (hasLookupTopicResponse()) { + if (!getLookupTopicResponse().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } memoizedIsInitialized = 1; return true; } @@ -12395,6 +15086,18 @@ public void writeTo(com.yahoo.pulsar.common.util.protobuf.ByteBufCodedOutputStre if (((bitField0_ & 0x00080000) == 0x00080000)) { output.writeMessage(20, redeliverUnacknowledgedMessages_); } + if (((bitField0_ & 0x00100000) == 0x00100000)) { + output.writeMessage(21, partitionMetadata_); + } + if (((bitField0_ & 0x00200000) == 0x00200000)) { + output.writeMessage(22, partitionMetadataResponse_); + } + if (((bitField0_ & 0x00400000) == 0x00400000)) { + output.writeMessage(23, lookupTopic_); + } + if (((bitField0_ & 0x00800000) == 0x00800000)) { + output.writeMessage(24, lookupTopicResponse_); + } } private int memoizedSerializedSize = -1; @@ -12483,6 +15186,22 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeMessageSize(20, redeliverUnacknowledgedMessages_); } + if (((bitField0_ & 0x00100000) == 0x00100000)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(21, partitionMetadata_); + } + if (((bitField0_ & 0x00200000) == 0x00200000)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(22, partitionMetadataResponse_); + } + if (((bitField0_ & 0x00400000) == 0x00400000)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(23, lookupTopic_); + } + if (((bitField0_ & 0x00800000) == 0x00800000)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(24, lookupTopicResponse_); + } memoizedSerializedSize = size; return size; } @@ -12636,6 +15355,14 @@ public Builder clear() { bitField0_ = (bitField0_ & ~0x00040000); redeliverUnacknowledgedMessages_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandRedeliverUnacknowledgedMessages.getDefaultInstance(); bitField0_ = (bitField0_ & ~0x00080000); + partitionMetadata_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata.getDefaultInstance(); + bitField0_ = (bitField0_ & ~0x00100000); + partitionMetadataResponse_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.getDefaultInstance(); + bitField0_ = (bitField0_ & ~0x00200000); + lookupTopic_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic.getDefaultInstance(); + bitField0_ = (bitField0_ & ~0x00400000); + lookupTopicResponse_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.getDefaultInstance(); + bitField0_ = (bitField0_ & ~0x00800000); return this; } @@ -12749,6 +15476,22 @@ public com.yahoo.pulsar.common.api.proto.PulsarApi.BaseCommand buildPartial() { to_bitField0_ |= 0x00080000; } result.redeliverUnacknowledgedMessages_ = redeliverUnacknowledgedMessages_; + if (((from_bitField0_ & 0x00100000) == 0x00100000)) { + to_bitField0_ |= 0x00100000; + } + result.partitionMetadata_ = partitionMetadata_; + if (((from_bitField0_ & 0x00200000) == 0x00200000)) { + to_bitField0_ |= 0x00200000; + } + result.partitionMetadataResponse_ = partitionMetadataResponse_; + if (((from_bitField0_ & 0x00400000) == 0x00400000)) { + to_bitField0_ |= 0x00400000; + } + result.lookupTopic_ = lookupTopic_; + if (((from_bitField0_ & 0x00800000) == 0x00800000)) { + to_bitField0_ |= 0x00800000; + } + result.lookupTopicResponse_ = lookupTopicResponse_; result.bitField0_ = to_bitField0_; return result; } @@ -12815,6 +15558,18 @@ public Builder mergeFrom(com.yahoo.pulsar.common.api.proto.PulsarApi.BaseCommand if (other.hasRedeliverUnacknowledgedMessages()) { mergeRedeliverUnacknowledgedMessages(other.getRedeliverUnacknowledgedMessages()); } + if (other.hasPartitionMetadata()) { + mergePartitionMetadata(other.getPartitionMetadata()); + } + if (other.hasPartitionMetadataResponse()) { + mergePartitionMetadataResponse(other.getPartitionMetadataResponse()); + } + if (other.hasLookupTopic()) { + mergeLookupTopic(other.getLookupTopic()); + } + if (other.hasLookupTopicResponse()) { + mergeLookupTopicResponse(other.getLookupTopicResponse()); + } return this; } @@ -12925,6 +15680,30 @@ public final boolean isInitialized() { return false; } } + if (hasPartitionMetadata()) { + if (!getPartitionMetadata().isInitialized()) { + + return false; + } + } + if (hasPartitionMetadataResponse()) { + if (!getPartitionMetadataResponse().isInitialized()) { + + return false; + } + } + if (hasLookupTopic()) { + if (!getLookupTopic().isInitialized()) { + + return false; + } + } + if (hasLookupTopicResponse()) { + if (!getLookupTopicResponse().isInitialized()) { + + return false; + } + } return true; } @@ -13149,6 +15928,46 @@ public Builder mergeFrom( subBuilder.recycle(); break; } + case 170: { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata.Builder subBuilder = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata.newBuilder(); + if (hasPartitionMetadata()) { + subBuilder.mergeFrom(getPartitionMetadata()); + } + input.readMessage(subBuilder, extensionRegistry); + setPartitionMetadata(subBuilder.buildPartial()); + subBuilder.recycle(); + break; + } + case 178: { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.Builder subBuilder = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.newBuilder(); + if (hasPartitionMetadataResponse()) { + subBuilder.mergeFrom(getPartitionMetadataResponse()); + } + input.readMessage(subBuilder, extensionRegistry); + setPartitionMetadataResponse(subBuilder.buildPartial()); + subBuilder.recycle(); + break; + } + case 186: { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic.Builder subBuilder = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic.newBuilder(); + if (hasLookupTopic()) { + subBuilder.mergeFrom(getLookupTopic()); + } + input.readMessage(subBuilder, extensionRegistry); + setLookupTopic(subBuilder.buildPartial()); + subBuilder.recycle(); + break; + } + case 194: { + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.Builder subBuilder = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.newBuilder(); + if (hasLookupTopicResponse()) { + subBuilder.mergeFrom(getLookupTopicResponse()); + } + input.readMessage(subBuilder, extensionRegistry); + setLookupTopicResponse(subBuilder.buildPartial()); + subBuilder.recycle(); + break; + } } } } @@ -13996,6 +16815,178 @@ public Builder clearRedeliverUnacknowledgedMessages() { return this; } + // optional .com.yahoo.pulsar.common.api.proto.CommandPartitionedTopicMetadata partitionMetadata = 21; + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata partitionMetadata_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata.getDefaultInstance(); + public boolean hasPartitionMetadata() { + return ((bitField0_ & 0x00100000) == 0x00100000); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata getPartitionMetadata() { + return partitionMetadata_; + } + public Builder setPartitionMetadata(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata value) { + if (value == null) { + throw new NullPointerException(); + } + partitionMetadata_ = value; + + bitField0_ |= 0x00100000; + return this; + } + public Builder setPartitionMetadata( + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata.Builder builderForValue) { + partitionMetadata_ = builderForValue.build(); + + bitField0_ |= 0x00100000; + return this; + } + public Builder mergePartitionMetadata(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata value) { + if (((bitField0_ & 0x00100000) == 0x00100000) && + partitionMetadata_ != com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata.getDefaultInstance()) { + partitionMetadata_ = + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata.newBuilder(partitionMetadata_).mergeFrom(value).buildPartial(); + } else { + partitionMetadata_ = value; + } + + bitField0_ |= 0x00100000; + return this; + } + public Builder clearPartitionMetadata() { + partitionMetadata_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata.getDefaultInstance(); + + bitField0_ = (bitField0_ & ~0x00100000); + return this; + } + + // optional .com.yahoo.pulsar.common.api.proto.CommandPartitionedTopicMetadataResponse partitionMetadataResponse = 22; + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse partitionMetadataResponse_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.getDefaultInstance(); + public boolean hasPartitionMetadataResponse() { + return ((bitField0_ & 0x00200000) == 0x00200000); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse getPartitionMetadataResponse() { + return partitionMetadataResponse_; + } + public Builder setPartitionMetadataResponse(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse value) { + if (value == null) { + throw new NullPointerException(); + } + partitionMetadataResponse_ = value; + + bitField0_ |= 0x00200000; + return this; + } + public Builder setPartitionMetadataResponse( + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.Builder builderForValue) { + partitionMetadataResponse_ = builderForValue.build(); + + bitField0_ |= 0x00200000; + return this; + } + public Builder mergePartitionMetadataResponse(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse value) { + if (((bitField0_ & 0x00200000) == 0x00200000) && + partitionMetadataResponse_ != com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.getDefaultInstance()) { + partitionMetadataResponse_ = + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.newBuilder(partitionMetadataResponse_).mergeFrom(value).buildPartial(); + } else { + partitionMetadataResponse_ = value; + } + + bitField0_ |= 0x00200000; + return this; + } + public Builder clearPartitionMetadataResponse() { + partitionMetadataResponse_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadataResponse.getDefaultInstance(); + + bitField0_ = (bitField0_ & ~0x00200000); + return this; + } + + // optional .com.yahoo.pulsar.common.api.proto.CommandLookupTopic lookupTopic = 23; + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic lookupTopic_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic.getDefaultInstance(); + public boolean hasLookupTopic() { + return ((bitField0_ & 0x00400000) == 0x00400000); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic getLookupTopic() { + return lookupTopic_; + } + public Builder setLookupTopic(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic value) { + if (value == null) { + throw new NullPointerException(); + } + lookupTopic_ = value; + + bitField0_ |= 0x00400000; + return this; + } + public Builder setLookupTopic( + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic.Builder builderForValue) { + lookupTopic_ = builderForValue.build(); + + bitField0_ |= 0x00400000; + return this; + } + public Builder mergeLookupTopic(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic value) { + if (((bitField0_ & 0x00400000) == 0x00400000) && + lookupTopic_ != com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic.getDefaultInstance()) { + lookupTopic_ = + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic.newBuilder(lookupTopic_).mergeFrom(value).buildPartial(); + } else { + lookupTopic_ = value; + } + + bitField0_ |= 0x00400000; + return this; + } + public Builder clearLookupTopic() { + lookupTopic_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic.getDefaultInstance(); + + bitField0_ = (bitField0_ & ~0x00400000); + return this; + } + + // optional .com.yahoo.pulsar.common.api.proto.CommandLookupTopicResponse lookupTopicResponse = 24; + private com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse lookupTopicResponse_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.getDefaultInstance(); + public boolean hasLookupTopicResponse() { + return ((bitField0_ & 0x00800000) == 0x00800000); + } + public com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse getLookupTopicResponse() { + return lookupTopicResponse_; + } + public Builder setLookupTopicResponse(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse value) { + if (value == null) { + throw new NullPointerException(); + } + lookupTopicResponse_ = value; + + bitField0_ |= 0x00800000; + return this; + } + public Builder setLookupTopicResponse( + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.Builder builderForValue) { + lookupTopicResponse_ = builderForValue.build(); + + bitField0_ |= 0x00800000; + return this; + } + public Builder mergeLookupTopicResponse(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse value) { + if (((bitField0_ & 0x00800000) == 0x00800000) && + lookupTopicResponse_ != com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.getDefaultInstance()) { + lookupTopicResponse_ = + com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.newBuilder(lookupTopicResponse_).mergeFrom(value).buildPartial(); + } else { + lookupTopicResponse_ = value; + } + + bitField0_ |= 0x00800000; + return this; + } + public Builder clearLookupTopicResponse() { + lookupTopicResponse_ = com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.getDefaultInstance(); + + bitField0_ = (bitField0_ & ~0x00800000); + return this; + } + // @@protoc_insertion_point(builder_scope:com.yahoo.pulsar.common.api.proto.BaseCommand) } diff --git a/pulsar-common/src/main/java/com/yahoo/pulsar/common/lookup/data/LookupData.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/lookup/data/LookupData.java index b595b4476c715..c908c28cd1231 100644 --- a/pulsar-common/src/main/java/com/yahoo/pulsar/common/lookup/data/LookupData.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/lookup/data/LookupData.java @@ -21,15 +21,23 @@ public class LookupData { private String brokerUrl; private String brokerUrlTls; private String httpUrl; // Web service HTTP address + private String httpUrlTls; // Web service HTTPS address private String nativeUrl; public LookupData() { } - public LookupData(String brokerUrl, String brokerUrlTls, String httpUrl) { + public LookupData(String brokerUrl, String brokerUrlTls, String httpUrl, String httpUrlTls) { this.brokerUrl = brokerUrl; this.brokerUrlTls = brokerUrlTls; this.httpUrl = httpUrl; + this.httpUrlTls = httpUrlTls; + this.nativeUrl = brokerUrl; + } + + public LookupData(String brokerUrl, String brokerUrlTls, boolean redirect, boolean authoritative) { + this.brokerUrl = brokerUrl; + this.brokerUrlTls = brokerUrlTls; this.nativeUrl = brokerUrl; } @@ -44,6 +52,14 @@ public String getBrokerUrlTls() { public String getHttpUrl() { return httpUrl; } + + public String getHttpUrlTls() { + return httpUrlTls; + } + + public void setHttpUrlTls(String httpUrlTls) { + this.httpUrlTls = httpUrlTls; + } /** * Legacy name, but client libraries are still using it so it needs to be included in Json diff --git a/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/ClusterData.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/ClusterData.java index e57a5699c425e..388478404e058 100644 --- a/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/ClusterData.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/ClusterData.java @@ -20,6 +20,8 @@ public class ClusterData { private String serviceUrl; private String serviceUrlTls; + private String brokerServiceUrl; + private String brokerServiceUrlTls; public ClusterData() { } @@ -32,6 +34,13 @@ public ClusterData(String serviceUrl, String serviceUrlTls) { this.serviceUrl = serviceUrl; this.serviceUrlTls = serviceUrlTls; } + + public ClusterData(String serviceUrl, String serviceUrlTls, String brokerServiceUrl, String brokerServiceUrlTls) { + this.serviceUrl = serviceUrl; + this.serviceUrlTls = serviceUrlTls; + this.brokerServiceUrl = brokerServiceUrl; + this.brokerServiceUrlTls = brokerServiceUrlTls; + } public String getServiceUrl() { return serviceUrl; @@ -48,12 +57,30 @@ public void setServiceUrl(String serviceUrl) { public void setServiceUrlTls(String serviceUrlTls) { this.serviceUrlTls = serviceUrlTls; } + + public String getBrokerServiceUrl() { + return brokerServiceUrl; + } + + public void setBrokerServiceUrl(String brokerServiceUrl) { + this.brokerServiceUrl = brokerServiceUrl; + } + + public String getBrokerServiceUrlTls() { + return brokerServiceUrlTls; + } + + public void setBrokerServiceUrlTls(String brokerServiceUrlTls) { + this.brokerServiceUrlTls = brokerServiceUrlTls; + } @Override public boolean equals(Object obj) { if (obj instanceof ClusterData) { ClusterData other = (ClusterData) obj; - return Objects.equal(serviceUrl, other.serviceUrl) && Objects.equal(serviceUrlTls, other.serviceUrlTls); + return Objects.equal(serviceUrl, other.serviceUrl) && Objects.equal(serviceUrlTls, other.serviceUrlTls) + && Objects.equal(brokerServiceUrl, other.brokerServiceUrl) + && Objects.equal(brokerServiceUrlTls, other.brokerServiceUrlTls); } return false; @@ -61,19 +88,26 @@ public boolean equals(Object obj) { @Override public int hashCode() { - if (serviceUrlTls != null && !serviceUrlTls.isEmpty()) { - return Objects.hashCode(serviceUrl + serviceUrlTls); - } else { - return Objects.hashCode(serviceUrl); - } + return Objects.hashCode(this.toString()); } @Override public String toString() { - if (serviceUrlTls == null || serviceUrlTls.isEmpty()) { - return serviceUrl; - } else { - return serviceUrl + "," + serviceUrlTls; + StringBuilder str = new StringBuilder(); + str.append(serviceUrl); + if (serviceUrlTls != null && !serviceUrlTls.isEmpty()) { + str.append(","); + str.append(serviceUrlTls); + } + if (brokerServiceUrl != null && !brokerServiceUrl.isEmpty()) { + str.append(","); + str.append(brokerServiceUrl); + } + if (brokerServiceUrlTls != null && !brokerServiceUrlTls.isEmpty()) { + str.append(","); + str.append(brokerServiceUrlTls); } + return str.toString(); } + } diff --git a/pulsar-common/src/main/java/com/yahoo/pulsar/common/util/protobuf/ByteBufCodedOutputStream.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/util/protobuf/ByteBufCodedOutputStream.java index 3fbdedab7f213..8086a7dffef3e 100644 --- a/pulsar-common/src/main/java/com/yahoo/pulsar/common/util/protobuf/ByteBufCodedOutputStream.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/util/protobuf/ByteBufCodedOutputStream.java @@ -127,7 +127,18 @@ public void writeUInt64(final int fieldNumber, final long value) throws IOExcept writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); writeUInt64NoTag(value); } + + /** Write a {@code bool} field, including tag, to the stream. */ + public void writeBool(final int fieldNumber, final boolean value) throws IOException { + writeTag(fieldNumber, WireFormat.WIRETYPE_VARINT); + writeBoolNoTag(value); + } + /** Write a {@code bool} field to the stream. */ + public void writeBoolNoTag(final boolean value) throws IOException { + writeRawByte(value ? 1 : 0); + } + /** Write a {@code uint64} field to the stream. */ public void writeUInt64NoTag(final long value) throws IOException { writeRawVarint64(value); diff --git a/pulsar-common/src/main/proto/PulsarApi.proto b/pulsar-common/src/main/proto/PulsarApi.proto index 6ec3f57c92731..f68c23bfc4b17 100644 --- a/pulsar-common/src/main/proto/PulsarApi.proto +++ b/pulsar-common/src/main/proto/PulsarApi.proto @@ -125,6 +125,50 @@ message CommandSubscribe { optional string consumer_name = 6; } +message CommandPartitionedTopicMetadata { + required string topic = 1; + required uint64 request_id = 2; +} + +message CommandPartitionedTopicMetadataResponse { + enum LookupType { + Redirect = 0; + Success = 1; + Failed = 2; + } + optional uint32 partitions = 1; // Optional in case of error + required uint64 request_id = 2; + optional LookupType response = 3; + optional string brokerServiceUrl = 4; // Present in case of redirect + optional string brokerServiceUrlTls = 5; + optional ServerError error = 6; + optional string message = 7; + +} + +message CommandLookupTopic { + required string topic = 1; + required uint64 request_id = 2; + optional bool authoritative = 3 [default = false]; +} + + +message CommandLookupTopicResponse { + enum LookupType { + Redirect = 0; + Connect = 1; + Failed = 2; + } + + optional string brokerServiceUrl = 1; // Optional in case of error + optional string brokerServiceUrlTls = 2; + optional LookupType response = 3; + required uint64 request_id = 4; + optional bool authoritative = 5 [default = false]; + optional ServerError error = 6; + optional string message = 7; +} + /// Create a new Producer on a topic, assigning the given producer_id, /// all messages sent with this producer_id will be persisted on the topic message CommandProducer { @@ -266,6 +310,12 @@ message BaseCommand { PONG = 19; REDELIVER_UNACKNOWLEDGED_MESSAGES = 20; + + PARTITIONED_METADATA = 21; + PARTITIONED_METADATA_RESPONSE = 22; + + LOOKUP = 23; + LOOKUP_RESPONSE = 24; } required Type type = 1; @@ -293,4 +343,10 @@ message BaseCommand { optional CommandPing ping = 18; optional CommandPong pong = 19; optional CommandRedeliverUnacknowledgedMessages redeliverUnacknowledgedMessages = 20; + + optional CommandPartitionedTopicMetadata partitionMetadata = 21; + optional CommandPartitionedTopicMetadataResponse partitionMetadataResponse = 22; + + optional CommandLookupTopic lookupTopic = 23; + optional CommandLookupTopicResponse lookupTopicResponse = 24; } diff --git a/pulsar-common/src/test/java/com/yahoo/pulsar/common/lookup/data/LookupDataTest.java b/pulsar-common/src/test/java/com/yahoo/pulsar/common/lookup/data/LookupDataTest.java index 5be68f9edd4a9..69ae0f9a92619 100644 --- a/pulsar-common/src/test/java/com/yahoo/pulsar/common/lookup/data/LookupDataTest.java +++ b/pulsar-common/src/test/java/com/yahoo/pulsar/common/lookup/data/LookupDataTest.java @@ -30,7 +30,8 @@ public class LookupDataTest { @Test void withConstructor() { - LookupData data = new LookupData("pulsar://localhost:8888", "pulsar://localhost:8884", "http://localhost:8080"); + LookupData data = new LookupData("pulsar://localhost:8888", "pulsar://localhost:8884", "http://localhost:8080", + "http://localhost:8081"); assertEquals(data.getBrokerUrl(), "pulsar://localhost:8888"); assertEquals(data.getHttpUrl(), "http://localhost:8080"); } @@ -38,7 +39,8 @@ void withConstructor() { @SuppressWarnings("unchecked") @Test void serializeToJsonTest() throws Exception { - LookupData data = new LookupData("pulsar://localhost:8888", "pulsar://localhost:8884", "http://localhost:8080"); + LookupData data = new LookupData("pulsar://localhost:8888", "pulsar://localhost:8884", "http://localhost:8080", + "http://localhost:8081"); ObjectMapper mapper = ObjectMapperFactory.getThreadLocal(); String json = mapper.writeValueAsString(data); @@ -49,5 +51,6 @@ void serializeToJsonTest() throws Exception { assertEquals(jsonMap.get("brokerUrlSsl"), ""); assertEquals(jsonMap.get("nativeUrl"), "pulsar://localhost:8888"); assertEquals(jsonMap.get("httpUrl"), "http://localhost:8080"); + assertEquals(jsonMap.get("httpUrlTls"), "http://localhost:8081"); } } diff --git a/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/BrokerDiscoveryProvider.java b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/BrokerDiscoveryProvider.java new file mode 100644 index 0000000000000..5bbfe82e90fe7 --- /dev/null +++ b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/BrokerDiscoveryProvider.java @@ -0,0 +1,83 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.discovery.service; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport; +import com.yahoo.pulsar.discovery.service.server.ServiceConfig; +import com.yahoo.pulsar.discovery.service.web.ZookeeperCacheLoader; +import com.yahoo.pulsar.zookeeper.ZooKeeperClientFactory; +import static org.apache.bookkeeper.util.MathUtils.signSafeMod; + +/** + * Maintains available active broker list and returns next active broker in round-robin for discovery service. + * + */ +public class BrokerDiscoveryProvider { + + private ZookeeperCacheLoader zkCache; + private final String zookeeperServers; + private final AtomicInteger counter = new AtomicInteger(); + + public BrokerDiscoveryProvider(ServiceConfig config) { + this.zookeeperServers = config.getZookeeperServers(); + } + + /** + * starts {@link ZookeeperCacheLoader} to maintain active-broker list + * + * @param zkClientFactory + * @throws PulsarServerException + */ + public void start(ZooKeeperClientFactory zkClientFactory) throws PulsarServerException { + try { + zkCache = new ZookeeperCacheLoader(zkClientFactory, zookeeperServers); + } catch (Exception e) { + LOG.error("Failed to start Zookkeeper {}", e.getMessage(), e); + throw new PulsarServerException("Failed to start zookeeper :" + e.getMessage(), e); + } + } + + /** + * Find next broke {@link LoadReport} in round-robin fashion. + * + * @return + * @throws PulsarServerException + */ + LoadReport nextBroker() throws PulsarServerException { + List availableBrokers = zkCache.getAvailableBrokers(); + + if (availableBrokers.isEmpty()) { + throw new PulsarServerException("No active broker is available"); + } else { + int brokersCount = availableBrokers.size(); + int nextIdx = signSafeMod(counter.getAndIncrement(), brokersCount); + return availableBrokers.get(nextIdx); + } + } + + public void close() { + zkCache.close(); + } + + private static final Logger LOG = LoggerFactory.getLogger(BrokerDiscoveryProvider.class); + +} diff --git a/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/DiscoveryService.java b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/DiscoveryService.java new file mode 100644 index 0000000000000..e0b3a4dbbd573 --- /dev/null +++ b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/DiscoveryService.java @@ -0,0 +1,184 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.discovery.service; + +import java.net.InetAddress; + +import org.apache.commons.lang.SystemUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.yahoo.pulsar.discovery.service.server.ServiceConfig; +import com.yahoo.pulsar.zookeeper.ZooKeeperClientFactory; +import com.yahoo.pulsar.zookeeper.ZookeeperClientFactoryImpl; + +import io.netty.bootstrap.ServerBootstrap; +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.channel.AdaptiveRecvByteBufAllocator; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.epoll.EpollChannelOption; +import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.epoll.EpollMode; +import io.netty.channel.epoll.EpollServerSocketChannel; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.util.concurrent.DefaultThreadFactory; + +/** + * Main discovery-service which starts component to serve incoming discovery-request over binary-proto channel and + * redirects to one of the active broker + * + */ +public class DiscoveryService { + + private final ServiceConfig config; + private final String serviceUrl; + private final String serviceUrlTls; + private ZooKeeperClientFactory zkClientFactory = null; + private BrokerDiscoveryProvider discoveryProvider; + private final EventLoopGroup acceptorGroup; + private final EventLoopGroup workerGroup; + private final DefaultThreadFactory acceptorThreadFactory = new DefaultThreadFactory("pulsar-discovery-acceptor"); + private final DefaultThreadFactory workersThreadFactory = new DefaultThreadFactory("pulsar-discovery-io"); + private final int numThreads = Runtime.getRuntime().availableProcessors(); + + public DiscoveryService(ServiceConfig serviceConfig) { + + this.config = serviceConfig; + this.serviceUrl = serviceUrl(); + this.serviceUrlTls = serviceUrlTls(); + discoveryProvider = new BrokerDiscoveryProvider(serviceConfig); + EventLoopGroup acceptorEventLoop, workersEventLoop; + if (SystemUtils.IS_OS_LINUX) { + try { + acceptorEventLoop = new EpollEventLoopGroup(1, acceptorThreadFactory); + workersEventLoop = new EpollEventLoopGroup(numThreads, workersThreadFactory); + } catch (UnsatisfiedLinkError e) { + acceptorEventLoop = new NioEventLoopGroup(1, acceptorThreadFactory); + workersEventLoop = new NioEventLoopGroup(numThreads, workersThreadFactory); + } + } else { + acceptorEventLoop = new NioEventLoopGroup(1, acceptorThreadFactory); + workersEventLoop = new NioEventLoopGroup(numThreads, workersThreadFactory); + } + this.acceptorGroup = acceptorEventLoop; + this.workerGroup = workersEventLoop; + } + + /** + * Starts discovery service by initializing zookkeeper and server + * @throws Exception + */ + public void start() throws Exception { + discoveryProvider.start(getZooKeeperClientFactory()); + startServer(); + } + + /** + * starts server to handle discovery-request from client-channel + * + * @throws Exception + */ + public void startServer() throws Exception { + + ServerBootstrap bootstrap = new ServerBootstrap(); + bootstrap.childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT); + bootstrap.group(acceptorGroup, workerGroup); + bootstrap.childOption(ChannelOption.TCP_NODELAY, true); + bootstrap.childOption(ChannelOption.RCVBUF_ALLOCATOR, + new AdaptiveRecvByteBufAllocator(1024, 16 * 1024, 1 * 1024 * 1024)); + + if (workerGroup instanceof EpollEventLoopGroup) { + bootstrap.channel(EpollServerSocketChannel.class); + bootstrap.childOption(EpollChannelOption.EPOLL_MODE, EpollMode.LEVEL_TRIGGERED); + } else { + bootstrap.channel(NioServerSocketChannel.class); + } + + bootstrap.childHandler(new ServiceChannelInitializer(this, config, false)); + // Bind and start to accept incoming connections. + bootstrap.bind(config.getServicePort()).sync(); + LOG.info("Started Pulsar Broker service on port {}", config.getWebServicePort()); + + if (config.isTlsEnabled()) { + ServerBootstrap tlsBootstrap = bootstrap.clone(); + tlsBootstrap.childHandler(new ServiceChannelInitializer(this, config, true)); + tlsBootstrap.bind(config.getServicePortTls()).sync(); + LOG.info("Started Pulsar Broker TLS service on port {}", config.getWebServicePortTls()); + } + } + + public ZooKeeperClientFactory getZooKeeperClientFactory() { + if (zkClientFactory == null) { + zkClientFactory = new ZookeeperClientFactoryImpl(); + } + // Return default factory + return zkClientFactory; + } + + public BrokerDiscoveryProvider getDiscoveryProvider() { + return discoveryProvider; + } + + public void close() { + discoveryProvider.close(); + acceptorGroup.shutdownGracefully(); + workerGroup.shutdownGracefully(); + } + + /** + * Derive the host + * + * @param isBindOnLocalhost + * @return + */ + public String host() { + try { + if (!config.isBindOnLocalhost()) { + return InetAddress.getLocalHost().getHostName(); + } else { + return "localhost"; + } + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new IllegalStateException("failed to find host", e); + } + } + + public String serviceUrl() { + return new StringBuilder("pulsar://").append(host()).append(":").append(config.getServicePort()).toString(); + } + + public String serviceUrlTls() { + if (config.isTlsEnabled()) { + return new StringBuilder("pulsar://").append(host()).append(":").append(config.getServicePortTls()) + .toString(); + } else { + return ""; + } + } + + public String getServiceUrl() { + return serviceUrl; + } + + public String getServiceUrlTls() { + return serviceUrlTls; + } + + private static final Logger LOG = LoggerFactory.getLogger(DiscoveryService.class); +} \ No newline at end of file diff --git a/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/PulsarServerException.java b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/PulsarServerException.java new file mode 100644 index 0000000000000..1e57fe01a9b82 --- /dev/null +++ b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/PulsarServerException.java @@ -0,0 +1,35 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.discovery.service; + +import java.io.IOException; + +public class PulsarServerException extends IOException { + private static final long serialVersionUID = 1; + + public PulsarServerException(String message) { + super(message); + } + + public PulsarServerException(Throwable cause) { + super(cause); + } + + public PulsarServerException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/ServerConnection.java b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/ServerConnection.java new file mode 100644 index 0000000000000..e32fe05a414a3 --- /dev/null +++ b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/ServerConnection.java @@ -0,0 +1,107 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.discovery.service; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType.Redirect; + +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.yahoo.pulsar.common.api.Commands; +import com.yahoo.pulsar.common.api.PulsarHandler; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandConnect; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandLookupTopic; +import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandPartitionedTopicMetadata; +import com.yahoo.pulsar.common.api.proto.PulsarApi.ServerError; +import com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport; + +/** + * Handles incoming discovery request from client and sends appropriate response back to client + * + */ +public class ServerConnection extends PulsarHandler { + + private BrokerDiscoveryProvider discoveryProvider; + private State state; + + enum State { + Start, Connected + } + + public ServerConnection(BrokerDiscoveryProvider discoveryhandler) { + super(0, TimeUnit.SECONDS); // discovery-service doesn't need to run keepAlive task + this.discoveryProvider = discoveryhandler; + this.state = State.Start; + } + + /** + * handles connect request and sends {@code State.Connected} ack to client + */ + @Override + protected void handleConnect(CommandConnect connect) { + checkArgument(state == State.Start); + if (LOG.isDebugEnabled()) { + LOG.debug("Received CONNECT from {}", remoteAddress); + } + ctx.writeAndFlush(Commands.newConnected(connect)); + state = State.Connected; + remoteEndpointProtocolVersion = connect.getProtocolVersion(); + } + + @Override + protected void handlePartitionMetadataRequest(CommandPartitionedTopicMetadata partitionMetadata) { + checkArgument(state == State.Connected); + if (LOG.isDebugEnabled()) { + LOG.debug("Received PartitionMetadataLookup from {}", remoteAddress); + } + sendLookupResponse(partitionMetadata.getRequestId()); + } + + /** + * handles discovery request from client ands sends next active broker address + */ + @Override + protected void handleLookup(CommandLookupTopic lookup) { + checkArgument(state == State.Connected); + if (LOG.isDebugEnabled()) { + LOG.debug("Received Lookup from {}", remoteAddress); + } + sendLookupResponse(lookup.getRequestId()); + } + + private void sendLookupResponse(long requestId) { + try { + LoadReport availableBroker = discoveryProvider.nextBroker(); + ctx.writeAndFlush(Commands.newLookupResponse(availableBroker.getPulsarServiceUrl(), + availableBroker.getPulsarServieUrlTls(), false, Redirect, requestId)); + } catch (PulsarServerException e) { + LOG.warn("[{}] Failed to get next active broker {}", remoteAddress, e.getMessage(), e); + ctx.writeAndFlush( + Commands.newLookupResponse(ServerError.ServiceNotReady, e.getMessage(), requestId)); + } + } + + @Override + protected boolean isHandshakeCompleted() { + return state == State.Connected; + } + + private static final Logger LOG = LoggerFactory.getLogger(ServerConnection.class); + +} diff --git a/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/ServiceChannelInitializer.java b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/ServiceChannelInitializer.java new file mode 100644 index 0000000000000..23063c8babb45 --- /dev/null +++ b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/ServiceChannelInitializer.java @@ -0,0 +1,64 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.discovery.service; + +import java.io.File; + +import com.yahoo.pulsar.common.api.PulsarDecoder; +import com.yahoo.pulsar.common.api.PulsarLengthFieldFrameDecoder; +import com.yahoo.pulsar.discovery.service.server.ServiceConfig; + +import io.netty.channel.ChannelInitializer; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.ssl.ClientAuth; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; + +/** + * Initialize service channel handlers. + * + */ +public class ServiceChannelInitializer extends ChannelInitializer { + + public static final String TLS_HANDLER = "tls"; + private ServiceConfig serviceConfig; + private DiscoveryService discoveryService; + private boolean enableTLS; + + public ServiceChannelInitializer(DiscoveryService discoveryService, ServiceConfig serviceConfig, boolean enableTLS) { + super(); + this.serviceConfig = serviceConfig; + this.discoveryService = discoveryService; + this.enableTLS = enableTLS; + } + + @Override + protected void initChannel(SocketChannel ch) throws Exception { + if (enableTLS) { + File tlsCert = new File(serviceConfig.getTlsCertificateFilePath()); + File tlsKey = new File(serviceConfig.getTlsKeyFilePath()); + SslContextBuilder builder = SslContextBuilder.forServer(tlsCert, tlsKey); + // allows insecure connection + builder.trustManager(InsecureTrustManagerFactory.INSTANCE); + SslContext sslCtx = builder.clientAuth(ClientAuth.OPTIONAL).build(); + ch.pipeline().addLast(TLS_HANDLER, sslCtx.newHandler(ch.alloc())); + } + ch.pipeline().addLast("frameDecoder", + new PulsarLengthFieldFrameDecoder(PulsarDecoder.MaxFrameSize, 0, 4, 0, 4)); + ch.pipeline().addLast("handler", new ServerConnection(discoveryService.getDiscoveryProvider())); + } +} diff --git a/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/server/DiscoveryServiceStarter.java b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/server/DiscoveryServiceStarter.java index a5c541fef4548..07d8b1e805ade 100644 --- a/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/server/DiscoveryServiceStarter.java +++ b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/server/DiscoveryServiceStarter.java @@ -33,6 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.yahoo.pulsar.discovery.service.DiscoveryService; import com.yahoo.pulsar.discovery.service.web.DiscoveryServiceServlet; /** @@ -52,19 +53,29 @@ public static void init(String configFile) throws Exception { // load config file final ServiceConfig config = load(configFile); + + // create broker service + DiscoveryService discoveryService = new DiscoveryService(config); // create a web-service final ServerManager server = new ServerManager(config); + Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { + discoveryService.close(); server.stop(); } catch (Exception e) { log.warn("server couldn't stop gracefully {}", e.getMessage(), e); } } }); + + discoveryService.start(); + startWebService(server, config); + } + protected static void startWebService(ServerManager server, ServiceConfig config) throws Exception { // add servlet Map initParameters = new TreeMap<>(); initParameters.put("zookeeperServers", config.getZookeeperServers()); diff --git a/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/server/ServerManager.java b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/server/ServerManager.java index 2014aa56b3c77..0154c31743c8a 100644 --- a/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/server/ServerManager.java +++ b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/server/ServerManager.java @@ -133,6 +133,10 @@ public void stop() throws Exception { webServiceExecutor.shutdown(); log.info("Server stopped successfully"); } + + public boolean isStarted() { + return server.isStarted(); + } private static final Logger log = LoggerFactory.getLogger(ServerManager.class); } diff --git a/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/server/ServiceConfig.java b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/server/ServiceConfig.java index 1ec9f2e75e47a..95d9c25873a80 100644 --- a/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/server/ServiceConfig.java +++ b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/server/ServiceConfig.java @@ -25,10 +25,17 @@ public class ServiceConfig { // Zookeeper quorum connection string private String zookeeperServers; + // Port to use to server binary-proto request + private int servicePort = 5000; + // Port to use to server binary-proto-tls request + private int servicePortTls = 5001; // Port to use to server HTTP request private int webServicePort = 8080; // Port to use to server HTTPS request private int webServicePortTls = 8443; + // Control whether to bind directly on localhost rather than on normal + // hostname + private boolean bindOnLocalhost = false; /***** --- TLS --- ****/ // Enable TLS @@ -46,6 +53,22 @@ public void setZookeeperServers(String zookeeperServers) { this.zookeeperServers = zookeeperServers; } + public int getServicePort() { + return servicePort; + } + + public void setServicePort(int servicePort) { + this.servicePort = servicePort; + } + + public int getServicePortTls() { + return servicePortTls; + } + + public void setServicePortTls(int servicePortTls) { + this.servicePortTls = servicePortTls; + } + public int getWebServicePort() { return webServicePort; } @@ -86,4 +109,12 @@ public void setTlsKeyFilePath(String tlsKeyFilePath) { this.tlsKeyFilePath = tlsKeyFilePath; } + public boolean isBindOnLocalhost() { + return bindOnLocalhost; + } + + public void setBindOnLocalhost(boolean bindOnLocalhost) { + this.bindOnLocalhost = bindOnLocalhost; + } + } diff --git a/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/web/DiscoveryServiceServlet.java b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/web/DiscoveryServiceServlet.java index 242dd8908d997..89031a1ffbaf6 100644 --- a/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/web/DiscoveryServiceServlet.java +++ b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/web/DiscoveryServiceServlet.java @@ -130,7 +130,7 @@ private void redirect(HttpServletRequest request, HttpServletResponse response) location.append('?').append(request.getQueryString()); } if (log.isDebugEnabled()) { - log.info("Redirecting to {}", location); + log.info("Redirecting to {}", location); } response.sendRedirect(location.toString()); } catch (URISyntaxException e) { diff --git a/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/web/ZookeeperCacheLoader.java b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/web/ZookeeperCacheLoader.java index df4a16c41d725..df7c611ce2304 100644 --- a/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/web/ZookeeperCacheLoader.java +++ b/pulsar-discovery-service/src/main/java/com/yahoo/pulsar/discovery/service/web/ZookeeperCacheLoader.java @@ -49,7 +49,7 @@ public class ZookeeperCacheLoader implements Closeable { private final OrderedSafeExecutor orderedExecutor = new OrderedSafeExecutor(8, "pulsar-discovery"); - static final String LOADBALANCE_BROKERS_ROOT = "/loadbalance/brokers"; + public static final String LOADBALANCE_BROKERS_ROOT = "/loadbalance/brokers"; private static final int zooKeeperSessionTimeoutMillis = 30_000; diff --git a/pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/BaseDiscoveryTestSetup.java b/pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/BaseDiscoveryTestSetup.java new file mode 100644 index 0000000000000..c6b07e55e8603 --- /dev/null +++ b/pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/BaseDiscoveryTestSetup.java @@ -0,0 +1,86 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.discovery.service; + +import static com.yahoo.pulsar.discovery.service.web.ZookeeperCacheLoader.LOADBALANCE_BROKERS_ROOT; +import static org.apache.bookkeeper.test.PortManager.nextFreePort; + +import java.util.concurrent.CompletableFuture; + +import org.apache.bookkeeper.util.ZkUtils; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.MockZooKeeper; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; + +import com.google.common.util.concurrent.MoreExecutors; +import com.yahoo.pulsar.discovery.service.server.ServiceConfig; +import com.yahoo.pulsar.zookeeper.ZooKeeperClientFactory; +import com.yahoo.pulsar.zookeeper.ZookeeperClientFactoryImpl; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; + + +public class BaseDiscoveryTestSetup { + + protected ServiceConfig config; + protected DiscoveryService service; + protected MockZooKeeper mockZookKeeper; + private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt"; + private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key"; + + protected void setup() throws Exception { + config = new ServiceConfig(); + config.setServicePort(nextFreePort()); + config.setServicePortTls(nextFreePort()); + config.setBindOnLocalhost(true); + + config.setTlsEnabled(true); + config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); + config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); + + mockZookKeeper = createMockZooKeeper(); + service = spy(new DiscoveryService(config)); + doReturn(mockZooKeeperClientFactory).when(service).getZooKeeperClientFactory(); + service.start(); + + } + + protected void cleanup() throws Exception { + mockZookKeeper.shutdown(); + service.close(); + } + + protected MockZooKeeper createMockZooKeeper() throws Exception { + MockZooKeeper zk = MockZooKeeper.newInstance(MoreExecutors.sameThreadExecutor()); + + ZkUtils.createFullPathOptimistic(zk, LOADBALANCE_BROKERS_ROOT, + "".getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + return zk; + } + + protected ZooKeeperClientFactory mockZooKeeperClientFactory = new ZooKeeperClientFactory() { + + @Override + public CompletableFuture create(String serverList, SessionType sessionType, + int zkSessionTimeoutMillis) { + // Always return the same instance (so that we don't loose the mock ZK content on broker restart + return CompletableFuture.completedFuture(mockZookKeeper); + } + }; + +} diff --git a/pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/DiscoveryServiceTest.java b/pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/DiscoveryServiceTest.java new file mode 100644 index 0000000000000..62d19cf1d02b8 --- /dev/null +++ b/pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/DiscoveryServiceTest.java @@ -0,0 +1,208 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.discovery.service; + +import static com.yahoo.pulsar.discovery.service.web.ZookeeperCacheLoader.LOADBALANCE_BROKERS_ROOT; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.bookkeeper.util.ZkUtils; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.ZooDefs; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.yahoo.pulsar.common.api.Commands; +import com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport; +import com.yahoo.pulsar.common.util.ObjectMapperFactory; +import com.yahoo.pulsar.common.util.SecurityUtility; +import com.yahoo.pulsar.zookeeper.ZookeeperClientFactoryImpl; + +import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; + +public class DiscoveryServiceTest extends BaseDiscoveryTestSetup { + + private final static String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt"; + private final static String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key"; + + @BeforeMethod + private void init() throws Exception { + super.setup(); + } + + @AfterMethod + private void clean() throws Exception { + super.cleanup(); + } + + /** + * Verifies: Discovery-service returns broker is round-robin manner + * + * @throws Exception + */ + @Test + public void testBrokerDiscoveryRoundRobin() throws Exception { + addBrokerToZk(5); + String prevUrl = null; + for (int i = 0; i < 10; i++) { + String current = service.getDiscoveryProvider().nextBroker().getPulsarServiceUrl(); + assertNotEquals(prevUrl, current); + prevUrl = current; + } + } + + /** + * It verifies: client connects to Discovery-service and receives discovery response successfully. + * + * @throws Exception + */ + @Test + public void testClientServerConnection() throws Exception { + addBrokerToZk(2); + // 1. client connects to DiscoveryService, 2. Client receive service-lookup response + final int messageTransfer = 2; + final CountDownLatch latch = new CountDownLatch(messageTransfer); + NioEventLoopGroup workerGroup = connectToService(service.getServiceUrl(), latch, false); + try { + assertTrue(latch.await(1, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + fail("should have received lookup response message from server", e); + } + workerGroup.shutdownGracefully(); + } + + @Test(enabled = true) + public void testClientServerConnectionTls() throws Exception { + addBrokerToZk(2); + // 1. client connects to DiscoveryService, 2. Client receive service-lookup response + final int messageTransfer = 2; + final CountDownLatch latch = new CountDownLatch(messageTransfer); + NioEventLoopGroup workerGroup = connectToService(service.getServiceUrlTls(), latch, true); + try { + assertTrue(latch.await(1, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + fail("should have received lookup response message from server", e); + } + workerGroup.shutdownGracefully(); + } + + /** + * creates ClientHandler channel to connect and communicate with server + * + * @param serviceUrl + * @param latch + * @return + * @throws URISyntaxException + */ + public static NioEventLoopGroup connectToService(String serviceUrl, CountDownLatch latch, boolean tls) + throws URISyntaxException { + NioEventLoopGroup workerGroup = new NioEventLoopGroup(); + Bootstrap b = new Bootstrap(); + b.group(workerGroup); + b.channel(NioSocketChannel.class); + + b.handler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) throws Exception { + if(tls) { + SslContextBuilder builder = SslContextBuilder.forClient(); + builder.trustManager(InsecureTrustManagerFactory.INSTANCE); + X509Certificate[] certificates = SecurityUtility.loadCertificatesFromPemFile(TLS_CLIENT_CERT_FILE_PATH); + PrivateKey privateKey = SecurityUtility.loadPrivateKeyFromPemFile(TLS_CLIENT_KEY_FILE_PATH); + builder.keyManager(privateKey, (X509Certificate[]) certificates); + SslContext sslCtx = builder.build(); + ch.pipeline().addLast("tls", sslCtx.newHandler(ch.alloc())); + } + ch.pipeline().addLast(new ClientHandler(latch)); + } + }); + URI uri = new URI(serviceUrl); + InetSocketAddress serviceAddress = new InetSocketAddress(uri.getHost(), uri.getPort()); + b.connect(serviceAddress).addListener((ChannelFuture future) -> { + if(!future.isSuccess()) { + throw new IllegalStateException(future.cause()); + } + }); + return workerGroup; + } + + static class ClientHandler extends ChannelInboundHandlerAdapter { + + final CountDownLatch latch; + + public ClientHandler(CountDownLatch latch) { + this.latch = latch; + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + ByteBuf buffer = (ByteBuf) msg; + buffer.release(); + latch.countDown(); + ctx.close(); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + // Close the connection when an exception is raised. + cause.printStackTrace(); + ctx.close(); + } + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + super.channelActive(ctx); + ctx.writeAndFlush(Commands.newConnect("", "")); + latch.countDown(); + } + + } + + private void addBrokerToZk(int number) throws Exception { + + for (int i = 0; i < number; i++) { + LoadReport report = new LoadReport(null, null, "pulsar://broker-:15000" + i, null); + String reportData = ObjectMapperFactory.getThreadLocal().writeValueAsString(report); + ZkUtils.createFullPathOptimistic(mockZookKeeper, LOADBALANCE_BROKERS_ROOT + "/" + "broker-" + i, + reportData.getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + } + + Thread.sleep(100); // wait to get cache updated + } + +} diff --git a/pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/server/DiscoveryServiceWebTest.java b/pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/server/DiscoveryServiceWebTest.java new file mode 100644 index 0000000000000..48cce09034555 --- /dev/null +++ b/pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/server/DiscoveryServiceWebTest.java @@ -0,0 +1,58 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.discovery.service.server; + +import static org.apache.bookkeeper.test.PortManager.nextFreePort; +import static org.testng.Assert.assertTrue; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; + +import org.testng.annotations.Test; + +/** + * 1. starts discovery service a. loads broker list from zk 2. http-client calls multiple http request: GET, PUT and + * POST. 3. discovery service redirects to appropriate brokers in round-robin 4. client receives unknown host exception + * with redirected broker + * + */ +public class DiscoveryServiceWebTest { + + + @Test + public void testWebDiscoveryServiceStarter() throws Exception { + + int port = nextFreePort(); + File testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties"); + if (testConfigFile.exists()) { + testConfigFile.delete(); + } + PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(testConfigFile))); + printWriter.println("zookeeperServers=z1.pulsar.com,z2.pulsar.com,z3.pulsar.com"); + printWriter.println("webServicePort=" + port); + printWriter.close(); + testConfigFile.deleteOnExit(); + final ServiceConfig config = DiscoveryServiceStarter.load(testConfigFile.getAbsolutePath()); + final ServerManager server = new ServerManager(config); + DiscoveryServiceStarter.startWebService(server, config); + assertTrue(server.isStarted()); + server.stop(); + testConfigFile.delete(); + } + +} diff --git a/pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/web/DiscoveryServiceWebTest.java b/pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/web/DiscoveryServiceWebTest.java index 0ee072713ed8e..35417c31bbe80 100644 --- a/pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/web/DiscoveryServiceWebTest.java +++ b/pulsar-discovery-service/src/test/java/com/yahoo/pulsar/discovery/service/web/DiscoveryServiceWebTest.java @@ -104,9 +104,11 @@ public void testNextBroker() throws Exception { } catch (KeeperException.NodeExistsException ne) { // Ok } catch (KeeperException | InterruptedException e) { - fail("failed while creating broker znodes", e); + e.printStackTrace(); + fail("failed while creating broker znodes"); } catch (JsonProcessingException e) { - fail("failed while creating broker znodes", e); + e.printStackTrace(); + fail("failed while creating broker znodes"); } }); @@ -153,9 +155,11 @@ public void testRiderectUrlWithServerStarted() throws Exception { } catch (KeeperException.NodeExistsException ne) { // Ok } catch (KeeperException | InterruptedException e) { - fail("failed while creating broker znodes", e); + e.printStackTrace(); + fail("failed while creating broker znodes"); } catch (JsonProcessingException e) { - fail("failed while creating broker znodes", e); + e.printStackTrace(); + fail("failed while creating broker znodes"); } }); @@ -214,8 +218,10 @@ public void testTlsEnable() throws Exception { } catch (KeeperException.NodeExistsException ne) { // Ok } catch (KeeperException | InterruptedException e) { - fail("failed while creating broker znodes", e); + e.printStackTrace(); + fail("failed while creating broker znodes"); } catch (JsonProcessingException e) { + e.printStackTrace(); fail("failed while creating broker znodes"); } }); diff --git a/pulsar-discovery-service/src/test/resources/certificate/client.crt b/pulsar-discovery-service/src/test/resources/certificate/client.crt new file mode 100644 index 0000000000000..2d7d156866a86 --- /dev/null +++ b/pulsar-discovery-service/src/test/resources/certificate/client.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDVjCCAj4CCQCtw/UnTFDT7DANBgkqhkiG9w0BAQUFADBtMQswCQYDVQQGEwJB +VTETMBEGA1UECAwKU29tZS1TdGF0ZTEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxDzANBgNVBAMMBmNsaWVu +dDAeFw0xNjA2MjAwMTQ1NDZaFw0yNjA2MTgwMTQ1NDZaMG0xCzAJBgNVBAYTAkFV +MRMwEQYDVQQIDApTb21lLVN0YXRlMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxITAf +BgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50 +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqQV5F3Au9FWXIYPdWqiX +Rk5gdVmVkDuuFK4ZoOd8inoJpB3PPkpmpgoVkKQHDFhgx3ODGWIUgo+n6QDsJxY4 +ygHfVeggQgek8iUfteYVsIcHS0bjkhIij/3ihC301FkiqbrV069oLvUXLKcv3zxG +mdBAiz0k4xGZhFieVRvQCLY9syUUxmQ/3Cv42lDY8a1gTw4CRRx/hCfDvXCKhOT4 +bMwUIDZfHB3JoDh3Thp8FLz0nTrRF75mSQJ/OdcafIm0Xoz2Otp/CSxLS+U1lLvG +05crWTDe0om7NW4mK4CqGCFq5gUw7eIzaeO7Q5Qez9XGTMzkgIDTMvNYGGEeJhhm +NQIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQAKXy4g6hljY5MpO8mbZh+uJHq6NEUs +4dr7OKDDWc39AROZsGf2eFUmHOjmRSw7VHpguGKI+rFRELVffpg/VvMh5apu+DBf +jhxtDNceAyh5uugPNUJHXyeikBDYW8bAzUU3DmMldPkTZWcGjurmyhDQ1TtK2YJe +RMFBXw5aAzdJMNi6OfXDH/ZX32hrb482yghDZj+ndnm0FefmLbFTQRMF8/fIHb1W +kqNHwIaapZwH6j/MJy/TRFYcJunrBUYT9zVjY46k3GU0ex/Bn7T4pg9gzgFGZJhn +jQQFKliIC84thCzdlPkrLduLY8tmlDKpLXatbEQ+s1MmNOURm6irPp6g +-----END CERTIFICATE----- diff --git a/pulsar-discovery-service/src/test/resources/certificate/client.key b/pulsar-discovery-service/src/test/resources/certificate/client.key new file mode 100644 index 0000000000000..34fc701c5257d --- /dev/null +++ b/pulsar-discovery-service/src/test/resources/certificate/client.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCpBXkXcC70VZch +g91aqJdGTmB1WZWQO64Urhmg53yKegmkHc8+SmamChWQpAcMWGDHc4MZYhSCj6fp +AOwnFjjKAd9V6CBCB6TyJR+15hWwhwdLRuOSEiKP/eKELfTUWSKputXTr2gu9Rcs +py/fPEaZ0ECLPSTjEZmEWJ5VG9AItj2zJRTGZD/cK/jaUNjxrWBPDgJFHH+EJ8O9 +cIqE5PhszBQgNl8cHcmgOHdOGnwUvPSdOtEXvmZJAn851xp8ibRejPY62n8JLEtL +5TWUu8bTlytZMN7Sibs1biYrgKoYIWrmBTDt4jNp47tDlB7P1cZMzOSAgNMy81gY +YR4mGGY1AgMBAAECggEAcJj3yVhvv0/BhY8+CCYl2K1f7u1GCLbpSleNNTbhLbMM +9yrwo/OWnGg9Y4USOPQrTNOz81X2id+/oSZ/K67PGCvVJ3qi+rny9WkrzdbAfkAF +6O0Jr4arRbeBjkK7Rjc3M1EHH6VLx3R5AsNBzfpuogss5FVQXICd/5+1oscLeLEx +/Fn+51IEn9FUg5vr7ElG51f+zPxexcWHLNoqGjTEIGGtI8/CfTzD9tBV4sIjf/Nc +Zzfs9XYrChfcrS0U1zDa+L7c5gYfoN6M08sBiuZlhyyO9wgzPlp+XnsrSFv6hUta +0scjAbN4bh+orQn6zgFN/sjkQnraWXW7pKFLyTR/IQKBgQDVju4IbhE9XRweNgXi +s3BuGV+HsuFffEf0904/zCuCUcScGb5WCz5+KtlFJ//YxfocHVZajH+4GdCGbWim +m+H3XvRpWgfK/aBNOXu5ueLbnPYyPjTrcpKRsomeoiV+Jz1tv5PQElwzCiCzVvQf +fMyhQT16YIsFQAGJzQMBEHWODQKBgQDKnKps3sKSR3ycUtIxCVXUir7p52qst0Pm +bPO8JrcRKZP2z8MJB96+DcQFzrxj7t5DDktkYEsFOPPuIeUsYXsY+MKHs4hEQVCz +hpDJJNQ8s+SV8TLzKpinZEmLIjslLbn2rQrpqybPg84VxqX3qqM8IrXhMf77aGj6 +QHqvQwHWyQKBgQDF1RVO+9++j82ncvY6z22coKath5leIjxqgtqbISFBJUxUK0j2 +Xo4yxLDnbqmE/8m1V7wSP8tlGYzhquLiTM+kn/Mc0Ukc0503TMQABmJQfXRYkOXn +IwkCLXltWdoPpnwyeeGNRCTjJ0OpvyiBLtRFobE498xxPZzvMdrRlpS/1QKBgQCo +wmMleUnBQ2/kWQugMnFeLg6kjs+IesFAnYFKN0kGL4aB7j06OWbrEFY0rCS4bA6O +9coQGjCCchSjRXI4TB2XCCQnmX8nsuuADNZt45Iv2XrM9XEFn3Y0/tBO5j0zU2nw +r+NGC/uwns050BMPPf7mqNarctQ6HZZK0wgdEQfoGQKBgC+pbkQv9cn68TsiaJ3w +tvNRTXCIAAH4Vtn9Cp+63ao+kXn94BJqQF99i58kJpG4ol6wbCHUoC6fHgxUh5HB +JB0HjC2eCMgn4acAQg0sPW6l35KX36yYxtrL7eosB/yBYum0XAwmboNjEhlCZkOs +YOpSsn61g7xqqrt40Spb5vUn +-----END PRIVATE KEY----- diff --git a/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/BaseDiscoveryTestSetup.java b/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/BaseDiscoveryTestSetup.java new file mode 100644 index 0000000000000..c6b07e55e8603 --- /dev/null +++ b/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/BaseDiscoveryTestSetup.java @@ -0,0 +1,86 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.discovery.service; + +import static com.yahoo.pulsar.discovery.service.web.ZookeeperCacheLoader.LOADBALANCE_BROKERS_ROOT; +import static org.apache.bookkeeper.test.PortManager.nextFreePort; + +import java.util.concurrent.CompletableFuture; + +import org.apache.bookkeeper.util.ZkUtils; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.MockZooKeeper; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; + +import com.google.common.util.concurrent.MoreExecutors; +import com.yahoo.pulsar.discovery.service.server.ServiceConfig; +import com.yahoo.pulsar.zookeeper.ZooKeeperClientFactory; +import com.yahoo.pulsar.zookeeper.ZookeeperClientFactoryImpl; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; + + +public class BaseDiscoveryTestSetup { + + protected ServiceConfig config; + protected DiscoveryService service; + protected MockZooKeeper mockZookKeeper; + private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt"; + private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key"; + + protected void setup() throws Exception { + config = new ServiceConfig(); + config.setServicePort(nextFreePort()); + config.setServicePortTls(nextFreePort()); + config.setBindOnLocalhost(true); + + config.setTlsEnabled(true); + config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); + config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); + + mockZookKeeper = createMockZooKeeper(); + service = spy(new DiscoveryService(config)); + doReturn(mockZooKeeperClientFactory).when(service).getZooKeeperClientFactory(); + service.start(); + + } + + protected void cleanup() throws Exception { + mockZookKeeper.shutdown(); + service.close(); + } + + protected MockZooKeeper createMockZooKeeper() throws Exception { + MockZooKeeper zk = MockZooKeeper.newInstance(MoreExecutors.sameThreadExecutor()); + + ZkUtils.createFullPathOptimistic(zk, LOADBALANCE_BROKERS_ROOT, + "".getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + return zk; + } + + protected ZooKeeperClientFactory mockZooKeeperClientFactory = new ZooKeeperClientFactory() { + + @Override + public CompletableFuture create(String serverList, SessionType sessionType, + int zkSessionTimeoutMillis) { + // Always return the same instance (so that we don't loose the mock ZK content on broker restart + return CompletableFuture.completedFuture(mockZookKeeper); + } + }; + +} diff --git a/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/DiscoveryServiceTest.java b/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/DiscoveryServiceTest.java new file mode 100644 index 0000000000000..62d19cf1d02b8 --- /dev/null +++ b/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/DiscoveryServiceTest.java @@ -0,0 +1,208 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.discovery.service; + +import static com.yahoo.pulsar.discovery.service.web.ZookeeperCacheLoader.LOADBALANCE_BROKERS_ROOT; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.bookkeeper.util.ZkUtils; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.ZooDefs; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.yahoo.pulsar.common.api.Commands; +import com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport; +import com.yahoo.pulsar.common.util.ObjectMapperFactory; +import com.yahoo.pulsar.common.util.SecurityUtility; +import com.yahoo.pulsar.zookeeper.ZookeeperClientFactoryImpl; + +import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; + +public class DiscoveryServiceTest extends BaseDiscoveryTestSetup { + + private final static String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt"; + private final static String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key"; + + @BeforeMethod + private void init() throws Exception { + super.setup(); + } + + @AfterMethod + private void clean() throws Exception { + super.cleanup(); + } + + /** + * Verifies: Discovery-service returns broker is round-robin manner + * + * @throws Exception + */ + @Test + public void testBrokerDiscoveryRoundRobin() throws Exception { + addBrokerToZk(5); + String prevUrl = null; + for (int i = 0; i < 10; i++) { + String current = service.getDiscoveryProvider().nextBroker().getPulsarServiceUrl(); + assertNotEquals(prevUrl, current); + prevUrl = current; + } + } + + /** + * It verifies: client connects to Discovery-service and receives discovery response successfully. + * + * @throws Exception + */ + @Test + public void testClientServerConnection() throws Exception { + addBrokerToZk(2); + // 1. client connects to DiscoveryService, 2. Client receive service-lookup response + final int messageTransfer = 2; + final CountDownLatch latch = new CountDownLatch(messageTransfer); + NioEventLoopGroup workerGroup = connectToService(service.getServiceUrl(), latch, false); + try { + assertTrue(latch.await(1, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + fail("should have received lookup response message from server", e); + } + workerGroup.shutdownGracefully(); + } + + @Test(enabled = true) + public void testClientServerConnectionTls() throws Exception { + addBrokerToZk(2); + // 1. client connects to DiscoveryService, 2. Client receive service-lookup response + final int messageTransfer = 2; + final CountDownLatch latch = new CountDownLatch(messageTransfer); + NioEventLoopGroup workerGroup = connectToService(service.getServiceUrlTls(), latch, true); + try { + assertTrue(latch.await(1, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + fail("should have received lookup response message from server", e); + } + workerGroup.shutdownGracefully(); + } + + /** + * creates ClientHandler channel to connect and communicate with server + * + * @param serviceUrl + * @param latch + * @return + * @throws URISyntaxException + */ + public static NioEventLoopGroup connectToService(String serviceUrl, CountDownLatch latch, boolean tls) + throws URISyntaxException { + NioEventLoopGroup workerGroup = new NioEventLoopGroup(); + Bootstrap b = new Bootstrap(); + b.group(workerGroup); + b.channel(NioSocketChannel.class); + + b.handler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) throws Exception { + if(tls) { + SslContextBuilder builder = SslContextBuilder.forClient(); + builder.trustManager(InsecureTrustManagerFactory.INSTANCE); + X509Certificate[] certificates = SecurityUtility.loadCertificatesFromPemFile(TLS_CLIENT_CERT_FILE_PATH); + PrivateKey privateKey = SecurityUtility.loadPrivateKeyFromPemFile(TLS_CLIENT_KEY_FILE_PATH); + builder.keyManager(privateKey, (X509Certificate[]) certificates); + SslContext sslCtx = builder.build(); + ch.pipeline().addLast("tls", sslCtx.newHandler(ch.alloc())); + } + ch.pipeline().addLast(new ClientHandler(latch)); + } + }); + URI uri = new URI(serviceUrl); + InetSocketAddress serviceAddress = new InetSocketAddress(uri.getHost(), uri.getPort()); + b.connect(serviceAddress).addListener((ChannelFuture future) -> { + if(!future.isSuccess()) { + throw new IllegalStateException(future.cause()); + } + }); + return workerGroup; + } + + static class ClientHandler extends ChannelInboundHandlerAdapter { + + final CountDownLatch latch; + + public ClientHandler(CountDownLatch latch) { + this.latch = latch; + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + ByteBuf buffer = (ByteBuf) msg; + buffer.release(); + latch.countDown(); + ctx.close(); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + // Close the connection when an exception is raised. + cause.printStackTrace(); + ctx.close(); + } + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + super.channelActive(ctx); + ctx.writeAndFlush(Commands.newConnect("", "")); + latch.countDown(); + } + + } + + private void addBrokerToZk(int number) throws Exception { + + for (int i = 0; i < number; i++) { + LoadReport report = new LoadReport(null, null, "pulsar://broker-:15000" + i, null); + String reportData = ObjectMapperFactory.getThreadLocal().writeValueAsString(report); + ZkUtils.createFullPathOptimistic(mockZookKeeper, LOADBALANCE_BROKERS_ROOT + "/" + "broker-" + i, + reportData.getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + } + + Thread.sleep(100); // wait to get cache updated + } + +} diff --git a/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/server/DiscoveryServiceWebTest.java b/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/server/DiscoveryServiceWebTest.java new file mode 100644 index 0000000000000..48cce09034555 --- /dev/null +++ b/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/server/DiscoveryServiceWebTest.java @@ -0,0 +1,58 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.discovery.service.server; + +import static org.apache.bookkeeper.test.PortManager.nextFreePort; +import static org.testng.Assert.assertTrue; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; + +import org.testng.annotations.Test; + +/** + * 1. starts discovery service a. loads broker list from zk 2. http-client calls multiple http request: GET, PUT and + * POST. 3. discovery service redirects to appropriate brokers in round-robin 4. client receives unknown host exception + * with redirected broker + * + */ +public class DiscoveryServiceWebTest { + + + @Test + public void testWebDiscoveryServiceStarter() throws Exception { + + int port = nextFreePort(); + File testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties"); + if (testConfigFile.exists()) { + testConfigFile.delete(); + } + PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(testConfigFile))); + printWriter.println("zookeeperServers=z1.pulsar.com,z2.pulsar.com,z3.pulsar.com"); + printWriter.println("webServicePort=" + port); + printWriter.close(); + testConfigFile.deleteOnExit(); + final ServiceConfig config = DiscoveryServiceStarter.load(testConfigFile.getAbsolutePath()); + final ServerManager server = new ServerManager(config); + DiscoveryServiceStarter.startWebService(server, config); + assertTrue(server.isStarted()); + server.stop(); + testConfigFile.delete(); + } + +} diff --git a/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/web/BaseZKStarterTest.java b/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/web/BaseZKStarterTest.java new file mode 100644 index 0000000000000..0ac792ec9c2b1 --- /dev/null +++ b/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/web/BaseZKStarterTest.java @@ -0,0 +1,68 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.discovery.service.web; + +import static com.yahoo.pulsar.discovery.service.web.ZookeeperCacheLoader.LOADBALANCE_BROKERS_ROOT; + +import java.util.concurrent.CompletableFuture; + +import org.apache.bookkeeper.util.ZkUtils; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.MockZooKeeper; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; + +import com.google.common.util.concurrent.MoreExecutors; +import com.yahoo.pulsar.zookeeper.ZooKeeperClientFactory; +import com.yahoo.pulsar.zookeeper.ZookeeperClientFactoryImpl; + +public class BaseZKStarterTest { + + protected MockZooKeeper mockZookKeeper; + + protected void start() throws Exception { + mockZookKeeper = createMockZooKeeper(); + } + + protected void close() throws Exception { + mockZookKeeper.shutdown(); + } + + /** + * Create MockZookeeper instance + * @return + * @throws Exception + */ + protected MockZooKeeper createMockZooKeeper() throws Exception { + MockZooKeeper zk = MockZooKeeper.newInstance(MoreExecutors.sameThreadExecutor()); + + ZkUtils.createFullPathOptimistic(zk, LOADBALANCE_BROKERS_ROOT, + "".getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + return zk; + } + + protected static class DiscoveryZooKeeperClientFactoryImpl implements ZooKeeperClientFactory { + static ZooKeeper zk; + + @Override + public CompletableFuture create(String serverList, SessionType sessionType, + int zkSessionTimeoutMillis) { + return CompletableFuture.completedFuture(zk); + } + } + +} diff --git a/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/web/DiscoveryServiceWebTest.java b/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/web/DiscoveryServiceWebTest.java new file mode 100644 index 0000000000000..5fc43e6b644bb --- /dev/null +++ b/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/web/DiscoveryServiceWebTest.java @@ -0,0 +1,317 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.discovery.service.web; + +import static com.yahoo.pulsar.discovery.service.web.ZookeeperCacheLoader.LOADBALANCE_BROKERS_ROOT; +import static javax.ws.rs.core.Response.Status.BAD_GATEWAY; +import static javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR; +import static org.apache.bookkeeper.test.PortManager.nextFreePort; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.lang.reflect.Field; +import java.net.InetAddress; +import java.net.URL; +import java.net.UnknownHostException; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.KeyManager; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.ws.rs.HttpMethod; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.Invocation; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.bookkeeper.util.ZkUtils; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.MockZooKeeper; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.ACL; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.filter.LoggingFilter; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.MoreExecutors; +import com.yahoo.pulsar.common.policies.data.BundlesData; +import com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport; +import com.yahoo.pulsar.common.util.ObjectMapperFactory; +import com.yahoo.pulsar.discovery.service.server.DiscoveryServiceStarter; +import com.yahoo.pulsar.discovery.service.server.ServerManager; +import com.yahoo.pulsar.discovery.service.server.ServiceConfig; +import com.yahoo.pulsar.discovery.service.web.DiscoveryServiceServlet; +import com.yahoo.pulsar.discovery.service.web.RestException; +import com.yahoo.pulsar.discovery.service.web.ZookeeperCacheLoader; +import com.yahoo.pulsar.zookeeper.ZooKeeperClientFactory; +import com.yahoo.pulsar.zookeeper.ZookeeperClientFactoryImpl; + +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; + +/** + * 1. starts discovery service a. loads broker list from zk 2. http-client calls multiple http request: GET, PUT and + * POST. 3. discovery service redirects to appropriate brokers in round-robin 4. client receives unknown host exception + * with redirected broker + * + */ +public class DiscoveryServiceWebTest extends BaseZKStarterTest{ + + private Client client = ClientBuilder.newClient(new ClientConfig().register(LoggingFilter.class)); + private static final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt"; + private static final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key"; + + + @BeforeMethod + private void init() throws Exception { + start(); + } + + @AfterMethod + private void cleanup() throws Exception { + close(); + } + + @Test + public void testNextBroker() throws Exception { + + // 1. create znode for each broker + List brokers = Lists.newArrayList("broker-1", "broker-2", "broker-3"); + brokers.stream().forEach(broker -> { + try { + LoadReport report = new LoadReport(broker, null, null, null); + String reportData = ObjectMapperFactory.getThreadLocal().writeValueAsString(report); + ZkUtils.createFullPathOptimistic(mockZookKeeper, LOADBALANCE_BROKERS_ROOT + "/" + broker, + reportData.getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + } catch (KeeperException.NodeExistsException ne) { + // Ok + } catch (KeeperException | InterruptedException e) { + e.printStackTrace(); + fail("failed while creating broker znodes"); + } catch (JsonProcessingException e) { + e.printStackTrace(); + fail("failed while creating broker znodes"); + } + }); + + // 2. Setup discovery-zkcache + DiscoveryServiceServlet discovery = new DiscoveryServiceServlet(); + DiscoveryZooKeeperClientFactoryImpl.zk = mockZookKeeper; + Field zkCacheField = DiscoveryServiceServlet.class.getDeclaredField("zkCache"); + zkCacheField.setAccessible(true); + ZookeeperCacheLoader zkCache = new ZookeeperCacheLoader(new DiscoveryZooKeeperClientFactoryImpl(), + "zk-test-servers"); + zkCacheField.set(discovery, zkCache); + + // 3. verify nextBroker functionality : round-robin in broker list + for (String broker : brokers) { + assertEquals(broker, discovery.nextBroker().getWebServiceUrl()); + } + } + + @Test + public void testRiderectUrlWithServerStarted() throws Exception { + + // 1. start server + int port = nextFreePort(); + ServiceConfig config = new ServiceConfig(); + config.setWebServicePort(port); + ServerManager server = new ServerManager(config); + DiscoveryZooKeeperClientFactoryImpl.zk = mockZookKeeper; + Map params = new TreeMap<>(); + params.put("zookeeperServers", "dummy-value"); + params.put("zookeeperClientFactoryClass", DiscoveryZooKeeperClientFactoryImpl.class.getName()); + server.addServlet("/", DiscoveryServiceServlet.class, params); + server.start(); + + // 2. create znode for each broker + List brokers = Lists.newArrayList("broker-1", "broker-2", "broker-3"); + brokers.stream().forEach(b -> { + try { + final String broker = b + ":15000"; + LoadReport report = new LoadReport("http://" + broker, null, null, null); + String reportData = ObjectMapperFactory.getThreadLocal().writeValueAsString(report); + ZkUtils.createFullPathOptimistic(mockZookKeeper, LOADBALANCE_BROKERS_ROOT + "/" + broker, + reportData.getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + } catch (KeeperException.NodeExistsException ne) { + // Ok + } catch (KeeperException | InterruptedException e) { + e.printStackTrace(); + fail("failed while creating broker znodes"); + } catch (JsonProcessingException e) { + e.printStackTrace(); + fail("failed while creating broker znodes"); + } + }); + + String serviceUrl = server.getServiceUri().toString(); + String requestUrl = serviceUrl + "admin/namespaces/p1/c1/n1"; + + /** + * 3. verify : every time when vip receives a request: it redirects to above brokers sequentially and client + * must get unknown host exception with above brokers in a sequential manner. + **/ + + assertEquals(brokers, validateRequest(brokers, HttpMethod.PUT, requestUrl, new BundlesData(1)), + "redirection failed"); + assertEquals(brokers, validateRequest(brokers, HttpMethod.GET, requestUrl, null), "redirection failed"); + assertEquals(brokers, validateRequest(brokers, HttpMethod.POST, requestUrl, new BundlesData(1)), + "redirection failed"); + + server.stop(); + + } + + + @Test + public void testTlsEnable() throws Exception { + + // 1. start server with tls enable + int port = nextFreePort(); + int tlsPort = nextFreePort(); + ServiceConfig config = new ServiceConfig(); + config.setWebServicePort(port); + config.setWebServicePortTls(tlsPort); + config.setTlsEnabled(true); + config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); + config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); + ServerManager server = new ServerManager(config); + DiscoveryZooKeeperClientFactoryImpl.zk = mockZookKeeper; + Map params = new TreeMap<>(); + params.put("zookeeperServers", "dummy-value"); + params.put("zookeeperClientFactoryClass", DiscoveryZooKeeperClientFactoryImpl.class.getName()); + server.addServlet("/", DiscoveryServiceServlet.class, params); + server.start(); + + // 2. get ZookeeperCacheLoader to add more brokers + final String redirect_broker_host = "broker-1"; + List brokers = Lists.newArrayList(redirect_broker_host); + brokers.stream().forEach(b -> { + try { + final String brokerUrl = b + ":" + port; + final String brokerUrlTls = b + ":" + tlsPort; + + LoadReport report = new LoadReport("http://" + brokerUrl, "https://" + brokerUrlTls, null, null); + String reportData = ObjectMapperFactory.getThreadLocal().writeValueAsString(report); + ZkUtils.createFullPathOptimistic(mockZookKeeper, LOADBALANCE_BROKERS_ROOT + "/" + brokerUrl, + reportData.getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT); + } catch (KeeperException.NodeExistsException ne) { + // Ok + } catch (KeeperException | InterruptedException e) { + e.printStackTrace(); + fail("failed while creating broker znodes"); + } catch (JsonProcessingException e) { + e.printStackTrace(); + fail("failed while creating broker znodes"); + } + }); + + // 3. https request with tls enable at server side + String serviceUrl = String.format("https://localhost:%s/", tlsPort); + String requestUrl = serviceUrl + "admin/namespaces/p1/c1/n1"; + + KeyManager[] keyManagers = null; + TrustManager[] trustManagers = InsecureTrustManagerFactory.INSTANCE.getTrustManagers(); + SSLContext sslCtx = SSLContext.getInstance("TLS"); + sslCtx.init(keyManagers, trustManagers, new SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory()); + try { + InputStream response = new URL(requestUrl).openStream(); + fail("it should give unknown host exception as: discovery service redirects request to: " + + redirect_broker_host); + } catch (Exception e) { + // 4. Verify: server accepts https request and redirected to one of the available broker host defined into + // zk. and as broker-service is not up: it should give "UnknownHostException with host=broker-url" + String host = e.getLocalizedMessage(); + assertEquals(e.getClass(), UnknownHostException.class); + assertTrue(host.startsWith(redirect_broker_host)); + } + + server.stop(); + } + + @Test + public void testException() { + RestException exception1 = new RestException(BAD_GATEWAY, "test-msg"); + assertTrue(exception1.getMessage().contains(BAD_GATEWAY.toString())); + RestException exception2 = new RestException(BAD_GATEWAY.getStatusCode(), "test-msg"); + assertTrue(exception2.getMessage().contains(BAD_GATEWAY.toString())); + RestException exception3 = new RestException(exception2); + assertTrue(exception3.getMessage().contains(INTERNAL_SERVER_ERROR.toString())); + assertTrue(RestException.getExceptionData(exception2).contains(BAD_GATEWAY.toString())); + } + + public List validateRequest(List brokers, String method, String url, BundlesData bundle) { + + List redirectBrokers = brokers.stream().map(broker -> { + + String redirectedBroker = null; + try { + WebTarget webTarget = client.target(url); + Invocation.Builder invocationBuilder = webTarget.request(MediaType.APPLICATION_JSON); + if (HttpMethod.PUT.equals(method)) { + invocationBuilder.put(Entity.entity(bundle, MediaType.APPLICATION_JSON)); + fail(); + } else if (HttpMethod.GET.equals(method)) { + invocationBuilder.get(); + fail(); + } else if (HttpMethod.POST.equals(method)) { + invocationBuilder.post(Entity.entity(bundle, MediaType.APPLICATION_JSON)); + fail(); + } else { + fail("Unsupported http method"); + } + } catch (Exception e) { + + if (e.getCause() instanceof UnknownHostException) { + redirectedBroker = e.getCause().getMessage().split(":")[0]; + } else { + // fail + fail(); + } + } + return redirectedBroker; + }).collect(Collectors.toList()); + + return redirectBrokers; + } + +} diff --git a/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/web/ZookeeperCacheLoaderTest.java b/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/web/ZookeeperCacheLoaderTest.java new file mode 100644 index 0000000000000..530c35ea130fa --- /dev/null +++ b/pulsar-discovery-service/src/test/test/java/com/yahoo/pulsar/discovery/service/web/ZookeeperCacheLoaderTest.java @@ -0,0 +1,101 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.discovery.service.web; + +import static com.yahoo.pulsar.discovery.service.web.ZookeeperCacheLoader.LOADBALANCE_BROKERS_ROOT; +import static org.testng.Assert.fail; + +import java.io.IOException; +import java.util.List; + +import org.apache.bookkeeper.util.ZkUtils; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.beust.jcommander.internal.Lists; +import com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport; +import com.yahoo.pulsar.discovery.service.web.ZookeeperCacheLoader; +import com.yahoo.pulsar.discovery.service.web.BaseZKStarterTest.DiscoveryZooKeeperClientFactoryImpl; +import com.yahoo.pulsar.zookeeper.MockedZooKeeperClientFactoryImpl; +import com.yahoo.pulsar.zookeeper.ZooKeeperClientFactory; +import com.yahoo.pulsar.zookeeper.ZookeeperClientFactoryImpl; + +public class ZookeeperCacheLoaderTest extends BaseZKStarterTest { + + @BeforeMethod + private void init() throws Exception { + start(); + } + + @AfterMethod + private void cleanup() throws Exception { + close(); + } + + /** + * Create znode for available broker in ZooKeeper and updates it again to verify ZooKeeper cache update + * + * @throws InterruptedException + * @throws KeeperException + * @throws IOException + */ + @Test + public void testZookeeperCacheLoader() throws InterruptedException, KeeperException, Exception { + + DiscoveryZooKeeperClientFactoryImpl.zk = mockZookKeeper; + + ZookeeperCacheLoader zkLoader = new ZookeeperCacheLoader(new DiscoveryZooKeeperClientFactoryImpl(), ""); + + List brokers = Lists.newArrayList("broker-1:15000", "broker-2:15000", "broker-3:15000"); + // 1. create znode for each broker + brokers.stream().forEach(b -> { + try { + zkLoader.getLocalZkCache().getZooKeeper().create(LOADBALANCE_BROKERS_ROOT + "/" + b, new byte[0], + ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } catch (KeeperException | InterruptedException e) { + fail("failed while creating broker znodes"); + } + }); + + Thread.sleep(100); // wait for 100 msec: to get cache updated + + // 2. get available brokers from ZookeeperCacheLoader + List list = zkLoader.getAvailableBrokers(); + + // 3. verify retrieved broker list + Assert.assertTrue(brokers.containsAll(list)); + + // 4.a add new broker + zkLoader.getLocalZkCache().getZooKeeper().create(LOADBALANCE_BROKERS_ROOT + "/" + "broker-4:15000", new byte[0], + ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + brokers.add("broker-4:15000"); + + Thread.sleep(100); // wait for 100 msec: to get cache updated + + // 4.b. get available brokers from ZookeeperCacheLoader + list = zkLoader.getAvailableBrokers(); + + // 4.c. verify retrieved broker list + Assert.assertTrue(brokers.containsAll(list)); + + } + +} diff --git a/pulsar-zookeeper-utils/src/main/java/com/yahoo/pulsar/zookeeper/ZooKeeperCache.java b/pulsar-zookeeper-utils/src/main/java/com/yahoo/pulsar/zookeeper/ZooKeeperCache.java index b65290b462823..a67b96dc367eb 100644 --- a/pulsar-zookeeper-utils/src/main/java/com/yahoo/pulsar/zookeeper/ZooKeeperCache.java +++ b/pulsar-zookeeper-utils/src/main/java/com/yahoo/pulsar/zookeeper/ZooKeeperCache.java @@ -204,6 +204,17 @@ public Optional getData(final String path, final Deserializer deserial return getData(path, this, deserializer).map(e -> e.getKey()); } + public CompletableFuture> getDataAsync(final String path, final Deserializer deserializer) { + CompletableFuture> future = new CompletableFuture<>(); + getDataAsync(path, this, deserializer).thenAccept(data -> { + future.complete(data.map(e -> e.getKey())); + }).exceptionally(ex -> { + future.complete(Optional.empty()); + return null; + }); + return future; + } + /** * Cache that implements automatic reloading on update will pass a different Watcher object to reload cache entry * automatically