From a95fddb02af816c2df9aad85bf281328c34d6021 Mon Sep 17 00:00:00 2001 From: "gao.ran" Date: Sat, 1 Feb 2020 04:12:28 +0800 Subject: [PATCH 01/11] add namespace offload policies --- .../mledger/LedgerOffloaderFactory.java | 5 +- .../apache/pulsar/broker/PulsarService.java | 39 +++++-- .../broker/admin/impl/NamespacesBase.java | 45 ++++++++ .../pulsar/broker/admin/v2/Namespaces.java | 27 +++++ .../pulsar/broker/service/BrokerService.java | 2 +- .../service/persistent/PersistentTopic.java | 2 + .../broker/admin/AdminApiOffloadTest.java | 18 ++- .../pulsar/client/admin/Namespaces.java | 71 ++++++++++++ .../client/admin/internal/NamespacesImpl.java | 23 ++++ .../pulsar/admin/cli/PulsarAdminToolTest.java | 9 ++ .../pulsar/admin/cli/CmdNamespaces.java | 107 ++++++++++++++++++ .../common/policies/data/OffloadPolicies.java | 90 +++++++++++++++ .../pulsar/common/policies/data/Policies.java | 12 +- .../sql/presto/PulsarConnectorCache.java | 26 +++-- .../pulsar/sql/presto/PulsarRecordCursor.java | 3 +- .../apache/pulsar/sql/presto/PulsarSplit.java | 13 ++- .../pulsar/sql/presto/PulsarSplitManager.java | 28 +++-- .../sql/presto/TestPulsarConnector.java | 2 +- .../sql/presto/TestPulsarSplitManager.java | 26 ++--- site2/docs/reference-pulsar-admin.md | 26 +++++ .../FileSystemLedgerOffloaderFactory.java | 4 +- .../jcloud/JCloudLedgerOffloaderFactory.java | 7 +- .../TieredStorageConfigurationData.java | 28 +++++ 23 files changed, 564 insertions(+), 49 deletions(-) create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java index f0a6890aeb272..18886d4003493 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java @@ -24,6 +24,7 @@ import org.apache.bookkeeper.common.annotation.InterfaceAudience.LimitedPrivate; import org.apache.bookkeeper.common.annotation.InterfaceStability.Evolving; import org.apache.bookkeeper.common.util.OrderedScheduler; +import org.apache.pulsar.common.policies.data.OffloadPolicies; /** * Factory to create {@link LedgerOffloader} to offload ledgers into long-term storage. @@ -46,12 +47,14 @@ public interface LedgerOffloaderFactory { * @param properties service configuration * @param userMetadata user metadata * @param scheduler scheduler + * @param overrideOffloadPolicies offloadPolicies for override * @return the offloader instance * @throws IOException when fail to create an offloader */ T create(Properties properties, Map userMetadata, - OrderedScheduler scheduler) + OrderedScheduler scheduler, + OffloadPolicies overrideOffloadPolicies) throws IOException; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index fd169fe1d1d59..fd85b9d490797 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -88,6 +89,7 @@ import org.apache.pulsar.broker.web.WebService; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.common.conf.InternalConfigurationData; @@ -98,6 +100,7 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TenantInfo; @@ -162,7 +165,8 @@ public class PulsarService implements AutoCloseable { private ScheduledExecutorService compactorExecutor; private OrderedScheduler offloaderScheduler; private Offloaders offloaderManager = new Offloaders(); - private LedgerOffloader offloader; + private LedgerOffloader defaultOffloader; + private Map ledgerOffloaderMap = new ConcurrentHashMap<>(); private ScheduledFuture loadReportTask = null; private ScheduledFuture loadSheddingTask = null; private ScheduledFuture loadResourceQuotaTask = null; @@ -398,7 +402,7 @@ public void start() throws PulsarServerException { // Start load management service (even if load balancing is disabled) this.loadManager.set(LoadManager.create(this)); - this.offloader = createManagedLedgerOffloader(this.getConfiguration()); + this.defaultOffloader = createManagedLedgerOffloader(this.getConfiguration(), null); brokerService.start(); @@ -760,11 +764,31 @@ public ManagedLedgerClientFactory getManagedLedgerClientFactory() { return managedLedgerClientFactory; } - public LedgerOffloader getManagedLedgerOffloader() { - return offloader; + public LedgerOffloader getManagedLedgerOffloaderByPolicies(OffloadPolicies offloadPolicies) { + if (offloadPolicies == null) { + return getDefaultOffloader(); + } + return ledgerOffloaderMap.computeIfAbsent(offloadPolicies, op -> { + try { + return createManagedLedgerOffloader(getConfiguration(), offloadPolicies); + } catch (PulsarServerException e) { + LOG.error("create ledgerOffloader failed for {}", offloadPolicies.toString(), e); + return new NullLedgerOffloader(); + } + }); + } + + public LedgerOffloader getManagedLedgerOffloader(NamespaceName namespaceName) { + try { + OffloadPolicies offloadPolicies = getAdminClient().namespaces().getOffload(namespaceName.toString()); + return getManagedLedgerOffloaderByPolicies(offloadPolicies); + } catch (Exception e) { + LOG.error("Failed to get ledgerOffloader for namespaceName {}", namespaceName, e); + return new NullLedgerOffloader(); + } } - public synchronized LedgerOffloader createManagedLedgerOffloader(ServiceConfiguration conf) + public synchronized LedgerOffloader createManagedLedgerOffloader(ServiceConfiguration conf, OffloadPolicies offloadPolicies) throws PulsarServerException { try { if (StringUtils.isNotBlank(conf.getManagedLedgerOffloadDriver())) { @@ -781,12 +805,13 @@ public synchronized LedgerOffloader createManagedLedgerOffloader(ServiceConfigur LedgerOffloader.METADATA_SOFTWARE_VERSION_KEY.toLowerCase(), PulsarVersion.getVersion(), LedgerOffloader.METADATA_SOFTWARE_GITSHA_KEY.toLowerCase(), PulsarVersion.getGitSha() ), - getOffloaderScheduler(conf)); + getOffloaderScheduler(conf), + offloadPolicies); } catch (IOException ioe) { throw new PulsarServerException(ioe.getMessage(), ioe.getCause()); } } else { - LOG.info("No ledger offloader configured, using NULL instance"); + LOG.warn("No ledger offloader configured, using NULL instance"); return NullLedgerOffloader.INSTANCE; } } catch (Throwable t) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index adc7bd4d502b6..3d32fa608b74e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -76,6 +76,7 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.LocalPolicies; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; @@ -2142,5 +2143,49 @@ private void mutatePolicy(Function policyTransformation, } } + protected void internalSetOffload(OffloadPolicies offload) { + validateAdminAccessForTenant(namespaceName.getTenant()); + validatePoliciesReadOnlyAccess(); + + try { + Stat nodeStat = new Stat(); + final String path = path(POLICIES, namespaceName.toString()); + byte[] content = globalZk().getData(path, null, nodeStat); + Policies policies = jsonMapper().readValue(content, Policies.class); + if (StringUtils.isEmpty(offload.getBucket())) { + log.warn("[{}] Failed to update offload configuration for namespace {}: bucket must be specified", + clientAppId(), namespaceName); + throw new RestException(Status.PRECONDITION_FAILED, + "The bucket must be specified for namespace offload."); + } + policies.offload_policies = offload; + globalZk().setData(path, jsonMapper().writeValueAsBytes(policies), nodeStat.getVersion()); + policiesCache().invalidate(path(POLICIES, namespaceName.toString())); + log.info("[{}] Successfully updated offload configuration: namespace={}, map={}", clientAppId(), + namespaceName, jsonMapper().writeValueAsString(policies.offload_policies)); + } catch (KeeperException.NoNodeException e) { + log.warn("[{}] Failed to update offload configuration for namespace {}: does not exist", clientAppId(), + namespaceName); + throw new RestException(Status.NOT_FOUND, "Namespace does not exist"); + } catch (KeeperException.BadVersionException e) { + log.warn("[{}] Failed to update offload configuration for namespace {}: concurrent modification", + clientAppId(), namespaceName); + throw new RestException(Status.CONFLICT, "Concurrent modification"); + } catch (RestException pfe) { + throw pfe; + } catch (Exception e) { + log.error("[{}] Failed to update offload configuration for namespace {}", clientAppId(), namespaceName, + e); + throw new RestException(e); + } + } + + protected OffloadPolicies internalGetOffload() { + validateAdminAccessForTenant(namespaceName.getTenant()); + + Policies policies = getNamespacePolicies(namespaceName); + return policies.offload_policies; + } + private static final Logger log = LoggerFactory.getLogger(NamespacesBase.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 5bc71fc84ebf1..4a3d27e0b39cf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -51,6 +51,7 @@ import org.apache.pulsar.common.policies.data.BookieAffinityGroupData; import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.DispatchRate; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; @@ -1052,5 +1053,31 @@ public void setSchemaValidtionEnforced(@PathParam("tenant") String tenant, internalSetSchemaValidationEnforced(schemaValidationEnforced); } + @POST + @Path("/{tenant}/{namespace}/offload") + @ApiOperation(value = " Set offload configuration on a namespace.") + @ApiResponses(value = { + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Namespace does not exist"), + @ApiResponse(code = 409, message = "Concurrent modification"), + @ApiResponse(code = 412, message = "Bucket must be specified") }) + public void setOffload(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace, + OffloadPolicies offload) { + validateNamespaceName(tenant, namespace); + internalSetOffload(offload); + } + + @GET + @Path("/{tenant}/{namespace}/offload") + @ApiOperation(value = "Get offload config on a namespace.") + @ApiResponses(value = { + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Namespace does not exist") }) + public OffloadPolicies getOffload(@PathParam("tenant") String tenant, + @PathParam("namespace") String namespace) { + validateNamespaceName(tenant, namespace); + return internalGetOffload(); + } + private static final Logger log = LoggerFactory.getLogger(Namespaces.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index eb8a05fd60ad6..72227ef67308e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -981,7 +981,7 @@ public CompletableFuture getManagedLedgerConfig(TopicName t managedLedgerConfig.setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES); managedLedgerConfig.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB()); - managedLedgerConfig.setLedgerOffloader(pulsar.getManagedLedgerOffloader()); + managedLedgerConfig.setLedgerOffloader(pulsar.getManagedLedgerOffloader(namespace)); policies.ifPresent(p -> { long lag = serviceConfig.getManagedLedgerOffloadDeletionLagMs(); if (p.offload_deletion_lag_ms != null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 34cad879e6631..96ab8c08d6a46 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1740,6 +1740,8 @@ public CompletableFuture onPoliciesUpdate(Policies data) { if (this.subscribeRateLimiter.isPresent()) { subscribeRateLimiter.get().onPoliciesUpdate(data); } + getManagedLedger().getConfig().setLedgerOffloader( + brokerService.pulsar().getManagedLedgerOffloaderByPolicies(data.offload_policies)); return CompletableFuture.allOf(replicationFuture, dedupFuture, persistentPoliciesFuture); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java index 7f56a3fc02e9b..f455423103130 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java @@ -38,6 +38,7 @@ import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.TenantInfo; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -71,7 +72,8 @@ private void testOffload(String topicName, String mlName) throws Exception { LedgerOffloader offloader = mock(LedgerOffloader.class); when(offloader.getOffloadDriverName()).thenReturn("mock"); - doReturn(offloader).when(pulsar).getManagedLedgerOffloader(); + doReturn(offloader).when(pulsar).getManagedLedgerOffloaderByPolicies(any()); + doReturn(offloader).when(pulsar).getManagedLedgerOffloader(any()); CompletableFuture promise = new CompletableFuture<>(); doReturn(promise).when(offloader).offload(any(), any(), any()); @@ -138,4 +140,18 @@ public void testOffloadV1() throws Exception { String mlName = "prop-xyz/test/ns1/persistent/topic2"; testOffload(topicName, mlName); } + + @Test + public void testOffloadPolicies() throws Exception { + String namespaceName = "prop-xyz/ns1"; + String endpoint = "test-endpoint"; + String region = "test-region"; + String bucket = "test-bucket"; + + OffloadPolicies offload1 = new OffloadPolicies(endpoint, region, bucket); + admin.namespaces().setOffload(namespaceName, offload1); + OffloadPolicies offload2 = admin.namespaces().getOffload(namespaceName); + Assert.assertEquals(offload1, offload2); + } + } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java index 8b858ba53f93f..446643dc37563 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java @@ -33,6 +33,7 @@ import org.apache.pulsar.common.policies.data.BookieAffinityGroupData; import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.DispatchRate; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; @@ -1591,4 +1592,74 @@ boolean getIsAllowAutoUpdateSchema(String namespace) */ void setIsAllowAutoUpdateSchema(String namespace, boolean isAllowAutoUpdateSchema) throws PulsarAdminException; + + /** + * Set the offload configuration for all the topics on a namespace. + *

+ * Set the offload configuration on a namespace. This operation requires Pulsar super-user access. + *

+ * Request parameter example: + *

+ * + * 

+     * 
+     * {
+     *     "region" : "us-east-2",                   // The long term storage region
+     *     "bucket" : "bucket",                      // Bucket to place offloaded ledger into
+     *     "endpoint" : "endpoint",                  // Alternative endpoint to connect to
+     *     "maxBlockSize" : 1024,                    // Max Block Size, default 64MB
+     *     "readBufferSize" : 1024,                  // Read Buffer Size, default 1MB
+     * }
+     * 
+     *
+ * + * @param namespace + * Namespace name + * @param offloadPolicies + * Offload configuration + * + * @throws NotAuthorizedException + * Don't have admin permission + * @throws NotFoundException + * Namespace does not exist + * @throws ConflictException + * Concurrent modification + * @throws PulsarAdminException + * Unexpected error + */ + void setOffload(String namespace, OffloadPolicies offloadPolicies) throws PulsarAdminException; + + /** + * Get the offload configuration for a namespace. + *

+ * Get the offload configuration for a namespace. + *

+ * Response example: + *

+ * + * 

+     * 
+     * {
+     *     "region" : "us-east-2",                   // The long term storage region
+     *     "bucket" : "bucket",                      // Bucket to place offloaded ledger into
+     *     "endpoint" : "endpoint",                  // Alternative endpoint to connect to
+     *     "maxBlockSize" : 1024,                    // Max Block Size, default 64MB
+     *     "readBufferSize" : 1024,                  // Read Buffer Size, default 1MB
+     * }
+     * 
+     *
+ * + * @param namespace + * Namespace name + * @throws NotAuthorizedException + * Don't have admin permission + * @throws NotFoundException + * Namespace does not exist + * @throws ConflictException + * Concurrent modification + * @throws PulsarAdminException + * Unexpected error + */ + OffloadPolicies getOffload(String namespace) throws PulsarAdminException; + } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java index 7ab7762c7612d..85342c3c770dc 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java @@ -43,6 +43,7 @@ import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.ErrorData; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; @@ -1006,6 +1007,28 @@ public void setIsAllowAutoUpdateSchema(String namespace, boolean isAllowAutoUpda } } + @Override + public void setOffload(String namespace, OffloadPolicies offloadPolicies) throws PulsarAdminException { + try { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "offload"); + request(path).post(Entity.entity(offloadPolicies, MediaType.APPLICATION_JSON), ErrorData.class); + } catch (Exception e) { + throw getApiException(e); + } + } + + @Override + public OffloadPolicies getOffload(String namespace) throws PulsarAdminException { + try { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "offload"); + return request(path).get(OffloadPolicies.class); + } catch (Exception e) { + throw getApiException(e); + } + } + private WebTarget namespacePath(NamespaceName namespace, String... parts) { final WebTarget base = namespace.isV2() ? adminV2Namespaces : adminNamespaces; WebTarget namespacePath = base.path(namespace.toString()); diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index b62e4f473ffde..0fbbdea3310b0 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -63,6 +63,7 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.FailureDomain; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; @@ -480,6 +481,14 @@ void namespaces() throws Exception { namespaces.run(split("clear-offload-deletion-lag myprop/clust/ns1")); verify(mockNamespaces).clearOffloadDeleteLag("myprop/clust/ns1"); + + namespaces.run(split("set-offload myprop/clust/ns1 -r test-region -b test-bucket -e http://test.endpoint -mbs 32M -rbs 5M")); + verify(mockNamespaces).setOffload("myprop/clust/ns1", + new OffloadPolicies("test-region", "test-bucket", "http://test.endpoint", + 32 * 1024 * 1024, 5 * 1024 * 1024)); + + namespaces.run(split("get-offload myprop/clust/ns1")); + verify(mockNamespaces).getOffload("myprop/clust/ns1"); } @Test diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 3c56c732033ae..7389b54008704 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -22,6 +22,7 @@ import com.beust.jcommander.ParameterException; import com.beust.jcommander.Parameters; import com.beust.jcommander.converters.CommaParameterSplitter; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -32,6 +33,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.admin.cli.utils.IOUtils; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -40,6 +42,7 @@ import org.apache.pulsar.common.policies.data.BookieAffinityGroupData; import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.DispatchRate; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; @@ -1246,6 +1249,106 @@ void run() throws PulsarAdminException { } } + @Parameters(commandDescription = "Set the offload policies for a namespace") + private class SetOffload extends CliCommand { + @Parameter(description = "tenant/namespace", required = true) + private java.util.List params; + + @Parameter( + names = {"--region", "-r"}, + description = "The long term storage region, " + + "default is s3ManagedLedgerOffloadRegion or gcsManagedLedgerOffloadRegion in broker.conf", + required = false) + private String region; + + @Parameter( + names = {"--bucket", "-b"}, + description = "Bucket to place offloaded ledger into", + required = true) + private String bucket; + + @Parameter( + names = {"--endpoint", "-e"}, + description = "Alternative endpoint to connect to, " + + "s3 default is s3ManagedLedgerOffloadServiceEndpoint in broker.conf", + required = false) + private String endpoint; + + @Parameter( + names = {"--maxBlockSize", "-mbs"}, + description = "Max block size (eg: 32M, 64M), default is 64MB", + required = false) + private String maxBlockSizeStr; + + @Parameter( + names = {"--readBufferSize", "-rbs"}, + description = "Read buffer size (eg: 1M, 5M), default is 1MB", + required = false) + private String readBufferSizeStr; + + private final String[] DRIVER_NAMES = {"S3", "aws-s3", "google-cloud-storage"}; + + public boolean isS3Driver(String driver) { + if (StringUtils.isEmpty(driver)) { + return false; + } + return driver.equalsIgnoreCase(DRIVER_NAMES[0]) || driver.equalsIgnoreCase(DRIVER_NAMES[1]); + } + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(params); + + String offloadDriver = admin.brokers().getRuntimeConfigurations().get("managedLedgerOffloadDriver"); + if (isS3Driver(offloadDriver) && Strings.isNullOrEmpty(region) && Strings.isNullOrEmpty(endpoint)) { + throw new ParameterException( + "Either s3ManagedLedgerOffloadRegion or s3ManagedLedgerOffloadServiceEndpoint must be set" + + " if s3 offload enabled"); + } + + long maxBlockSize = 0; + if (StringUtils.isNotEmpty(maxBlockSizeStr)) { + maxBlockSize = validateSizeString(maxBlockSizeStr); + } + long readBufferSize = 0; + if (StringUtils.isNotEmpty(readBufferSizeStr)) { + readBufferSize = validateSizeString(readBufferSizeStr); + } + + final int maxBlockSizeInBytes; + if (maxBlockSize > 0 && maxBlockSize <= Integer.MAX_VALUE) { + maxBlockSizeInBytes = new Long(maxBlockSize).intValue(); + } else { + maxBlockSizeInBytes = OffloadPolicies.MAX_BLOCK_SIZE_IN_BYTES; + } + + final int readBufferSizeInBytes; + if (readBufferSize > 0 && readBufferSize <= Integer.MAX_VALUE) { + readBufferSizeInBytes = new Long(readBufferSize).intValue(); + } else { + readBufferSizeInBytes = OffloadPolicies.READ_BUFFER_SIZE_IN_BYTES; + } + + admin.namespaces().setOffload(namespace, new OffloadPolicies( + region == null ? "" : region, + bucket, + endpoint == null ? "" : endpoint, + maxBlockSizeInBytes, readBufferSizeInBytes)); + } + } + + @Parameters(commandDescription = "Get the offload policies for a namespace") + private class GetOffload extends CliCommand { + @Parameter(description = "tenant/namespace\n", required = true) + private java.util.List params; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(params); + print(admin.namespaces().getOffload(namespace)); + } + } + public CmdNamespaces(PulsarAdmin admin) { super("namespaces", admin); jcommander.addCommand("list", new GetNamespacesPerProperty()); @@ -1345,5 +1448,9 @@ public CmdNamespaces(PulsarAdmin admin) { jcommander.addCommand("get-schema-validation-enforce", new GetSchemaValidationEnforced()); jcommander.addCommand("set-schema-validation-enforce", new SetSchemaValidationEnforced()); + + jcommander.addCommand("set-offload", new SetOffload()); + jcommander.addCommand("get-offload", new GetOffload()); } + } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java new file mode 100644 index 0000000000000..563d357996d4b --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java @@ -0,0 +1,90 @@ +package org.apache.pulsar.common.policies.data; + +import com.google.common.base.MoreObjects; + +import java.util.Objects; + +/** + * Definition of the offload policies. + */ +public class OffloadPolicies { + + public final static int MAX_BLOCK_SIZE_IN_BYTES = 64 * 1024 * 1024; // 64MB + public final static int READ_BUFFER_SIZE_IN_BYTES = 1024 * 1024; // 1MB + + private String region; + private String bucket; + private String endpoint; + private int maxBlockSizeInBytes; + private int readBufferSizeInBytes; + + public OffloadPolicies() { + + } + + public OffloadPolicies(String region, String bucket, String endpoint) { + this(region, bucket, endpoint, MAX_BLOCK_SIZE_IN_BYTES, READ_BUFFER_SIZE_IN_BYTES); + } + + public OffloadPolicies(String region, String bucket, String endpoint, + int maxBlockSizeInBytes, int readBufferSizeInBytes) { + this.region = region; + this.bucket = bucket; + this.endpoint = endpoint; + this.maxBlockSizeInBytes = maxBlockSizeInBytes; + this.readBufferSizeInBytes = readBufferSizeInBytes; + } + + public String getRegion() { + return region; + } + + public String getBucket() { + return bucket; + } + + public String getEndpoint() { + return endpoint; + } + + public int getMaxBlockSizeInBytes() { + return maxBlockSizeInBytes; + } + + public int getReadBufferSizeInBytes() { + return readBufferSizeInBytes; + } + + @Override + public int hashCode() { + return Objects.hash(region, bucket, endpoint, maxBlockSizeInBytes, readBufferSizeInBytes); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + OffloadPolicies other = (OffloadPolicies) obj; + return Objects.equals(region, other.getRegion()) + && Objects.equals(bucket, other.getBucket()) + && Objects.equals(endpoint, other.getEndpoint()) + && Objects.equals(maxBlockSizeInBytes, other.getMaxBlockSizeInBytes()) + && Objects.equals(readBufferSizeInBytes, other.getReadBufferSizeInBytes()); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("region", region) + .add("bucket", bucket) + .add("endpoint", endpoint) + .add("maxBlockSizeInBytes", maxBlockSizeInBytes) + .add("readBufferSizeInBytes", readBufferSizeInBytes) + .toString(); + } + +} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java index 8e55280fc8de9..bf597e6eac1f2 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java @@ -94,6 +94,9 @@ public class Policies { @SuppressWarnings("checkstyle:MemberName") public boolean schema_validation_enforced = false; + @SuppressWarnings("checkstyle:MemberName") + public OffloadPolicies offload_policies = null; + @Override public int hashCode() { return Objects.hash(auth_policies, replication_clusters, @@ -110,7 +113,8 @@ public int hashCode() { schema_auto_update_compatibility_strategy, schema_validation_enforced, schema_compatibility_strategy, - is_allow_auto_update_schema); + is_allow_auto_update_schema, + offload_policies); } @Override @@ -143,7 +147,8 @@ public boolean equals(Object obj) { && schema_auto_update_compatibility_strategy == other.schema_auto_update_compatibility_strategy && schema_validation_enforced == other.schema_validation_enforced && schema_compatibility_strategy == other.schema_compatibility_strategy - && is_allow_auto_update_schema == other.is_allow_auto_update_schema; + && is_allow_auto_update_schema == other.is_allow_auto_update_schema + && offload_policies == other.offload_policies; } return false; @@ -191,6 +196,7 @@ public String toString() { .add("schema_auto_update_compatibility_strategy", schema_auto_update_compatibility_strategy) .add("schema_validation_enforced", schema_validation_enforced) .add("schema_compatibility_Strategy", schema_compatibility_strategy) - .add("is_allow_auto_update_Schema", is_allow_auto_update_schema).toString(); + .add("is_allow_auto_update_Schema", is_allow_auto_update_schema) + .add("offload_policies", offload_policies).toString(); } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java index 1af86a2225a06..7bf4d3b4e6ef2 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java @@ -39,6 +39,7 @@ import org.apache.bookkeeper.stats.StatsProvider; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.PulsarVersion; +import org.apache.pulsar.common.policies.data.OffloadPolicies; /** * Implementation of a cache for the Pulsar connector. @@ -55,7 +56,8 @@ public class PulsarConnectorCache { private final StatsProvider statsProvider; private OrderedScheduler offloaderScheduler; private Offloaders offloaderManager; - private LedgerOffloader offloader; + private LedgerOffloader defaultOffloader; + private Map offloaderMap; private static final String OFFLOADERS_DIRECTOR = "offloadersDirectory"; private static final String MANAGED_LEDGER_OFFLOAD_DRIVER = "managedLedgerOffloadDriver"; @@ -74,7 +76,7 @@ private PulsarConnectorCache(PulsarConnectorConfig pulsarConnectorConfig) throws this.statsProvider.start(clientConfiguration); - this.offloader = initManagedLedgerOffloader(pulsarConnectorConfig); + this.defaultOffloader = initManagedLedgerOffloader(pulsarConnectorConfig, null); } public static PulsarConnectorCache getConnectorCache(PulsarConnectorConfig pulsarConnectorConfig) throws Exception { @@ -108,10 +110,17 @@ private static ManagedLedgerFactory initManagedLedgerFactory(PulsarConnectorConf return new ManagedLedgerFactoryImpl(bkClientConfiguration, managedLedgerFactoryConfig); } - public ManagedLedgerConfig getManagedLedgerConfig() { - - return new ManagedLedgerConfig() - .setLedgerOffloader(this.offloader); + public ManagedLedgerConfig getManagedLedgerConfig(PulsarConnectorConfig pulsarConnectorConfig, + OffloadPolicies offloadPolicies) { + ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); + if (offloadPolicies == null) { + managedLedgerConfig.setLedgerOffloader(this.defaultOffloader); + } else { + LedgerOffloader ledgerOffloader = offloaderMap.computeIfAbsent(offloadPolicies, + op -> initManagedLedgerOffloader(pulsarConnectorConfig, offloadPolicies)); + managedLedgerConfig.setLedgerOffloader(ledgerOffloader); + } + return managedLedgerConfig; } private synchronized OrderedScheduler getOffloaderScheduler(PulsarConnectorConfig pulsarConnectorConfig) { @@ -123,7 +132,7 @@ private synchronized OrderedScheduler getOffloaderScheduler(PulsarConnectorConfi return this.offloaderScheduler; } - private LedgerOffloader initManagedLedgerOffloader(PulsarConnectorConfig conf) { + private LedgerOffloader initManagedLedgerOffloader(PulsarConnectorConfig conf, OffloadPolicies offloadPolicies) { try { if (StringUtils.isNotBlank(conf.getManagedLedgerOffloadDriver())) { @@ -147,7 +156,8 @@ private LedgerOffloader initManagedLedgerOffloader(PulsarConnectorConfig conf) { LedgerOffloader.METADATA_SOFTWARE_VERSION_KEY.toLowerCase(), PulsarVersion.getVersion(), LedgerOffloader.METADATA_SOFTWARE_GITSHA_KEY.toLowerCase(), PulsarVersion.getGitSha() ), - getOffloaderScheduler(conf)); + getOffloaderScheduler(conf), + offloadPolicies); } catch (IOException ioe) { log.error("Failed to create offloader: ", ioe); throw new RuntimeException(ioe.getMessage(), ioe.getCause()); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index 4d5a25db836d6..3c36f01b1b0f5 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -111,7 +111,8 @@ public PulsarRecordCursor(List columnHandles, PulsarSplit pu throw new RuntimeException(e); } initialize(columnHandles, pulsarSplit, pulsarConnectorConfig, - pulsarConnectorCache.getManagedLedgerFactory(), pulsarConnectorCache.getManagedLedgerConfig(), + pulsarConnectorCache.getManagedLedgerFactory(), + pulsarConnectorCache.getManagedLedgerConfig(pulsarConnectorConfig, pulsarSplit.getOffloadPolicies()), new PulsarConnectorMetricsTracker(pulsarConnectorCache.getStatsProvider())); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplit.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplit.java index eeebbd174dd3f..dbebbf54a3eba 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplit.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplit.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; @@ -55,6 +56,8 @@ public class PulsarSplit implements ConnectorSplit { private final PositionImpl startPosition; private final PositionImpl endPosition; + private final OffloadPolicies offloadPolicies; + @JsonCreator public PulsarSplit( @JsonProperty("splitId") long splitId, @@ -69,7 +72,8 @@ public PulsarSplit( @JsonProperty("startPositionLedgerId") long startPositionLedgerId, @JsonProperty("endPositionLedgerId") long endPositionLedgerId, @JsonProperty("tupleDomain") TupleDomain tupleDomain, - @JsonProperty("properties") Map schemaInfoProperties) { + @JsonProperty("properties") Map schemaInfoProperties, + @JsonProperty("offloadPolicies") OffloadPolicies offloadPolicies) { this.splitId = splitId; requireNonNull(schemaName, "schema name is null"); this.schemaInfo = SchemaInfo.builder() @@ -91,6 +95,7 @@ public PulsarSplit( this.tupleDomain = requireNonNull(tupleDomain, "tupleDomain is null"); this.startPosition = PositionImpl.get(startPositionLedgerId, startPositionEntryId); this.endPosition = PositionImpl.get(endPositionLedgerId, endPositionEntryId); + this.offloadPolicies = offloadPolicies; } @JsonProperty @@ -161,6 +166,11 @@ public PositionImpl getEndPosition() { return endPosition; } + @JsonProperty + public OffloadPolicies getOffloadPolicies() { + return offloadPolicies; + } + @Override public boolean isRemotelyAccessible() { return true; @@ -190,6 +200,7 @@ public String toString() { + ", endPositionEntryId=" + endPositionEntryId + ", startPositionLedgerId=" + startPositionLedgerId + ", endPositionLedgerId=" + endPositionLedgerId + + (offloadPolicies == null ? "" : offloadPolicies.toString()) + '}'; } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java index 06bc1200146f7..b093b1d824c6d 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java @@ -59,6 +59,7 @@ import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.schema.SchemaInfo; /** @@ -122,11 +123,14 @@ public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHand Collection splits; try { + OffloadPolicies offloadPolicies = this.pulsarAdmin.namespaces().getOffload(topicName.getNamespace()); if (!PulsarConnectorUtils.isPartitionedTopic(topicName, this.pulsarAdmin)) { - splits = getSplitsNonPartitionedTopic(numSplits, topicName, tableHandle, schemaInfo, tupleDomain); + splits = getSplitsNonPartitionedTopic( + numSplits, topicName, tableHandle, schemaInfo, tupleDomain, offloadPolicies); log.debug("Splits for non-partitioned topic %s: %s", topicName, splits); } else { - splits = getSplitsPartitionedTopic(numSplits, topicName, tableHandle, schemaInfo, tupleDomain); + splits = getSplitsPartitionedTopic( + numSplits, topicName, tableHandle, schemaInfo, tupleDomain, offloadPolicies); log.debug("Splits for partitioned topic %s: %s", topicName, splits); } } catch (Exception e) { @@ -138,7 +142,8 @@ public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHand @VisibleForTesting Collection getSplitsPartitionedTopic(int numSplits, TopicName topicName, PulsarTableHandle - tableHandle, SchemaInfo schemaInfo, TupleDomain tupleDomain) throws Exception { + tableHandle, SchemaInfo schemaInfo, TupleDomain tupleDomain, + OffloadPolicies offloadPolicies) throws Exception { List predicatedPartitions = getPredicatedPartitions(topicName, tupleDomain); if (log.isDebugEnabled()) { @@ -165,7 +170,8 @@ Collection getSplitsPartitionedTopic(int numSplits, TopicName topic tableHandle, schemaInfo, topicName.getPartition(predicatedPartitions.get(i)).getLocalName(), - tupleDomain)); + tupleDomain, + offloadPolicies)); } return splits; } @@ -219,8 +225,8 @@ private List getPredicatedPartitions(TopicName topicName, TupleDomain getSplitsNonPartitionedTopic(int numSplits, TopicName topicName, - PulsarTableHandle tableHandle, SchemaInfo schemaInfo, TupleDomain tupleDomain) - throws Exception { + PulsarTableHandle tableHandle, SchemaInfo schemaInfo, TupleDomain tupleDomain, + OffloadPolicies offloadPolicies) throws Exception { ManagedLedgerFactory managedLedgerFactory = PulsarConnectorCache.getConnectorCache(pulsarConnectorConfig) .getManagedLedgerFactory(); @@ -230,7 +236,9 @@ Collection getSplitsNonPartitionedTopic(int numSplits, TopicName to numSplits, tableHandle, schemaInfo, - tableHandle.getTableName(), tupleDomain); + tableHandle.getTableName(), + tupleDomain, + offloadPolicies); } @VisibleForTesting @@ -239,7 +247,8 @@ Collection getSplitsForTopic(String topicNamePersistenceEncoding, int numSplits, PulsarTableHandle tableHandle, SchemaInfo schemaInfo, String tableName, - TupleDomain tupleDomain) + TupleDomain tupleDomain, + OffloadPolicies offloadPolicies) throws ManagedLedgerException, InterruptedException { ReadOnlyCursor readOnlyCursor = null; @@ -296,7 +305,8 @@ Collection getSplitsForTopic(String topicNamePersistenceEncoding, startPosition.getLedgerId(), endPosition.getLedgerId(), tupleDomain, - schemaInfo.getProperties())); + schemaInfo.getProperties(), + offloadPolicies)); } return splits; } finally { diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java index cd81442db6b8b..6c911a407541a 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java @@ -561,7 +561,7 @@ public static class Boo { new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema()), topicsToSchemas.get(topicName.getSchemaName()).getType(), 0, topicsToNumEntries.get(topicName.getSchemaName()), - 0, 0, TupleDomain.all(), new HashMap<>())); + 0, 0, TupleDomain.all(), new HashMap<>(), null)); } } diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarSplitManager.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarSplitManager.java index ef361487a054f..5ecbffc307764 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarSplitManager.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarSplitManager.java @@ -87,7 +87,7 @@ public void testTopic(String delimiter) throws Exception { PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, TupleDomain.all()); final ResultCaptor> resultCaptor = new ResultCaptor<>(); - doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any()); + doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits( @@ -95,7 +95,7 @@ public void testTopic(String delimiter) throws Exception { pulsarTableLayoutHandle, null); verify(this.pulsarSplitManager, times(1)) - .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any()); + .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); int totalSize = 0; for (PulsarSplit pulsarSplit : resultCaptor.getResult()) { @@ -134,13 +134,13 @@ public void testPartitionedTopic(String delimiter) throws Exception { PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, TupleDomain.all()); final ResultCaptor> resultCaptor = new ResultCaptor<>(); - doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any()); + doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); this.pulsarSplitManager.getSplits(mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), pulsarTableLayoutHandle, null); verify(this.pulsarSplitManager, times(1)) - .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any()); + .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); int partitions = partitionedTopicsToPartitions.get(topicName.toString()); @@ -200,8 +200,8 @@ public void testPublishTimePredicatePushdown(String delimiter) throws Exception PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, tupleDomain); final ResultCaptor> resultCaptor = new ResultCaptor<>(); - doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsNonPartitionedTopic(anyInt(), any(), any(), any - (), any()); + doAnswer(resultCaptor).when(this.pulsarSplitManager) + .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits( mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), @@ -209,7 +209,7 @@ public void testPublishTimePredicatePushdown(String delimiter) throws Exception verify(this.pulsarSplitManager, times(1)) - .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any()); + .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); int totalSize = 0; int initalStart = 1; @@ -258,7 +258,7 @@ public void testPublishTimePredicatePushdownPartitionedTopic(String delimiter) t final ResultCaptor> resultCaptor = new ResultCaptor<>(); doAnswer(resultCaptor).when(this.pulsarSplitManager) - .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any()); + .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits( mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), @@ -266,7 +266,7 @@ public void testPublishTimePredicatePushdownPartitionedTopic(String delimiter) t verify(this.pulsarSplitManager, times(1)) - .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any()); + .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); int partitions = partitionedTopicsToPartitions.get(topicName.toString()); @@ -315,7 +315,7 @@ public void testPartitionFilter(String delimiter) throws Exception { domainMap.put(PulsarInternalColumn.PARTITION.getColumnHandle(pulsarConnectorId.toString(), false), domain); TupleDomain tupleDomain = TupleDomain.withColumnDomains(domainMap); Collection splits = this.pulsarSplitManager.getSplitsPartitionedTopic(2, topicName, pulsarTableHandle, - schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain); + schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain, null); if (topicsToNumEntries.get(topicName.getSchemaName()) > 1) { Assert.assertEquals(splits.size(), 2); } @@ -332,7 +332,7 @@ public void testPartitionFilter(String delimiter) throws Exception { domainMap.put(PulsarInternalColumn.PARTITION.getColumnHandle(pulsarConnectorId.toString(), false), domain); tupleDomain = TupleDomain.withColumnDomains(domainMap); splits = this.pulsarSplitManager.getSplitsPartitionedTopic(1, topicName, pulsarTableHandle, - schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain); + schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain, null); if (topicsToNumEntries.get(topicName.getSchemaName()) > 1) { Assert.assertEquals(splits.size(), 2); } @@ -348,7 +348,7 @@ public void testPartitionFilter(String delimiter) throws Exception { domainMap.put(PulsarInternalColumn.PARTITION.getColumnHandle(pulsarConnectorId.toString(), false), domain); tupleDomain = TupleDomain.withColumnDomains(domainMap); splits = this.pulsarSplitManager.getSplitsPartitionedTopic(2, topicName, pulsarTableHandle, - schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain); + schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain, null); if (topicsToNumEntries.get(topicName.getSchemaName()) > 1) { Assert.assertEquals(splits.size(), 3); } @@ -367,7 +367,7 @@ public void testPartitionFilter(String delimiter) throws Exception { domainMap.put(PulsarInternalColumn.PARTITION.getColumnHandle(pulsarConnectorId.toString(), false), domain); tupleDomain = TupleDomain.withColumnDomains(domainMap); splits = this.pulsarSplitManager.getSplitsPartitionedTopic(2, topicName, pulsarTableHandle, - schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain); + schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain, null); if (topicsToNumEntries.get(topicName.getSchemaName()) > 1) { Assert.assertEquals(splits.size(), 4); } diff --git a/site2/docs/reference-pulsar-admin.md b/site2/docs/reference-pulsar-admin.md index af8c88b50ad69..7612054a71770 100644 --- a/site2/docs/reference-pulsar-admin.md +++ b/site2/docs/reference-pulsar-admin.md @@ -876,6 +876,8 @@ Subcommands * `clear-offload-deletion-lag` * `get-schema-autoupdate-strategy` * `set-schema-autoupdate-strategy` +* `set-offload` +* `get-offload` ### `list` @@ -2210,3 +2212,27 @@ Options |`-t`, `--type`|The type of the schema (avro or json)|| +### `get-offload` +Get the offload policy for a namespace + +Usage +```bash +$ pulsar-admin namespaces get-offload tenant/namespace +``` + +### `set-offload` +Set the offload policy for a namespace + +Usage +```bash +$ pulsar-admin namespaces set-offload tenant/namespace +``` + +Options +|Flag|Description|Default| +|----|---|---| +|`-r`, `--region`|The long term storage region|s3ManagedLedgerOffloadRegion or gcsManagedLedgerOffloadRegion in broker.conf +|`-b`, `--bucket`|Bucket to place offloaded ledger into|-- +|`-e`, `--endpoint`|Alternative endpoint to connect to|s3 is s3ManagedLedgerOffloadServiceEndpoint in broker.conf +|`-mbs`, `--maxBlockSize`|Max block size|64MB +|`-rbs`, `--readBufferSize`|Read buffer size|1MB diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java index cd52197a59bb5..026f08ccf6dd3 100644 --- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java +++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java @@ -21,6 +21,7 @@ import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.LedgerOffloaderFactory; import org.apache.bookkeeper.mledger.offload.filesystem.impl.FileSystemManagedLedgerOffloader; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import java.io.IOException; import java.util.Map; @@ -33,7 +34,8 @@ public boolean isDriverSupported(String driverName) { } @Override - public FileSystemManagedLedgerOffloader create(Properties properties, Map userMetadata, OrderedScheduler scheduler) throws IOException { + public FileSystemManagedLedgerOffloader create(Properties properties, Map userMetadata, OrderedScheduler scheduler, + OffloadPolicies overrideOffloadPolicies) throws IOException { FileSystemConfigurationData data = FileSystemConfigurationData.create(properties); return FileSystemManagedLedgerOffloader.create(data, scheduler); } diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java index dffe253f94a98..58c5e901c8a9c 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java @@ -24,6 +24,7 @@ import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.LedgerOffloaderFactory; import org.apache.bookkeeper.mledger.offload.jcloud.impl.BlobStoreManagedLedgerOffloader; +import org.apache.pulsar.common.policies.data.OffloadPolicies; /** * A jcloud based offloader factory. @@ -44,8 +45,10 @@ public boolean isDriverSupported(String driverName) { @Override public BlobStoreManagedLedgerOffloader create(Properties properties, Map userMetadata, - OrderedScheduler scheduler) throws IOException { + OrderedScheduler scheduler, + OffloadPolicies overrideOffloadPolicies) throws IOException { TieredStorageConfigurationData data = TieredStorageConfigurationData.create(properties); - return BlobStoreManagedLedgerOffloader.create(data, userMetadata, scheduler); + return BlobStoreManagedLedgerOffloader.create( + data.overridePolicies(overrideOffloadPolicies), userMetadata, scheduler); } } diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/TieredStorageConfigurationData.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/TieredStorageConfigurationData.java index a4c5cf4fa8b2e..1761285c90486 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/TieredStorageConfigurationData.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/TieredStorageConfigurationData.java @@ -29,6 +29,9 @@ import java.util.Arrays; import java.util.Properties; import lombok.Data; +import org.apache.bookkeeper.mledger.offload.jcloud.impl.BlobStoreManagedLedgerOffloader; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.common.policies.data.OffloadPolicies; /** * Configuration for tiered storage. @@ -118,4 +121,29 @@ public static TieredStorageConfigurationData create(Properties properties) { return data; } + public TieredStorageConfigurationData overridePolicies(OffloadPolicies offloadPolicies) { + if (offloadPolicies == null) { + return this; + } + if (BlobStoreManagedLedgerOffloader.isGcsDriver(managedLedgerOffloadDriver)) { + this.gcsManagedLedgerOffloadBucket = offloadPolicies.getBucket(); + if (StringUtils.isNotEmpty(offloadPolicies.getRegion())) { + this.gcsManagedLedgerOffloadRegion = offloadPolicies.getRegion(); + } + this.gcsManagedLedgerOffloadMaxBlockSizeInBytes = offloadPolicies.getMaxBlockSizeInBytes(); + this.gcsManagedLedgerOffloadReadBufferSizeInBytes = offloadPolicies.getReadBufferSizeInBytes(); + } else if (BlobStoreManagedLedgerOffloader.isS3Driver(managedLedgerOffloadDriver)) { + this.s3ManagedLedgerOffloadBucket = offloadPolicies.getBucket(); + if (StringUtils.isNotEmpty(offloadPolicies.getRegion())) { + this.s3ManagedLedgerOffloadRegion = offloadPolicies.getRegion(); + } + if (StringUtils.isNotEmpty(offloadPolicies.getEndpoint())) { + this.s3ManagedLedgerOffloadServiceEndpoint = offloadPolicies.getEndpoint(); + } + this.s3ManagedLedgerOffloadMaxBlockSizeInBytes = offloadPolicies.getMaxBlockSizeInBytes(); + this.s3ManagedLedgerOffloadReadBufferSizeInBytes = offloadPolicies.getReadBufferSizeInBytes(); + } + return this; + } + } From 72a76cbee1d0a8a87f46da1857ff61aaf4132e16 Mon Sep 17 00:00:00 2001 From: "gao.ran" Date: Sat, 1 Feb 2020 04:24:29 +0800 Subject: [PATCH 02/11] fix --- .../main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java | 2 +- .../java/org/apache/pulsar/common/policies/data/Policies.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 4a3d27e0b39cf..e7201eb44b60a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -1069,7 +1069,7 @@ public void setOffload(@PathParam("tenant") String tenant, @PathParam("namespace @GET @Path("/{tenant}/{namespace}/offload") - @ApiOperation(value = "Get offload config on a namespace.") + @ApiOperation(value = "Get offload configuration on a namespace.") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java index bf597e6eac1f2..7d0e67d771b18 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java @@ -148,7 +148,7 @@ public boolean equals(Object obj) { && schema_validation_enforced == other.schema_validation_enforced && schema_compatibility_strategy == other.schema_compatibility_strategy && is_allow_auto_update_schema == other.is_allow_auto_update_schema - && offload_policies == other.offload_policies; + && Objects.equals(offload_policies, other.offload_policies); } return false; From e5caac00317de3e111aeaeda01abdc43da38f917 Mon Sep 17 00:00:00 2001 From: "gao.ran" Date: Sat, 1 Feb 2020 04:12:28 +0800 Subject: [PATCH 03/11] add namespace offload policies --- .../mledger/LedgerOffloaderFactory.java | 5 +- .../apache/pulsar/broker/PulsarService.java | 39 +++++-- .../broker/admin/impl/NamespacesBase.java | 45 ++++++++ .../pulsar/broker/admin/v2/Namespaces.java | 27 +++++ .../pulsar/broker/service/BrokerService.java | 2 +- .../service/persistent/PersistentTopic.java | 2 + .../broker/admin/AdminApiOffloadTest.java | 18 ++- .../pulsar/client/admin/Namespaces.java | 71 ++++++++++++ .../client/admin/internal/NamespacesImpl.java | 23 ++++ .../pulsar/admin/cli/PulsarAdminToolTest.java | 9 ++ .../pulsar/admin/cli/CmdNamespaces.java | 106 ++++++++++++++++++ .../common/policies/data/OffloadPolicies.java | 90 +++++++++++++++ .../pulsar/common/policies/data/Policies.java | 12 +- .../sql/presto/PulsarConnectorCache.java | 26 +++-- .../pulsar/sql/presto/PulsarRecordCursor.java | 3 +- .../apache/pulsar/sql/presto/PulsarSplit.java | 13 ++- .../pulsar/sql/presto/PulsarSplitManager.java | 28 +++-- .../sql/presto/TestPulsarConnector.java | 2 +- .../sql/presto/TestPulsarSplitManager.java | 26 ++--- site2/docs/reference-pulsar-admin.md | 26 +++++ .../FileSystemLedgerOffloaderFactory.java | 4 +- .../jcloud/JCloudLedgerOffloaderFactory.java | 7 +- .../TieredStorageConfigurationData.java | 28 +++++ 23 files changed, 563 insertions(+), 49 deletions(-) create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java index f0a6890aeb272..18886d4003493 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java @@ -24,6 +24,7 @@ import org.apache.bookkeeper.common.annotation.InterfaceAudience.LimitedPrivate; import org.apache.bookkeeper.common.annotation.InterfaceStability.Evolving; import org.apache.bookkeeper.common.util.OrderedScheduler; +import org.apache.pulsar.common.policies.data.OffloadPolicies; /** * Factory to create {@link LedgerOffloader} to offload ledgers into long-term storage. @@ -46,12 +47,14 @@ public interface LedgerOffloaderFactory { * @param properties service configuration * @param userMetadata user metadata * @param scheduler scheduler + * @param overrideOffloadPolicies offloadPolicies for override * @return the offloader instance * @throws IOException when fail to create an offloader */ T create(Properties properties, Map userMetadata, - OrderedScheduler scheduler) + OrderedScheduler scheduler, + OffloadPolicies overrideOffloadPolicies) throws IOException; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index ff7ba47728a9c..52a120f060193 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -88,6 +89,7 @@ import org.apache.pulsar.broker.web.WebService; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.common.conf.InternalConfigurationData; @@ -98,6 +100,7 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TenantInfo; @@ -162,7 +165,8 @@ public class PulsarService implements AutoCloseable { private ScheduledExecutorService compactorExecutor; private OrderedScheduler offloaderScheduler; private Offloaders offloaderManager = new Offloaders(); - private LedgerOffloader offloader; + private LedgerOffloader defaultOffloader; + private Map ledgerOffloaderMap = new ConcurrentHashMap<>(); private ScheduledFuture loadReportTask = null; private ScheduledFuture loadSheddingTask = null; private ScheduledFuture loadResourceQuotaTask = null; @@ -398,7 +402,7 @@ public void start() throws PulsarServerException { // Start load management service (even if load balancing is disabled) this.loadManager.set(LoadManager.create(this)); - this.offloader = createManagedLedgerOffloader(this.getConfiguration()); + this.defaultOffloader = createManagedLedgerOffloader(this.getConfiguration(), null); brokerService.start(); @@ -764,11 +768,31 @@ public ManagedLedgerClientFactory getManagedLedgerClientFactory() { return managedLedgerClientFactory; } - public LedgerOffloader getManagedLedgerOffloader() { - return offloader; + public LedgerOffloader getManagedLedgerOffloaderByPolicies(OffloadPolicies offloadPolicies) { + if (offloadPolicies == null) { + return getDefaultOffloader(); + } + return ledgerOffloaderMap.computeIfAbsent(offloadPolicies, op -> { + try { + return createManagedLedgerOffloader(getConfiguration(), offloadPolicies); + } catch (PulsarServerException e) { + LOG.error("create ledgerOffloader failed for {}", offloadPolicies.toString(), e); + return new NullLedgerOffloader(); + } + }); + } + + public LedgerOffloader getManagedLedgerOffloader(NamespaceName namespaceName) { + try { + OffloadPolicies offloadPolicies = getAdminClient().namespaces().getOffload(namespaceName.toString()); + return getManagedLedgerOffloaderByPolicies(offloadPolicies); + } catch (Exception e) { + LOG.error("Failed to get ledgerOffloader for namespaceName {}", namespaceName, e); + return new NullLedgerOffloader(); + } } - public synchronized LedgerOffloader createManagedLedgerOffloader(ServiceConfiguration conf) + public synchronized LedgerOffloader createManagedLedgerOffloader(ServiceConfiguration conf, OffloadPolicies offloadPolicies) throws PulsarServerException { try { if (StringUtils.isNotBlank(conf.getManagedLedgerOffloadDriver())) { @@ -785,12 +809,13 @@ public synchronized LedgerOffloader createManagedLedgerOffloader(ServiceConfigur LedgerOffloader.METADATA_SOFTWARE_VERSION_KEY.toLowerCase(), PulsarVersion.getVersion(), LedgerOffloader.METADATA_SOFTWARE_GITSHA_KEY.toLowerCase(), PulsarVersion.getGitSha() ), - getOffloaderScheduler(conf)); + getOffloaderScheduler(conf), + offloadPolicies); } catch (IOException ioe) { throw new PulsarServerException(ioe.getMessage(), ioe.getCause()); } } else { - LOG.info("No ledger offloader configured, using NULL instance"); + LOG.warn("No ledger offloader configured, using NULL instance"); return NullLedgerOffloader.INSTANCE; } } catch (Throwable t) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 11409cde663db..9ccbccc8a3f55 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -77,6 +77,7 @@ import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; import org.apache.pulsar.common.policies.data.LocalPolicies; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; @@ -2188,5 +2189,49 @@ private void mutatePolicy(Function policyTransformation, } } + protected void internalSetOffload(OffloadPolicies offload) { + validateAdminAccessForTenant(namespaceName.getTenant()); + validatePoliciesReadOnlyAccess(); + + try { + Stat nodeStat = new Stat(); + final String path = path(POLICIES, namespaceName.toString()); + byte[] content = globalZk().getData(path, null, nodeStat); + Policies policies = jsonMapper().readValue(content, Policies.class); + if (StringUtils.isEmpty(offload.getBucket())) { + log.warn("[{}] Failed to update offload configuration for namespace {}: bucket must be specified", + clientAppId(), namespaceName); + throw new RestException(Status.PRECONDITION_FAILED, + "The bucket must be specified for namespace offload."); + } + policies.offload_policies = offload; + globalZk().setData(path, jsonMapper().writeValueAsBytes(policies), nodeStat.getVersion()); + policiesCache().invalidate(path(POLICIES, namespaceName.toString())); + log.info("[{}] Successfully updated offload configuration: namespace={}, map={}", clientAppId(), + namespaceName, jsonMapper().writeValueAsString(policies.offload_policies)); + } catch (KeeperException.NoNodeException e) { + log.warn("[{}] Failed to update offload configuration for namespace {}: does not exist", clientAppId(), + namespaceName); + throw new RestException(Status.NOT_FOUND, "Namespace does not exist"); + } catch (KeeperException.BadVersionException e) { + log.warn("[{}] Failed to update offload configuration for namespace {}: concurrent modification", + clientAppId(), namespaceName); + throw new RestException(Status.CONFLICT, "Concurrent modification"); + } catch (RestException pfe) { + throw pfe; + } catch (Exception e) { + log.error("[{}] Failed to update offload configuration for namespace {}", clientAppId(), namespaceName, + e); + throw new RestException(e); + } + } + + protected OffloadPolicies internalGetOffload() { + validateAdminAccessForTenant(namespaceName.getTenant()); + + Policies policies = getNamespacePolicies(namespaceName); + return policies.offload_policies; + } + private static final Logger log = LoggerFactory.getLogger(NamespacesBase.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 6bc5c234adf6e..57e2f1bf5fa0a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -51,6 +51,7 @@ import org.apache.pulsar.common.policies.data.BookieAffinityGroupData; import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.DispatchRate; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; @@ -1076,5 +1077,31 @@ public void setSchemaValidtionEnforced(@PathParam("tenant") String tenant, internalSetSchemaValidationEnforced(schemaValidationEnforced); } + @POST + @Path("/{tenant}/{namespace}/offload") + @ApiOperation(value = " Set offload configuration on a namespace.") + @ApiResponses(value = { + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Namespace does not exist"), + @ApiResponse(code = 409, message = "Concurrent modification"), + @ApiResponse(code = 412, message = "Bucket must be specified") }) + public void setOffload(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace, + OffloadPolicies offload) { + validateNamespaceName(tenant, namespace); + internalSetOffload(offload); + } + + @GET + @Path("/{tenant}/{namespace}/offload") + @ApiOperation(value = "Get offload config on a namespace.") + @ApiResponses(value = { + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Namespace does not exist") }) + public OffloadPolicies getOffload(@PathParam("tenant") String tenant, + @PathParam("namespace") String namespace) { + validateNamespaceName(tenant, namespace); + return internalGetOffload(); + } + private static final Logger log = LoggerFactory.getLogger(Namespaces.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 21a7d74e15a1e..83e65191f6629 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -982,7 +982,7 @@ public CompletableFuture getManagedLedgerConfig(TopicName t managedLedgerConfig.setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES); managedLedgerConfig.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB()); - managedLedgerConfig.setLedgerOffloader(pulsar.getManagedLedgerOffloader()); + managedLedgerConfig.setLedgerOffloader(pulsar.getManagedLedgerOffloader(namespace)); policies.ifPresent(p -> { long lag = serviceConfig.getManagedLedgerOffloadDeletionLagMs(); if (p.offload_deletion_lag_ms != null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index b177b116a735b..f6f977597892b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1776,6 +1776,8 @@ public CompletableFuture onPoliciesUpdate(Policies data) { if (this.subscribeRateLimiter.isPresent()) { subscribeRateLimiter.get().onPoliciesUpdate(data); } + getManagedLedger().getConfig().setLedgerOffloader( + brokerService.pulsar().getManagedLedgerOffloaderByPolicies(data.offload_policies)); return CompletableFuture.allOf(replicationFuture, dedupFuture, persistentPoliciesFuture); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java index 7f56a3fc02e9b..f455423103130 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java @@ -38,6 +38,7 @@ import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.TenantInfo; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -71,7 +72,8 @@ private void testOffload(String topicName, String mlName) throws Exception { LedgerOffloader offloader = mock(LedgerOffloader.class); when(offloader.getOffloadDriverName()).thenReturn("mock"); - doReturn(offloader).when(pulsar).getManagedLedgerOffloader(); + doReturn(offloader).when(pulsar).getManagedLedgerOffloaderByPolicies(any()); + doReturn(offloader).when(pulsar).getManagedLedgerOffloader(any()); CompletableFuture promise = new CompletableFuture<>(); doReturn(promise).when(offloader).offload(any(), any(), any()); @@ -138,4 +140,18 @@ public void testOffloadV1() throws Exception { String mlName = "prop-xyz/test/ns1/persistent/topic2"; testOffload(topicName, mlName); } + + @Test + public void testOffloadPolicies() throws Exception { + String namespaceName = "prop-xyz/ns1"; + String endpoint = "test-endpoint"; + String region = "test-region"; + String bucket = "test-bucket"; + + OffloadPolicies offload1 = new OffloadPolicies(endpoint, region, bucket); + admin.namespaces().setOffload(namespaceName, offload1); + OffloadPolicies offload2 = admin.namespaces().getOffload(namespaceName); + Assert.assertEquals(offload1, offload2); + } + } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java index 19dea3ea455f4..df6b719321618 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java @@ -34,6 +34,7 @@ import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; @@ -1654,4 +1655,74 @@ boolean getIsAllowAutoUpdateSchema(String namespace) */ void setIsAllowAutoUpdateSchema(String namespace, boolean isAllowAutoUpdateSchema) throws PulsarAdminException; + + /** + * Set the offload configuration for all the topics on a namespace. + *

+ * Set the offload configuration on a namespace. This operation requires Pulsar super-user access. + *

+ * Request parameter example: + *

+ * + * 

+     * 
+     * {
+     *     "region" : "us-east-2",                   // The long term storage region
+     *     "bucket" : "bucket",                      // Bucket to place offloaded ledger into
+     *     "endpoint" : "endpoint",                  // Alternative endpoint to connect to
+     *     "maxBlockSize" : 1024,                    // Max Block Size, default 64MB
+     *     "readBufferSize" : 1024,                  // Read Buffer Size, default 1MB
+     * }
+     * 
+     *
+ * + * @param namespace + * Namespace name + * @param offloadPolicies + * Offload configuration + * + * @throws NotAuthorizedException + * Don't have admin permission + * @throws NotFoundException + * Namespace does not exist + * @throws ConflictException + * Concurrent modification + * @throws PulsarAdminException + * Unexpected error + */ + void setOffload(String namespace, OffloadPolicies offloadPolicies) throws PulsarAdminException; + + /** + * Get the offload configuration for a namespace. + *

+ * Get the offload configuration for a namespace. + *

+ * Response example: + *

+ * + * 

+     * 
+     * {
+     *     "region" : "us-east-2",                   // The long term storage region
+     *     "bucket" : "bucket",                      // Bucket to place offloaded ledger into
+     *     "endpoint" : "endpoint",                  // Alternative endpoint to connect to
+     *     "maxBlockSize" : 1024,                    // Max Block Size, default 64MB
+     *     "readBufferSize" : 1024,                  // Read Buffer Size, default 1MB
+     * }
+     * 
+     *
+ * + * @param namespace + * Namespace name + * @throws NotAuthorizedException + * Don't have admin permission + * @throws NotFoundException + * Namespace does not exist + * @throws ConflictException + * Concurrent modification + * @throws PulsarAdminException + * Unexpected error + */ + OffloadPolicies getOffload(String namespace) throws PulsarAdminException; + } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java index 1d237eb5a0352..8ce5807f5bebd 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java @@ -44,6 +44,7 @@ import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; import org.apache.pulsar.common.policies.data.ErrorData; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; @@ -1029,6 +1030,28 @@ public void setIsAllowAutoUpdateSchema(String namespace, boolean isAllowAutoUpda } } + @Override + public void setOffload(String namespace, OffloadPolicies offloadPolicies) throws PulsarAdminException { + try { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "offload"); + request(path).post(Entity.entity(offloadPolicies, MediaType.APPLICATION_JSON), ErrorData.class); + } catch (Exception e) { + throw getApiException(e); + } + } + + @Override + public OffloadPolicies getOffload(String namespace) throws PulsarAdminException { + try { + NamespaceName ns = NamespaceName.get(namespace); + WebTarget path = namespacePath(ns, "offload"); + return request(path).get(OffloadPolicies.class); + } catch (Exception e) { + throw getApiException(e); + } + } + private WebTarget namespacePath(NamespaceName namespace, String... parts) { final WebTarget base = namespace.isV2() ? adminV2Namespaces : adminNamespaces; WebTarget namespacePath = base.path(namespace.toString()); diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index 30c52cd7b2aed..b383757899740 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -64,6 +64,7 @@ import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; import org.apache.pulsar.common.policies.data.FailureDomain; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; @@ -487,6 +488,14 @@ void namespaces() throws Exception { namespaces.run(split("clear-offload-deletion-lag myprop/clust/ns1")); verify(mockNamespaces).clearOffloadDeleteLag("myprop/clust/ns1"); + + namespaces.run(split("set-offload myprop/clust/ns1 -r test-region -b test-bucket -e http://test.endpoint -mbs 32M -rbs 5M")); + verify(mockNamespaces).setOffload("myprop/clust/ns1", + new OffloadPolicies("test-region", "test-bucket", "http://test.endpoint", + 32 * 1024 * 1024, 5 * 1024 * 1024)); + + namespaces.run(split("get-offload myprop/clust/ns1")); + verify(mockNamespaces).getOffload("myprop/clust/ns1"); } @Test diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 0913d651c28a2..e30672f11c7ff 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -22,6 +22,7 @@ import com.beust.jcommander.ParameterException; import com.beust.jcommander.Parameters; import com.beust.jcommander.converters.CommaParameterSplitter; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -32,6 +33,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.admin.cli.utils.IOUtils; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -41,6 +43,7 @@ import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; @@ -1287,6 +1290,106 @@ void run() throws PulsarAdminException { } } + @Parameters(commandDescription = "Set the offload policies for a namespace") + private class SetOffload extends CliCommand { + @Parameter(description = "tenant/namespace", required = true) + private java.util.List params; + + @Parameter( + names = {"--region", "-r"}, + description = "The long term storage region, " + + "default is s3ManagedLedgerOffloadRegion or gcsManagedLedgerOffloadRegion in broker.conf", + required = false) + private String region; + + @Parameter( + names = {"--bucket", "-b"}, + description = "Bucket to place offloaded ledger into", + required = true) + private String bucket; + + @Parameter( + names = {"--endpoint", "-e"}, + description = "Alternative endpoint to connect to, " + + "s3 default is s3ManagedLedgerOffloadServiceEndpoint in broker.conf", + required = false) + private String endpoint; + + @Parameter( + names = {"--maxBlockSize", "-mbs"}, + description = "Max block size (eg: 32M, 64M), default is 64MB", + required = false) + private String maxBlockSizeStr; + + @Parameter( + names = {"--readBufferSize", "-rbs"}, + description = "Read buffer size (eg: 1M, 5M), default is 1MB", + required = false) + private String readBufferSizeStr; + + private final String[] DRIVER_NAMES = {"S3", "aws-s3", "google-cloud-storage"}; + + public boolean isS3Driver(String driver) { + if (StringUtils.isEmpty(driver)) { + return false; + } + return driver.equalsIgnoreCase(DRIVER_NAMES[0]) || driver.equalsIgnoreCase(DRIVER_NAMES[1]); + } + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(params); + + String offloadDriver = admin.brokers().getRuntimeConfigurations().get("managedLedgerOffloadDriver"); + if (isS3Driver(offloadDriver) && Strings.isNullOrEmpty(region) && Strings.isNullOrEmpty(endpoint)) { + throw new ParameterException( + "Either s3ManagedLedgerOffloadRegion or s3ManagedLedgerOffloadServiceEndpoint must be set" + + " if s3 offload enabled"); + } + + long maxBlockSize = 0; + if (StringUtils.isNotEmpty(maxBlockSizeStr)) { + maxBlockSize = validateSizeString(maxBlockSizeStr); + } + long readBufferSize = 0; + if (StringUtils.isNotEmpty(readBufferSizeStr)) { + readBufferSize = validateSizeString(readBufferSizeStr); + } + + final int maxBlockSizeInBytes; + if (maxBlockSize > 0 && maxBlockSize <= Integer.MAX_VALUE) { + maxBlockSizeInBytes = new Long(maxBlockSize).intValue(); + } else { + maxBlockSizeInBytes = OffloadPolicies.MAX_BLOCK_SIZE_IN_BYTES; + } + + final int readBufferSizeInBytes; + if (readBufferSize > 0 && readBufferSize <= Integer.MAX_VALUE) { + readBufferSizeInBytes = new Long(readBufferSize).intValue(); + } else { + readBufferSizeInBytes = OffloadPolicies.READ_BUFFER_SIZE_IN_BYTES; + } + + admin.namespaces().setOffload(namespace, new OffloadPolicies( + region == null ? "" : region, + bucket, + endpoint == null ? "" : endpoint, + maxBlockSizeInBytes, readBufferSizeInBytes)); + } + } + + @Parameters(commandDescription = "Get the offload policies for a namespace") + private class GetOffload extends CliCommand { + @Parameter(description = "tenant/namespace\n", required = true) + private java.util.List params; + + @Override + void run() throws PulsarAdminException { + String namespace = validateNamespace(params); + print(admin.namespaces().getOffload(namespace)); + } + } + public CmdNamespaces(PulsarAdmin admin) { super("namespaces", admin); jcommander.addCommand("list", new GetNamespacesPerProperty()); @@ -1389,5 +1492,8 @@ public CmdNamespaces(PulsarAdmin admin) { jcommander.addCommand("get-schema-validation-enforce", new GetSchemaValidationEnforced()); jcommander.addCommand("set-schema-validation-enforce", new SetSchemaValidationEnforced()); + + jcommander.addCommand("set-offload", new SetOffload()); + jcommander.addCommand("get-offload", new GetOffload()); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java new file mode 100644 index 0000000000000..563d357996d4b --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java @@ -0,0 +1,90 @@ +package org.apache.pulsar.common.policies.data; + +import com.google.common.base.MoreObjects; + +import java.util.Objects; + +/** + * Definition of the offload policies. + */ +public class OffloadPolicies { + + public final static int MAX_BLOCK_SIZE_IN_BYTES = 64 * 1024 * 1024; // 64MB + public final static int READ_BUFFER_SIZE_IN_BYTES = 1024 * 1024; // 1MB + + private String region; + private String bucket; + private String endpoint; + private int maxBlockSizeInBytes; + private int readBufferSizeInBytes; + + public OffloadPolicies() { + + } + + public OffloadPolicies(String region, String bucket, String endpoint) { + this(region, bucket, endpoint, MAX_BLOCK_SIZE_IN_BYTES, READ_BUFFER_SIZE_IN_BYTES); + } + + public OffloadPolicies(String region, String bucket, String endpoint, + int maxBlockSizeInBytes, int readBufferSizeInBytes) { + this.region = region; + this.bucket = bucket; + this.endpoint = endpoint; + this.maxBlockSizeInBytes = maxBlockSizeInBytes; + this.readBufferSizeInBytes = readBufferSizeInBytes; + } + + public String getRegion() { + return region; + } + + public String getBucket() { + return bucket; + } + + public String getEndpoint() { + return endpoint; + } + + public int getMaxBlockSizeInBytes() { + return maxBlockSizeInBytes; + } + + public int getReadBufferSizeInBytes() { + return readBufferSizeInBytes; + } + + @Override + public int hashCode() { + return Objects.hash(region, bucket, endpoint, maxBlockSizeInBytes, readBufferSizeInBytes); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + OffloadPolicies other = (OffloadPolicies) obj; + return Objects.equals(region, other.getRegion()) + && Objects.equals(bucket, other.getBucket()) + && Objects.equals(endpoint, other.getEndpoint()) + && Objects.equals(maxBlockSizeInBytes, other.getMaxBlockSizeInBytes()) + && Objects.equals(readBufferSizeInBytes, other.getReadBufferSizeInBytes()); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("region", region) + .add("bucket", bucket) + .add("endpoint", endpoint) + .add("maxBlockSizeInBytes", maxBlockSizeInBytes) + .add("readBufferSizeInBytes", readBufferSizeInBytes) + .toString(); + } + +} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java index 8f247a682fc08..87dfabfbfa876 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java @@ -96,6 +96,9 @@ public class Policies { @SuppressWarnings("checkstyle:MemberName") public boolean schema_validation_enforced = false; + @SuppressWarnings("checkstyle:MemberName") + public OffloadPolicies offload_policies = null; + @Override public int hashCode() { return Objects.hash(auth_policies, replication_clusters, @@ -113,7 +116,8 @@ public int hashCode() { schema_auto_update_compatibility_strategy, schema_validation_enforced, schema_compatibility_strategy, - is_allow_auto_update_schema); + is_allow_auto_update_schema, + offload_policies); } @Override @@ -147,7 +151,8 @@ public boolean equals(Object obj) { && schema_auto_update_compatibility_strategy == other.schema_auto_update_compatibility_strategy && schema_validation_enforced == other.schema_validation_enforced && schema_compatibility_strategy == other.schema_compatibility_strategy - && is_allow_auto_update_schema == other.is_allow_auto_update_schema; + && is_allow_auto_update_schema == other.is_allow_auto_update_schema + && offload_policies == other.offload_policies; } return false; @@ -196,6 +201,7 @@ public String toString() { .add("schema_auto_update_compatibility_strategy", schema_auto_update_compatibility_strategy) .add("schema_validation_enforced", schema_validation_enforced) .add("schema_compatibility_Strategy", schema_compatibility_strategy) - .add("is_allow_auto_update_Schema", is_allow_auto_update_schema).toString(); + .add("is_allow_auto_update_Schema", is_allow_auto_update_schema) + .add("offload_policies", offload_policies).toString(); } } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java index 1af86a2225a06..7bf4d3b4e6ef2 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java @@ -39,6 +39,7 @@ import org.apache.bookkeeper.stats.StatsProvider; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.PulsarVersion; +import org.apache.pulsar.common.policies.data.OffloadPolicies; /** * Implementation of a cache for the Pulsar connector. @@ -55,7 +56,8 @@ public class PulsarConnectorCache { private final StatsProvider statsProvider; private OrderedScheduler offloaderScheduler; private Offloaders offloaderManager; - private LedgerOffloader offloader; + private LedgerOffloader defaultOffloader; + private Map offloaderMap; private static final String OFFLOADERS_DIRECTOR = "offloadersDirectory"; private static final String MANAGED_LEDGER_OFFLOAD_DRIVER = "managedLedgerOffloadDriver"; @@ -74,7 +76,7 @@ private PulsarConnectorCache(PulsarConnectorConfig pulsarConnectorConfig) throws this.statsProvider.start(clientConfiguration); - this.offloader = initManagedLedgerOffloader(pulsarConnectorConfig); + this.defaultOffloader = initManagedLedgerOffloader(pulsarConnectorConfig, null); } public static PulsarConnectorCache getConnectorCache(PulsarConnectorConfig pulsarConnectorConfig) throws Exception { @@ -108,10 +110,17 @@ private static ManagedLedgerFactory initManagedLedgerFactory(PulsarConnectorConf return new ManagedLedgerFactoryImpl(bkClientConfiguration, managedLedgerFactoryConfig); } - public ManagedLedgerConfig getManagedLedgerConfig() { - - return new ManagedLedgerConfig() - .setLedgerOffloader(this.offloader); + public ManagedLedgerConfig getManagedLedgerConfig(PulsarConnectorConfig pulsarConnectorConfig, + OffloadPolicies offloadPolicies) { + ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); + if (offloadPolicies == null) { + managedLedgerConfig.setLedgerOffloader(this.defaultOffloader); + } else { + LedgerOffloader ledgerOffloader = offloaderMap.computeIfAbsent(offloadPolicies, + op -> initManagedLedgerOffloader(pulsarConnectorConfig, offloadPolicies)); + managedLedgerConfig.setLedgerOffloader(ledgerOffloader); + } + return managedLedgerConfig; } private synchronized OrderedScheduler getOffloaderScheduler(PulsarConnectorConfig pulsarConnectorConfig) { @@ -123,7 +132,7 @@ private synchronized OrderedScheduler getOffloaderScheduler(PulsarConnectorConfi return this.offloaderScheduler; } - private LedgerOffloader initManagedLedgerOffloader(PulsarConnectorConfig conf) { + private LedgerOffloader initManagedLedgerOffloader(PulsarConnectorConfig conf, OffloadPolicies offloadPolicies) { try { if (StringUtils.isNotBlank(conf.getManagedLedgerOffloadDriver())) { @@ -147,7 +156,8 @@ private LedgerOffloader initManagedLedgerOffloader(PulsarConnectorConfig conf) { LedgerOffloader.METADATA_SOFTWARE_VERSION_KEY.toLowerCase(), PulsarVersion.getVersion(), LedgerOffloader.METADATA_SOFTWARE_GITSHA_KEY.toLowerCase(), PulsarVersion.getGitSha() ), - getOffloaderScheduler(conf)); + getOffloaderScheduler(conf), + offloadPolicies); } catch (IOException ioe) { log.error("Failed to create offloader: ", ioe); throw new RuntimeException(ioe.getMessage(), ioe.getCause()); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index 4d5a25db836d6..3c36f01b1b0f5 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -111,7 +111,8 @@ public PulsarRecordCursor(List columnHandles, PulsarSplit pu throw new RuntimeException(e); } initialize(columnHandles, pulsarSplit, pulsarConnectorConfig, - pulsarConnectorCache.getManagedLedgerFactory(), pulsarConnectorCache.getManagedLedgerConfig(), + pulsarConnectorCache.getManagedLedgerFactory(), + pulsarConnectorCache.getManagedLedgerConfig(pulsarConnectorConfig, pulsarSplit.getOffloadPolicies()), new PulsarConnectorMetricsTracker(pulsarConnectorCache.getStatsProvider())); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplit.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplit.java index eeebbd174dd3f..dbebbf54a3eba 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplit.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplit.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; @@ -55,6 +56,8 @@ public class PulsarSplit implements ConnectorSplit { private final PositionImpl startPosition; private final PositionImpl endPosition; + private final OffloadPolicies offloadPolicies; + @JsonCreator public PulsarSplit( @JsonProperty("splitId") long splitId, @@ -69,7 +72,8 @@ public PulsarSplit( @JsonProperty("startPositionLedgerId") long startPositionLedgerId, @JsonProperty("endPositionLedgerId") long endPositionLedgerId, @JsonProperty("tupleDomain") TupleDomain tupleDomain, - @JsonProperty("properties") Map schemaInfoProperties) { + @JsonProperty("properties") Map schemaInfoProperties, + @JsonProperty("offloadPolicies") OffloadPolicies offloadPolicies) { this.splitId = splitId; requireNonNull(schemaName, "schema name is null"); this.schemaInfo = SchemaInfo.builder() @@ -91,6 +95,7 @@ public PulsarSplit( this.tupleDomain = requireNonNull(tupleDomain, "tupleDomain is null"); this.startPosition = PositionImpl.get(startPositionLedgerId, startPositionEntryId); this.endPosition = PositionImpl.get(endPositionLedgerId, endPositionEntryId); + this.offloadPolicies = offloadPolicies; } @JsonProperty @@ -161,6 +166,11 @@ public PositionImpl getEndPosition() { return endPosition; } + @JsonProperty + public OffloadPolicies getOffloadPolicies() { + return offloadPolicies; + } + @Override public boolean isRemotelyAccessible() { return true; @@ -190,6 +200,7 @@ public String toString() { + ", endPositionEntryId=" + endPositionEntryId + ", startPositionLedgerId=" + startPositionLedgerId + ", endPositionLedgerId=" + endPositionLedgerId + + (offloadPolicies == null ? "" : offloadPolicies.toString()) + '}'; } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java index 06bc1200146f7..b093b1d824c6d 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java @@ -59,6 +59,7 @@ import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.schema.SchemaInfo; /** @@ -122,11 +123,14 @@ public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHand Collection splits; try { + OffloadPolicies offloadPolicies = this.pulsarAdmin.namespaces().getOffload(topicName.getNamespace()); if (!PulsarConnectorUtils.isPartitionedTopic(topicName, this.pulsarAdmin)) { - splits = getSplitsNonPartitionedTopic(numSplits, topicName, tableHandle, schemaInfo, tupleDomain); + splits = getSplitsNonPartitionedTopic( + numSplits, topicName, tableHandle, schemaInfo, tupleDomain, offloadPolicies); log.debug("Splits for non-partitioned topic %s: %s", topicName, splits); } else { - splits = getSplitsPartitionedTopic(numSplits, topicName, tableHandle, schemaInfo, tupleDomain); + splits = getSplitsPartitionedTopic( + numSplits, topicName, tableHandle, schemaInfo, tupleDomain, offloadPolicies); log.debug("Splits for partitioned topic %s: %s", topicName, splits); } } catch (Exception e) { @@ -138,7 +142,8 @@ public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHand @VisibleForTesting Collection getSplitsPartitionedTopic(int numSplits, TopicName topicName, PulsarTableHandle - tableHandle, SchemaInfo schemaInfo, TupleDomain tupleDomain) throws Exception { + tableHandle, SchemaInfo schemaInfo, TupleDomain tupleDomain, + OffloadPolicies offloadPolicies) throws Exception { List predicatedPartitions = getPredicatedPartitions(topicName, tupleDomain); if (log.isDebugEnabled()) { @@ -165,7 +170,8 @@ Collection getSplitsPartitionedTopic(int numSplits, TopicName topic tableHandle, schemaInfo, topicName.getPartition(predicatedPartitions.get(i)).getLocalName(), - tupleDomain)); + tupleDomain, + offloadPolicies)); } return splits; } @@ -219,8 +225,8 @@ private List getPredicatedPartitions(TopicName topicName, TupleDomain getSplitsNonPartitionedTopic(int numSplits, TopicName topicName, - PulsarTableHandle tableHandle, SchemaInfo schemaInfo, TupleDomain tupleDomain) - throws Exception { + PulsarTableHandle tableHandle, SchemaInfo schemaInfo, TupleDomain tupleDomain, + OffloadPolicies offloadPolicies) throws Exception { ManagedLedgerFactory managedLedgerFactory = PulsarConnectorCache.getConnectorCache(pulsarConnectorConfig) .getManagedLedgerFactory(); @@ -230,7 +236,9 @@ Collection getSplitsNonPartitionedTopic(int numSplits, TopicName to numSplits, tableHandle, schemaInfo, - tableHandle.getTableName(), tupleDomain); + tableHandle.getTableName(), + tupleDomain, + offloadPolicies); } @VisibleForTesting @@ -239,7 +247,8 @@ Collection getSplitsForTopic(String topicNamePersistenceEncoding, int numSplits, PulsarTableHandle tableHandle, SchemaInfo schemaInfo, String tableName, - TupleDomain tupleDomain) + TupleDomain tupleDomain, + OffloadPolicies offloadPolicies) throws ManagedLedgerException, InterruptedException { ReadOnlyCursor readOnlyCursor = null; @@ -296,7 +305,8 @@ Collection getSplitsForTopic(String topicNamePersistenceEncoding, startPosition.getLedgerId(), endPosition.getLedgerId(), tupleDomain, - schemaInfo.getProperties())); + schemaInfo.getProperties(), + offloadPolicies)); } return splits; } finally { diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java index cd81442db6b8b..6c911a407541a 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java @@ -561,7 +561,7 @@ public static class Boo { new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema()), topicsToSchemas.get(topicName.getSchemaName()).getType(), 0, topicsToNumEntries.get(topicName.getSchemaName()), - 0, 0, TupleDomain.all(), new HashMap<>())); + 0, 0, TupleDomain.all(), new HashMap<>(), null)); } } diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarSplitManager.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarSplitManager.java index ef361487a054f..5ecbffc307764 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarSplitManager.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarSplitManager.java @@ -87,7 +87,7 @@ public void testTopic(String delimiter) throws Exception { PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, TupleDomain.all()); final ResultCaptor> resultCaptor = new ResultCaptor<>(); - doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any()); + doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits( @@ -95,7 +95,7 @@ public void testTopic(String delimiter) throws Exception { pulsarTableLayoutHandle, null); verify(this.pulsarSplitManager, times(1)) - .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any()); + .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); int totalSize = 0; for (PulsarSplit pulsarSplit : resultCaptor.getResult()) { @@ -134,13 +134,13 @@ public void testPartitionedTopic(String delimiter) throws Exception { PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, TupleDomain.all()); final ResultCaptor> resultCaptor = new ResultCaptor<>(); - doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any()); + doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); this.pulsarSplitManager.getSplits(mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), pulsarTableLayoutHandle, null); verify(this.pulsarSplitManager, times(1)) - .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any()); + .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); int partitions = partitionedTopicsToPartitions.get(topicName.toString()); @@ -200,8 +200,8 @@ public void testPublishTimePredicatePushdown(String delimiter) throws Exception PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, tupleDomain); final ResultCaptor> resultCaptor = new ResultCaptor<>(); - doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsNonPartitionedTopic(anyInt(), any(), any(), any - (), any()); + doAnswer(resultCaptor).when(this.pulsarSplitManager) + .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits( mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), @@ -209,7 +209,7 @@ public void testPublishTimePredicatePushdown(String delimiter) throws Exception verify(this.pulsarSplitManager, times(1)) - .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any()); + .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); int totalSize = 0; int initalStart = 1; @@ -258,7 +258,7 @@ public void testPublishTimePredicatePushdownPartitionedTopic(String delimiter) t final ResultCaptor> resultCaptor = new ResultCaptor<>(); doAnswer(resultCaptor).when(this.pulsarSplitManager) - .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any()); + .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits( mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class), @@ -266,7 +266,7 @@ public void testPublishTimePredicatePushdownPartitionedTopic(String delimiter) t verify(this.pulsarSplitManager, times(1)) - .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any()); + .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any()); int partitions = partitionedTopicsToPartitions.get(topicName.toString()); @@ -315,7 +315,7 @@ public void testPartitionFilter(String delimiter) throws Exception { domainMap.put(PulsarInternalColumn.PARTITION.getColumnHandle(pulsarConnectorId.toString(), false), domain); TupleDomain tupleDomain = TupleDomain.withColumnDomains(domainMap); Collection splits = this.pulsarSplitManager.getSplitsPartitionedTopic(2, topicName, pulsarTableHandle, - schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain); + schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain, null); if (topicsToNumEntries.get(topicName.getSchemaName()) > 1) { Assert.assertEquals(splits.size(), 2); } @@ -332,7 +332,7 @@ public void testPartitionFilter(String delimiter) throws Exception { domainMap.put(PulsarInternalColumn.PARTITION.getColumnHandle(pulsarConnectorId.toString(), false), domain); tupleDomain = TupleDomain.withColumnDomains(domainMap); splits = this.pulsarSplitManager.getSplitsPartitionedTopic(1, topicName, pulsarTableHandle, - schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain); + schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain, null); if (topicsToNumEntries.get(topicName.getSchemaName()) > 1) { Assert.assertEquals(splits.size(), 2); } @@ -348,7 +348,7 @@ public void testPartitionFilter(String delimiter) throws Exception { domainMap.put(PulsarInternalColumn.PARTITION.getColumnHandle(pulsarConnectorId.toString(), false), domain); tupleDomain = TupleDomain.withColumnDomains(domainMap); splits = this.pulsarSplitManager.getSplitsPartitionedTopic(2, topicName, pulsarTableHandle, - schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain); + schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain, null); if (topicsToNumEntries.get(topicName.getSchemaName()) > 1) { Assert.assertEquals(splits.size(), 3); } @@ -367,7 +367,7 @@ public void testPartitionFilter(String delimiter) throws Exception { domainMap.put(PulsarInternalColumn.PARTITION.getColumnHandle(pulsarConnectorId.toString(), false), domain); tupleDomain = TupleDomain.withColumnDomains(domainMap); splits = this.pulsarSplitManager.getSplitsPartitionedTopic(2, topicName, pulsarTableHandle, - schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain); + schemas.getSchemaInfo(topicName.getSchemaName()), tupleDomain, null); if (topicsToNumEntries.get(topicName.getSchemaName()) > 1) { Assert.assertEquals(splits.size(), 4); } diff --git a/site2/docs/reference-pulsar-admin.md b/site2/docs/reference-pulsar-admin.md index 3cd80c6b1640c..19b4bb8f57566 100644 --- a/site2/docs/reference-pulsar-admin.md +++ b/site2/docs/reference-pulsar-admin.md @@ -878,6 +878,8 @@ Subcommands * `clear-offload-deletion-lag` * `get-schema-autoupdate-strategy` * `set-schema-autoupdate-strategy` +* `set-offload` +* `get-offload` ### `list` @@ -2244,3 +2246,27 @@ Options |`-t`, `--type`|The type of the schema (avro or json)|| +### `get-offload` +Get the offload policy for a namespace + +Usage +```bash +$ pulsar-admin namespaces get-offload tenant/namespace +``` + +### `set-offload` +Set the offload policy for a namespace + +Usage +```bash +$ pulsar-admin namespaces set-offload tenant/namespace +``` + +Options +|Flag|Description|Default| +|----|---|---| +|`-r`, `--region`|The long term storage region|s3ManagedLedgerOffloadRegion or gcsManagedLedgerOffloadRegion in broker.conf +|`-b`, `--bucket`|Bucket to place offloaded ledger into|-- +|`-e`, `--endpoint`|Alternative endpoint to connect to|s3 is s3ManagedLedgerOffloadServiceEndpoint in broker.conf +|`-mbs`, `--maxBlockSize`|Max block size|64MB +|`-rbs`, `--readBufferSize`|Read buffer size|1MB diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java index cd52197a59bb5..026f08ccf6dd3 100644 --- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java +++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java @@ -21,6 +21,7 @@ import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.LedgerOffloaderFactory; import org.apache.bookkeeper.mledger.offload.filesystem.impl.FileSystemManagedLedgerOffloader; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import java.io.IOException; import java.util.Map; @@ -33,7 +34,8 @@ public boolean isDriverSupported(String driverName) { } @Override - public FileSystemManagedLedgerOffloader create(Properties properties, Map userMetadata, OrderedScheduler scheduler) throws IOException { + public FileSystemManagedLedgerOffloader create(Properties properties, Map userMetadata, OrderedScheduler scheduler, + OffloadPolicies overrideOffloadPolicies) throws IOException { FileSystemConfigurationData data = FileSystemConfigurationData.create(properties); return FileSystemManagedLedgerOffloader.create(data, scheduler); } diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java index dffe253f94a98..58c5e901c8a9c 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java @@ -24,6 +24,7 @@ import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.LedgerOffloaderFactory; import org.apache.bookkeeper.mledger.offload.jcloud.impl.BlobStoreManagedLedgerOffloader; +import org.apache.pulsar.common.policies.data.OffloadPolicies; /** * A jcloud based offloader factory. @@ -44,8 +45,10 @@ public boolean isDriverSupported(String driverName) { @Override public BlobStoreManagedLedgerOffloader create(Properties properties, Map userMetadata, - OrderedScheduler scheduler) throws IOException { + OrderedScheduler scheduler, + OffloadPolicies overrideOffloadPolicies) throws IOException { TieredStorageConfigurationData data = TieredStorageConfigurationData.create(properties); - return BlobStoreManagedLedgerOffloader.create(data, userMetadata, scheduler); + return BlobStoreManagedLedgerOffloader.create( + data.overridePolicies(overrideOffloadPolicies), userMetadata, scheduler); } } diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/TieredStorageConfigurationData.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/TieredStorageConfigurationData.java index a4c5cf4fa8b2e..1761285c90486 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/TieredStorageConfigurationData.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/TieredStorageConfigurationData.java @@ -29,6 +29,9 @@ import java.util.Arrays; import java.util.Properties; import lombok.Data; +import org.apache.bookkeeper.mledger.offload.jcloud.impl.BlobStoreManagedLedgerOffloader; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.common.policies.data.OffloadPolicies; /** * Configuration for tiered storage. @@ -118,4 +121,29 @@ public static TieredStorageConfigurationData create(Properties properties) { return data; } + public TieredStorageConfigurationData overridePolicies(OffloadPolicies offloadPolicies) { + if (offloadPolicies == null) { + return this; + } + if (BlobStoreManagedLedgerOffloader.isGcsDriver(managedLedgerOffloadDriver)) { + this.gcsManagedLedgerOffloadBucket = offloadPolicies.getBucket(); + if (StringUtils.isNotEmpty(offloadPolicies.getRegion())) { + this.gcsManagedLedgerOffloadRegion = offloadPolicies.getRegion(); + } + this.gcsManagedLedgerOffloadMaxBlockSizeInBytes = offloadPolicies.getMaxBlockSizeInBytes(); + this.gcsManagedLedgerOffloadReadBufferSizeInBytes = offloadPolicies.getReadBufferSizeInBytes(); + } else if (BlobStoreManagedLedgerOffloader.isS3Driver(managedLedgerOffloadDriver)) { + this.s3ManagedLedgerOffloadBucket = offloadPolicies.getBucket(); + if (StringUtils.isNotEmpty(offloadPolicies.getRegion())) { + this.s3ManagedLedgerOffloadRegion = offloadPolicies.getRegion(); + } + if (StringUtils.isNotEmpty(offloadPolicies.getEndpoint())) { + this.s3ManagedLedgerOffloadServiceEndpoint = offloadPolicies.getEndpoint(); + } + this.s3ManagedLedgerOffloadMaxBlockSizeInBytes = offloadPolicies.getMaxBlockSizeInBytes(); + this.s3ManagedLedgerOffloadReadBufferSizeInBytes = offloadPolicies.getReadBufferSizeInBytes(); + } + return this; + } + } From f23cd7918191e5bbe8660584e76ac09b92dea2dc Mon Sep 17 00:00:00 2001 From: "gao.ran" Date: Sat, 1 Feb 2020 04:24:29 +0800 Subject: [PATCH 04/11] fix --- .../main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java | 2 +- .../java/org/apache/pulsar/common/policies/data/Policies.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 57e2f1bf5fa0a..278b2ffe475aa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -1093,7 +1093,7 @@ public void setOffload(@PathParam("tenant") String tenant, @PathParam("namespace @GET @Path("/{tenant}/{namespace}/offload") - @ApiOperation(value = "Get offload config on a namespace.") + @ApiOperation(value = "Get offload configuration on a namespace.") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java index 87dfabfbfa876..37221c8eefa12 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/Policies.java @@ -152,7 +152,7 @@ public boolean equals(Object obj) { && schema_validation_enforced == other.schema_validation_enforced && schema_compatibility_strategy == other.schema_compatibility_strategy && is_allow_auto_update_schema == other.is_allow_auto_update_schema - && offload_policies == other.offload_policies; + && Objects.equals(offload_policies, other.offload_policies); } return false; From b43a5f3c47466e8d06c7b431acc661d455377bf8 Mon Sep 17 00:00:00 2001 From: "gao.ran" Date: Wed, 5 Feb 2020 14:05:45 +0800 Subject: [PATCH 05/11] 1. Change the LedgerOffloader cache in PulsarService.java; 2. Add methods for interface LedgerOffloader, add the getOffloadPolicies method, add close method; 2. Unified the offload policies configuration type, use the OffloadPolicies instead of TieredStorageConfigurationData and FileSystemConfigurationData. --- .../bookkeeper/mledger/LedgerOffloader.java | 13 ++ .../mledger/LedgerOffloaderFactory.java | 8 +- .../mledger/impl/NullLedgerOffloader.java | 11 + .../mledger/impl/OffloadPrefixReadTest.java | 11 + .../mledger/impl/OffloadPrefixTest.java | 11 + .../apache/pulsar/broker/PulsarService.java | 65 +++--- .../broker/admin/impl/NamespacesBase.java | 9 +- .../pulsar/broker/service/BrokerService.java | 4 +- .../service/persistent/PersistentTopic.java | 4 +- .../broker/admin/AdminApiOffloadTest.java | 9 +- .../pulsar/client/admin/Namespaces.java | 5 +- .../pulsar/admin/cli/PulsarAdminToolTest.java | 6 +- .../pulsar/admin/cli/CmdNamespaces.java | 74 ++++--- .../common/policies/data/OffloadPolicies.java | 201 +++++++++++++----- .../sql/presto/PulsarConnectorCache.java | 54 +++-- .../pulsar/sql/presto/PulsarRecordCursor.java | 12 +- site2/docs/reference-pulsar-admin.md | 11 +- .../FileSystemConfigurationData.java | 68 ------ .../FileSystemLedgerOffloaderFactory.java | 7 +- .../FileSystemManagedLedgerOffloader.java | 28 ++- .../offload/filesystem/FileStoreTestBase.java | 3 +- .../offload/jcloud/CredentialsUtil.java | 25 +++ .../jcloud/JCloudLedgerOffloaderFactory.java | 10 +- .../TieredStorageConfigurationData.java | 149 ------------- .../impl/BlobStoreManagedLedgerOffloader.java | 32 ++- .../offload/jcloud/BlobStoreTestBase.java | 16 ++ .../BlobStoreManagedLedgerOffloaderTest.java | 69 +++--- 27 files changed, 484 insertions(+), 431 deletions(-) delete mode 100644 tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemConfigurationData.java create mode 100644 tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/CredentialsUtil.java delete mode 100644 tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/TieredStorageConfigurationData.java diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java index c85fe9fd79107..9402e82ff2215 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java @@ -26,6 +26,7 @@ import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.client.api.ReadHandle; +import org.apache.pulsar.common.policies.data.OffloadPolicies; /** * Interface for offloading ledgers to long-term storage @@ -112,5 +113,17 @@ CompletableFuture readOffloaded(long ledgerId, UUID uid, */ CompletableFuture deleteOffloaded(long ledgerId, UUID uid, Map offloadDriverMetadata); + + /** + * Get offload policies of this LedgerOffloader + * + * @return offload policies + */ + OffloadPolicies getOffloadPolicies(); + + /** + * Close the resources if necessary + */ + void close(); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java index 18886d4003493..7a0e6dc38d0d9 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java @@ -44,17 +44,15 @@ public interface LedgerOffloaderFactory { /** * Create a ledger offloader with the provided configuration, user-metadata and scheduler. * - * @param properties service configuration + * @param offloadPolicies offload policies * @param userMetadata user metadata * @param scheduler scheduler - * @param overrideOffloadPolicies offloadPolicies for override * @return the offloader instance * @throws IOException when fail to create an offloader */ - T create(Properties properties, + T create(OffloadPolicies offloadPolicies, Map userMetadata, - OrderedScheduler scheduler, - OffloadPolicies overrideOffloadPolicies) + OrderedScheduler scheduler) throws IOException; } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NullLedgerOffloader.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NullLedgerOffloader.java index 3401f1bbbb647..aa3aae71d1db7 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NullLedgerOffloader.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NullLedgerOffloader.java @@ -24,6 +24,7 @@ import org.apache.bookkeeper.client.api.ReadHandle; import org.apache.bookkeeper.mledger.LedgerOffloader; +import org.apache.pulsar.common.policies.data.OffloadPolicies; /** * Null implementation that throws an error on any invokation. @@ -60,4 +61,14 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, promise.completeExceptionally(new UnsupportedOperationException()); return promise; } + + @Override + public OffloadPolicies getOffloadPolicies() { + return null; + } + + @Override + public void close() { + + } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java index 17d54fa6e71f3..0a96a1fdc894e 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java @@ -54,6 +54,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.net.BookieSocketAddress; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.testng.Assert; import org.testng.annotations.Test; @@ -145,6 +146,16 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uuid, offloads.remove(uuid); return CompletableFuture.completedFuture(null); }; + + @Override + public OffloadPolicies getOffloadPolicies() { + return null; + } + + @Override + public void close() { + + } } static class MockOffloadReadHandle implements ReadHandle { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java index bf2f2e845d66b..997f3f66f3310 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java @@ -51,6 +51,7 @@ import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1025,6 +1026,16 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uuid, } return promise; }; + + @Override + public OffloadPolicies getOffloadPolicies() { + return null; + } + + @Override + public void close() { + + } } static class ErroringMockLedgerOffloader extends MockLedgerOffloader { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 52a120f060193..fe848b5e8b466 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -38,6 +38,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CompletableFuture; @@ -89,7 +90,6 @@ import org.apache.pulsar.broker.web.WebService; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; -import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.common.conf.InternalConfigurationData; @@ -112,7 +112,6 @@ import org.apache.pulsar.functions.worker.WorkerService; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreProvider; -import org.apache.pulsar.transaction.coordinator.impl.InMemTransactionMetadataStoreProvider; import org.apache.pulsar.websocket.WebSocketConsumerServlet; import org.apache.pulsar.websocket.WebSocketProducerServlet; import org.apache.pulsar.websocket.WebSocketReaderServlet; @@ -166,7 +165,7 @@ public class PulsarService implements AutoCloseable { private OrderedScheduler offloaderScheduler; private Offloaders offloaderManager = new Offloaders(); private LedgerOffloader defaultOffloader; - private Map ledgerOffloaderMap = new ConcurrentHashMap<>(); + private Map ledgerOffloaderMap = new ConcurrentHashMap<>(); private ScheduledFuture loadReportTask = null; private ScheduledFuture loadSheddingTask = null; private ScheduledFuture loadResourceQuotaTask = null; @@ -402,7 +401,8 @@ public void start() throws PulsarServerException { // Start load management service (even if load balancing is disabled) this.loadManager.set(LoadManager.create(this)); - this.defaultOffloader = createManagedLedgerOffloader(this.getConfiguration(), null); + this.defaultOffloader = createManagedLedgerOffloader( + OffloadPolicies.create(this.getConfiguration().getProperties())); brokerService.start(); @@ -768,54 +768,59 @@ public ManagedLedgerClientFactory getManagedLedgerClientFactory() { return managedLedgerClientFactory; } - public LedgerOffloader getManagedLedgerOffloaderByPolicies(OffloadPolicies offloadPolicies) { + /** + * First, get LedgerOffloader from local map cache, create new LedgerOffloader if not in cache or + * the OffloadPolicies changed, return the LedgerOffloader directly if exist in cache + * and the OffloadPolicies not changed. + * + * @param namespaceName NamespaceName + * @param offloadPolicies the OffloadPolicies + * @return LedgerOffloader + */ + public LedgerOffloader getManagedLedgerOffloader(NamespaceName namespaceName, OffloadPolicies offloadPolicies) { if (offloadPolicies == null) { return getDefaultOffloader(); } - return ledgerOffloaderMap.computeIfAbsent(offloadPolicies, op -> { + return ledgerOffloaderMap.compute(namespaceName, (ns, offloader) -> { try { - return createManagedLedgerOffloader(getConfiguration(), offloadPolicies); + if (offloader != null && Objects.equals(offloader.getOffloadPolicies(), offloadPolicies)) { + return offloader; + } else { + if (offloader != null) { + offloader.close(); + } + return createManagedLedgerOffloader(offloadPolicies); + } } catch (PulsarServerException e) { - LOG.error("create ledgerOffloader failed for {}", offloadPolicies.toString(), e); + LOG.error("create ledgerOffloader failed for namespace {}", namespaceName.toString(), e); return new NullLedgerOffloader(); } }); } - public LedgerOffloader getManagedLedgerOffloader(NamespaceName namespaceName) { - try { - OffloadPolicies offloadPolicies = getAdminClient().namespaces().getOffload(namespaceName.toString()); - return getManagedLedgerOffloaderByPolicies(offloadPolicies); - } catch (Exception e) { - LOG.error("Failed to get ledgerOffloader for namespaceName {}", namespaceName, e); - return new NullLedgerOffloader(); - } - } - - public synchronized LedgerOffloader createManagedLedgerOffloader(ServiceConfiguration conf, OffloadPolicies offloadPolicies) + public synchronized LedgerOffloader createManagedLedgerOffloader(OffloadPolicies offloadPolicies) throws PulsarServerException { try { - if (StringUtils.isNotBlank(conf.getManagedLedgerOffloadDriver())) { - checkNotNull(conf.getOffloadersDirectory(), + if (StringUtils.isNotBlank(offloadPolicies.getManagedLedgerOffloadDriver())) { + checkNotNull(offloadPolicies.getOffloadersDirectory(), "Offloader driver is configured to be '%s' but no offloaders directory is configured.", - conf.getManagedLedgerOffloadDriver()); - this.offloaderManager = OffloaderUtils.searchForOffloaders(conf.getOffloadersDirectory()); + offloadPolicies.getManagedLedgerOffloadDriver()); + this.offloaderManager = OffloaderUtils.searchForOffloaders(offloadPolicies.getOffloadersDirectory()); LedgerOffloaderFactory offloaderFactory = this.offloaderManager.getOffloaderFactory( - conf.getManagedLedgerOffloadDriver()); + offloadPolicies.getManagedLedgerOffloadDriver()); try { return offloaderFactory.create( - conf.getProperties(), + offloadPolicies, ImmutableMap.of( LedgerOffloader.METADATA_SOFTWARE_VERSION_KEY.toLowerCase(), PulsarVersion.getVersion(), LedgerOffloader.METADATA_SOFTWARE_GITSHA_KEY.toLowerCase(), PulsarVersion.getGitSha() ), - getOffloaderScheduler(conf), - offloadPolicies); + getOffloaderScheduler(offloadPolicies)); } catch (IOException ioe) { throw new PulsarServerException(ioe.getMessage(), ioe.getCause()); } } else { - LOG.warn("No ledger offloader configured, using NULL instance"); + LOG.info("No ledger offloader configured, using NULL instance"); return NullLedgerOffloader.INSTANCE; } } catch (Throwable t) { @@ -887,10 +892,10 @@ public synchronized Compactor getCompactor() throws PulsarServerException { return this.compactor; } - protected synchronized OrderedScheduler getOffloaderScheduler(ServiceConfiguration conf) { + protected synchronized OrderedScheduler getOffloaderScheduler(OffloadPolicies offloadPolicies) { if (this.offloaderScheduler == null) { this.offloaderScheduler = OrderedScheduler.newSchedulerBuilder() - .numThreads(conf.getManagedLedgerOffloadMaxThreads()) + .numThreads(offloadPolicies.getManagedLedgerOffloadMaxThreads()) .name("offloader").build(); } return this.offloaderScheduler; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 9ccbccc8a3f55..51f3afd893806 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -2198,7 +2198,14 @@ protected void internalSetOffload(OffloadPolicies offload) { final String path = path(POLICIES, namespaceName.toString()); byte[] content = globalZk().getData(path, null, nodeStat); Policies policies = jsonMapper().readValue(content, Policies.class); - if (StringUtils.isEmpty(offload.getBucket())) { + if (StringUtils.isEmpty(offload.getManagedLedgerOffloadDriver())) { + log.warn("[{}] Failed to update offload configuration for namespace {}: driver must be specified", + clientAppId(), namespaceName); + throw new RestException(Status.PRECONDITION_FAILED, + "The driver must be specified for namespace offload."); + } + if (StringUtils.isEmpty(offload.getS3ManagedLedgerOffloadBucket()) + && StringUtils.isEmpty(offload.getGcsManagedLedgerOffloadBucket())) { log.warn("[{}] Failed to update offload configuration for namespace {}: bucket must be specified", clientAppId(), namespaceName); throw new RestException(Status.PRECONDITION_FAILED, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 83e65191f6629..46f322ff3729b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -121,6 +121,7 @@ import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.LocalPolicies; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats; import org.apache.pulsar.common.policies.data.Policies; @@ -982,7 +983,8 @@ public CompletableFuture getManagedLedgerConfig(TopicName t managedLedgerConfig.setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES); managedLedgerConfig.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB()); - managedLedgerConfig.setLedgerOffloader(pulsar.getManagedLedgerOffloader(namespace)); + OffloadPolicies offloadPolicies = policies.map(p -> p.offload_policies).orElse(null); + managedLedgerConfig.setLedgerOffloader(pulsar.getManagedLedgerOffloader(namespace, offloadPolicies)); policies.ifPresent(p -> { long lag = serviceConfig.getManagedLedgerOffloadDeletionLagMs(); if (p.offload_deletion_lag_ms != null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index f6f977597892b..463b6e2dcc011 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -99,7 +99,6 @@ import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BacklogQuota; -import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; import org.apache.pulsar.common.policies.data.ConsumerStats; import org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; @@ -1777,7 +1776,8 @@ public CompletableFuture onPoliciesUpdate(Policies data) { subscribeRateLimiter.get().onPoliciesUpdate(data); } getManagedLedger().getConfig().setLedgerOffloader( - brokerService.pulsar().getManagedLedgerOffloaderByPolicies(data.offload_policies)); + brokerService.pulsar().getManagedLedgerOffloader( + TopicName.get(topic).getNamespaceObject(), data.offload_policies)); return CompletableFuture.allOf(replicationFuture, dedupFuture, persistentPoliciesFuture); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java index f455423103130..f85fc55d621da 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java @@ -72,8 +72,7 @@ private void testOffload(String topicName, String mlName) throws Exception { LedgerOffloader offloader = mock(LedgerOffloader.class); when(offloader.getOffloadDriverName()).thenReturn("mock"); - doReturn(offloader).when(pulsar).getManagedLedgerOffloaderByPolicies(any()); - doReturn(offloader).when(pulsar).getManagedLedgerOffloader(any()); + doReturn(offloader).when(pulsar).getManagedLedgerOffloader(any(), any()); CompletableFuture promise = new CompletableFuture<>(); doReturn(promise).when(offloader).offload(any(), any(), any()); @@ -144,11 +143,13 @@ public void testOffloadV1() throws Exception { @Test public void testOffloadPolicies() throws Exception { String namespaceName = "prop-xyz/ns1"; - String endpoint = "test-endpoint"; + String driver = "aws-s3"; String region = "test-region"; String bucket = "test-bucket"; + String endpoint = "test-endpoint"; - OffloadPolicies offload1 = new OffloadPolicies(endpoint, region, bucket); + OffloadPolicies offload1 = OffloadPolicies.create( + driver, region, bucket, endpoint, 100, 100); admin.namespaces().setOffload(namespaceName, offload1); OffloadPolicies offload2 = admin.namespaces().getOffload(namespaceName); Assert.assertEquals(offload1, offload2); diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java index bb862eb15fdf6..b64f624b3774c 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java @@ -35,7 +35,6 @@ import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; import org.apache.pulsar.common.policies.data.OffloadPolicies; -import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; @@ -1658,9 +1657,9 @@ void setIsAllowAutoUpdateSchema(String namespace, boolean isAllowAutoUpdateSchem throws PulsarAdminException; /** - * Set the offload configuration for all the topics on a namespace. + * Set the offload configuration for all the topics in a namespace. *

- * Set the offload configuration on a namespace. This operation requires Pulsar super-user access. + * Set the offload configuration in a namespace. This operation requires Pulsar super-user access. *

* Request parameter example: *

diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index b383757899740..28a22cff97e82 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -489,10 +489,10 @@ void namespaces() throws Exception { namespaces.run(split("clear-offload-deletion-lag myprop/clust/ns1")); verify(mockNamespaces).clearOffloadDeleteLag("myprop/clust/ns1"); - namespaces.run(split("set-offload myprop/clust/ns1 -r test-region -b test-bucket -e http://test.endpoint -mbs 32M -rbs 5M")); + namespaces.run(split("set-offload myprop/clust/ns1 -r test-region -d aws-s3 -b test-bucket -e http://test.endpoint -mbs 32M -rbs 5M")); verify(mockNamespaces).setOffload("myprop/clust/ns1", - new OffloadPolicies("test-region", "test-bucket", "http://test.endpoint", - 32 * 1024 * 1024, 5 * 1024 * 1024)); + OffloadPolicies.create("aws-s3", "test-region", "test-bucket", + "http://test.endpoint", 32 * 1024 * 1024, 5 * 1024 * 1024)); namespaces.run(split("get-offload myprop/clust/ns1")); verify(mockNamespaces).getOffload("myprop/clust/ns1"); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 4ac4703537f2b..c24506773198a 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -44,7 +44,6 @@ import org.apache.pulsar.common.policies.data.DispatchRate; import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; import org.apache.pulsar.common.policies.data.OffloadPolicies; -import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.common.policies.data.PersistencePolicies; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.PublishRate; @@ -1296,6 +1295,13 @@ private class SetOffload extends CliCommand { @Parameter(description = "tenant/namespace", required = true) private java.util.List params; + @Parameter( + names = {"--driver", "-d"}, + description = "Driver to use to offload old data to long term storage, " + + "(Possible values: S3, aws-s3, google-cloud-storage)", + required = true) + private String driver; + @Parameter( names = {"--region", "-r"}, description = "The long term storage region, " + @@ -1330,6 +1336,10 @@ private class SetOffload extends CliCommand { private final String[] DRIVER_NAMES = {"S3", "aws-s3", "google-cloud-storage"}; + public boolean driverSupported(String driver) { + return Arrays.stream(DRIVER_NAMES).anyMatch(d -> d.equalsIgnoreCase(driver)); + } + public boolean isS3Driver(String driver) { if (StringUtils.isEmpty(driver)) { return false; @@ -1337,45 +1347,57 @@ public boolean isS3Driver(String driver) { return driver.equalsIgnoreCase(DRIVER_NAMES[0]) || driver.equalsIgnoreCase(DRIVER_NAMES[1]); } + public boolean positiveCheck(String paramName, long value) { + if (value <= 0) { + throw new ParameterException(paramName + " is not be negative or 0!"); + } + return true; + } + + public boolean maxValueCheck(String paramName, long value, long maxValue) { + if (value > maxValue) { + throw new ParameterException(paramName + " is not bigger than " + maxValue + "!"); + } + return true; + } + @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - String offloadDriver = admin.brokers().getRuntimeConfigurations().get("managedLedgerOffloadDriver"); - if (isS3Driver(offloadDriver) && Strings.isNullOrEmpty(region) && Strings.isNullOrEmpty(endpoint)) { + if (!driverSupported(driver)) { + throw new ParameterException( + "No offloader found for driver '" + driver + "'." + " Please make sure " + + "you dropped the offloader nar packages under `${PULSAR_HOME}/offloaders`."); + } + + if (isS3Driver(driver) && Strings.isNullOrEmpty(region) && Strings.isNullOrEmpty(endpoint)) { throw new ParameterException( "Either s3ManagedLedgerOffloadRegion or s3ManagedLedgerOffloadServiceEndpoint must be set" + " if s3 offload enabled"); } - long maxBlockSize = 0; + int maxBlockSizeInBytes = OffloadPolicies.DEFAULT_MAX_BLOCK_SIZE_IN_BYTES; if (StringUtils.isNotEmpty(maxBlockSizeStr)) { - maxBlockSize = validateSizeString(maxBlockSizeStr); - } - long readBufferSize = 0; - if (StringUtils.isNotEmpty(readBufferSizeStr)) { - readBufferSize = validateSizeString(readBufferSizeStr); - } - - final int maxBlockSizeInBytes; - if (maxBlockSize > 0 && maxBlockSize <= Integer.MAX_VALUE) { - maxBlockSizeInBytes = new Long(maxBlockSize).intValue(); - } else { - maxBlockSizeInBytes = OffloadPolicies.MAX_BLOCK_SIZE_IN_BYTES; + long maxBlockSize = validateSizeString(maxBlockSizeStr); + if (positiveCheck("MaxBlockSize", maxBlockSize) + && maxValueCheck("MaxBlockSize", maxBlockSize, Integer.MAX_VALUE)) { + maxBlockSizeInBytes = new Long(maxBlockSize).intValue(); + } } - final int readBufferSizeInBytes; - if (readBufferSize > 0 && readBufferSize <= Integer.MAX_VALUE) { - readBufferSizeInBytes = new Long(readBufferSize).intValue(); - } else { - readBufferSizeInBytes = OffloadPolicies.READ_BUFFER_SIZE_IN_BYTES; + int readBufferSizeInBytes = OffloadPolicies.DEFAULT_READ_BUFFER_SIZE_IN_BYTES; + if (StringUtils.isNotEmpty(readBufferSizeStr) ) { + long readBufferSize = validateSizeString(readBufferSizeStr); + if (positiveCheck("ReadBufferSize", readBufferSize) + && maxValueCheck("ReadBufferSize", readBufferSize, Integer.MAX_VALUE)) { + readBufferSizeInBytes = new Long(readBufferSize).intValue(); + } } - admin.namespaces().setOffload(namespace, new OffloadPolicies( - region == null ? "" : region, - bucket, - endpoint == null ? "" : endpoint, - maxBlockSizeInBytes, readBufferSizeInBytes)); + OffloadPolicies offloadPolicies = OffloadPolicies.create(driver, region, bucket, endpoint, + maxBlockSizeInBytes, readBufferSizeInBytes); + admin.namespaces().setOffload(namespace, offloadPolicies); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java index 563d357996d4b..b8ff00c3c3318 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java @@ -1,63 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.pulsar.common.policies.data; -import com.google.common.base.MoreObjects; +import static org.apache.pulsar.common.util.FieldParser.value; +import com.google.common.base.MoreObjects; +import java.lang.reflect.Field; +import java.util.Arrays; import java.util.Objects; +import java.util.Properties; +import lombok.Data; /** * Definition of the offload policies. */ +@Data public class OffloadPolicies { - public final static int MAX_BLOCK_SIZE_IN_BYTES = 64 * 1024 * 1024; // 64MB - public final static int READ_BUFFER_SIZE_IN_BYTES = 1024 * 1024; // 1MB - - private String region; - private String bucket; - private String endpoint; - private int maxBlockSizeInBytes; - private int readBufferSizeInBytes; - - public OffloadPolicies() { - - } - - public OffloadPolicies(String region, String bucket, String endpoint) { - this(region, bucket, endpoint, MAX_BLOCK_SIZE_IN_BYTES, READ_BUFFER_SIZE_IN_BYTES); - } - - public OffloadPolicies(String region, String bucket, String endpoint, - int maxBlockSizeInBytes, int readBufferSizeInBytes) { - this.region = region; - this.bucket = bucket; - this.endpoint = endpoint; - this.maxBlockSizeInBytes = maxBlockSizeInBytes; - this.readBufferSizeInBytes = readBufferSizeInBytes; - } - - public String getRegion() { - return region; - } - - public String getBucket() { - return bucket; - } - - public String getEndpoint() { - return endpoint; - } - - public int getMaxBlockSizeInBytes() { - return maxBlockSizeInBytes; + public final static int DEFAULT_MAX_BLOCK_SIZE_IN_BYTES = 64 * 1024 * 1024; // 64MB + public final static int DEFAULT_READ_BUFFER_SIZE_IN_BYTES = 1024 * 1024; // 1MB + public final static int DEFAULT_OFFLOAD_MAX_THREADS = 2; + public final static String[] DRIVER_NAMES = {"S3", "aws-s3", "google-cloud-storage", "filesystem"}; + public final static String DEFAULT_OFFLOADER_DIRECTORY = "./offloaders"; + + // common config + private String offloadersDirectory = DEFAULT_OFFLOADER_DIRECTORY; + private String managedLedgerOffloadDriver = null; + private int managedLedgerOffloadMaxThreads = DEFAULT_OFFLOAD_MAX_THREADS; + + // s3 config, set by service configuration or cli + private String s3ManagedLedgerOffloadRegion = null; + private String s3ManagedLedgerOffloadBucket = null; + private String s3ManagedLedgerOffloadServiceEndpoint = null; + private int s3ManagedLedgerOffloadMaxBlockSizeInBytes = DEFAULT_MAX_BLOCK_SIZE_IN_BYTES; + private int s3ManagedLedgerOffloadReadBufferSizeInBytes = DEFAULT_READ_BUFFER_SIZE_IN_BYTES; + // s3 config, set by service configuration + private String s3ManagedLedgerOffloadRole = null; + private String s3ManagedLedgerOffloadRoleSessionName = "pulsar-s3-offload"; + + // gcs config, set by service configuration or cli + private String gcsManagedLedgerOffloadRegion = null; + private String gcsManagedLedgerOffloadBucket = null; + private int gcsManagedLedgerOffloadMaxBlockSizeInBytes = DEFAULT_MAX_BLOCK_SIZE_IN_BYTES; + private int gcsManagedLedgerOffloadReadBufferSizeInBytes = DEFAULT_READ_BUFFER_SIZE_IN_BYTES; + // gcs config, set by service configuration + private String gcsManagedLedgerOffloadServiceAccountKeyFile = null; + + // file system config, set by service configuration + private String fileSystemProfilePath = null; + private String fileSystemURI = null; + + public static OffloadPolicies create(String driver, String region, String bucket, String endpoint, + int maxBlockSizeInBytes, int readBufferSizeInBytes) { + OffloadPolicies offloadPolicies = new OffloadPolicies(); + offloadPolicies.setManagedLedgerOffloadDriver(driver); + if (driver.equalsIgnoreCase(DRIVER_NAMES[0]) || driver.equalsIgnoreCase(DRIVER_NAMES[1])) { + offloadPolicies.setS3ManagedLedgerOffloadRegion(region); + offloadPolicies.setS3ManagedLedgerOffloadBucket(bucket); + offloadPolicies.setS3ManagedLedgerOffloadServiceEndpoint(endpoint); + offloadPolicies.setS3ManagedLedgerOffloadMaxBlockSizeInBytes(maxBlockSizeInBytes); + offloadPolicies.setS3ManagedLedgerOffloadReadBufferSizeInBytes(readBufferSizeInBytes); + } else if (driver.equalsIgnoreCase(DRIVER_NAMES[2])) { + offloadPolicies.setGcsManagedLedgerOffloadRegion(region); + offloadPolicies.setGcsManagedLedgerOffloadBucket(bucket); + offloadPolicies.setGcsManagedLedgerOffloadMaxBlockSizeInBytes(maxBlockSizeInBytes); + offloadPolicies.setGcsManagedLedgerOffloadReadBufferSizeInBytes(readBufferSizeInBytes); + } + return offloadPolicies; } - public int getReadBufferSizeInBytes() { - return readBufferSizeInBytes; + public static OffloadPolicies create(Properties properties) { + OffloadPolicies data = new OffloadPolicies(); + Field[] fields = OffloadPolicies.class.getDeclaredFields(); + Arrays.stream(fields).forEach(f -> { + if (properties.containsKey(f.getName())) { + try { + f.setAccessible(true); + f.set(data, value((String) properties.get(f.getName()), f)); + } catch (Exception e) { + throw new IllegalArgumentException( + String.format("failed to initialize %s field while setting value %s", + f.getName(), properties.get(f.getName())), e); + } + } + }); + return data; } @Override public int hashCode() { - return Objects.hash(region, bucket, endpoint, maxBlockSizeInBytes, readBufferSizeInBytes); + return Objects.hash( + managedLedgerOffloadDriver, + managedLedgerOffloadMaxThreads, + s3ManagedLedgerOffloadRegion, + s3ManagedLedgerOffloadBucket, + s3ManagedLedgerOffloadServiceEndpoint, + s3ManagedLedgerOffloadMaxBlockSizeInBytes, + s3ManagedLedgerOffloadReadBufferSizeInBytes, + s3ManagedLedgerOffloadRole, + s3ManagedLedgerOffloadRoleSessionName, + gcsManagedLedgerOffloadRegion, + gcsManagedLedgerOffloadBucket, + gcsManagedLedgerOffloadMaxBlockSizeInBytes, + gcsManagedLedgerOffloadReadBufferSizeInBytes, + gcsManagedLedgerOffloadServiceAccountKeyFile, + fileSystemProfilePath, + fileSystemURI); } @Override @@ -69,21 +133,50 @@ public boolean equals(Object obj) { return false; } OffloadPolicies other = (OffloadPolicies) obj; - return Objects.equals(region, other.getRegion()) - && Objects.equals(bucket, other.getBucket()) - && Objects.equals(endpoint, other.getEndpoint()) - && Objects.equals(maxBlockSizeInBytes, other.getMaxBlockSizeInBytes()) - && Objects.equals(readBufferSizeInBytes, other.getReadBufferSizeInBytes()); + return Objects.equals(managedLedgerOffloadDriver, other.getManagedLedgerOffloadDriver()) + && Objects.equals(managedLedgerOffloadMaxThreads, other.getManagedLedgerOffloadMaxThreads()) + && Objects.equals(s3ManagedLedgerOffloadRegion, other.getS3ManagedLedgerOffloadRegion()) + && Objects.equals(s3ManagedLedgerOffloadBucket, other.getS3ManagedLedgerOffloadBucket()) + && Objects.equals(s3ManagedLedgerOffloadServiceEndpoint, + other.getS3ManagedLedgerOffloadServiceEndpoint()) + && Objects.equals(s3ManagedLedgerOffloadMaxBlockSizeInBytes, + other.getS3ManagedLedgerOffloadMaxBlockSizeInBytes()) + && Objects.equals(s3ManagedLedgerOffloadReadBufferSizeInBytes, + other.getS3ManagedLedgerOffloadReadBufferSizeInBytes()) + && Objects.equals(s3ManagedLedgerOffloadRole, other.getS3ManagedLedgerOffloadRole()) + && Objects.equals(s3ManagedLedgerOffloadRoleSessionName, + other.getS3ManagedLedgerOffloadRoleSessionName()) + && Objects.equals(gcsManagedLedgerOffloadRegion, other.getGcsManagedLedgerOffloadRegion()) + && Objects.equals(gcsManagedLedgerOffloadBucket, other.getGcsManagedLedgerOffloadBucket()) + && Objects.equals(gcsManagedLedgerOffloadMaxBlockSizeInBytes, + other.getGcsManagedLedgerOffloadMaxBlockSizeInBytes()) + && Objects.equals(gcsManagedLedgerOffloadReadBufferSizeInBytes, + other.getGcsManagedLedgerOffloadReadBufferSizeInBytes()) + && Objects.equals(gcsManagedLedgerOffloadServiceAccountKeyFile, + other.getGcsManagedLedgerOffloadServiceAccountKeyFile()) + && Objects.equals(fileSystemProfilePath, other.getFileSystemProfilePath()) + && Objects.equals(fileSystemURI, other.getFileSystemURI()); } @Override public String toString() { return MoreObjects.toStringHelper(this) - .add("region", region) - .add("bucket", bucket) - .add("endpoint", endpoint) - .add("maxBlockSizeInBytes", maxBlockSizeInBytes) - .add("readBufferSizeInBytes", readBufferSizeInBytes) + .add("managedLedgerOffloadDriver", managedLedgerOffloadDriver) + .add("managedLedgerOffloadMaxThreads", managedLedgerOffloadMaxThreads) + .add("s3ManagedLedgerOffloadRegion", s3ManagedLedgerOffloadRegion) + .add("s3ManagedLedgerOffloadBucket", s3ManagedLedgerOffloadBucket) + .add("s3ManagedLedgerOffloadServiceEndpoint", s3ManagedLedgerOffloadServiceEndpoint) + .add("s3ManagedLedgerOffloadMaxBlockSizeInBytes", s3ManagedLedgerOffloadMaxBlockSizeInBytes) + .add("s3ManagedLedgerOffloadReadBufferSizeInBytes", s3ManagedLedgerOffloadReadBufferSizeInBytes) + .add("s3ManagedLedgerOffloadRole", s3ManagedLedgerOffloadRole) + .add("s3ManagedLedgerOffloadRoleSessionName", s3ManagedLedgerOffloadRoleSessionName) + .add("gcsManagedLedgerOffloadRegion", gcsManagedLedgerOffloadRegion) + .add("gcsManagedLedgerOffloadBucket", gcsManagedLedgerOffloadBucket) + .add("gcsManagedLedgerOffloadMaxBlockSizeInBytes", gcsManagedLedgerOffloadMaxBlockSizeInBytes) + .add("gcsManagedLedgerOffloadReadBufferSizeInBytes", gcsManagedLedgerOffloadReadBufferSizeInBytes) + .add("gcsManagedLedgerOffloadServiceAccountKeyFile", gcsManagedLedgerOffloadServiceAccountKeyFile) + .add("fileSystemProfilePath", fileSystemProfilePath) + .add("fileSystemURI", fileSystemURI) .toString(); } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java index 7bf4d3b4e6ef2..a29aabd155879 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java @@ -25,6 +25,9 @@ import io.airlift.log.Logger; import java.io.IOException; import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.mledger.LedgerOffloader; @@ -37,8 +40,10 @@ import org.apache.bookkeeper.mledger.offload.OffloaderUtils; import org.apache.bookkeeper.mledger.offload.Offloaders; import org.apache.bookkeeper.stats.StatsProvider; +import org.apache.commons.beanutils.BeanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.PulsarVersion; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.OffloadPolicies; /** @@ -57,7 +62,7 @@ public class PulsarConnectorCache { private OrderedScheduler offloaderScheduler; private Offloaders offloaderManager; private LedgerOffloader defaultOffloader; - private Map offloaderMap; + private Map offloaderMap = new ConcurrentHashMap<>(); private static final String OFFLOADERS_DIRECTOR = "offloadersDirectory"; private static final String MANAGED_LEDGER_OFFLOAD_DRIVER = "managedLedgerOffloadDriver"; @@ -76,7 +81,9 @@ private PulsarConnectorCache(PulsarConnectorConfig pulsarConnectorConfig) throws this.statsProvider.start(clientConfiguration); - this.defaultOffloader = initManagedLedgerOffloader(pulsarConnectorConfig, null); + OffloadPolicies offloadPolicies = new OffloadPolicies(); + BeanUtils.copyProperties(offloadPolicies, pulsarConnectorConfig); + this.defaultOffloader = initManagedLedgerOffloader(offloadPolicies); } public static PulsarConnectorCache getConnectorCache(PulsarConnectorConfig pulsarConnectorConfig) throws Exception { @@ -110,54 +117,55 @@ private static ManagedLedgerFactory initManagedLedgerFactory(PulsarConnectorConf return new ManagedLedgerFactoryImpl(bkClientConfiguration, managedLedgerFactoryConfig); } - public ManagedLedgerConfig getManagedLedgerConfig(PulsarConnectorConfig pulsarConnectorConfig, - OffloadPolicies offloadPolicies) { + public ManagedLedgerConfig getManagedLedgerConfig(NamespaceName namespaceName, OffloadPolicies offloadPolicies) { ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); if (offloadPolicies == null) { managedLedgerConfig.setLedgerOffloader(this.defaultOffloader); } else { - LedgerOffloader ledgerOffloader = offloaderMap.computeIfAbsent(offloadPolicies, - op -> initManagedLedgerOffloader(pulsarConnectorConfig, offloadPolicies)); + LedgerOffloader ledgerOffloader = offloaderMap.compute(namespaceName, + (ns, offloader) -> { + if (offloader != null && Objects.equals(offloader.getOffloadPolicies(), offloadPolicies)) { + return offloader; + } else { + if (offloader != null) { + offloader.close(); + } + return initManagedLedgerOffloader(offloadPolicies); + } + }); managedLedgerConfig.setLedgerOffloader(ledgerOffloader); } return managedLedgerConfig; } - private synchronized OrderedScheduler getOffloaderScheduler(PulsarConnectorConfig pulsarConnectorConfig) { + private synchronized OrderedScheduler getOffloaderScheduler(OffloadPolicies offloadPolicies) { if (this.offloaderScheduler == null) { this.offloaderScheduler = OrderedScheduler.newSchedulerBuilder() - .numThreads(pulsarConnectorConfig.getManagedLedgerOffloadMaxThreads()) + .numThreads(offloadPolicies.getManagedLedgerOffloadMaxThreads()) .name("pulsar-offloader").build(); } return this.offloaderScheduler; } - private LedgerOffloader initManagedLedgerOffloader(PulsarConnectorConfig conf, OffloadPolicies offloadPolicies) { + private LedgerOffloader initManagedLedgerOffloader(OffloadPolicies offloadPolicies) { try { - if (StringUtils.isNotBlank(conf.getManagedLedgerOffloadDriver())) { - checkNotNull(conf.getOffloadersDirectory(), + if (StringUtils.isNotBlank(offloadPolicies.getManagedLedgerOffloadDriver())) { + checkNotNull(offloadPolicies.getOffloadersDirectory(), "Offloader driver is configured to be '%s' but no offloaders directory is configured.", - conf.getManagedLedgerOffloadDriver()); - this.offloaderManager = OffloaderUtils.searchForOffloaders(conf.getOffloadersDirectory()); + offloadPolicies.getManagedLedgerOffloadDriver()); + this.offloaderManager = OffloaderUtils.searchForOffloaders(offloadPolicies.getOffloadersDirectory()); LedgerOffloaderFactory offloaderFactory = this.offloaderManager.getOffloaderFactory( - conf.getManagedLedgerOffloadDriver()); - - Map offloaderProperties = conf.getOffloaderProperties(); - offloaderProperties.put(OFFLOADERS_DIRECTOR, conf.getOffloadersDirectory()); - offloaderProperties.put(MANAGED_LEDGER_OFFLOAD_DRIVER, conf.getManagedLedgerOffloadDriver()); - offloaderProperties - .put(MANAGED_LEDGER_OFFLOAD_MAX_THREADS, String.valueOf(conf.getManagedLedgerOffloadMaxThreads())); + offloadPolicies.getManagedLedgerOffloadDriver()); try { return offloaderFactory.create( - PulsarConnectorUtils.getProperties(offloaderProperties), + offloadPolicies, ImmutableMap.of( LedgerOffloader.METADATA_SOFTWARE_VERSION_KEY.toLowerCase(), PulsarVersion.getVersion(), LedgerOffloader.METADATA_SOFTWARE_GITSHA_KEY.toLowerCase(), PulsarVersion.getGitSha() ), - getOffloaderScheduler(conf), - offloadPolicies); + getOffloaderScheduler(offloadPolicies)); } catch (IOException ioe) { log.error("Failed to create offloader: ", ioe); throw new RuntimeException(ioe.getMessage(), ioe.getCause()); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index 3c36f01b1b0f5..eca99af4dbb4b 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -57,6 +57,7 @@ import org.apache.pulsar.common.api.raw.RawMessage; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.sql.presto.PulsarInternalColumn.PartitionColumn; import org.jctools.queues.MessagePassingQueue; import org.jctools.queues.SpscArrayQueue; @@ -110,9 +111,18 @@ public PulsarRecordCursor(List columnHandles, PulsarSplit pu close(); throw new RuntimeException(e); } + + OffloadPolicies offloadPolicies = pulsarSplit.getOffloadPolicies(); + if (offloadPolicies != null) { + offloadPolicies.setOffloadersDirectory(pulsarConnectorConfig.getOffloadersDirectory()); + offloadPolicies.setManagedLedgerOffloadMaxThreads( + pulsarConnectorConfig.getManagedLedgerOffloadMaxThreads()); + } initialize(columnHandles, pulsarSplit, pulsarConnectorConfig, pulsarConnectorCache.getManagedLedgerFactory(), - pulsarConnectorCache.getManagedLedgerConfig(pulsarConnectorConfig, pulsarSplit.getOffloadPolicies()), + pulsarConnectorCache.getManagedLedgerConfig( + TopicName.get("persistent", NamespaceName.get(pulsarSplit.getSchemaName()), + pulsarSplit.getTableName()).getNamespaceObject(), offloadPolicies), new PulsarConnectorMetricsTracker(pulsarConnectorCache.getStatsProvider())); } diff --git a/site2/docs/reference-pulsar-admin.md b/site2/docs/reference-pulsar-admin.md index 19b4bb8f57566..7ca1790d35f45 100644 --- a/site2/docs/reference-pulsar-admin.md +++ b/site2/docs/reference-pulsar-admin.md @@ -2265,8 +2265,9 @@ $ pulsar-admin namespaces set-offload tenant/namespace Options |Flag|Description|Default| |----|---|---| -|`-r`, `--region`|The long term storage region|s3ManagedLedgerOffloadRegion or gcsManagedLedgerOffloadRegion in broker.conf -|`-b`, `--bucket`|Bucket to place offloaded ledger into|-- -|`-e`, `--endpoint`|Alternative endpoint to connect to|s3 is s3ManagedLedgerOffloadServiceEndpoint in broker.conf -|`-mbs`, `--maxBlockSize`|Max block size|64MB -|`-rbs`, `--readBufferSize`|Read buffer size|1MB +|`-d`, `--driver`|Driver to use to offload old data to long term storage,(Possible values: S3, aws-s3, google-cloud-storage)|| +|`-r`, `--region`|The long term storage region|| +|`-b`, `--bucket`|Bucket to place offloaded ledger into|| +|`-e`, `--endpoint`|Alternative endpoint to connect to|| +|`-mbs`, `--maxBlockSize`|Max block size|64MB| +|`-rbs`, `--readBufferSize`|Read buffer size|1MB| diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemConfigurationData.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemConfigurationData.java deleted file mode 100644 index 899887b6f8fbf..0000000000000 --- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemConfigurationData.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bookkeeper.mledger.offload.filesystem; - -import lombok.Data; - -import java.io.Serializable; -import java.lang.reflect.Field; -import java.util.Arrays; -import java.util.Properties; - -import static org.apache.pulsar.common.util.FieldParser.value; - -/** - * Configuration for file system. - */ -@Data -public class FileSystemConfigurationData implements Serializable, Cloneable { - - /**** --- Ledger Offloading --- ****/ - // Driver to use to offload old data to long term storage - private String managedLedgerOffloadDriver = null; - - private String fileSystemProfilePath = null; - - private String fileSystemURI = null; - - private int managedLedgerOffloadMaxThreads = 2; - - /** - * Create a tiered storage configuration from the provided properties. - * - * @param properties the configuration properties - * @return tiered storage configuration - */ - public static FileSystemConfigurationData create(Properties properties) { - FileSystemConfigurationData data = new FileSystemConfigurationData(); - Field[] fields = FileSystemConfigurationData.class.getDeclaredFields(); - Arrays.stream(fields).forEach(f -> { - if (properties.containsKey(f.getName())) { - try { - f.setAccessible(true); - f.set(data, value((String) properties.get(f.getName()), f)); - } catch (Exception e) { - throw new IllegalArgumentException(String.format("failed to initialize %s field while setting value %s", - f.getName(), properties.get(f.getName())), e); - } - } - }); - return data; - } -} diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java index 026f08ccf6dd3..cf86f38e1bedb 100644 --- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java +++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/FileSystemLedgerOffloaderFactory.java @@ -34,9 +34,8 @@ public boolean isDriverSupported(String driverName) { } @Override - public FileSystemManagedLedgerOffloader create(Properties properties, Map userMetadata, OrderedScheduler scheduler, - OffloadPolicies overrideOffloadPolicies) throws IOException { - FileSystemConfigurationData data = FileSystemConfigurationData.create(properties); - return FileSystemManagedLedgerOffloader.create(data, scheduler); + public FileSystemManagedLedgerOffloader create(OffloadPolicies offloadPolicies, + Map userMetadata, OrderedScheduler scheduler) throws IOException { + return FileSystemManagedLedgerOffloader.create(offloadPolicies, scheduler); } } diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java index 309076c4c3605..eda747ded8712 100644 --- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java +++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java @@ -29,7 +29,6 @@ import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.LedgerOffloader; import org.apache.bookkeeper.mledger.offload.filesystem.FileSystemLedgerOffloaderFactory; -import org.apache.bookkeeper.mledger.offload.filesystem.FileSystemConfigurationData; import org.apache.bookkeeper.net.BookieSocketAddress; import org.apache.bookkeeper.proto.DataFormats; import org.apache.hadoop.conf.Configuration; @@ -39,6 +38,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.MapFile; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,6 +69,8 @@ public class FileSystemManagedLedgerOffloader implements LedgerOffloader { private OrderedScheduler scheduler; private static final long ENTRIES_PER_READ = 100; private OrderedScheduler assignmentScheduler; + private OffloadPolicies offloadPolicies; + public static boolean driverSupported(String driver) { return DRIVER_NAMES.equals(driver); } @@ -77,11 +79,12 @@ public String getOffloadDriverName() { return driverName; } - public static FileSystemManagedLedgerOffloader create(FileSystemConfigurationData conf, OrderedScheduler scheduler) throws IOException { + public static FileSystemManagedLedgerOffloader create(OffloadPolicies conf, OrderedScheduler scheduler) throws IOException { return new FileSystemManagedLedgerOffloader(conf, scheduler); } - private FileSystemManagedLedgerOffloader(FileSystemConfigurationData conf, OrderedScheduler scheduler) throws IOException { + private FileSystemManagedLedgerOffloader(OffloadPolicies conf, OrderedScheduler scheduler) throws IOException { + this.offloadPolicies = conf; this.configuration = new Configuration(); if (conf.getFileSystemProfilePath() != null) { String[] paths = conf.getFileSystemProfilePath().split(","); @@ -110,7 +113,8 @@ private FileSystemManagedLedgerOffloader(FileSystemConfigurationData conf, Order .name("offload-assignment").build(); } @VisibleForTesting - public FileSystemManagedLedgerOffloader(FileSystemConfigurationData conf, OrderedScheduler scheduler, String testHDFSPath, String baseDir) throws IOException { + public FileSystemManagedLedgerOffloader(OffloadPolicies conf, OrderedScheduler scheduler, String testHDFSPath, String baseDir) throws IOException { + this.offloadPolicies = conf; this.configuration = new Configuration(); this.configuration.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); this.configuration.set("fs.defaultFS", testHDFSPath); @@ -328,4 +332,20 @@ private static byte[] buildLedgerMetadataFormat(LedgerMetadata metadata) { return builder.build().toByteArray(); } + + @Override + public OffloadPolicies getOffloadPolicies() { + return offloadPolicies; + } + + @Override + public void close() { + if (fileSystem != null) { + try { + fileSystem.close(); + } catch (Exception e) { + log.error("FileSystemManagedLedgerOffloader close failed!", e); + } + } + } } diff --git a/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/FileStoreTestBase.java b/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/FileStoreTestBase.java index 4abff16443090..b014fd23860b6 100644 --- a/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/FileStoreTestBase.java +++ b/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/FileStoreTestBase.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -48,7 +49,7 @@ public void start() throws Exception { hdfsURI = "hdfs://localhost:"+ hdfsCluster.getNameNodePort() + "/"; Properties properties = new Properties(); fileSystemManagedLedgerOffloader = new FileSystemManagedLedgerOffloader( - FileSystemConfigurationData.create(properties), + OffloadPolicies.create(properties), scheduler, hdfsURI, basePath); } diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/CredentialsUtil.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/CredentialsUtil.java new file mode 100644 index 0000000000000..6b5bcd597b32b --- /dev/null +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/CredentialsUtil.java @@ -0,0 +1,25 @@ +package org.apache.bookkeeper.mledger.offload.jcloud; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; +import com.google.common.base.Strings; +import org.apache.pulsar.common.policies.data.OffloadPolicies; + +public class CredentialsUtil { + + /** + * Builds an AWS credential provider based on the offload options + * @return aws credential provider + */ + public static AWSCredentialsProvider getAWSCredentialProvider(OffloadPolicies offloadPolicies) { + if (Strings.isNullOrEmpty(offloadPolicies.getS3ManagedLedgerOffloadRole())) { + return DefaultAWSCredentialsProviderChain.getInstance(); + } else { + String roleName = offloadPolicies.getS3ManagedLedgerOffloadRole(); + String roleSessionName = offloadPolicies.getS3ManagedLedgerOffloadRoleSessionName(); + return new STSAssumeRoleSessionCredentialsProvider.Builder(roleName, roleSessionName).build(); + } + } + +} diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java index 58c5e901c8a9c..20fac68f1f822 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/JCloudLedgerOffloaderFactory.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.Map; -import java.util.Properties; import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.LedgerOffloaderFactory; import org.apache.bookkeeper.mledger.offload.jcloud.impl.BlobStoreManagedLedgerOffloader; @@ -43,12 +42,9 @@ public boolean isDriverSupported(String driverName) { } @Override - public BlobStoreManagedLedgerOffloader create(Properties properties, + public BlobStoreManagedLedgerOffloader create(OffloadPolicies offloadPolicies, Map userMetadata, - OrderedScheduler scheduler, - OffloadPolicies overrideOffloadPolicies) throws IOException { - TieredStorageConfigurationData data = TieredStorageConfigurationData.create(properties); - return BlobStoreManagedLedgerOffloader.create( - data.overridePolicies(overrideOffloadPolicies), userMetadata, scheduler); + OrderedScheduler scheduler) throws IOException { + return BlobStoreManagedLedgerOffloader.create(offloadPolicies, userMetadata, scheduler); } } diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/TieredStorageConfigurationData.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/TieredStorageConfigurationData.java deleted file mode 100644 index 1761285c90486..0000000000000 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/TieredStorageConfigurationData.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bookkeeper.mledger.offload.jcloud; - -import static org.apache.pulsar.common.util.FieldParser.value; - -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; -import com.google.common.base.Strings; -import java.io.Serializable; -import java.lang.reflect.Field; -import java.util.Arrays; -import java.util.Properties; -import lombok.Data; -import org.apache.bookkeeper.mledger.offload.jcloud.impl.BlobStoreManagedLedgerOffloader; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.common.policies.data.OffloadPolicies; - -/** - * Configuration for tiered storage. - */ -@Data -public class TieredStorageConfigurationData implements Serializable, Cloneable { - - /**** --- Ledger Offloading --- ****/ - // Driver to use to offload old data to long term storage - private String managedLedgerOffloadDriver = null; - - // Maximum number of thread pool threads for ledger offloading - private int managedLedgerOffloadMaxThreads = 2; - - // For Amazon S3 ledger offload, AWS region - private String s3ManagedLedgerOffloadRegion = null; - - // For Amazon S3 ledger offload, Bucket to place offloaded ledger into - private String s3ManagedLedgerOffloadBucket = null; - - // For Amazon S3 ledger offload, Alternative endpoint to connect to (useful for testing) - private String s3ManagedLedgerOffloadServiceEndpoint = null; - - // For Amazon S3 ledger offload, Max block size in bytes. - private int s3ManagedLedgerOffloadMaxBlockSizeInBytes = 64 * 1024 * 1024; // 64MB - - // For Amazon S3 ledger offload, Read buffer size in bytes. - private int s3ManagedLedgerOffloadReadBufferSizeInBytes = 1024 * 1024; // 1MB - - // For Amazon S3 ledger offload, provide a role to assume before writing to s3 - private String s3ManagedLedgerOffloadRole = null; - - // For Amazon S3 ledger offload, provide a role session name when using a role - private String s3ManagedLedgerOffloadRoleSessionName = "pulsar-s3-offload"; - - // For Google Cloud Storage ledger offload, region where offload bucket is located. - // reference this page for more details: https://cloud.google.com/storage/docs/bucket-locations - private String gcsManagedLedgerOffloadRegion = null; - - // For Google Cloud Storage ledger offload, Bucket to place offloaded ledger into - private String gcsManagedLedgerOffloadBucket = null; - - // For Google Cloud Storage ledger offload, Max block size in bytes. - private int gcsManagedLedgerOffloadMaxBlockSizeInBytes = 64 * 1024 * 1024; // 64MB - - // For Google Cloud Storage ledger offload, Read buffer size in bytes. - private int gcsManagedLedgerOffloadReadBufferSizeInBytes = 1024 * 1024; // 1MB - - // For Google Cloud Storage, path to json file containing service account credentials. - // For more details, see the "Service Accounts" section of https://support.google.com/googleapi/answer/6158849 - private String gcsManagedLedgerOffloadServiceAccountKeyFile = null; - - /** - * Builds an AWS credential provider based on the offload options - * @return aws credential provider - */ - public AWSCredentialsProvider getAWSCredentialProvider() { - if (Strings.isNullOrEmpty(this.getS3ManagedLedgerOffloadRole())) { - return DefaultAWSCredentialsProviderChain.getInstance(); - } else { - String roleName = this.getS3ManagedLedgerOffloadRole(); - String roleSessionName = this.getS3ManagedLedgerOffloadRoleSessionName(); - return new STSAssumeRoleSessionCredentialsProvider.Builder(roleName, roleSessionName).build(); - } - } - - /** - * Create a tiered storage configuration from the provided properties. - * - * @param properties the configuration properties - * @return tiered storage configuration - */ - public static TieredStorageConfigurationData create(Properties properties) { - TieredStorageConfigurationData data = new TieredStorageConfigurationData(); - Field[] fields = TieredStorageConfigurationData.class.getDeclaredFields(); - Arrays.stream(fields).forEach(f -> { - if (properties.containsKey(f.getName())) { - try { - f.setAccessible(true); - f.set(data, value((String) properties.get(f.getName()), f)); - } catch (Exception e) { - throw new IllegalArgumentException(String.format("failed to initialize %s field while setting value %s", - f.getName(), properties.get(f.getName())), e); - } - } - }); - return data; - } - - public TieredStorageConfigurationData overridePolicies(OffloadPolicies offloadPolicies) { - if (offloadPolicies == null) { - return this; - } - if (BlobStoreManagedLedgerOffloader.isGcsDriver(managedLedgerOffloadDriver)) { - this.gcsManagedLedgerOffloadBucket = offloadPolicies.getBucket(); - if (StringUtils.isNotEmpty(offloadPolicies.getRegion())) { - this.gcsManagedLedgerOffloadRegion = offloadPolicies.getRegion(); - } - this.gcsManagedLedgerOffloadMaxBlockSizeInBytes = offloadPolicies.getMaxBlockSizeInBytes(); - this.gcsManagedLedgerOffloadReadBufferSizeInBytes = offloadPolicies.getReadBufferSizeInBytes(); - } else if (BlobStoreManagedLedgerOffloader.isS3Driver(managedLedgerOffloadDriver)) { - this.s3ManagedLedgerOffloadBucket = offloadPolicies.getBucket(); - if (StringUtils.isNotEmpty(offloadPolicies.getRegion())) { - this.s3ManagedLedgerOffloadRegion = offloadPolicies.getRegion(); - } - if (StringUtils.isNotEmpty(offloadPolicies.getEndpoint())) { - this.s3ManagedLedgerOffloadServiceEndpoint = offloadPolicies.getEndpoint(); - } - this.s3ManagedLedgerOffloadMaxBlockSizeInBytes = offloadPolicies.getMaxBlockSizeInBytes(); - this.s3ManagedLedgerOffloadReadBufferSizeInBytes = offloadPolicies.getReadBufferSizeInBytes(); - } - return this; - } - -} diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java index 7f7acaf25ccd6..0c92b96695e72 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java @@ -44,10 +44,11 @@ import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.LedgerOffloader; import org.apache.bookkeeper.mledger.offload.jcloud.BlockAwareSegmentInputStream; +import org.apache.bookkeeper.mledger.offload.jcloud.CredentialsUtil; import org.apache.bookkeeper.mledger.offload.jcloud.OffloadIndexBlock; -import org.apache.bookkeeper.mledger.offload.jcloud.TieredStorageConfigurationData; import org.apache.bookkeeper.mledger.offload.jcloud.OffloadIndexBlockBuilder; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.jcloud.shade.com.google.common.base.Supplier; import org.jclouds.Constants; import org.jclouds.ContextBuilder; @@ -181,16 +182,19 @@ private static Pair createBlobStore(String driver, // offload driver metadata to be stored as part of the original ledger metadata private final String offloadDriverName; + private static OffloadPolicies offloadPolicies; + @VisibleForTesting - static BlobStoreManagedLedgerOffloader create(TieredStorageConfigurationData conf, + static BlobStoreManagedLedgerOffloader create(OffloadPolicies conf, OrderedScheduler scheduler) throws IOException { return create(conf, Maps.newHashMap(), scheduler); } - public static BlobStoreManagedLedgerOffloader create(TieredStorageConfigurationData conf, + public static BlobStoreManagedLedgerOffloader create(OffloadPolicies conf, Map userMetadata, OrderedScheduler scheduler) throws IOException { + offloadPolicies = conf; String driver = conf.getManagedLedgerOffloadDriver(); if (!driverSupported(driver)) { throw new IOException( @@ -232,7 +236,8 @@ public static BlobStoreManagedLedgerOffloader create(TieredStorageConfigurationD maxBlockSize, readBufferSize, endpoint, region, credentials, userMetadata); } - public static Supplier getCredentials(String driver, TieredStorageConfigurationData conf) throws IOException { + public static Supplier getCredentials(String driver, + OffloadPolicies conf) throws IOException { // credentials: // for s3, get by DefaultAWSCredentialsProviderChain. // for gcs, use downloaded file 'google_creds.json', which contains service account key by @@ -252,7 +257,7 @@ public static Supplier getCredentials(String driver, TieredStorageC throw new IOException(ioe); } } else if (isS3Driver(driver)) { - AWSCredentialsProvider credsChain = conf.getAWSCredentialProvider(); + AWSCredentialsProvider credsChain = CredentialsUtil.getAWSCredentialProvider(conf); // try and get creds before starting... if we can't fetch // creds on boot, we want to fail try { @@ -587,6 +592,23 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, public interface VersionCheck { void check(String key, Blob blob) throws IOException; } + + @Override + public OffloadPolicies getOffloadPolicies() { + return offloadPolicies; + } + + @Override + public void close() { + if (writeBlobStore != null) { + writeBlobStore.getContext().close(); + } + for (BlobStore readBlobStore : readBlobStores.values()) { + if (readBlobStore != null) { + readBlobStore.getContext().close(); + } + } + } } diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreTestBase.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreTestBase.java index 183424177e9b7..2725c051d1269 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreTestBase.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreTestBase.java @@ -21,11 +21,21 @@ import org.jclouds.ContextBuilder; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.BlobStoreContext; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.IObjectFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.ObjectFactory; + +@PrepareForTest({CredentialsUtil.class}) +@PowerMockIgnore({ + "org.apache.logging.log4j.*", + "org.apache.pulsar.jcloud.shade.com.google.common.*", + "org.jclouds.*"}) public class BlobStoreTestBase { private static final Logger log = LoggerFactory.getLogger(BlobStoreTestBase.class); @@ -79,4 +89,10 @@ public void tearDown() { } } + @ObjectFactory + // Necessary to make PowerMockito.mockStatic work with TestNG. + public IObjectFactory getObjectFactory() { + return new org.powermock.modules.testng.PowerMockObjectFactory(); + } + } diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java index a3c4972365007..8aa977a0e8366 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java @@ -31,7 +31,6 @@ import com.google.common.util.concurrent.MoreExecutors; import java.io.File; import java.io.IOException; -import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -53,8 +52,9 @@ import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.mledger.LedgerOffloader; import org.apache.bookkeeper.mledger.offload.jcloud.BlobStoreTestBase; -import org.apache.bookkeeper.mledger.offload.jcloud.TieredStorageConfigurationData; +import org.apache.bookkeeper.mledger.offload.jcloud.CredentialsUtil; import org.apache.bookkeeper.util.ZkUtils; +import org.apache.pulsar.common.policies.data.OffloadPolicies; import org.apache.pulsar.jcloud.shade.com.google.common.base.Supplier; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.MockZooKeeper; @@ -64,13 +64,14 @@ import org.jclouds.blobstore.options.CopyOptions; import org.jclouds.domain.Credentials; import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.Test; import org.testng.collections.Maps; -class BlobStoreManagedLedgerOffloaderTest extends BlobStoreTestBase { +public class BlobStoreManagedLedgerOffloaderTest extends BlobStoreTestBase { private static final Logger log = LoggerFactory.getLogger(BlobStoreManagedLedgerOffloaderTest.class); private static MockZooKeeper createMockZooKeeper() throws Exception { @@ -90,7 +91,7 @@ private static MockZooKeeper createMockZooKeeper() throws Exception { final OrderedScheduler scheduler; final PulsarMockBookKeeper bk; - BlobStoreManagedLedgerOffloaderTest() throws Exception { + public BlobStoreManagedLedgerOffloaderTest() throws Exception { scheduler = OrderedScheduler.newSchedulerBuilder().numThreads(1).name("offloader").build(); bk = new PulsarMockBookKeeper(createMockZooKeeper(), scheduler.chooseThread(this)); } @@ -154,7 +155,7 @@ public void testBucketDoesNotExist() throws Exception { @Test public void testNoRegionConfigured() throws Exception { - TieredStorageConfigurationData conf = new TieredStorageConfigurationData(); + OffloadPolicies conf = new OffloadPolicies(); conf.setManagedLedgerOffloadDriver("s3"); conf.setS3ManagedLedgerOffloadBucket(BUCKET); @@ -168,7 +169,7 @@ public void testNoRegionConfigured() throws Exception { @Test public void testNoBucketConfigured() throws Exception { - TieredStorageConfigurationData conf = new TieredStorageConfigurationData(); + OffloadPolicies conf = new OffloadPolicies(); conf.setManagedLedgerOffloadDriver("s3"); conf.setS3ManagedLedgerOffloadRegion("eu-west-1"); @@ -182,7 +183,7 @@ public void testNoBucketConfigured() throws Exception { @Test public void testSmallBlockSizeConfigured() throws Exception { - TieredStorageConfigurationData conf = new TieredStorageConfigurationData(); + OffloadPolicies conf = new OffloadPolicies(); conf.setManagedLedgerOffloadDriver("s3"); conf.setS3ManagedLedgerOffloadRegion("eu-west-1"); conf.setS3ManagedLedgerOffloadBucket(BUCKET); @@ -198,7 +199,7 @@ public void testSmallBlockSizeConfigured() throws Exception { @Test public void testGcsNoKeyPath() throws Exception { - TieredStorageConfigurationData conf = new TieredStorageConfigurationData(); + OffloadPolicies conf = new OffloadPolicies(); conf.setManagedLedgerOffloadDriver("google-cloud-storage"); conf.setGcsManagedLedgerOffloadBucket(BUCKET); @@ -213,7 +214,7 @@ public void testGcsNoKeyPath() throws Exception { @Test public void testGcsNoBucketConfigured() throws Exception { - TieredStorageConfigurationData conf = new TieredStorageConfigurationData(); + OffloadPolicies conf = new OffloadPolicies(); conf.setManagedLedgerOffloadDriver("google-cloud-storage"); File tmpKeyFile = File.createTempFile("gcsOffload", "json"); conf.setGcsManagedLedgerOffloadServiceAccountKeyFile(tmpKeyFile.getAbsolutePath()); @@ -229,7 +230,7 @@ public void testGcsNoBucketConfigured() throws Exception { @Test public void testGcsSmallBlockSizeConfigured() throws Exception { - TieredStorageConfigurationData conf = new TieredStorageConfigurationData(); + OffloadPolicies conf = new OffloadPolicies(); conf.setManagedLedgerOffloadDriver("google-cloud-storage"); File tmpKeyFile = File.createTempFile("gcsOffload", "json"); conf.setGcsManagedLedgerOffloadServiceAccountKeyFile(tmpKeyFile.getAbsolutePath()); @@ -247,37 +248,35 @@ public void testGcsSmallBlockSizeConfigured() throws Exception { @Test public void testS3DriverConfiguredWell() throws Exception { - TieredStorageConfigurationData conf = new TieredStorageConfigurationData() { + PowerMockito.mockStatic(CredentialsUtil.class); + PowerMockito.when(CredentialsUtil.getAWSCredentialProvider(any())).thenReturn(new AWSCredentialsProvider() { @Override - public AWSCredentialsProvider getAWSCredentialProvider() { - return new AWSCredentialsProvider() { + public AWSCredentials getCredentials() { + return new AWSSessionCredentials() { @Override - public AWSCredentials getCredentials() { - return new AWSSessionCredentials() { - @Override - public String getSessionToken() { - return "token"; - } - - @Override - public String getAWSAccessKeyId() { - return "access"; - } - - @Override - public String getAWSSecretKey() { - return "secret"; - } - }; + public String getSessionToken() { + return "token"; } @Override - public void refresh() { + public String getAWSAccessKeyId() { + return "access"; + } + @Override + public String getAWSSecretKey() { + return "secret"; } }; } - }; + + @Override + public void refresh() { + + } + }); + + OffloadPolicies conf = new OffloadPolicies(); conf.setManagedLedgerOffloadDriver("s3"); conf.setS3ManagedLedgerOffloadBucket(BUCKET); conf.setS3ManagedLedgerOffloadServiceEndpoint("http://fake.s3.end.point"); @@ -644,8 +643,8 @@ public void testReadUnknownIndexVersion() throws Exception { @Test public void testSessionCredentialSupplier() throws Exception { - TieredStorageConfigurationData mock = mock(TieredStorageConfigurationData.class); - Mockito.when(mock.getAWSCredentialProvider()).thenReturn(new AWSCredentialsProvider() { + PowerMockito.mockStatic(CredentialsUtil.class); + PowerMockito.when(CredentialsUtil.getAWSCredentialProvider(any())).thenReturn(new AWSCredentialsProvider() { @Override public AWSCredentials getCredentials() { return new AWSSessionCredentials() { @@ -672,7 +671,7 @@ public void refresh() { } }); - Supplier creds = BlobStoreManagedLedgerOffloader.getCredentials("aws-s3", mock); + Supplier creds = BlobStoreManagedLedgerOffloader.getCredentials("aws-s3", any()); Assert.assertTrue(creds.get() instanceof SessionCredentials); SessionCredentials sessCreds = (SessionCredentials) creds.get(); From 9874009aab24e4d0a5c8dd6bf16cfc82d44f8600 Mon Sep 17 00:00:00 2001 From: "gao.ran" Date: Wed, 5 Feb 2020 21:23:46 +0800 Subject: [PATCH 06/11] add licensed header for CredentialsUtil --- .../offload/jcloud/CredentialsUtil.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/CredentialsUtil.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/CredentialsUtil.java index 6b5bcd597b32b..4971a6d0e1b70 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/CredentialsUtil.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/CredentialsUtil.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.bookkeeper.mledger.offload.jcloud; import com.amazonaws.auth.AWSCredentialsProvider; From a898f240997d4e2746a2f9d8dbc468149502b3f3 Mon Sep 17 00:00:00 2001 From: "gao.ran" Date: Wed, 5 Feb 2020 22:29:45 +0800 Subject: [PATCH 07/11] 1. modify the `set-offload` to `set-offload-policies`; 2. modify the `get-offload` to `get-offload-policies`; 3. add the OffloadPolicies validate in NamespacesBase. --- .../broker/admin/impl/NamespacesBase.java | 42 +++++++++++------- .../pulsar/broker/admin/v2/Namespaces.java | 12 ++--- .../broker/admin/AdminApiOffloadTest.java | 4 +- .../pulsar/client/admin/Namespaces.java | 6 +-- .../client/admin/internal/NamespacesImpl.java | 8 ++-- .../pulsar/admin/cli/PulsarAdminToolTest.java | 8 ++-- .../pulsar/admin/cli/CmdNamespaces.java | 13 +++--- .../common/policies/data/OffloadPolicies.java | 44 +++++++++++++++++++ .../pulsar/sql/presto/PulsarSplitManager.java | 2 +- site2/docs/reference-pulsar-admin.md | 12 ++--- 10 files changed, 102 insertions(+), 49 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 51f3afd893806..80025746f8a95 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -2189,29 +2189,17 @@ private void mutatePolicy(Function policyTransformation, } } - protected void internalSetOffload(OffloadPolicies offload) { + protected void internalSetOffloadPolicies(OffloadPolicies offloadPolicies) { validateAdminAccessForTenant(namespaceName.getTenant()); validatePoliciesReadOnlyAccess(); + validateOffloadPolicies(offloadPolicies); try { Stat nodeStat = new Stat(); final String path = path(POLICIES, namespaceName.toString()); byte[] content = globalZk().getData(path, null, nodeStat); Policies policies = jsonMapper().readValue(content, Policies.class); - if (StringUtils.isEmpty(offload.getManagedLedgerOffloadDriver())) { - log.warn("[{}] Failed to update offload configuration for namespace {}: driver must be specified", - clientAppId(), namespaceName); - throw new RestException(Status.PRECONDITION_FAILED, - "The driver must be specified for namespace offload."); - } - if (StringUtils.isEmpty(offload.getS3ManagedLedgerOffloadBucket()) - && StringUtils.isEmpty(offload.getGcsManagedLedgerOffloadBucket())) { - log.warn("[{}] Failed to update offload configuration for namespace {}: bucket must be specified", - clientAppId(), namespaceName); - throw new RestException(Status.PRECONDITION_FAILED, - "The bucket must be specified for namespace offload."); - } - policies.offload_policies = offload; + policies.offload_policies = offloadPolicies; globalZk().setData(path, jsonMapper().writeValueAsBytes(policies), nodeStat.getVersion()); policiesCache().invalidate(path(POLICIES, namespaceName.toString())); log.info("[{}] Successfully updated offload configuration: namespace={}, map={}", clientAppId(), @@ -2233,7 +2221,29 @@ protected void internalSetOffload(OffloadPolicies offload) { } } - protected OffloadPolicies internalGetOffload() { + private void validateOffloadPolicies(OffloadPolicies offloadPolicies) { + if (offloadPolicies == null) { + log.warn("[{}] Failed to update offload configuration for namespace {}: offloadPolicies is null", + clientAppId(), namespaceName); + throw new RestException(Status.PRECONDITION_FAILED, + "The offloadPolicies must be specified for namespace offload."); + } + if (!offloadPolicies.driverSupported()) { + log.warn("[{}] Failed to update offload configuration for namespace {}: " + + "driver is not supported, support value: {}", + clientAppId(), namespaceName, OffloadPolicies.getSupportedDriverNames()); + throw new RestException(Status.PRECONDITION_FAILED, + "The driver is not supported, support value: " + OffloadPolicies.getSupportedDriverNames()); + } + if (!offloadPolicies.bucketValid()) { + log.warn("[{}] Failed to update offload configuration for namespace {}: bucket must be specified", + clientAppId(), namespaceName); + throw new RestException(Status.PRECONDITION_FAILED, + "The bucket must be specified for namespace offload."); + } + } + + protected OffloadPolicies internalGetOffloadPolicies() { validateAdminAccessForTenant(namespaceName.getTenant()); Policies policies = getNamespacePolicies(namespaceName); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index 278b2ffe475aa..c0da9a889fe08 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -1078,29 +1078,29 @@ public void setSchemaValidtionEnforced(@PathParam("tenant") String tenant, } @POST - @Path("/{tenant}/{namespace}/offload") + @Path("/{tenant}/{namespace}/offloadPolicies") @ApiOperation(value = " Set offload configuration on a namespace.") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), @ApiResponse(code = 412, message = "Bucket must be specified") }) - public void setOffload(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace, + public void setOffloadPolicies(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace, OffloadPolicies offload) { validateNamespaceName(tenant, namespace); - internalSetOffload(offload); + internalSetOffloadPolicies(offload); } @GET - @Path("/{tenant}/{namespace}/offload") + @Path("/{tenant}/{namespace}/offloadPolicies") @ApiOperation(value = "Get offload configuration on a namespace.") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) - public OffloadPolicies getOffload(@PathParam("tenant") String tenant, + public OffloadPolicies getOffloadPolicies(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace) { validateNamespaceName(tenant, namespace); - return internalGetOffload(); + return internalGetOffloadPolicies(); } private static final Logger log = LoggerFactory.getLogger(Namespaces.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java index f85fc55d621da..25c45dfd29a7d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java @@ -150,8 +150,8 @@ public void testOffloadPolicies() throws Exception { OffloadPolicies offload1 = OffloadPolicies.create( driver, region, bucket, endpoint, 100, 100); - admin.namespaces().setOffload(namespaceName, offload1); - OffloadPolicies offload2 = admin.namespaces().getOffload(namespaceName); + admin.namespaces().setOffloadPolicies(namespaceName, offload1); + OffloadPolicies offload2 = admin.namespaces().getOffloadPolicies(namespaceName); Assert.assertEquals(offload1, offload2); } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java index b64f624b3774c..c2d9f7f360222 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Namespaces.java @@ -1659,7 +1659,7 @@ void setIsAllowAutoUpdateSchema(String namespace, boolean isAllowAutoUpdateSchem /** * Set the offload configuration for all the topics in a namespace. *

- * Set the offload configuration in a namespace. This operation requires Pulsar super-user access. + * Set the offload configuration in a namespace. This operation requires pulsar tenant access. *

* Request parameter example: *

@@ -1690,7 +1690,7 @@ void setIsAllowAutoUpdateSchema(String namespace, boolean isAllowAutoUpdateSchem * @throws PulsarAdminException * Unexpected error */ - void setOffload(String namespace, OffloadPolicies offloadPolicies) throws PulsarAdminException; + void setOffloadPolicies(String namespace, OffloadPolicies offloadPolicies) throws PulsarAdminException; /** * Get the offload configuration for a namespace. @@ -1723,6 +1723,6 @@ void setIsAllowAutoUpdateSchema(String namespace, boolean isAllowAutoUpdateSchem * @throws PulsarAdminException * Unexpected error */ - OffloadPolicies getOffload(String namespace) throws PulsarAdminException; + OffloadPolicies getOffloadPolicies(String namespace) throws PulsarAdminException; } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java index 8ce5807f5bebd..bfcdfe576249a 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/NamespacesImpl.java @@ -1031,10 +1031,10 @@ public void setIsAllowAutoUpdateSchema(String namespace, boolean isAllowAutoUpda } @Override - public void setOffload(String namespace, OffloadPolicies offloadPolicies) throws PulsarAdminException { + public void setOffloadPolicies(String namespace, OffloadPolicies offloadPolicies) throws PulsarAdminException { try { NamespaceName ns = NamespaceName.get(namespace); - WebTarget path = namespacePath(ns, "offload"); + WebTarget path = namespacePath(ns, "offloadPolicies"); request(path).post(Entity.entity(offloadPolicies, MediaType.APPLICATION_JSON), ErrorData.class); } catch (Exception e) { throw getApiException(e); @@ -1042,10 +1042,10 @@ public void setOffload(String namespace, OffloadPolicies offloadPolicies) throws } @Override - public OffloadPolicies getOffload(String namespace) throws PulsarAdminException { + public OffloadPolicies getOffloadPolicies(String namespace) throws PulsarAdminException { try { NamespaceName ns = NamespaceName.get(namespace); - WebTarget path = namespacePath(ns, "offload"); + WebTarget path = namespacePath(ns, "offloadPolicies"); return request(path).get(OffloadPolicies.class); } catch (Exception e) { throw getApiException(e); diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index 28a22cff97e82..74fda36ed29c1 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -489,13 +489,13 @@ void namespaces() throws Exception { namespaces.run(split("clear-offload-deletion-lag myprop/clust/ns1")); verify(mockNamespaces).clearOffloadDeleteLag("myprop/clust/ns1"); - namespaces.run(split("set-offload myprop/clust/ns1 -r test-region -d aws-s3 -b test-bucket -e http://test.endpoint -mbs 32M -rbs 5M")); - verify(mockNamespaces).setOffload("myprop/clust/ns1", + namespaces.run(split("set-offload-policies myprop/clust/ns1 -r test-region -d aws-s3 -b test-bucket -e http://test.endpoint -mbs 32M -rbs 5M")); + verify(mockNamespaces).setOffloadPolicies("myprop/clust/ns1", OffloadPolicies.create("aws-s3", "test-region", "test-bucket", "http://test.endpoint", 32 * 1024 * 1024, 5 * 1024 * 1024)); - namespaces.run(split("get-offload myprop/clust/ns1")); - verify(mockNamespaces).getOffload("myprop/clust/ns1"); + namespaces.run(split("get-offload-policies myprop/clust/ns1")); + verify(mockNamespaces).getOffloadPolicies("myprop/clust/ns1"); } @Test diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index c24506773198a..dd3f9afb60569 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -30,7 +30,6 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; @@ -1291,7 +1290,7 @@ void run() throws PulsarAdminException { } @Parameters(commandDescription = "Set the offload policies for a namespace") - private class SetOffload extends CliCommand { + private class SetOffloadPolicies extends CliCommand { @Parameter(description = "tenant/namespace", required = true) private java.util.List params; @@ -1397,19 +1396,19 @@ && maxValueCheck("ReadBufferSize", readBufferSize, Integer.MAX_VALUE)) { OffloadPolicies offloadPolicies = OffloadPolicies.create(driver, region, bucket, endpoint, maxBlockSizeInBytes, readBufferSizeInBytes); - admin.namespaces().setOffload(namespace, offloadPolicies); + admin.namespaces().setOffloadPolicies(namespace, offloadPolicies); } } @Parameters(commandDescription = "Get the offload policies for a namespace") - private class GetOffload extends CliCommand { + private class GetOffloadPolicies extends CliCommand { @Parameter(description = "tenant/namespace\n", required = true) private java.util.List params; @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - print(admin.namespaces().getOffload(namespace)); + print(admin.namespaces().getOffloadPolicies(namespace)); } } @@ -1516,7 +1515,7 @@ public CmdNamespaces(PulsarAdmin admin) { jcommander.addCommand("get-schema-validation-enforce", new GetSchemaValidationEnforced()); jcommander.addCommand("set-schema-validation-enforce", new SetSchemaValidationEnforced()); - jcommander.addCommand("set-offload", new SetOffload()); - jcommander.addCommand("get-offload", new GetOffload()); + jcommander.addCommand("set-offload-policies", new SetOffloadPolicies()); + jcommander.addCommand("get-offload-policies", new GetOffloadPolicies()); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java index b8ff00c3c3318..a248476a50c5e 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java @@ -26,6 +26,7 @@ import java.util.Objects; import java.util.Properties; import lombok.Data; +import org.apache.commons.lang3.StringUtils; /** * Definition of the offload policies. @@ -103,6 +104,49 @@ public static OffloadPolicies create(Properties properties) { return data; } + public boolean driverSupported() { + return Arrays.stream(DRIVER_NAMES).anyMatch(d -> d.equalsIgnoreCase(this.managedLedgerOffloadDriver)); + } + + public static String getSupportedDriverNames() { + return StringUtils.join(DRIVER_NAMES, ","); + } + + public boolean isS3Driver() { + if (managedLedgerOffloadDriver == null) { + return false; + } + return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES[0]) || managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES[1]); + } + + public boolean isGcsDriver() { + if (managedLedgerOffloadDriver == null) { + return false; + } + return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES[2]); + } + + public boolean isFileSystemDriver() { + if (managedLedgerOffloadDriver == null) { + return false; + } + return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES[3]); + } + + public boolean bucketValid() { + if (managedLedgerOffloadDriver == null) { + return false; + } + if (isS3Driver()) { + return StringUtils.isNotEmpty(s3ManagedLedgerOffloadBucket); + } else if (isGcsDriver()) { + return StringUtils.isNotEmpty(gcsManagedLedgerOffloadBucket); + } else if (isFileSystemDriver()) { + return true; + } + return false; + } + @Override public int hashCode() { return Objects.hash( diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java index b093b1d824c6d..f6060526b6874 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java @@ -123,7 +123,7 @@ public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHand Collection splits; try { - OffloadPolicies offloadPolicies = this.pulsarAdmin.namespaces().getOffload(topicName.getNamespace()); + OffloadPolicies offloadPolicies = this.pulsarAdmin.namespaces().getOffloadPolicies(topicName.getNamespace()); if (!PulsarConnectorUtils.isPartitionedTopic(topicName, this.pulsarAdmin)) { splits = getSplitsNonPartitionedTopic( numSplits, topicName, tableHandle, schemaInfo, tupleDomain, offloadPolicies); diff --git a/site2/docs/reference-pulsar-admin.md b/site2/docs/reference-pulsar-admin.md index 7ca1790d35f45..e97acc0d178ad 100644 --- a/site2/docs/reference-pulsar-admin.md +++ b/site2/docs/reference-pulsar-admin.md @@ -878,8 +878,8 @@ Subcommands * `clear-offload-deletion-lag` * `get-schema-autoupdate-strategy` * `set-schema-autoupdate-strategy` -* `set-offload` -* `get-offload` +* `set-offload-policies` +* `get-offload-policies` ### `list` @@ -2246,20 +2246,20 @@ Options |`-t`, `--type`|The type of the schema (avro or json)|| -### `get-offload` +### `get-offload-policies` Get the offload policy for a namespace Usage ```bash -$ pulsar-admin namespaces get-offload tenant/namespace +$ pulsar-admin namespaces get-offload-policies tenant/namespace ``` -### `set-offload` +### `set-offload-policies` Set the offload policy for a namespace Usage ```bash -$ pulsar-admin namespaces set-offload tenant/namespace +$ pulsar-admin namespaces set-offload-policies tenant/namespace ``` Options From 0a4d81ee6c58551ee6029f9381931eac5fa8646e Mon Sep 17 00:00:00 2001 From: "gao.ran" Date: Thu, 6 Feb 2020 11:48:45 +0800 Subject: [PATCH 08/11] checkstyle fix --- .../main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java | 4 ++-- .../apache/pulsar/common/policies/data/OffloadPolicies.java | 3 ++- .../java/org/apache/pulsar/sql/presto/PulsarSplitManager.java | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index dd3f9afb60569..2d569eac24d2a 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -1366,8 +1366,8 @@ void run() throws PulsarAdminException { if (!driverSupported(driver)) { throw new ParameterException( - "No offloader found for driver '" + driver + "'." + " Please make sure " + - "you dropped the offloader nar packages under `${PULSAR_HOME}/offloaders`."); + "The driver " + driver + " is not supported, " + + "(Possible values: S3, aws-s3, google-cloud-storage)."); } if (isS3Driver(driver) && Strings.isNullOrEmpty(region) && Strings.isNullOrEmpty(endpoint)) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java index a248476a50c5e..f46b44f031fbe 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPolicies.java @@ -116,7 +116,8 @@ public boolean isS3Driver() { if (managedLedgerOffloadDriver == null) { return false; } - return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES[0]) || managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES[1]); + return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES[0]) + || managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES[1]); } public boolean isGcsDriver() { diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java index f6060526b6874..848b9ccd1789c 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSplitManager.java @@ -123,7 +123,8 @@ public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHand Collection splits; try { - OffloadPolicies offloadPolicies = this.pulsarAdmin.namespaces().getOffloadPolicies(topicName.getNamespace()); + OffloadPolicies offloadPolicies = this.pulsarAdmin.namespaces() + .getOffloadPolicies(topicName.getNamespace()); if (!PulsarConnectorUtils.isPartitionedTopic(topicName, this.pulsarAdmin)) { splits = getSplitsNonPartitionedTopic( numSplits, topicName, tableHandle, schemaInfo, tupleDomain, offloadPolicies); From 4300e2242d5ab5260e402c01e2847cd117afeec7 Mon Sep 17 00:00:00 2001 From: "gao.ran" Date: Sat, 8 Feb 2020 22:59:09 +0800 Subject: [PATCH 09/11] 1. modify internalSetOffloadPolicies method to asynchronous; 2. modify setOffloadPolicies api error message. --- .../broker/admin/impl/NamespacesBase.java | 22 +++++++++++++------ .../pulsar/broker/admin/v2/Namespaces.java | 14 ++++++++---- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 80025746f8a95..437260322c6ab 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -2189,7 +2189,7 @@ private void mutatePolicy(Function policyTransformation, } } - protected void internalSetOffloadPolicies(OffloadPolicies offloadPolicies) { + protected void internalSetOffloadPolicies(AsyncResponse asyncResponse, OffloadPolicies offloadPolicies) { validateAdminAccessForTenant(namespaceName.getTenant()); validatePoliciesReadOnlyAccess(); validateOffloadPolicies(offloadPolicies); @@ -2200,24 +2200,32 @@ protected void internalSetOffloadPolicies(OffloadPolicies offloadPolicies) { byte[] content = globalZk().getData(path, null, nodeStat); Policies policies = jsonMapper().readValue(content, Policies.class); policies.offload_policies = offloadPolicies; - globalZk().setData(path, jsonMapper().writeValueAsBytes(policies), nodeStat.getVersion()); - policiesCache().invalidate(path(POLICIES, namespaceName.toString())); + globalZk().setData(path, jsonMapper().writeValueAsBytes(policies), nodeStat.getVersion(), + (rc, path1, ctx, stat) -> { + if (rc == KeeperException.Code.OK.intValue()) { + policiesCache().invalidate(path(POLICIES, namespaceName.toString())); + } else { + asyncResponse.resume(KeeperException + .create(KeeperException.Code.get(rc), "failed to create update partitions")); + } + }, null); log.info("[{}] Successfully updated offload configuration: namespace={}, map={}", clientAppId(), namespaceName, jsonMapper().writeValueAsString(policies.offload_policies)); + asyncResponse.resume(Response.noContent().build()); } catch (KeeperException.NoNodeException e) { log.warn("[{}] Failed to update offload configuration for namespace {}: does not exist", clientAppId(), namespaceName); - throw new RestException(Status.NOT_FOUND, "Namespace does not exist"); + asyncResponse.resume(new RestException(Status.NOT_FOUND, "Namespace does not exist")); } catch (KeeperException.BadVersionException e) { log.warn("[{}] Failed to update offload configuration for namespace {}: concurrent modification", clientAppId(), namespaceName); - throw new RestException(Status.CONFLICT, "Concurrent modification"); + asyncResponse.resume(new RestException(Status.CONFLICT, "Concurrent modification")); } catch (RestException pfe) { - throw pfe; + asyncResponse.resume(pfe); } catch (Exception e) { log.error("[{}] Failed to update offload configuration for namespace {}", clientAppId(), namespaceName, e); - throw new RestException(e); + asyncResponse.resume(new RestException(e)); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index c0da9a889fe08..9b313b3e4e6a2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -1084,11 +1084,17 @@ public void setSchemaValidtionEnforced(@PathParam("tenant") String tenant, @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist"), @ApiResponse(code = 409, message = "Concurrent modification"), - @ApiResponse(code = 412, message = "Bucket must be specified") }) + @ApiResponse(code = 412, message = "OffloadPolicies is empty or driver is not supported or bucket is not valid") }) public void setOffloadPolicies(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace, - OffloadPolicies offload) { - validateNamespaceName(tenant, namespace); - internalSetOffloadPolicies(offload); + OffloadPolicies offload, @Suspended final AsyncResponse asyncResponse) { + try { + validateNamespaceName(tenant, namespace); + internalSetOffloadPolicies(asyncResponse, offload); + } catch (WebApplicationException wae) { + asyncResponse.resume(wae); + } catch (Exception e) { + asyncResponse.resume(new RestException(e)); + } } @GET From 9fc437eec73d73d79cc531b511bf7c12c8f04827 Mon Sep 17 00:00:00 2001 From: "gao.ran" Date: Sun, 9 Feb 2020 12:04:32 +0800 Subject: [PATCH 10/11] exception process fix --- .../broker/admin/impl/NamespacesBase.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 7623aefc8b75e..63bb7602896ed 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -2287,23 +2287,23 @@ protected void internalSetOffloadPolicies(AsyncResponse asyncResponse, OffloadPo if (rc == KeeperException.Code.OK.intValue()) { policiesCache().invalidate(path(POLICIES, namespaceName.toString())); } else { - asyncResponse.resume(KeeperException - .create(KeeperException.Code.get(rc), "failed to create update partitions")); + String errorMsg = String.format( + "[%s] Failed to update offload configuration for namespace %s", + clientAppId(), namespaceName); + if (rc == KeeperException.Code.NONODE.intValue()) { + log.warn("{} : does not exist", errorMsg); + asyncResponse.resume(new RestException(Status.NOT_FOUND, "Namespace does not exist")); + } else if (rc == KeeperException.Code.BADVERSION.intValue()) { + log.warn("{} : concurrent modification", errorMsg); + asyncResponse.resume(new RestException(Status.CONFLICT, "Concurrent modification")); + } else { + asyncResponse.resume(KeeperException.create(KeeperException.Code.get(rc), errorMsg)); + } } }, null); log.info("[{}] Successfully updated offload configuration: namespace={}, map={}", clientAppId(), namespaceName, jsonMapper().writeValueAsString(policies.offload_policies)); asyncResponse.resume(Response.noContent().build()); - } catch (KeeperException.NoNodeException e) { - log.warn("[{}] Failed to update offload configuration for namespace {}: does not exist", clientAppId(), - namespaceName); - asyncResponse.resume(new RestException(Status.NOT_FOUND, "Namespace does not exist")); - } catch (KeeperException.BadVersionException e) { - log.warn("[{}] Failed to update offload configuration for namespace {}: concurrent modification", - clientAppId(), namespaceName); - asyncResponse.resume(new RestException(Status.CONFLICT, "Concurrent modification")); - } catch (RestException pfe) { - asyncResponse.resume(pfe); } catch (Exception e) { log.error("[{}] Failed to update offload configuration for namespace {}", clientAppId(), namespaceName, e); From 998fcb530cc129c3aa6accd2793cde79618c170c Mon Sep 17 00:00:00 2001 From: "gao.ran" Date: Mon, 10 Feb 2020 17:52:17 +0800 Subject: [PATCH 11/11] fix unit test --- .../mledger/offload/jcloud/BlobStoreBackedInputStreamTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreBackedInputStreamTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreBackedInputStreamTest.java index 16784a553be9e..baefb95a13b00 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreBackedInputStreamTest.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/BlobStoreBackedInputStreamTest.java @@ -44,7 +44,7 @@ import org.testng.annotations.Test; @Slf4j -class BlobStoreBackedInputStreamTest extends BlobStoreTestBase { +public class BlobStoreBackedInputStreamTest extends BlobStoreTestBase { private static final Logger log = LoggerFactory.getLogger(BlobStoreBackedInputStreamTest.class); class RandomInputStream extends InputStream {