From 098c566d61e41a9f2e40244f4e6d8f909dd97495 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Fri, 24 Jun 2022 11:33:22 +0200 Subject: [PATCH 01/26] Store preprocess function info in Sink FunctionDetails --- .../apache/pulsar/common/io/SinkConfig.java | 3 + .../org/apache/pulsar/admin/cli/CmdSinks.java | 14 ++ .../apache/pulsar/functions/LocalRunner.java | 12 +- .../proto/src/main/proto/Function.proto | 1 + .../functions/utils/FunctionCommon.java | 29 ++-- .../functions/utils/FunctionConfigUtils.java | 8 +- .../functions/utils/SinkConfigUtils.java | 55 +++++-- .../functions/utils/SinkConfigUtilsTest.java | 24 +-- .../worker/rest/api/ComponentImpl.java | 139 +++++++++++++++--- .../worker/rest/api/FunctionsImpl.java | 110 ++------------ .../functions/worker/rest/api/SinksImpl.java | 137 ++++++++--------- .../worker/rest/api/SourcesImpl.java | 89 ++--------- .../rest/api/v3/SinkApiV3ResourceTest.java | 2 +- 13 files changed, 310 insertions(+), 313 deletions(-) diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SinkConfig.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SinkConfig.java index f1844e6090c97..ebb2adf2e825c 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SinkConfig.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SinkConfig.java @@ -91,4 +91,7 @@ public class SinkConfig { // to change behavior at runtime. Currently, this primarily used by the KubernetesManifestCustomizer // interface private String customRuntimeOptions; + private String preprocessFunction; + private String preprocessFunctionClassName; + private String preprocessFunctionConfig; } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java index f89dc5b8cc0d9..019861fa5172c 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java @@ -398,6 +398,12 @@ abstract class SinkDetailsCommand extends BaseCommand { @Parameter(names = "--secrets", description = "The map of secretName to an object that encapsulates " + "how the secret is fetched by the underlying secrets provider") protected String secretsString; + @Parameter(names = "--preprocess-function", description = "Preprocess function applied before the Sink") + protected String preprocessFunction; + @Parameter(names = "--preprocess-function-config", description = "Configuration of the preprocess function applied before the Sink") + protected String preprocessFunctionConfig; + + protected SinkConfig sinkConfig; @@ -578,6 +584,14 @@ void processArguments() throws Exception { sinkConfig.setSecrets(secretsMap); } + if (preprocessFunction != null) { + sinkConfig.setPreprocessFunction(preprocessFunction); + } + + if (preprocessFunctionConfig != null) { + sinkConfig.setPreprocessFunctionConfig(preprocessFunctionConfig); + } + // check if configs are valid validateSinkConfigs(sinkConfig); } diff --git a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java index 2b9b25ed74349..6b7d547e6f5bd 100644 --- a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java +++ b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java @@ -439,17 +439,19 @@ public void start(boolean blocking) throws Exception { ClassLoader builtInSinkClassLoader = userCodeFile != null ? isBuiltInSink(userCodeFile) : null; if (builtInSinkClassLoader != null) { + //TODO: handle preprocess function functionDetails = SinkConfigUtils.convert( sinkConfig, SinkConfigUtils.validateAndExtractDetails( - sinkConfig, builtInSinkClassLoader, true)); + sinkConfig, builtInSinkClassLoader, null,true)); userCodeClassLoader = builtInSinkClassLoader; } else if (Utils.isFunctionPackageUrlSupported(userCodeFile)) { File file = FunctionCommon.extractFileFromPkgURL(userCodeFile); ClassLoader sinkClassLoader = FunctionCommon.getClassLoaderFromPackage( Function.FunctionDetails.ComponentType.SINK, sinkConfig.getClassName(), file, narExtractionDirectory); + //TODO: handle preprocess function functionDetails = SinkConfigUtils.convert( - sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, true)); + sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, null,true)); userCodeClassLoader = sinkClassLoader; userCodeClassLoaderCreated = true; } else if (userCodeFile != null) { @@ -460,17 +462,19 @@ public void start(boolean blocking) throws Exception { ClassLoader sinkClassLoader = FunctionCommon.getClassLoaderFromPackage( Function.FunctionDetails.ComponentType.SINK, sinkConfig.getClassName(), file, narExtractionDirectory); + //TODO: handle preprocess function functionDetails = SinkConfigUtils.convert( - sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, true)); + sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, null,true)); userCodeClassLoader = sinkClassLoader; userCodeClassLoaderCreated = true; } else { if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) { throw new IllegalStateException("The archive property must be specified in SourceConfig."); } + //TODO: handle preprocess function functionDetails = SinkConfigUtils.convert( sinkConfig, SinkConfigUtils.validateAndExtractDetails( - sinkConfig, Thread.currentThread().getContextClassLoader(), true)); + sinkConfig, Thread.currentThread().getContextClassLoader(), null,true)); } } else { throw new IllegalArgumentException("Must specify Function, Source or Sink config"); diff --git a/pulsar-functions/proto/src/main/proto/Function.proto b/pulsar-functions/proto/src/main/proto/Function.proto index 09ff9cfc8ebc9..65b5e0f0fc076 100644 --- a/pulsar-functions/proto/src/main/proto/Function.proto +++ b/pulsar-functions/proto/src/main/proto/Function.proto @@ -211,6 +211,7 @@ message FunctionMetaData { uint64 createTime = 4; map instanceStates = 5; FunctionAuthenticationSpec functionAuthSpec = 6; + PackageLocationMetaData extraFunctionPackageLocation = 7; } message FunctionAuthenticationSpec { diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java index 49b71afcad795..2bd24f0b7b0d0 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java @@ -56,6 +56,7 @@ import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.api.WindowFunction; +import org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType; import org.apache.pulsar.functions.proto.Function.FunctionDetails.Runtime; import org.apache.pulsar.functions.utils.functions.FunctionUtils; import org.apache.pulsar.functions.utils.io.ConnectorUtils; @@ -401,7 +402,7 @@ public static double roundDecimal(double value, int places) { } public static ClassLoader getClassLoaderFromPackage( - org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType componentType, + ComponentType componentType, String className, File packageFile, String narExtractionDirectory) { @@ -437,11 +438,9 @@ public static ClassLoader getClassLoaderFromPackage( narClassLoaderException); } try { - if (componentType - == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.FUNCTION) { - connectorClassName = FunctionUtils.getFunctionClass((NarClassLoader) narClassLoader); - } else if (componentType - == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SOURCE) { + if (componentType == ComponentType.FUNCTION) { + connectorClassName = FunctionUtils.getFunctionClass(narClassLoader); + } else if (componentType == ComponentType.SOURCE) { connectorClassName = ConnectorUtils.getIOSourceClass((NarClassLoader) narClassLoader); } else { connectorClassName = ConnectorUtils.getIOSinkClass((NarClassLoader) narClassLoader); @@ -532,26 +531,28 @@ public static String capFirstLetter(Enum en) { } public static boolean isFunctionCodeBuiltin( - org.apache.pulsar.functions.proto.Function.FunctionDetailsOrBuilder functionDetails) { - if (functionDetails.hasSource()) { + org.apache.pulsar.functions.proto.Function.FunctionDetailsOrBuilder functionDetail) { + return isFunctionCodeBuiltin(functionDetail, functionDetail.getComponentType()); + } + + public static boolean isFunctionCodeBuiltin( + org.apache.pulsar.functions.proto.Function.FunctionDetailsOrBuilder functionDetails, + ComponentType componentType) { + if (componentType == ComponentType.SOURCE && functionDetails.hasSource()) { org.apache.pulsar.functions.proto.Function.SourceSpec sourceSpec = functionDetails.getSource(); if (!isEmpty(sourceSpec.getBuiltin())) { return true; } } - if (functionDetails.hasSink()) { + if (componentType == ComponentType.SINK && functionDetails.hasSink()) { org.apache.pulsar.functions.proto.Function.SinkSpec sinkSpec = functionDetails.getSink(); if (!isEmpty(sinkSpec.getBuiltin())) { return true; } } - if (!isEmpty(functionDetails.getBuiltin())) { - return true; - } - - return false; + return componentType == ComponentType.FUNCTION && !isEmpty(functionDetails.getBuiltin()); } public static SubscriptionInitialPosition convertFromFunctionDetailsSubscriptionPosition( diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java index d24542185e57e..cd86fedafbf5b 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java @@ -33,6 +33,9 @@ import java.io.IOException; import java.lang.reflect.Type; import java.net.MalformedURLException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collection; import java.util.HashMap; import java.util.LinkedList; @@ -43,11 +46,13 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.common.functions.ConsumerConfig; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.ProducerConfig; import org.apache.pulsar.common.functions.Resources; +import org.apache.pulsar.common.functions.Utils; import org.apache.pulsar.common.functions.WindowConfig; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.ObjectMapperFactory; @@ -562,7 +567,6 @@ public static void inferMissingArguments(FunctionConfig functionConfig, } else if (functionConfig.getGo() != null) { functionConfig.setRuntime(FunctionConfig.Runtime.GO); } - WindowConfig windowConfig = functionConfig.getWindowConfig(); if (windowConfig != null) { WindowConfigUtils.inferMissingArguments(windowConfig); @@ -570,7 +574,7 @@ public static void inferMissingArguments(FunctionConfig functionConfig, } } - private static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfig, ClassLoader clsLoader) { + public static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfig, ClassLoader clsLoader) { String functionClassName = functionConfig.getClassName(); Class functionClass; diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java index 31b147a1caf28..b1ccc95002e3f 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java @@ -22,6 +22,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsSubscriptionPosition; import static org.apache.pulsar.functions.utils.FunctionCommon.convertProcessingGuarantee; +import static org.apache.pulsar.functions.utils.FunctionCommon.getFunctionTypes; import static org.apache.pulsar.functions.utils.FunctionCommon.getSinkType; import com.fasterxml.jackson.core.type.TypeReference; import com.google.gson.Gson; @@ -53,6 +54,7 @@ import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.Function.FunctionDetails; +import org.apache.pulsar.functions.utils.functions.FunctionUtils; import org.apache.pulsar.functions.utils.io.ConnectorUtils; @Slf4j @@ -64,6 +66,7 @@ public class SinkConfigUtils { public static class ExtractedSinkDetails { private String sinkClassName; private String typeArg; + private String functionClassName; } public static FunctionDetails convert(SinkConfig sinkConfig, ExtractedSinkDetails sinkDetails) throws IOException { @@ -88,7 +91,14 @@ public static FunctionDetails convert(SinkConfig sinkConfig, ExtractedSinkDetail } else { functionDetailsBuilder.setParallelism(1); } - functionDetailsBuilder.setClassName(IdentityFunction.class.getName()); + if (sinkDetails.getFunctionClassName() != null) { + functionDetailsBuilder.setClassName(sinkDetails.getFunctionClassName()); + } else { + functionDetailsBuilder.setClassName(IdentityFunction.class.getName()); + } + if (sinkConfig.getPreprocessFunctionConfig() != null) { + functionDetailsBuilder.setUserConfig(sinkConfig.getPreprocessFunctionConfig()); + } if (sinkConfig.getProcessingGuarantees() != null) { functionDetailsBuilder.setProcessingGuarantees( convertProcessingGuarantee(sinkConfig.getProcessingGuarantees())); @@ -374,6 +384,7 @@ public static SinkConfig convertFromDetails(FunctionDetails functionDetails) { public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConfig, ClassLoader sinkClassLoader, + ClassLoader functionClassLoader, boolean validateConnectorConfig) { if (isEmpty(sinkConfig.getTenant())) { throw new IllegalArgumentException("Sink tenant cannot be null"); @@ -429,18 +440,44 @@ public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConf String.format("Sink class %s not found in class loader", sinkClassName), e); } - // extract type from sink class - Class typeArg = getSinkType(sinkClass); + String functionClassName = sinkConfig.getPreprocessFunctionClassName(); + Class typeArg; + ClassLoader inputClassLoader; + if (functionClassLoader != null) { + // if function class name in sink config is not set, this should be a built-in function + // thus we should try to find it class name in the NAR service definition + if (functionClassName == null) { + try { + functionClassName = FunctionUtils.getFunctionClass(functionClassLoader); + } catch (IOException e) { + throw new IllegalArgumentException("Failed to extract sink class from archive", e); + } + } + Class functionClass; + try { + functionClass = functionClassLoader.loadClass(functionClassName); + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException( + String.format("Function class %s not found in class loader", functionClassName), e); + } + // extract type from preprocess function class + typeArg = getFunctionTypes(functionClass, false)[0]; + inputClassLoader = functionClassLoader; + } else { + // extract type from sink class + typeArg = getSinkType(sinkClass); + inputClassLoader = sinkClassLoader; + } if (sinkConfig.getTopicToSerdeClassName() != null) { for (String serdeClassName : sinkConfig.getTopicToSerdeClassName().values()) { - ValidatorUtils.validateSerde(serdeClassName, typeArg, sinkClassLoader, true); + ValidatorUtils.validateSerde(serdeClassName, typeArg, inputClassLoader, true); } } if (sinkConfig.getTopicToSchemaType() != null) { for (String schemaType : sinkConfig.getTopicToSchemaType().values()) { - ValidatorUtils.validateSchema(schemaType, typeArg, sinkClassLoader, true); + ValidatorUtils.validateSchema(schemaType, typeArg, inputClassLoader, true); } } @@ -453,13 +490,13 @@ public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConf throw new IllegalArgumentException("Only one of serdeClassName or schemaType should be set"); } if (!isEmpty(consumerSpec.getSerdeClassName())) { - ValidatorUtils.validateSerde(consumerSpec.getSerdeClassName(), typeArg, sinkClassLoader, true); + ValidatorUtils.validateSerde(consumerSpec.getSerdeClassName(), typeArg, inputClassLoader, true); } if (!isEmpty(consumerSpec.getSchemaType())) { - ValidatorUtils.validateSchema(consumerSpec.getSchemaType(), typeArg, sinkClassLoader, true); + ValidatorUtils.validateSchema(consumerSpec.getSchemaType(), typeArg, inputClassLoader, true); } if (consumerSpec.getCryptoConfig() != null) { - ValidatorUtils.validateCryptoKeyReader(consumerSpec.getCryptoConfig(), sinkClassLoader, false); + ValidatorUtils.validateCryptoKeyReader(consumerSpec.getCryptoConfig(), inputClassLoader, false); } } } @@ -469,7 +506,7 @@ public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConf validateSinkConfig(sinkConfig, (NarClassLoader) sinkClassLoader); } - return new ExtractedSinkDetails(sinkClassName, typeArg.getName()); + return new ExtractedSinkDetails(sinkClassName, typeArg.getName(), functionClassName); } private static Collection collectAllInputTopics(SinkConfig sinkConfig) { diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java index 60f7d30052ec3..b670b9b583e3c 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java @@ -111,7 +111,7 @@ public void testConvertBackFidelity() throws IOException { sinkConfig.setResources(Resources.getDefaultResources()); - Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null)); + Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); assertEquals(Function.SubscriptionType.SHARED, functionDetails.getSource().getSubscriptionType()); SinkConfig convertedConfig = SinkConfigUtils.convertFromDetails(functionDetails); assertEquals( @@ -122,7 +122,7 @@ public void testConvertBackFidelity() throws IOException { sinkConfig.setRetainOrdering(true); sinkConfig.setRetainKeyOrdering(false); - functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null)); + functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); assertEquals(Function.SubscriptionType.FAILOVER, functionDetails.getSource().getSubscriptionType()); convertedConfig = SinkConfigUtils.convertFromDetails(functionDetails); assertEquals( @@ -132,7 +132,7 @@ public void testConvertBackFidelity() throws IOException { sinkConfig.setRetainOrdering(false); sinkConfig.setRetainKeyOrdering(true); - functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null)); + functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); assertEquals(Function.SubscriptionType.KEY_SHARED, functionDetails.getSource().getSubscriptionType()); convertedConfig = SinkConfigUtils.convertFromDetails(functionDetails); assertEquals( @@ -146,7 +146,7 @@ public void testParseRetainOrderingField() throws IOException { for (Boolean testcase : testcases) { SinkConfig sinkConfig = createSinkConfig(); sinkConfig.setRetainOrdering(testcase); - Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null)); + Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); SinkConfig result = SinkConfigUtils.convertFromDetails(functionDetails); assertEquals(result.getRetainOrdering(), testcase != null ? testcase : Boolean.valueOf(false)); } @@ -158,7 +158,7 @@ public void testParseKeyRetainOrderingField() throws IOException { for (Boolean testcase : testcases) { SinkConfig sinkConfig = createSinkConfig(); sinkConfig.setRetainKeyOrdering(testcase); - Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null)); + Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); SinkConfig result = SinkConfigUtils.convertFromDetails(functionDetails); assertEquals(result.getRetainKeyOrdering(), testcase != null ? testcase : Boolean.valueOf(false)); } @@ -176,7 +176,7 @@ public void testParseProcessingGuaranteesField() throws IOException { for (FunctionConfig.ProcessingGuarantees testcase : testcases) { SinkConfig sinkConfig = createSinkConfig(); sinkConfig.setProcessingGuarantees(testcase); - Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null)); + Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); SinkConfig result = SinkConfigUtils.convertFromDetails(functionDetails); assertEquals(result.getProcessingGuarantees(), testcase == null ? ATLEAST_ONCE : testcase); } @@ -189,7 +189,7 @@ public void testCleanSubscriptionField() throws IOException { for (Boolean testcase : testcases) { SinkConfig sinkConfig = createSinkConfig(); sinkConfig.setCleanupSubscription(testcase); - Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null)); + Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); SinkConfig result = SinkConfigUtils.convertFromDetails(functionDetails); assertEquals(result.getCleanupSubscription(), testcase == null ? Boolean.valueOf(true) : testcase); } @@ -478,7 +478,7 @@ private SinkConfig createUpdatedSinkConfig(String fieldName, Object fieldValue) @Test public void testPoolMessages() throws IOException { SinkConfig sinkConfig = createSinkConfig(); - Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null)); + Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); assertFalse(functionDetails.getSource().getInputSpecsMap().get("test-input").getPoolMessages()); SinkConfig convertedConfig = SinkConfigUtils.convertFromDetails(functionDetails); assertFalse(convertedConfig.getInputSpecs().get("test-input").isPoolMessages()); @@ -488,7 +488,7 @@ public void testPoolMessages() throws IOException { .poolMessages(true).build()); sinkConfig.setInputSpecs(inputSpecs); - functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null)); + functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); assertTrue(functionDetails.getSource().getInputSpecsMap().get("test-input").getPoolMessages()); convertedConfig = SinkConfigUtils.convertFromDetails(functionDetails); assertTrue(convertedConfig.getInputSpecs().get("test-input").isPoolMessages()); @@ -499,13 +499,13 @@ public void testAllowDisableSinkTimeout() { SinkConfig sinkConfig = createSinkConfig(); sinkConfig.setInputSpecs(null); sinkConfig.setTopicsPattern("my-topic-*"); - SinkConfigUtils.validateAndExtractDetails(sinkConfig, this.getClass().getClassLoader(), + SinkConfigUtils.validateAndExtractDetails(sinkConfig, this.getClass().getClassLoader(), null, true); sinkConfig.setTimeoutMs(null); - SinkConfigUtils.validateAndExtractDetails(sinkConfig, this.getClass().getClassLoader(), + SinkConfigUtils.validateAndExtractDetails(sinkConfig, this.getClass().getClassLoader(), null, true); sinkConfig.setTimeoutMs(0L); - SinkConfigUtils.validateAndExtractDetails(sinkConfig, this.getClass().getClassLoader(), + SinkConfigUtils.validateAndExtractDetails(sinkConfig, this.getClass().getClassLoader(), null, true); } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index f3ebef89b92b6..8dc78a9663c6c 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -26,9 +26,7 @@ import static org.apache.pulsar.functions.utils.FunctionCommon.createPkgTempFile; import static org.apache.pulsar.functions.utils.FunctionCommon.getStateNamespace; import static org.apache.pulsar.functions.utils.FunctionCommon.getUniquePackageName; -import static org.apache.pulsar.functions.utils.FunctionCommon.isFunctionCodeBuiltin; import static org.apache.pulsar.functions.worker.rest.RestUtils.throwUnavailableException; -import static org.apache.pulsar.functions.worker.rest.api.FunctionsImpl.downloadPackageFile; import com.google.common.base.Utf8; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; @@ -42,6 +40,7 @@ import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Base64; import java.util.Collection; import java.util.LinkedList; @@ -67,6 +66,7 @@ import org.apache.bookkeeper.clients.exceptions.NamespaceNotFoundException; import org.apache.bookkeeper.clients.exceptions.StreamNotFoundException; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.internal.FunctionsImpl; @@ -292,16 +292,28 @@ boolean isWorkerServiceAvailable() { PackageLocationMetaData.Builder getFunctionPackageLocation(final FunctionMetaData functionMetaData, final String functionPkgUrl, final FormDataContentDisposition fileDetail, - final File uploadedInputStreamAsFile) throws Exception { + final File uploadedInputStreamAsFile) + throws Exception { + return getFunctionPackageLocation(functionMetaData, functionPkgUrl, fileDetail, uploadedInputStreamAsFile, + componentType, getFunctionCodeBuiltin(functionMetaData.getFunctionDetails(), componentType)); + } + + PackageLocationMetaData.Builder getFunctionPackageLocation(final FunctionMetaData functionMetaData, + final String functionPkgUrl, + final FormDataContentDisposition fileDetail, + final File uploadedInputStreamAsFile, + final FunctionDetails.ComponentType componentType, + final String builtin) + throws Exception { FunctionDetails functionDetails = functionMetaData.getFunctionDetails(); String tenant = functionDetails.getTenant(); String namespace = functionDetails.getNamespace(); String componentName = functionDetails.getName(); PackageLocationMetaData.Builder packageLocationMetaDataBuilder = PackageLocationMetaData.newBuilder(); - boolean isBuiltin = isFunctionCodeBuiltin(functionDetails); boolean isPkgUrlProvided = isNotBlank(functionPkgUrl); boolean isPackageManagementEnabled = worker().getWorkerConfig().isFunctionsWorkerEnablePackageManagement(); - PackageName packageName = PackageName.get(functionDetails.getComponentType().name(), + PackageName packageName = PackageName.get( + componentType.name(), tenant, namespace, componentName, @@ -311,7 +323,7 @@ PackageLocationMetaData.Builder getFunctionPackageLocation(final FunctionMetaDat if (worker().getFunctionRuntimeManager().getRuntimeFactory().externallyManaged()) { // For externally managed schedulers, the pkgUrl/builtin stuff can be copied to bk // if the function worker image does not include connectors - if (isBuiltin) { + if (!isEmpty(builtin)) { if (worker().getWorkerConfig().getUploadBuiltinSinksSources()) { File component; String archiveName; @@ -328,7 +340,7 @@ PackageLocationMetaData.Builder getFunctionPackageLocation(final FunctionMetaDat archiveName = functionDetails.getBuiltin(); component = worker().getFunctionsManager().getFunctionArchive(archiveName).toFile(); break; - } + } packageLocationMetaDataBuilder.setPackagePath(createPackagePath(tenant, namespace, componentName, component.getName())); packageLocationMetaDataBuilder.setOriginalFileName(component.getName()); @@ -337,6 +349,8 @@ PackageLocationMetaData.Builder getFunctionPackageLocation(final FunctionMetaDat worker().getBrokerAdmin().packages().upload(metadata, packageName.toString(), component.getAbsolutePath()); } else { + packageLocationMetaDataBuilder.setPackagePath(createPackagePath(tenant, namespace, + componentName, component.getName())); WorkerUtils.uploadFileToBookkeeper(packageLocationMetaDataBuilder.getPackagePath(), component, worker().getDlogNamespace()); } @@ -344,8 +358,7 @@ PackageLocationMetaData.Builder getFunctionPackageLocation(final FunctionMetaDat packageLocationMetaDataBuilder.getPackagePath()); } else { log.info("Skipping upload for the built-in package {}", ComponentTypeUtils.toString(componentType)); - packageLocationMetaDataBuilder - .setPackagePath("builtin://" + getFunctionCodeBuiltin(functionDetails)); + packageLocationMetaDataBuilder.setPackagePath("builtin://" + builtin); } } else if (isPkgUrlProvided) { packageLocationMetaDataBuilder.setOriginalFileName(uploadedInputStreamAsFile.getName()); @@ -398,8 +411,8 @@ PackageLocationMetaData.Builder getFunctionPackageLocation(final FunctionMetaDat } } else { // For pulsar managed schedulers, the pkgUrl/builtin stuff should be copied to bk - if (isBuiltin) { - packageLocationMetaDataBuilder.setPackagePath("builtin://" + getFunctionCodeBuiltin(functionDetails)); + if (!isEmpty(builtin)) { + packageLocationMetaDataBuilder.setPackagePath("builtin://" + builtin); } else if (isPkgUrlProvided) { packageLocationMetaDataBuilder.setPackagePath(functionPkgUrl); } else if (functionMetaData.getPackageLocation().getPackagePath().startsWith(Utils.HTTP) @@ -573,8 +586,7 @@ public FunctionConfig getFunctionInfo(final String tenant, throw new RestException(Status.NOT_FOUND, String.format(ComponentTypeUtils.toString(componentType) + " %s doesn't exist", componentName)); } - FunctionConfig config = FunctionConfigUtils.convertFromDetails(functionMetaData.getFunctionDetails()); - return config; + return FunctionConfigUtils.convertFromDetails(functionMetaData.getFunctionDetails()); } @Override @@ -1149,7 +1161,7 @@ public String triggerFunction(final String tenant, Reader reader = null; Producer producer = null; try { - if (outputTopic != null && !outputTopic.isEmpty()) { + if (!isEmpty(outputTopic)) { reader = worker().getClient().newReader() .topic(outputTopic) .startMessageId(MessageId.latest) @@ -1452,7 +1464,7 @@ public StreamingOutput downloadFunction(String tenant, String namespace, String } private StreamingOutput getStreamingOutput(String pkgPath) { - final StreamingOutput streamingOutput = output -> { + return output -> { if (pkgPath.startsWith(Utils.HTTP)) { URL url = URI.create(pkgPath).toURL(); try (InputStream inputStream = url.openStream()) { @@ -1493,7 +1505,6 @@ private StreamingOutput getStreamingOutput(String pkgPath) { WorkerUtils.downloadFromBookkeeper(worker().getDlogNamespace(), output, pkgPath); } }; - return streamingOutput; } @Override @@ -1605,23 +1616,26 @@ private void validateFunctionStateParams(final String tenant, } } - private String getFunctionCodeBuiltin(FunctionDetails functionDetails) { - if (functionDetails.hasSource()) { + private String getFunctionCodeBuiltin(FunctionDetails functionDetails, + FunctionDetails.ComponentType componentType) { + if (componentType == FunctionDetails.ComponentType.SOURCE && functionDetails.hasSource()) { SourceSpec sourceSpec = functionDetails.getSource(); if (!isEmpty(sourceSpec.getBuiltin())) { return sourceSpec.getBuiltin(); } } - if (functionDetails.hasSink()) { + if (componentType == FunctionDetails.ComponentType.SOURCE && functionDetails.hasSink()) { SinkSpec sinkSpec = functionDetails.getSink(); if (!isEmpty(sinkSpec.getBuiltin())) { return sinkSpec.getBuiltin(); } } - if (!isEmpty(functionDetails.getBuiltin())) { - return functionDetails.getBuiltin(); + if (componentType == FunctionDetails.ComponentType.FUNCTION) { + if (!isEmpty(functionDetails.getBuiltin())) { + return functionDetails.getBuiltin(); + } } return null; @@ -1837,4 +1851,87 @@ protected ClassLoader getClassLoaderFromPackage(String className, String narExtractionDirectory) { return FunctionCommon.getClassLoaderFromPackage(componentType, className, packageFile, narExtractionDirectory); } + + static File downloadPackageFile(PulsarWorkerService worker, String packageName) + throws IOException, PulsarAdminException { + Path tempDirectory; + if (worker.getWorkerConfig().getDownloadDirectory() != null) { + tempDirectory = Paths.get(worker.getWorkerConfig().getDownloadDirectory()); + } else { + // use the Nar extraction directory as a temporary directory for downloaded files + tempDirectory = Paths.get(worker.getWorkerConfig().getNarExtractionDirectory()); + } + Files.createDirectories(tempDirectory); + File file = Files.createTempFile(tempDirectory, "function", ".tmp").toFile(); + worker.getBrokerAdmin().packages().download(packageName, file.toString()); + return file; + } + + protected File getPackageFile(String functionPkgUrl, String existingPackagePath, InputStream uploadedInputStream) + throws IOException, PulsarAdminException { + File componentPackageFile = null; + if (isNotBlank(functionPkgUrl)) { + componentPackageFile = getPackageFile(functionPkgUrl); + } else if (existingPackagePath.startsWith(Utils.FILE) || existingPackagePath.startsWith(Utils.HTTP)) { + try { + componentPackageFile = FunctionCommon.extractFileFromPkgURL(existingPackagePath); + } catch (Exception e) { + throw new IllegalArgumentException(String.format("Encountered error \"%s\" " + + "when getting %s package from %s", e.getMessage(), + ComponentTypeUtils.toString(componentType), functionPkgUrl)); + } + } else if (uploadedInputStream != null) { + componentPackageFile = WorkerUtils.dumpToTmpFile(uploadedInputStream); + } else if (!existingPackagePath.startsWith(Utils.BUILTIN)) { + componentPackageFile = FunctionCommon.createPkgTempFile(); + componentPackageFile.deleteOnExit(); + if (worker().getWorkerConfig().isFunctionsWorkerEnablePackageManagement()) { + worker().getBrokerAdmin().packages().download( + existingPackagePath, + componentPackageFile.getAbsolutePath()); + } else { + WorkerUtils.downloadFromBookkeeper(worker().getDlogNamespace(), + componentPackageFile, existingPackagePath); + } + } + return componentPackageFile; + } + + protected File downloadPackageFile(String packageName) throws IOException, PulsarAdminException { + return downloadPackageFile(worker(), packageName); + } + + protected File getPackageFile(String functionPkgUrl) throws IOException, PulsarAdminException { + if (Utils.hasPackageTypePrefix(functionPkgUrl)) { + return downloadPackageFile(functionPkgUrl); + } else { + if (!Utils.isFunctionPackageUrlSupported(functionPkgUrl)) { + throw new IllegalArgumentException("Function Package url is not valid." + + "supported url (http/https/file)"); + } + try { + return FunctionCommon.extractFileFromPkgURL(functionPkgUrl); + } catch (Exception e) { + throw new IllegalArgumentException(String.format("Encountered error \"%s\" " + + "when getting %s package from %s", e.getMessage(), + ComponentTypeUtils.toString(componentType), functionPkgUrl), e); + } + } + } + + protected ClassLoader getBuiltinFunctionClassLoader(String archive) { + if (!StringUtils.isEmpty(archive)) { + if (archive.startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN)) { + archive = archive.replaceFirst("^builtin://", ""); + + FunctionArchive function = worker().getFunctionsManager().getFunction(archive); + // check if builtin connector exists + if (function == null) { + throw new IllegalArgumentException("Built-in " + componentType + " is not available"); + } + return function.getClassLoader(); + } + } + return null; + } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java index 3e7f0ab733911..799de49d3cc63 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java @@ -28,9 +28,6 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -57,7 +54,6 @@ import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.utils.ComponentTypeUtils; -import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.FunctionConfigUtils; import org.apache.pulsar.functions.utils.functions.FunctionArchive; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; @@ -149,29 +145,13 @@ public void registerFunction(final String tenant, } Function.FunctionDetails functionDetails; - boolean isPkgUrlProvided = isNotBlank(functionPkgUrl); File componentPackageFile = null; try { // validate parameters try { - if (isPkgUrlProvided) { - if (Utils.hasPackageTypePrefix(functionPkgUrl)) { - componentPackageFile = downloadPackageFile(functionPkgUrl); - } else { - if (!Utils.isFunctionPackageUrlSupported(functionPkgUrl)) { - throw new IllegalArgumentException("Function Package url is not valid." - + "supported url (http/https/file)"); - } - try { - - componentPackageFile = FunctionCommon.extractFileFromPkgURL(functionPkgUrl); - } catch (Exception e) { - throw new IllegalArgumentException(String.format("Encountered error \"%s\" " - + "when getting %s package from %s", e.getMessage(), - ComponentTypeUtils.toString(componentType), functionPkgUrl), e); - } - } + if (isNotBlank(functionPkgUrl)) { + componentPackageFile = getPackageFile(functionPkgUrl); functionDetails = validateUpdateRequestParams(tenant, namespace, functionName, functionConfig, componentPackageFile); } else { @@ -339,60 +319,17 @@ public void updateFunction(final String tenant, // validate parameters try { - if (isNotBlank(functionPkgUrl)) { - if (Utils.hasPackageTypePrefix(functionPkgUrl)) { - componentPackageFile = downloadPackageFile(functionPkgUrl); - } else { - try { - componentPackageFile = FunctionCommon.extractFileFromPkgURL(functionPkgUrl); - } catch (Exception e) { - throw new IllegalArgumentException(String.format("Encountered error \"%s\" " - + "when getting %s package from %s", e.getMessage(), - ComponentTypeUtils.toString(componentType), functionPkgUrl)); - } - } - functionDetails = validateUpdateRequestParams(tenant, namespace, functionName, - mergedConfig, componentPackageFile); - - } else if (existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.FILE) - || existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.HTTP)) { - try { - componentPackageFile = FunctionCommon.extractFileFromPkgURL( - existingComponent.getPackageLocation().getPackagePath()); - } catch (Exception e) { - throw new IllegalArgumentException(String.format("Encountered error \"%s\" " - + "when getting %s package from %s", e.getMessage(), - ComponentTypeUtils.toString(componentType), functionPkgUrl)); - } - functionDetails = validateUpdateRequestParams(tenant, namespace, functionName, - mergedConfig, componentPackageFile); - } else if (uploadedInputStream != null) { - - componentPackageFile = WorkerUtils.dumpToTmpFile(uploadedInputStream); - functionDetails = validateUpdateRequestParams(tenant, namespace, functionName, - mergedConfig, componentPackageFile); - - } else if (existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.BUILTIN)) { - functionDetails = validateUpdateRequestParams(tenant, namespace, functionName, - mergedConfig, componentPackageFile); - if (!isFunctionCodeBuiltin(functionDetails) - && (componentPackageFile == null || fileDetail == null)) { - throw new IllegalArgumentException(ComponentTypeUtils.toString(componentType) - + " Package is not provided"); - } - } else { - componentPackageFile = FunctionCommon.createPkgTempFile(); - componentPackageFile.deleteOnExit(); - if (worker().getWorkerConfig().isFunctionsWorkerEnablePackageManagement()) { - worker().getBrokerAdmin().packages().download( - existingComponent.getPackageLocation().getPackagePath(), - componentPackageFile.getAbsolutePath()); - } else { - WorkerUtils.downloadFromBookkeeper(worker().getDlogNamespace(), - componentPackageFile, existingComponent.getPackageLocation().getPackagePath()); - } - functionDetails = validateUpdateRequestParams(tenant, namespace, functionName, - mergedConfig, componentPackageFile); + componentPackageFile = getPackageFile( + functionPkgUrl, + existingComponent.getPackageLocation().getPackagePath(), + uploadedInputStream); + functionDetails = validateUpdateRequestParams(tenant, namespace, functionName, + mergedConfig, componentPackageFile); + if (existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.BUILTIN) + && !isFunctionCodeBuiltin(functionDetails) + && (componentPackageFile == null || fileDetail == null)) { + throw new IllegalArgumentException(ComponentTypeUtils.toString(componentType) + + " Package is not provided"); } } catch (Exception e) { log.error("Invalid update {} request @ /{}/{}/{}", ComponentTypeUtils.toString(componentType), @@ -821,7 +758,7 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant if (archive.startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN)) { archive = archive.replaceFirst("^builtin://", ""); - FunctionsManager functionsManager = this.worker().getFunctionsManager(); + FunctionsManager functionsManager = worker().getFunctionsManager(); FunctionArchive function = functionsManager.getFunction(archive); // check if builtin function exists @@ -860,23 +797,4 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant } } } - - private File downloadPackageFile(String packageName) throws IOException, PulsarAdminException { - return downloadPackageFile(worker(), packageName); - } - - static File downloadPackageFile(PulsarWorkerService worker, String packageName) - throws IOException, PulsarAdminException { - Path tempDirectory; - if (worker.getWorkerConfig().getDownloadDirectory() != null) { - tempDirectory = Paths.get(worker.getWorkerConfig().getDownloadDirectory()); - } else { - // use the Nar extraction directory as a temporary directory for downloaded files - tempDirectory = Paths.get(worker.getWorkerConfig().getNarExtractionDirectory()); - } - Files.createDirectories(tempDirectory); - File file = Files.createTempFile(tempDirectory, "function", ".tmp").toFile(); - worker.getBrokerAdmin().packages().download(packageName, file.toString()); - return file; - } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index 7b2c5a68ef2c9..d27dd4098fc28 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -54,7 +54,6 @@ import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.utils.ComponentTypeUtils; -import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.SinkConfigUtils; import org.apache.pulsar.functions.utils.io.Connector; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; @@ -144,29 +143,14 @@ public void registerSink(final String tenant, String.format("%s %s already exists", ComponentTypeUtils.toString(componentType), sinkName)); } - Function.FunctionDetails functionDetails = null; - boolean isPkgUrlProvided = isNotBlank(sinkPkgUrl); + Function.FunctionDetails functionDetails; File componentPackageFile = null; try { // validate parameters try { - if (isPkgUrlProvided) { - if (Utils.hasPackageTypePrefix(sinkPkgUrl)) { - componentPackageFile = downloadPackageFile(sinkPkgUrl); - } else { - if (!Utils.isFunctionPackageUrlSupported(sinkPkgUrl)) { - throw new IllegalArgumentException( - "Function Package url is not valid. supported url (http/https/file)"); - } - try { - componentPackageFile = FunctionCommon.extractFileFromPkgURL(sinkPkgUrl); - } catch (Exception e) { - throw new IllegalArgumentException( - String.format("Encountered error \"%s\" when getting %s package from %s", - e.getMessage(), ComponentTypeUtils.toString(componentType), sinkPkgUrl)); - } - } + if (isNotBlank(sinkPkgUrl)) { + componentPackageFile = getPackageFile(sinkPkgUrl); functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName, sinkConfig, componentPackageFile); } else { @@ -243,6 +227,34 @@ public void registerSink(final String tenant, } functionMetaDataBuilder.setPackageLocation(packageLocationMetaDataBuilder); + + File functionPackageFile = null; + String preprocessFunction = sinkConfig.getPreprocessFunction(); + if (isNotBlank(preprocessFunction)) { + try { + String builtin = null; + if (!preprocessFunction.startsWith(Utils.BUILTIN)) { + functionPackageFile = getPackageFile(preprocessFunction); + } else { + builtin = preprocessFunction.replaceFirst("^builtin://", ""); + } + Function.PackageLocationMetaData.Builder functionPackageLocation = + getFunctionPackageLocation(functionMetaDataBuilder.build(), + preprocessFunction, null, functionPackageFile, + Function.FunctionDetails.ComponentType.FUNCTION, builtin); + functionMetaDataBuilder.setExtraFunctionPackageLocation(functionPackageLocation); + } catch (Exception e) { + log.error("Failed process {} {}/{}/{} preprocess function package: ", + ComponentTypeUtils.toString(componentType), tenant, namespace, sinkName, e); + throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } finally { + if (functionPackageFile != null && functionPackageFile.exists()) { + if (!preprocessFunction.startsWith(Utils.FILE)) { + functionPackageFile.delete(); + } + } + } + } updateRequest(null, functionMetaDataBuilder.build()); } finally { if (componentPackageFile != null && componentPackageFile.exists()) { @@ -329,67 +341,24 @@ public void updateSink(final String tenant, throw new RestException(Response.Status.BAD_REQUEST, "Update contains no change"); } - Function.FunctionDetails functionDetails = null; + Function.FunctionDetails functionDetails; File componentPackageFile = null; try { // validate parameters try { - if (isNotBlank(sinkPkgUrl)) { - if (Utils.hasPackageTypePrefix(sinkPkgUrl)) { - componentPackageFile = downloadPackageFile(sinkPkgUrl); - } else { - try { - componentPackageFile = FunctionCommon.extractFileFromPkgURL(sinkPkgUrl); - } catch (Exception e) { - throw new IllegalArgumentException( - String.format("Encountered error \"%s\" when getting %s package from %s", - e.getMessage(), ComponentTypeUtils.toString(componentType), sinkPkgUrl)); - } - } - functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName, - mergedConfig, componentPackageFile); - - } else if (existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.FILE) - || existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.HTTP)) { - try { - componentPackageFile = FunctionCommon - .extractFileFromPkgURL(existingComponent.getPackageLocation().getPackagePath()); - } catch (Exception e) { - throw new IllegalArgumentException( - String.format("Encountered error \"%s\" when getting %s package from %s", - e.getMessage(), ComponentTypeUtils.toString(componentType), sinkPkgUrl)); - } - functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName, - mergedConfig, componentPackageFile); - } else if (uploadedInputStream != null) { - - componentPackageFile = WorkerUtils.dumpToTmpFile(uploadedInputStream); - functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName, - mergedConfig, componentPackageFile); - - } else if (existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.BUILTIN)) { - functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName, - mergedConfig, componentPackageFile); - if (!isFunctionCodeBuiltin(functionDetails) - && (componentPackageFile == null || fileDetail == null)) { + componentPackageFile = getPackageFile( + sinkPkgUrl, + existingComponent.getPackageLocation().getPackagePath(), + uploadedInputStream); + functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName, + mergedConfig, componentPackageFile); + if (existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.BUILTIN) + && !isFunctionCodeBuiltin(functionDetails) + && (componentPackageFile == null || fileDetail == null)) { throw new IllegalArgumentException( ComponentTypeUtils.toString(componentType) + " Package is not provided"); } - } else { - componentPackageFile = FunctionCommon.createPkgTempFile(); - componentPackageFile.deleteOnExit(); - if (worker().getWorkerConfig().isFunctionsWorkerEnablePackageManagement()) { - worker().getBrokerAdmin().packages().download( - existingComponent.getPackageLocation().getPackagePath(), - componentPackageFile.getAbsolutePath()); - } else { - WorkerUtils.downloadFromBookkeeper(worker().getDlogNamespace(), componentPackageFile, - existingComponent.getPackageLocation().getPackagePath()); - } - functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName, - mergedConfig, componentPackageFile); - } } catch (Exception e) { log.error("Invalid update {} request @ /{}/{}/{}", ComponentTypeUtils.toString(componentType), tenant, namespace, sinkName, e); @@ -758,7 +727,8 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant final String namespace, final String sinkName, final SinkConfig sinkConfig, - final File sinkPackageFile) throws IOException { + final File sinkPackageFile) + throws IOException, PulsarAdminException { // The rest end points take precedence over whatever is there in sinkConfig sinkConfig.setTenant(tenant); @@ -796,8 +766,23 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant throw new IllegalArgumentException("Sink package is not provided"); } + ClassLoader functionClassLoader = null; + if (isNotBlank(sinkConfig.getPreprocessFunction())) { + functionClassLoader = + getBuiltinFunctionClassLoader(sinkConfig.getPreprocessFunction()); + if (functionClassLoader == null) { + File functionPackageFile = getPackageFile(sinkConfig.getPreprocessFunction()); + functionClassLoader = getClassLoaderFromPackage(sinkConfig.getPreprocessFunctionClassName(), + functionPackageFile, worker().getWorkerConfig().getNarExtractionDirectory()); + } + if (functionClassLoader == null) { + throw new IllegalArgumentException("Pre-process Function package not found"); + } + } + SinkConfigUtils.ExtractedSinkDetails sinkDetails = SinkConfigUtils.validateAndExtractDetails( - sinkConfig, classLoader, worker().getWorkerConfig().getValidateConnectorConfig()); + sinkConfig, classLoader, functionClassLoader, worker().getWorkerConfig().getValidateConnectorConfig()); + return SinkConfigUtils.convert(sinkConfig, sinkDetails); } finally { if (shouldCloseClassLoader) { @@ -805,8 +790,4 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant } } } - - private File downloadPackageFile(String packageName) throws IOException, PulsarAdminException { - return FunctionsImpl.downloadPackageFile(worker(), packageName); - } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java index 82d2043e9a9c3..44199dd9917fa 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java @@ -25,7 +25,6 @@ import static org.apache.pulsar.functions.worker.rest.RestUtils.throwUnavailableException; import com.google.protobuf.ByteString; import java.io.File; -import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.util.ArrayList; @@ -54,7 +53,6 @@ import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.utils.ComponentTypeUtils; -import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.SourceConfigUtils; import org.apache.pulsar.functions.utils.io.Connector; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; @@ -152,21 +150,7 @@ public void registerSource(final String tenant, // validate parameters try { if (isPkgUrlProvided) { - if (Utils.hasPackageTypePrefix(sourcePkgUrl)) { - componentPackageFile = downloadPackageFile(sourcePkgUrl); - } else { - if (!Utils.isFunctionPackageUrlSupported(sourcePkgUrl)) { - throw new IllegalArgumentException( - "Function Package url is not valid. supported url (http/https/file)"); - } - try { - componentPackageFile = FunctionCommon.extractFileFromPkgURL(sourcePkgUrl); - } catch (Exception e) { - throw new IllegalArgumentException( - String.format("Encountered error \"%s\" when getting %s package from %s", - e.getMessage(), ComponentTypeUtils.toString(componentType), sourcePkgUrl)); - } - } + componentPackageFile = getPackageFile(sourcePkgUrl); functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName, sourceConfig, componentPackageFile); } else { @@ -329,66 +313,23 @@ public void updateSource(final String tenant, throw new RestException(Response.Status.BAD_REQUEST, "Update contains no change"); } - Function.FunctionDetails functionDetails = null; + Function.FunctionDetails functionDetails; File componentPackageFile = null; try { // validate parameters try { - if (isNotBlank(sourcePkgUrl)) { - if (Utils.hasPackageTypePrefix(sourcePkgUrl)) { - componentPackageFile = downloadPackageFile(sourcePkgUrl); - } else { - try { - componentPackageFile = FunctionCommon.extractFileFromPkgURL(sourcePkgUrl); - } catch (Exception e) { - throw new IllegalArgumentException( - String.format("Encountered error \"%s\" when getting %s package from %s", - e.getMessage(), ComponentTypeUtils.toString(componentType), sourcePkgUrl)); - } - } - functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName, - mergedConfig, componentPackageFile); - - } else if (existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.FILE) - || existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.HTTP)) { - try { - componentPackageFile = FunctionCommon - .extractFileFromPkgURL(existingComponent.getPackageLocation().getPackagePath()); - } catch (Exception e) { - throw new IllegalArgumentException( - String.format("Encountered error \"%s\" when getting %s package from %s", - e.getMessage(), ComponentTypeUtils.toString(componentType), sourcePkgUrl)); - } - functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName, - mergedConfig, componentPackageFile); - } else if (uploadedInputStream != null) { - - componentPackageFile = WorkerUtils.dumpToTmpFile(uploadedInputStream); - functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName, - mergedConfig, componentPackageFile); - - } else if (existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.BUILTIN)) { - functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName, - mergedConfig, componentPackageFile); - if (!isFunctionCodeBuiltin(functionDetails) - && (componentPackageFile == null || fileDetail == null)) { - throw new IllegalArgumentException( - ComponentTypeUtils.toString(componentType) + " Package is not provided"); - } - } else { - componentPackageFile = FunctionCommon.createPkgTempFile(); - componentPackageFile.deleteOnExit(); - if (worker().getWorkerConfig().isFunctionsWorkerEnablePackageManagement()) { - worker().getBrokerAdmin().packages().download( - existingComponent.getPackageLocation().getPackagePath(), - componentPackageFile.getAbsolutePath()); - } else { - WorkerUtils.downloadFromBookkeeper(worker().getDlogNamespace(), componentPackageFile, - existingComponent.getPackageLocation().getPackagePath()); - } - functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName, - mergedConfig, componentPackageFile); + componentPackageFile = getPackageFile( + sourcePkgUrl, + existingComponent.getPackageLocation().getPackagePath(), + uploadedInputStream); + functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName, + mergedConfig, componentPackageFile); + if (existingComponent.getPackageLocation().getPackagePath().startsWith(Utils.BUILTIN) + && !isFunctionCodeBuiltin(functionDetails) + && (componentPackageFile == null || fileDetail == null)) { + throw new IllegalArgumentException(ComponentTypeUtils.toString(componentType) + + " Package is not provided"); } } catch (Exception e) { log.error("Invalid update {} request @ /{}/{}/{}", ComponentTypeUtils.toString(componentType), tenant, @@ -805,8 +746,4 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant } } } - - private File downloadPackageFile(String packageName) throws IOException, PulsarAdminException { - return FunctionsImpl.downloadPackageFile(worker(), packageName); - } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java index 791273cf537d5..54063f1e4e063 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java @@ -1574,7 +1574,7 @@ private SinkConfig createDefaultSinkConfig() { private FunctionDetails createDefaultFunctionDetails() throws IOException { return SinkConfigUtils.convert(createDefaultSinkConfig(), - new SinkConfigUtils.ExtractedSinkDetails(null, null)); + new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); } /* From 361b9b9ec41dad169ea3389a6c5844c4553a3535 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Fri, 24 Jun 2022 19:21:46 +0200 Subject: [PATCH 02/26] Use a special name to store the function in package or BK --- .../pulsar/functions/worker/rest/api/ComponentImpl.java | 5 +++-- .../apache/pulsar/functions/worker/rest/api/SinksImpl.java | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index 8dc78a9663c6c..0dbed39e3ce6c 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -295,20 +295,21 @@ PackageLocationMetaData.Builder getFunctionPackageLocation(final FunctionMetaDat final File uploadedInputStreamAsFile) throws Exception { return getFunctionPackageLocation(functionMetaData, functionPkgUrl, fileDetail, uploadedInputStreamAsFile, - componentType, getFunctionCodeBuiltin(functionMetaData.getFunctionDetails(), componentType)); + functionMetaData.getFunctionDetails().getName(), componentType, + getFunctionCodeBuiltin(functionMetaData.getFunctionDetails(), componentType)); } PackageLocationMetaData.Builder getFunctionPackageLocation(final FunctionMetaData functionMetaData, final String functionPkgUrl, final FormDataContentDisposition fileDetail, final File uploadedInputStreamAsFile, + final String componentName, final FunctionDetails.ComponentType componentType, final String builtin) throws Exception { FunctionDetails functionDetails = functionMetaData.getFunctionDetails(); String tenant = functionDetails.getTenant(); String namespace = functionDetails.getNamespace(); - String componentName = functionDetails.getName(); PackageLocationMetaData.Builder packageLocationMetaDataBuilder = PackageLocationMetaData.newBuilder(); boolean isPkgUrlProvided = isNotBlank(functionPkgUrl); boolean isPackageManagementEnabled = worker().getWorkerConfig().isFunctionsWorkerEnablePackageManagement(); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index d27dd4098fc28..74898cc0815bf 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -241,6 +241,7 @@ public void registerSink(final String tenant, Function.PackageLocationMetaData.Builder functionPackageLocation = getFunctionPackageLocation(functionMetaDataBuilder.build(), preprocessFunction, null, functionPackageFile, + sinkName + "__sink-function", Function.FunctionDetails.ComponentType.FUNCTION, builtin); functionMetaDataBuilder.setExtraFunctionPackageLocation(functionPackageLocation); } catch (Exception e) { From 7a90b6d63b1d27043a605621e71d14e3ab376c30 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Tue, 28 Jun 2022 18:12:09 +0200 Subject: [PATCH 03/26] Add --extra-function-jar param to JavaInstanceStarter --- .../apache/pulsar/functions/LocalRunner.java | 6 ++ .../runtime/JavaInstanceStarter.java | 8 +++ .../functions/runtime/RuntimeFactory.java | 1 + .../functions/runtime/RuntimeSpawner.java | 3 + .../kubernetes/KubernetesRuntimeFactory.java | 2 + .../process/ProcessRuntimeFactory.java | 2 + .../runtime/thread/ThreadRuntime.java | 3 + .../runtime/thread/ThreadRuntimeFactory.java | 3 + .../kubernetes/KubernetesRuntimeTest.java | 24 ++++---- .../runtime/process/ProcessRuntimeTest.java | 5 +- .../functions/worker/FunctionActioner.java | 59 ++++++++++++------- .../worker/FunctionRuntimeManager.java | 7 ++- .../worker/FunctionActionerTest.java | 8 +-- .../worker/FunctionRuntimeManagerTest.java | 5 +- 14 files changed, 93 insertions(+), 43 deletions(-) diff --git a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java index 6b7d547e6f5bd..3a50faa1ea9fa 100644 --- a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java +++ b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java @@ -576,10 +576,13 @@ private void startProcessMode(org.apache.pulsar.functions.proto.Function.Functio } } + // TODO: support preprocessing function in LocalRunner RuntimeSpawner runtimeSpawner = new RuntimeSpawner( instanceConfig, userCodeFile, null, + null, + null, runtimeFactory, instanceLivenessCheck); spawners.add(runtimeSpawner); @@ -680,10 +683,13 @@ private void startThreadedMode(org.apache.pulsar.functions.proto.Function.Functi } } + // TODO: support preprocessing Function in LocalRunner RuntimeSpawner runtimeSpawner = new RuntimeSpawner( instanceConfig, userCodeFile, null, + null, + null, runtimeFactory, instanceLivenessCheck); spawners.add(runtimeSpawner); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java index e65a8e254e25a..9c02e2aec2e9a 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java @@ -61,6 +61,12 @@ public class JavaInstanceStarter implements AutoCloseable { listConverter = StringConverter.class) public String jarFile; + @Parameter( + names = "--extra-function-jar", + description = "Path to Extra Function Jar\n", + listConverter = StringConverter.class) + public String extraFunctionJarFile; + @Parameter(names = "--instance_id", description = "Instance Id\n", required = true) public int instanceId; @@ -220,6 +226,8 @@ public void start(String[] args, ClassLoader functionInstanceClassLoader, ClassL instanceConfig, jarFile, null, // we really dont use this in thread container + extraFunctionJarFile, + null, // we really dont use this in thread container containerFactory, expectedHealthCheckInterval * 1000); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeFactory.java index 208420957cc7a..04d750b02e567 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeFactory.java @@ -52,6 +52,7 @@ void initialize(WorkerConfig workerConfig, */ Runtime createContainer( InstanceConfig instanceConfig, String codeFile, String originalCodeFileName, + String extraFunctionFile, String originalExtraFunctionFileName, Long expectedHealthCheckInterval) throws Exception; default boolean externallyManaged() { diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java index a2b7bf4c254b0..02be49ee512b0 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java @@ -57,6 +57,8 @@ public class RuntimeSpawner implements AutoCloseable { public RuntimeSpawner(InstanceConfig instanceConfig, String codeFile, String originalCodeFileName, + String extraFunctionFile, + String originalExtraFunctionFileName, RuntimeFactory containerFactory, long instanceLivenessCheckFreqMs) { this.instanceConfig = instanceConfig; this.runtimeFactory = containerFactory; @@ -65,6 +67,7 @@ public RuntimeSpawner(InstanceConfig instanceConfig, this.instanceLivenessCheckFreqMs = instanceLivenessCheckFreqMs; try { this.runtime = runtimeFactory.createContainer(this.instanceConfig, codeFile, originalCodeFileName, + extraFunctionFile, originalExtraFunctionFileName, instanceLivenessCheckFreqMs / 1000); } catch (Exception e) { throw new RuntimeException(e); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java index aa000e789c287..229dee095a237 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java @@ -269,6 +269,8 @@ public void initialize(WorkerConfig workerConfig, AuthenticationConfig authentic @Override public KubernetesRuntime createContainer(InstanceConfig instanceConfig, String codePkgUrl, String originalCodeFileName, + String extraFunctionPkgUrl, + String originalExtraFunctionFileName, Long expectedHealthCheckInterval) throws Exception { String instanceFile = null; switch (instanceConfig.getFunctionDetails().getRuntime()) { diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeFactory.java index 36e09bafd34e1..c5c5b664d695e 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeFactory.java @@ -193,6 +193,8 @@ private void initialize(String pulsarServiceUrl, @Override public ProcessRuntime createContainer(InstanceConfig instanceConfig, String codeFile, String originalCodeFileName, + String extraFunctionFile, + String originalExtraFunctionFileName, Long expectedHealthCheckInterval) throws Exception { String instanceFile = null; switch (instanceConfig.getFunctionDetails().getRuntime()) { diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java index f2aa1ca6f9843..fdb125eb557f2 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java @@ -64,6 +64,7 @@ public class ThreadRuntime implements Runtime { private final ThreadGroup threadGroup; private final FunctionCacheManager fnCache; private final String jarFile; + private final String extraFunctionFile; private final ClientBuilder clientBuilder; private final PulsarClient pulsarClient; private final PulsarAdmin pulsarAdmin; @@ -79,6 +80,7 @@ public class ThreadRuntime implements Runtime { FunctionCacheManager fnCache, ThreadGroup threadGroup, String jarFile, + String extraFunctionFile, PulsarClient client, ClientBuilder clientBuilder, PulsarAdmin pulsarAdmin, @@ -97,6 +99,7 @@ public class ThreadRuntime implements Runtime { this.threadGroup = threadGroup; this.fnCache = fnCache; this.jarFile = jarFile; + this.extraFunctionFile = extraFunctionFile; this.clientBuilder = clientBuilder; this.pulsarClient = client; this.pulsarAdmin = pulsarAdmin; diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java index da8a9bb14dd6d..f53454e587182 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java @@ -177,6 +177,8 @@ public void initialize(WorkerConfig workerConfig, AuthenticationConfig authentic @Override public ThreadRuntime createContainer(InstanceConfig instanceConfig, String jarFile, String originalCodeFileName, + String extraFunctionFile, + String originalExtraFunctionFileName, Long expectedHealthCheckInterval) { SecretsProvider secretsProvider = defaultSecretsProvider; if (secretsProvider == null) { @@ -196,6 +198,7 @@ public ThreadRuntime createContainer(InstanceConfig instanceConfig, String jarFi fnCache, threadGroup, jarFile, + extraFunctionFile, pulsarClient, clientBuilder, pulsarAdmin, diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java index df5c369303c34..cdec1e96abd1d 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java @@ -304,7 +304,7 @@ private void verifyRamPadding(int percentMemoryPadding, long ram, long expectedR factory = createKubernetesRuntimeFactory(null, percentMemoryPadding, 1.0, 1.0); InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, true); - KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, 30l); + KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null, 30l); Function.Resources resources = Function.Resources.newBuilder().setRam(ram).build(); @@ -364,7 +364,7 @@ private void testResources(double userCpuRequest, long userMemoryRequest, double InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, false); factory = createKubernetesRuntimeFactory(null, 10, cpuOverCommitRatio, memoryOverCommitRatio); - KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, 30l); + KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null, 30l); List args = container.getProcessArgs(); // check padding and xmx @@ -387,7 +387,7 @@ private void verifyJavaInstance(InstanceConfig config, String depsDir, boolean s List args; try (MockedStatic systemUtils = Mockito.mockStatic(SystemUtils.class, Mockito.CALLS_REAL_METHODS)) { systemUtils.when(() -> SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)).thenReturn(true); - container = factory.createContainer(config, userJarFile, userJarFile, 30L); + container = factory.createContainer(config, userJarFile, userJarFile, null, null,30L); args = container.getProcessArgs(); } @@ -491,7 +491,7 @@ public void testPythonConstructorWithDeps() throws Exception { } private void verifyPythonInstance(InstanceConfig config, String extraDepsDir, boolean secretsAttached) throws Exception { - KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, 30l); + KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null, 30l); List args = container.getProcessArgs(); int totalArgs; @@ -582,7 +582,7 @@ public void testCreateFunctionLabels() throws Exception { InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, false); config.setFunctionDetails(functionDetails); - KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, 30l); + KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null, 30l); V1StatefulSet spec = container.createStatefulSet(); assertEquals(spec.getMetadata().getLabels().get("tenant"), "tenant"); @@ -741,7 +741,7 @@ public void testNoOpKubernetesManifestCustomizer() throws Exception { factory = createKubernetesRuntimeFactory(null, 10, 1.0, 1.0); verifyJavaInstance(config, pulsarRootDir + "/instances/deps", false); - KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, 30l); + KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null, 30l); V1Service serviceSpec = container.createService(); assertEquals(serviceSpec.getMetadata().getNamespace(), "default"); @@ -760,7 +760,7 @@ public void testBasicKubernetesManifestCustomizer() throws Exception { factory = createKubernetesRuntimeFactory(null, 10, 1.0, 1.0, Optional.of(new BasicKubernetesManifestCustomizer())); verifyJavaInstance(config, pulsarRootDir + "/instances/deps", false); - KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, 30l); + KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null, 30l); V1StatefulSet spec = container.createStatefulSet(); assertEquals(spec.getMetadata().getAnnotations().get("annotation"), "test"); assertEquals(spec.getMetadata().getLabels().get("label"), "test"); @@ -801,7 +801,7 @@ public void testCustomKubernetesManifestCustomizer() throws Exception { factory = createKubernetesRuntimeFactory(null, 10, 1.0, 1.0, Optional.of(new TestKubernetesCustomManifestCustomizer())); verifyJavaInstance(config, pulsarRootDir + "/instances/deps", false); - KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, 30l); + KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null, 30l); V1StatefulSet spec = container.createStatefulSet(); assertEquals(spec.getSpec().getTemplate().getSpec().getServiceAccountName(), "my-service-account"); } @@ -830,7 +830,7 @@ public void testGolangConstructor() throws Exception { } private void verifyGolangInstance(InstanceConfig config) throws Exception { - KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, 30l); + KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null, 30l); List args = container.getProcessArgs(); int totalArgs = 8; @@ -1007,7 +1007,7 @@ public void testBasicKubernetesManifestCustomizerWithRuntimeCustomizerConfig() t "org.apache.pulsar.functions.runtime.kubernetes.BasicKubernetesManifestCustomizer", configs); verifyJavaInstance(config, pulsarRootDir + "/instances/deps", false); - KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, 30l); + KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null, 30l); V1StatefulSet spec = container.createStatefulSet(); assertEquals(spec.getMetadata().getAnnotations().get("annotation"), "test"); assertEquals(spec.getMetadata().getLabels().get("label"), "test"); @@ -1055,7 +1055,7 @@ public void testBasicKubernetesManifestCustomizerWithRuntimeCustomizerConfigOver "org.apache.pulsar.functions.runtime.kubernetes.BasicKubernetesManifestCustomizer", configs); verifyJavaInstance(config, pulsarRootDir + "/instances/deps", false); - KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, 30l); + KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null, 30l); V1StatefulSet spec = container.createStatefulSet(); assertEquals(spec.getMetadata().getAnnotations().get("annotation"), "test"); assertEquals(spec.getMetadata().getLabels().get("label"), "test"); @@ -1141,7 +1141,7 @@ private void assertMetricsPortConfigured(Map functionRuntimeFact new DefaultSecretsProviderConfigurator(), Mockito.mock(ConnectorsManager.class), Mockito.mock(FunctionsManager.class), Optional.empty(), Optional.empty()); InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, true); - KubernetesRuntime container = kubernetesRuntimeFactory.createContainer(config, userJarFile, userJarFile, 30l); + KubernetesRuntime container = kubernetesRuntimeFactory.createContainer(config, userJarFile, userJarFile, null, null, 30l); V1PodTemplateSpec template = container.createStatefulSet().getSpec().getTemplate(); Map annotations = template.getMetadata().getAnnotations(); diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java index 6d80ec871b68c..59ea8c9c54f1f 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java @@ -289,11 +289,10 @@ private void verifyJavaInstance(InstanceConfig config, Path depsDir, String webS List args; try (MockedStatic systemUtils = Mockito.mockStatic(SystemUtils.class, Mockito.CALLS_REAL_METHODS)) { systemUtils.when(() -> SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)).thenReturn(true); - ProcessRuntime container = factory.createContainer(config, userJarFile, userJarFile, 30L); + ProcessRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null,30L); args = container.getProcessArgs(); } - String classpath = javaInstanceJarFile; String extraDepsEnv; int portArg; @@ -368,7 +367,7 @@ public void testPythonConstructorWithDeps() throws Exception { } private void verifyPythonInstance(InstanceConfig config, String extraDepsDir) throws Exception { - ProcessRuntime container = factory.createContainer(config, userJarFile, null, 30l); + ProcessRuntime container = factory.createContainer(config, userJarFile, null, null, null,30l); List args = container.getProcessArgs(); int totalArgs = 36; diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java index 01b240d7f97a1..22bde59b8cdaa 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java @@ -33,6 +33,7 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; @@ -106,37 +107,25 @@ public void startFunction(FunctionRuntimeInfo functionRuntimeInfo) { functionDetails.getName(), instanceId); String packageFile; + String extraFunctionPackageFile; String pkgLocation = functionMetaData.getPackageLocation().getPackagePath(); - boolean isPkgUrlProvided = isFunctionPackageUrlSupported(pkgLocation); + String extraFunctionPkgLocation = functionMetaData.getExtraFunctionPackageLocation().getPackagePath(); if (runtimeFactory.externallyManaged()) { packageFile = pkgLocation; + extraFunctionPackageFile = extraFunctionPkgLocation; } else { - if (isPkgUrlProvided && pkgLocation.startsWith(FILE)) { - URL url = new URL(pkgLocation); - File pkgFile = new File(url.toURI()); - packageFile = pkgFile.getAbsolutePath(); - } else if (FunctionCommon.isFunctionCodeBuiltin(functionDetails)) { - File pkgFile = getBuiltinArchive(FunctionDetails.newBuilder(functionMetaData.getFunctionDetails())); - packageFile = pkgFile.getAbsolutePath(); - } else { - File pkgDir = new File(workerConfig.getDownloadDirectory(), - getDownloadPackagePath(functionMetaData, instanceId)); - pkgDir.mkdirs(); - File pkgFile = new File( - pkgDir, - new File(getDownloadFileName(functionMetaData.getFunctionDetails(), - functionMetaData.getPackageLocation())).getName()); - downloadFile(pkgFile, isPkgUrlProvided, functionMetaData, instanceId); - packageFile = pkgFile.getAbsolutePath(); - } + packageFile = getPackageFile(functionMetaData, functionDetails, instanceId, pkgLocation); + extraFunctionPackageFile = + getPackageFile(functionMetaData, functionDetails, instanceId, extraFunctionPkgLocation); } // Setup for batch sources if necessary setupBatchSource(functionDetails); - RuntimeSpawner runtimeSpawner = getRuntimeSpawner(functionRuntimeInfo.getFunctionInstance(), packageFile); + RuntimeSpawner runtimeSpawner = getRuntimeSpawner(functionRuntimeInfo.getFunctionInstance(), + packageFile, extraFunctionPackageFile); functionRuntimeInfo.setRuntimeSpawner(runtimeSpawner); runtimeSpawner.start(); @@ -149,7 +138,33 @@ public void startFunction(FunctionRuntimeInfo functionRuntimeInfo) { } } - RuntimeSpawner getRuntimeSpawner(Function.Instance instance, String packageFile) { + private String getPackageFile(FunctionMetaData functionMetaData, FunctionDetails functionDetails, int instanceId, + String pkgLocation) + throws URISyntaxException, IOException, ClassNotFoundException, PulsarAdminException { + boolean isPkgUrlProvided = isFunctionPackageUrlSupported(pkgLocation); + String packageFile; + if (isPkgUrlProvided && pkgLocation.startsWith(FILE)) { + URL url = new URL(pkgLocation); + File pkgFile = new File(url.toURI()); + packageFile = pkgFile.getAbsolutePath(); + } else if (FunctionCommon.isFunctionCodeBuiltin(functionDetails)) { + File pkgFile = getBuiltinArchive(FunctionDetails.newBuilder(functionMetaData.getFunctionDetails())); + packageFile = pkgFile.getAbsolutePath(); + } else { + File pkgDir = new File(workerConfig.getDownloadDirectory(), + getDownloadPackagePath(functionMetaData, instanceId)); + pkgDir.mkdirs(); + File pkgFile = new File( + pkgDir, + new File(getDownloadFileName(functionMetaData.getFunctionDetails(), + functionMetaData.getPackageLocation())).getName()); + downloadFile(pkgFile, isPkgUrlProvided, functionMetaData, instanceId); + packageFile = pkgFile.getAbsolutePath(); + } + return packageFile; + } + + RuntimeSpawner getRuntimeSpawner(Function.Instance instance, String packageFile, String extraFunctionPackageFile) { FunctionMetaData functionMetaData = instance.getFunctionMetaData(); int instanceId = instance.getInstanceId(); @@ -170,6 +185,8 @@ RuntimeSpawner getRuntimeSpawner(Function.Instance instance, String packageFile) RuntimeSpawner runtimeSpawner = new RuntimeSpawner(instanceConfig, packageFile, functionMetaData.getPackageLocation().getOriginalFileName(), + extraFunctionPackageFile, + functionMetaData.getExtraFunctionPackageLocation().getOriginalFileName(), runtimeFactory, workerConfig.getInstanceLivenessCheckFreqMs()); return runtimeSpawner; diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java index 006437cee3be5..0f84e81844eb5 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java @@ -758,7 +758,12 @@ private void updateAssignment(Assignment assignment) { newFunctionRuntimeInfo.setFunctionInstance(assignment.getInstance()); RuntimeSpawner runtimeSpawner = functionActioner.getRuntimeSpawner( assignment.getInstance(), - assignment.getInstance().getFunctionMetaData().getPackageLocation().getPackagePath()); + assignment.getInstance().getFunctionMetaData().getPackageLocation().getPackagePath(), + assignment + .getInstance() + .getFunctionMetaData() + .getExtraFunctionPackageLocation() + .getPackagePath()); // re-initialize if necessary runtimeSpawner.getRuntime().reinitialize(); newFunctionRuntimeInfo.setRuntimeSpawner(runtimeSpawner); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java index 9ccdc09bd7db8..926b4fcf5f7e4 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java @@ -115,7 +115,7 @@ public void testStartFunctionWithPkgUrl() throws Exception { RuntimeFactory factory = mock(RuntimeFactory.class); Runtime runtime = mock(Runtime.class); - doReturn(runtime).when(factory).createContainer(any(), any(), any(), any()); + doReturn(runtime).when(factory).createContainer(any(), any(), any(), any(), any(), any()); doNothing().when(runtime).start(); Namespace dlogNamespace = mock(Namespace.class); final String exceptionMsg = "dl namespace not-found"; @@ -177,7 +177,7 @@ public void testFunctionAuthDisabled() throws Exception { RuntimeFactory factory = mock(RuntimeFactory.class); Runtime runtime = mock(Runtime.class); - doReturn(runtime).when(factory).createContainer(any(), any(), any(), any()); + doReturn(runtime).when(factory).createContainer(any(), any(), any(), any(), any(), any()); doNothing().when(runtime).start(); Namespace dlogNamespace = mock(Namespace.class); final String exceptionMsg = "dl namespace not-found"; @@ -198,7 +198,7 @@ public void testFunctionAuthDisabled() throws Exception { Function.Instance instance = Function.Instance.newBuilder() .setFunctionMetaData(functionMeta).build(); - RuntimeSpawner runtimeSpawner = spy(actioner.getRuntimeSpawner(instance, "foo")); + RuntimeSpawner runtimeSpawner = spy(actioner.getRuntimeSpawner(instance, "foo", "bar")); assertNull(runtimeSpawner.getInstanceConfig().getFunctionAuthenticationSpec()); @@ -236,7 +236,7 @@ public void testStartFunctionWithPackageUrl() throws Exception { RuntimeFactory factory = mock(RuntimeFactory.class); Runtime runtime = mock(Runtime.class); - doReturn(runtime).when(factory).createContainer(any(), any(), any(), any()); + doReturn(runtime).when(factory).createContainer(any(), any(), any(), any(), any(), any()); doNothing().when(runtime).start(); Namespace dlogNamespace = mock(Namespace.class); final String exceptionMsg = "dl namespace not-found"; diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java index ceda6f89eb8df..cc23c86c3590c 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java @@ -716,7 +716,7 @@ public void testExternallyManagedRuntimeUpdate() throws Exception { doReturn(true).when(kubernetesRuntimeFactory).externallyManaged(); KubernetesRuntime kubernetesRuntime = mock(KubernetesRuntime.class); - doReturn(kubernetesRuntime).when(kubernetesRuntimeFactory).createContainer(any(), any(), any(), any()); + doReturn(kubernetesRuntime).when(kubernetesRuntimeFactory).createContainer(any(), any(), any(), any(), any(), any()); FunctionActioner functionActioner = spy(new FunctionActioner( workerConfig, @@ -771,7 +771,8 @@ public void testExternallyManagedRuntimeUpdate() throws Exception { FunctionRuntimeInfo functionRuntimeInfo = new FunctionRuntimeInfo() .setFunctionInstance(instance) .setRuntimeSpawner(functionActioner - .getRuntimeSpawner(instance, function1.getPackageLocation().getPackagePath())); + .getRuntimeSpawner(instance, function1.getPackageLocation().getPackagePath(), + function1.getExtraFunctionPackageLocation().getPackagePath())); functionRuntimeManager.functionRuntimeInfos.put( "test-tenant/test-namespace/func-1:0", functionRuntimeInfo); From be1331752d3ee976aff5f5685635028b4ccf3b7c Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Thu, 30 Jun 2022 15:31:58 +0200 Subject: [PATCH 04/26] Get function from extra function package in JavaInstanceRunnable --- .../org/apache/pulsar/admin/cli/CmdSinks.java | 6 ++ .../functions/instance/InstanceConfig.java | 1 + .../instance/JavaInstanceRunnable.java | 32 +++++----- .../instance/JavaInstanceRunnableTest.java | 2 +- .../api/examples/RecordFunction.java | 1 + .../runtime/JavaInstanceStarter.java | 4 ++ .../functions/runtime/RuntimeUtils.java | 4 ++ .../runtime/thread/ThreadRuntime.java | 32 ++++++---- .../functions/worker/FunctionActioner.java | 63 +++++++++++-------- .../worker/rest/api/ComponentImpl.java | 2 +- .../functions/worker/rest/api/SinksImpl.java | 1 + .../worker/rest/api/FunctionsImplTest.java | 4 +- 12 files changed, 96 insertions(+), 56 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java index 019861fa5172c..cc370bbd47eee 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java @@ -400,6 +400,8 @@ abstract class SinkDetailsCommand extends BaseCommand { protected String secretsString; @Parameter(names = "--preprocess-function", description = "Preprocess function applied before the Sink") protected String preprocessFunction; + @Parameter(names = "--preprocess-function-classname", description = "The preprocess function class name") + protected String preprocessFunctionClassName; @Parameter(names = "--preprocess-function-config", description = "Configuration of the preprocess function applied before the Sink") protected String preprocessFunctionConfig; @@ -588,6 +590,10 @@ void processArguments() throws Exception { sinkConfig.setPreprocessFunction(preprocessFunction); } + if (preprocessFunctionClassName != null) { + sinkConfig.setPreprocessFunctionClassName(preprocessFunctionClassName); + } + if (preprocessFunctionConfig != null) { sinkConfig.setPreprocessFunctionConfig(preprocessFunctionConfig); } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java index 71f0c154af6f1..b66dbdda4f104 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java @@ -33,6 +33,7 @@ public class InstanceConfig { private int instanceId; private String functionId; + private String extraFunctionId; private String functionVersion; private FunctionDetails functionDetails; private int maxBufferedTuples; diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index 11e443e72e08d..85d22162c7efc 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -126,6 +126,7 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable { private final Map properties; private final ClassLoader instanceClassLoader; + private final ClassLoader componentClassLoader; private final ClassLoader functionClassLoader; // a flog to determine if member variables have been initialized as part of setup(). @@ -143,7 +144,8 @@ public JavaInstanceRunnable(InstanceConfig instanceConfig, String stateStorageServiceUrl, SecretsProvider secretsProvider, FunctionCollectorRegistry collectorRegistry, - ClassLoader functionClassLoader) throws PulsarClientException { + ClassLoader componentClassLoader, + ClassLoader extraFunctionClassLoader) throws PulsarClientException { this.instanceConfig = instanceConfig; this.clientBuilder = clientBuilder; this.client = (PulsarClientImpl) pulsarClient; @@ -151,7 +153,8 @@ public JavaInstanceRunnable(InstanceConfig instanceConfig, this.stateStorageImplClass = stateStorageImplClass; this.stateStorageServiceUrl = stateStorageServiceUrl; this.secretsProvider = secretsProvider; - this.functionClassLoader = functionClassLoader; + this.componentClassLoader = componentClassLoader; + this.functionClassLoader = extraFunctionClassLoader != null ? extraFunctionClassLoader : componentClassLoader; this.metricsLabels = new String[]{ instanceConfig.getFunctionDetails().getTenant(), String.format("%s/%s", instanceConfig.getFunctionDetails().getTenant(), @@ -260,8 +263,7 @@ public void run() { Thread currentThread = Thread.currentThread(); Consumer asyncErrorHandler = throwable -> currentThread.interrupt(); - AsyncResultConsumer asyncResultConsumer = - (record, javaExecutionResult) -> handleResult(record, javaExecutionResult); + AsyncResultConsumer asyncResultConsumer = this::handleResult; while (true) { currentRecord = readInput(); @@ -383,7 +385,7 @@ void handleResult(Record srcRecord, JavaExecutionResult result) throws Exception private void sendOutputMessage(Record srcRecord, Object output) throws Exception { if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SINK) { - Thread.currentThread().setContextClassLoader(functionClassLoader); + Thread.currentThread().setContextClassLoader(componentClassLoader); } AbstractSinkRecord sinkRecord; if (output instanceof Record) { @@ -407,7 +409,7 @@ private void sendOutputMessage(Record srcRecord, Object output) throws Exception private Record readInput() throws Exception { Record record; if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SOURCE) { - Thread.currentThread().setContextClassLoader(functionClassLoader); + Thread.currentThread().setContextClassLoader(componentClassLoader); } try { record = this.source.read(); @@ -446,7 +448,7 @@ public synchronized void close() { if (source != null) { if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SOURCE) { - Thread.currentThread().setContextClassLoader(functionClassLoader); + Thread.currentThread().setContextClassLoader(componentClassLoader); } try { source.close(); @@ -461,7 +463,7 @@ public synchronized void close() { if (sink != null) { if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SINK) { - Thread.currentThread().setContextClassLoader(functionClassLoader); + Thread.currentThread().setContextClassLoader(componentClassLoader); } try { sink.close(); @@ -753,11 +755,11 @@ private void setupInput(ContextImpl contextImpl) throws Exception { if (pulsarSourceConfig instanceof SingleConsumerPulsarSourceConfig) { object = new SingleConsumerPulsarSource(this.client, (SingleConsumerPulsarSourceConfig) pulsarSourceConfig, this.properties, - this.functionClassLoader); + this.componentClassLoader); } else { object = new MultiConsumerPulsarSource(this.client, (MultiConsumerPulsarSourceConfig) pulsarSourceConfig, - this.properties, this.functionClassLoader); + this.properties, this.componentClassLoader); } } else { @@ -769,7 +771,7 @@ private void setupInput(ContextImpl contextImpl) throws Exception { } else { object = Reflections.createInstance( sourceSpec.getClassName(), - this.functionClassLoader); + this.componentClassLoader); } } @@ -783,7 +785,7 @@ private void setupInput(ContextImpl contextImpl) throws Exception { this.source = (Source) object; if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SOURCE) { - Thread.currentThread().setContextClassLoader(this.functionClassLoader); + Thread.currentThread().setContextClassLoader(this.componentClassLoader); } try { if (sourceSpec.getConfigs().isEmpty()) { @@ -842,12 +844,12 @@ private void setupOutput(ContextImpl contextImpl) throws Exception { } object = new PulsarSink(this.client, pulsarSinkConfig, this.properties, this.stats, - this.functionClassLoader); + this.componentClassLoader); } } else { object = Reflections.createInstance( sinkSpec.getClassName(), - this.functionClassLoader); + this.componentClassLoader); } if (object instanceof Sink) { @@ -857,7 +859,7 @@ private void setupOutput(ContextImpl contextImpl) throws Exception { } if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SINK) { - Thread.currentThread().setContextClassLoader(this.functionClassLoader); + Thread.currentThread().setContextClassLoader(this.componentClassLoader); } try { if (sinkSpec.getConfigs().isEmpty()) { diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java index 5029a064311b6..0f8385fc6fa76 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceRunnableTest.java @@ -79,7 +79,7 @@ private JavaInstanceRunnable createRunnable(FunctionDetails functionDetails) thr when(clientBuilder.build()).thenReturn(null); InstanceConfig config = createInstanceConfig(functionDetails); JavaInstanceRunnable javaInstanceRunnable = new JavaInstanceRunnable( - config, clientBuilder, null, null, null, null, null, null, null); + config, clientBuilder, null, null, null, null, null, null, null, null); return javaInstanceRunnable; } diff --git a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/RecordFunction.java b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/RecordFunction.java index 55adf848da570..512c583e30138 100644 --- a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/RecordFunction.java +++ b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/RecordFunction.java @@ -38,6 +38,7 @@ public Record process(String input, Context context) throws Exception { return context.newOutputRecordBuilder(Schema.STRING) .destinationTopic(publishTopic) .value(output) + .schema(Schema.STRING) .properties(properties) .build(); } diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java index 9c02e2aec2e9a..a7baa1ceab028 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java @@ -73,6 +73,9 @@ public class JavaInstanceStarter implements AutoCloseable { @Parameter(names = "--function_id", description = "Function Id\n", required = true) public String functionId; + @Parameter(names = "--extra-function_id", description = "Extra Function Id\n", required = true) + public String extraFunctionId; + @Parameter(names = "--function_version", description = "Function Version\n", required = true) public String functionVersion; @@ -163,6 +166,7 @@ public void start(String[] args, ClassLoader functionInstanceClassLoader, ClassL InstanceConfig instanceConfig = new InstanceConfig(); instanceConfig.setFunctionId(functionId); + instanceConfig.setExtraFunctionId(extraFunctionId); instanceConfig.setFunctionVersion(functionVersion); instanceConfig.setInstanceId(instanceId); instanceConfig.setMaxBufferedTuples(maxBufferedTuples); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java index a1101e80c95f2..5a49bf4aeac33 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java @@ -382,6 +382,10 @@ public static List getCmd(InstanceConfig instanceConfig, args.add(shardId); args.add("--function_id"); args.add(instanceConfig.getFunctionId()); + if (instanceConfig.getExtraFunctionId() != null) { + args.add("--extra-function_id"); + args.add(instanceConfig.getExtraFunctionId()); + } args.add("--function_version"); args.add(instanceConfig.getFunctionVersion()); args.add("--function_details"); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java index fdb125eb557f2..25a0488c105a7 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java @@ -113,14 +113,15 @@ public class ThreadRuntime implements Runtime { } private static ClassLoader getFunctionClassLoader(InstanceConfig instanceConfig, + String functionId, String jarFile, String narExtractionDirectory, FunctionCacheManager fnCache, Optional connectorsManager, - Optional functionsManager) throws Exception { - if (FunctionCommon.isFunctionCodeBuiltin(instanceConfig.getFunctionDetails())) { - Function.FunctionDetails.ComponentType componentType = - InstanceUtils.calculateSubjectType(instanceConfig.getFunctionDetails()); + Optional functionsManager, + Function.FunctionDetails.ComponentType componentType) + throws Exception { + if (FunctionCommon.isFunctionCodeBuiltin(instanceConfig.getFunctionDetails(), componentType)) { if (componentType == Function.FunctionDetails.ComponentType.FUNCTION && functionsManager.isPresent()) { return functionsManager.get() .getFunction(instanceConfig.getFunctionDetails().getBuiltin()) @@ -137,11 +138,12 @@ private static ClassLoader getFunctionClassLoader(InstanceConfig instanceConfig, .getClassLoader(); } } - return loadJars(jarFile, instanceConfig, narExtractionDirectory, fnCache); + return loadJars(jarFile, instanceConfig, functionId, narExtractionDirectory, fnCache); } private static ClassLoader loadJars(String jarFile, InstanceConfig instanceConfig, + String functionId, String narExtractionDirectory, FunctionCacheManager fnCache) throws Exception { if (jarFile == null) { @@ -154,7 +156,7 @@ private static ClassLoader loadJars(String jarFile, log.info("Trying Loading file as NAR file: {}", jarFile); // Let's first try to treat it as a nar archive fnCache.registerFunctionInstanceWithArchive( - instanceConfig.getFunctionId(), + functionId, instanceConfig.getInstanceName(), jarFile, narExtractionDirectory); loadedAsNar = true; @@ -168,16 +170,16 @@ private static ClassLoader loadJars(String jarFile, log.info("Load file as simple JAR file: {}", jarFile); // create the function class loader fnCache.registerFunctionInstance( - instanceConfig.getFunctionId(), + functionId, instanceConfig.getInstanceName(), Arrays.asList(jarFile), Collections.emptyList()); } log.info("Initialize function class loader for function {} at function cache manager, functionClassLoader: {}", - instanceConfig.getFunctionDetails().getName(), fnCache.getClassLoader(instanceConfig.getFunctionId())); + instanceConfig.getFunctionDetails().getName(), fnCache.getClassLoader(functionId)); - fnClassLoader = fnCache.getClassLoader(instanceConfig.getFunctionId()); + fnClassLoader = fnCache.getClassLoader(functionId); if (null == fnClassLoader) { throw new Exception("No function class loader available."); } @@ -193,8 +195,13 @@ public void start() throws Exception { // extract class loader for function ClassLoader functionClassLoader = - getFunctionClassLoader(instanceConfig, jarFile, narExtractionDirectory, fnCache, connectorsManager, - functionsManager); + getFunctionClassLoader(instanceConfig, instanceConfig.getFunctionId(), jarFile, narExtractionDirectory, + fnCache, connectorsManager, functionsManager, + InstanceUtils.calculateSubjectType(instanceConfig.getFunctionDetails())); + + ClassLoader extraFunctionClassLoader = extraFunctionFile == null ? null : getFunctionClassLoader( + instanceConfig, instanceConfig.getExtraFunctionId(), extraFunctionFile, narExtractionDirectory, fnCache, + connectorsManager, functionsManager, Function.FunctionDetails.ComponentType.FUNCTION); // re-initialize JavaInstanceRunnable so that variables in constructor can be re-initialized this.javaInstanceRunnable = new JavaInstanceRunnable( @@ -206,7 +213,8 @@ public void start() throws Exception { stateStorageServiceUrl, secretsProvider, collectorRegistry, - functionClassLoader); + functionClassLoader, + extraFunctionClassLoader); log.info("ThreadContainer starting function with instanceId {} functionId {} namespace {}", instanceConfig.getInstanceId(), diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java index 22bde59b8cdaa..15d0398253b46 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java @@ -19,6 +19,7 @@ package org.apache.pulsar.functions.worker; import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isEmpty; import static org.apache.pulsar.common.functions.Utils.FILE; import static org.apache.pulsar.common.functions.Utils.HTTP; import static org.apache.pulsar.common.functions.Utils.hasPackageTypePrefix; @@ -30,7 +31,6 @@ import com.google.common.io.MoreFiles; import com.google.common.io.RecursiveDeleteOption; import java.io.File; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.net.URISyntaxException; @@ -107,18 +107,23 @@ public void startFunction(FunctionRuntimeInfo functionRuntimeInfo) { functionDetails.getName(), instanceId); String packageFile; - String extraFunctionPackageFile; + String extraFunctionPackageFile = null; - String pkgLocation = functionMetaData.getPackageLocation().getPackagePath(); - String extraFunctionPkgLocation = functionMetaData.getExtraFunctionPackageLocation().getPackagePath(); + Function.PackageLocationMetaData pkgLocation = functionMetaData.getPackageLocation(); + Function.PackageLocationMetaData extraFunctionPkgLocation = + functionMetaData.getExtraFunctionPackageLocation(); if (runtimeFactory.externallyManaged()) { - packageFile = pkgLocation; - extraFunctionPackageFile = extraFunctionPkgLocation; + packageFile = pkgLocation.getPackagePath(); + extraFunctionPackageFile = extraFunctionPkgLocation.getPackagePath(); } else { - packageFile = getPackageFile(functionMetaData, functionDetails, instanceId, pkgLocation); - extraFunctionPackageFile = - getPackageFile(functionMetaData, functionDetails, instanceId, extraFunctionPkgLocation); + packageFile = getPackageFile(functionMetaData, functionDetails, instanceId, pkgLocation, + functionDetails.getComponentType()); + if (!isEmpty(extraFunctionPkgLocation.getPackagePath())) { + extraFunctionPackageFile = + getPackageFile(functionMetaData, functionDetails, instanceId, extraFunctionPkgLocation, + FunctionDetails.ComponentType.FUNCTION); + } } // Setup for batch sources if necessary @@ -139,16 +144,20 @@ public void startFunction(FunctionRuntimeInfo functionRuntimeInfo) { } private String getPackageFile(FunctionMetaData functionMetaData, FunctionDetails functionDetails, int instanceId, - String pkgLocation) + Function.PackageLocationMetaData pkgLocation, + FunctionDetails.ComponentType componentType) throws URISyntaxException, IOException, ClassNotFoundException, PulsarAdminException { - boolean isPkgUrlProvided = isFunctionPackageUrlSupported(pkgLocation); + String packagePath = pkgLocation.getPackagePath(); + boolean isPkgUrlProvided = isFunctionPackageUrlSupported(packagePath); String packageFile; - if (isPkgUrlProvided && pkgLocation.startsWith(FILE)) { - URL url = new URL(pkgLocation); + if (isPkgUrlProvided && packagePath.startsWith(FILE)) { + URL url = new URL(packagePath); File pkgFile = new File(url.toURI()); packageFile = pkgFile.getAbsolutePath(); - } else if (FunctionCommon.isFunctionCodeBuiltin(functionDetails)) { - File pkgFile = getBuiltinArchive(FunctionDetails.newBuilder(functionMetaData.getFunctionDetails())); + } else if (FunctionCommon.isFunctionCodeBuiltin(functionDetails, componentType)) { + File pkgFile = getBuiltinArchive( + componentType, + FunctionDetails.newBuilder(functionMetaData.getFunctionDetails())); packageFile = pkgFile.getAbsolutePath(); } else { File pkgDir = new File(workerConfig.getDownloadDirectory(), @@ -157,8 +166,8 @@ private String getPackageFile(FunctionMetaData functionMetaData, FunctionDetails File pkgFile = new File( pkgDir, new File(getDownloadFileName(functionMetaData.getFunctionDetails(), - functionMetaData.getPackageLocation())).getName()); - downloadFile(pkgFile, isPkgUrlProvided, functionMetaData, instanceId); + pkgLocation)).getName()); + downloadFile(pkgFile, isPkgUrlProvided, functionMetaData, instanceId, pkgLocation); packageFile = pkgFile.getAbsolutePath(); } return packageFile; @@ -198,6 +207,7 @@ InstanceConfig createInstanceConfig(FunctionDetails functionDetails, Function.Fu instanceConfig.setFunctionDetails(functionDetails); // TODO: set correct function id and version when features implemented instanceConfig.setFunctionId(UUID.randomUUID().toString()); + instanceConfig.setExtraFunctionId(UUID.randomUUID().toString()); instanceConfig.setFunctionVersion(UUID.randomUUID().toString()); instanceConfig.setInstanceId(instanceId); instanceConfig.setMaxBufferedTuples(1024); @@ -214,7 +224,8 @@ InstanceConfig createInstanceConfig(FunctionDetails functionDetails, Function.Fu } private void downloadFile(File pkgFile, boolean isPkgUrlProvided, FunctionMetaData functionMetaData, - int instanceId) throws FileNotFoundException, IOException, PulsarAdminException { + int instanceId, Function.PackageLocationMetaData pkgLocation) + throws IOException, PulsarAdminException { FunctionDetails details = functionMetaData.getFunctionDetails(); File pkgDir = pkgFile.getParentFile(); @@ -230,12 +241,12 @@ private void downloadFile(File pkgFile, boolean isPkgUrlProvided, FunctionMetaDa pkgDir, pkgFile.getName() + "." + instanceId + "." + UUID.randomUUID()); } while (tempPkgFile.exists() || !tempPkgFile.createNewFile()); - String pkgLocationPath = functionMetaData.getPackageLocation().getPackagePath(); + String pkgLocationPath = pkgLocation.getPackagePath(); boolean downloadFromHttp = isPkgUrlProvided && pkgLocationPath.startsWith(HTTP); boolean downloadFromPackageManagementService = isPkgUrlProvided && hasPackageTypePrefix(pkgLocationPath); log.info("{}/{}/{} Function package file {} will be downloaded from {}", tempPkgFile, details.getTenant(), details.getNamespace(), details.getName(), - downloadFromHttp ? pkgLocationPath : functionMetaData.getPackageLocation()); + downloadFromHttp ? pkgLocationPath : pkgLocation); if (downloadFromHttp) { FunctionCommon.downloadFromHttpUrl(pkgLocationPath, tempPkgFile); @@ -264,7 +275,7 @@ private void downloadFile(File pkgFile, boolean isPkgUrlProvided, FunctionMetaDa } catch (FileAlreadyExistsException faee) { // file already exists log.warn("Function package has been downloaded from {} and saved at {}", - functionMetaData.getPackageLocation(), pkgFile); + pkgLocation, pkgFile); } } finally { tempPkgFile.delete(); @@ -502,8 +513,9 @@ private String getDownloadPackagePath(FunctionMetaData functionMetaData, int ins File.separatorChar); } - private File getBuiltinArchive(FunctionDetails.Builder functionDetails) throws IOException, ClassNotFoundException { - if (functionDetails.hasSource()) { + private File getBuiltinArchive(FunctionDetails.ComponentType componentType, FunctionDetails.Builder functionDetails) + throws IOException, ClassNotFoundException { + if (componentType == FunctionDetails.ComponentType.SOURCE && functionDetails.hasSource()) { SourceSpec sourceSpec = functionDetails.getSource(); if (!StringUtils.isEmpty(sourceSpec.getBuiltin())) { Connector connector = connectorsManager.getConnector(sourceSpec.getBuiltin()); @@ -518,7 +530,7 @@ private File getBuiltinArchive(FunctionDetails.Builder functionDetails) throws I } } - if (functionDetails.hasSink()) { + if (componentType == FunctionDetails.ComponentType.SINK && functionDetails.hasSink()) { SinkSpec sinkSpec = functionDetails.getSink(); if (!StringUtils.isEmpty(sinkSpec.getBuiltin())) { Connector connector = connectorsManager.getConnector(sinkSpec.getBuiltin()); @@ -534,7 +546,8 @@ private File getBuiltinArchive(FunctionDetails.Builder functionDetails) throws I } } - if (!StringUtils.isEmpty(functionDetails.getBuiltin())) { + if (componentType == FunctionDetails.ComponentType.FUNCTION + && !StringUtils.isEmpty(functionDetails.getBuiltin())) { return functionsManager.getFunctionArchive(functionDetails.getBuiltin()).toFile(); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index 0dbed39e3ce6c..5d6e0a7973c5f 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -1626,7 +1626,7 @@ private String getFunctionCodeBuiltin(FunctionDetails functionDetails, } } - if (componentType == FunctionDetails.ComponentType.SOURCE && functionDetails.hasSink()) { + if (componentType == FunctionDetails.ComponentType.SINK && functionDetails.hasSink()) { SinkSpec sinkSpec = functionDetails.getSink(); if (!isEmpty(sinkSpec.getBuiltin())) { return sinkSpec.getBuiltin(); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index 74898cc0815bf..0e93eabd56bb0 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -237,6 +237,7 @@ public void registerSink(final String tenant, functionPackageFile = getPackageFile(preprocessFunction); } else { builtin = preprocessFunction.replaceFirst("^builtin://", ""); + functionMetaDataBuilder.setFunctionDetails(functionDetails.toBuilder().setBuiltin(builtin)); } Function.PackageLocationMetaData.Builder functionPackageLocation = getFunctionPackageLocation(functionMetaDataBuilder.build(), diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java index 4b7e9940230f1..7ffebcb3216f8 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java @@ -152,7 +152,7 @@ public void setup() throws Exception { instanceConfig.setMaxBufferedTuples(1024); JavaInstanceRunnable javaInstanceRunnable = new JavaInstanceRunnable( - instanceConfig, null, null, null, null, null, null, null, null); + instanceConfig, null, null, null, null, null, null, null, null, null); CompletableFuture metricsDataCompletableFuture = new CompletableFuture(); metricsDataCompletableFuture.complete(javaInstanceRunnable.getMetrics()); @@ -207,7 +207,7 @@ public void testMetricsEmpty() throws PulsarClientException { instanceConfig.setMaxBufferedTuples(1024); JavaInstanceRunnable javaInstanceRunnable = new JavaInstanceRunnable( - instanceConfig, null, null, null, null, null, null, null, null); + instanceConfig, null, null, null, null, null, null, null, null, null); CompletableFuture completableFuture = new CompletableFuture(); completableFuture.complete(javaInstanceRunnable.getMetrics()); From 88fac50f2ee6940781218ef5308da7b017a7074d Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Thu, 30 Jun 2022 16:23:43 +0200 Subject: [PATCH 05/26] Log ClassCastException due to incompatible Function and Sink --- .../main/java/org/apache/pulsar/admin/cli/CmdSinks.java | 2 -- .../pulsar/functions/instance/JavaInstanceRunnable.java | 7 ++++++- .../pulsar/functions/runtime/JavaInstanceStarter.java | 4 ++-- .../org/apache/pulsar/functions/runtime/RuntimeUtils.java | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java index cc370bbd47eee..221998d779890 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java @@ -405,8 +405,6 @@ abstract class SinkDetailsCommand extends BaseCommand { @Parameter(names = "--preprocess-function-config", description = "Configuration of the preprocess function applied before the Sink") protected String preprocessFunctionConfig; - - protected SinkConfig sinkConfig; private void mergeArgs() { diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index 85d22162c7efc..edc1d9b711f4a 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -396,7 +396,12 @@ private void sendOutputMessage(Record srcRecord, Object output) throws Exception try { this.sink.write(sinkRecord); } catch (Exception e) { - log.info("Encountered exception in sink write: ", e); + if (e instanceof ClassCastException && functionClassLoader != componentClassLoader) { + log.info("Encountered ClassCastException in sink write. " + + "Is the Function return type compatible with the Sink type ? ", e); + } else { + log.info("Encountered exception in sink write: ", e); + } stats.incrSinkExceptions(e); // fail the source record srcRecord.fail(); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java index a7baa1ceab028..3cd26c7ffb520 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java @@ -62,7 +62,7 @@ public class JavaInstanceStarter implements AutoCloseable { public String jarFile; @Parameter( - names = "--extra-function-jar", + names = "--extra_function_jar", description = "Path to Extra Function Jar\n", listConverter = StringConverter.class) public String extraFunctionJarFile; @@ -73,7 +73,7 @@ public class JavaInstanceStarter implements AutoCloseable { @Parameter(names = "--function_id", description = "Function Id\n", required = true) public String functionId; - @Parameter(names = "--extra-function_id", description = "Extra Function Id\n", required = true) + @Parameter(names = "--extra_function_id", description = "Extra Function Id\n", required = true) public String extraFunctionId; @Parameter(names = "--function_version", description = "Function Version\n", required = true) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java index 5a49bf4aeac33..59f1434f0d6f2 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java @@ -383,7 +383,7 @@ public static List getCmd(InstanceConfig instanceConfig, args.add("--function_id"); args.add(instanceConfig.getFunctionId()); if (instanceConfig.getExtraFunctionId() != null) { - args.add("--extra-function_id"); + args.add("--extra_function_id"); args.add(instanceConfig.getExtraFunctionId()); } args.add("--function_version"); From ab825c72edcb8f1fb4dfb62488e8ab0d884f378d Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Fri, 1 Jul 2022 09:59:15 +0200 Subject: [PATCH 06/26] Support ProcessRuntime for sink preprocess function --- .../functions/runtime/RuntimeUtils.java | 18 ++++++++------ .../runtime/kubernetes/KubernetesRuntime.java | 24 +++++++++++++++---- .../kubernetes/KubernetesRuntimeFactory.java | 1 + .../runtime/process/ProcessRuntime.java | 2 ++ .../process/ProcessRuntimeFactory.java | 1 + .../functions/runtime/RuntimeUtilsTest.java | 1 + 6 files changed, 35 insertions(+), 12 deletions(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java index 59f1434f0d6f2..60296e8368bfe 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java @@ -20,6 +20,7 @@ import static org.apache.commons.lang3.StringUtils.isEmpty; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.commons.lang3.StringUtils.isNotEmpty; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.protobuf.util.JsonFormat; import io.prometheus.client.hotspot.BufferPoolsExports; @@ -36,6 +37,7 @@ import java.net.HttpURLConnection; import java.net.InetAddress; import java.net.URL; +import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -66,6 +68,7 @@ public static List composeCmd(InstanceConfig instanceConfig, String extraDependenciesDir, /* extra dependencies for running instances */ String logDirectory, String originalCodeFileName, + String originalExtraFunctionFileName, String pulsarServiceUrl, String stateStorageServiceUrl, AuthenticationConfig authConfig, @@ -85,7 +88,7 @@ public static List composeCmd(InstanceConfig instanceConfig, final List cmd = getArgsBeforeCmd(instanceConfig, extraDependenciesDir); cmd.addAll(getCmd(instanceConfig, instanceFile, extraDependenciesDir, logDirectory, - originalCodeFileName, pulsarServiceUrl, stateStorageServiceUrl, + originalCodeFileName, originalExtraFunctionFileName, pulsarServiceUrl, stateStorageServiceUrl, authConfig, shardId, grpcPort, expectedHealthCheckInterval, logConfigFile, secretsProviderClassName, secretsProviderConfig, installUserCodeDependencies, pythonDependencyRepository, @@ -262,6 +265,7 @@ public static List getCmd(InstanceConfig instanceConfig, String extraDependenciesDir, /* extra dependencies for running instances */ String logDirectory, String originalCodeFileName, + String originalExtraFunctionFileName, String pulsarServiceUrl, String stateStorageServiceUrl, AuthenticationConfig authConfig, @@ -332,9 +336,7 @@ public static List getCmd(InstanceConfig instanceConfig, } if (!isEmpty(instanceConfig.getFunctionDetails().getRuntimeFlags())) { - for (String runtimeFlagArg : splitRuntimeArgs(instanceConfig.getFunctionDetails().getRuntimeFlags())) { - args.add(runtimeFlagArg); - } + Collections.addAll(args, splitRuntimeArgs(instanceConfig.getFunctionDetails().getRuntimeFlags())); } if (instanceConfig.getFunctionDetails().getResources() != null) { Function.Resources resources = instanceConfig.getFunctionDetails().getResources(); @@ -346,12 +348,14 @@ public static List getCmd(InstanceConfig instanceConfig, args.add("--jar"); args.add(originalCodeFileName); + if (isNotEmpty(originalExtraFunctionFileName)) { + args.add("--extra_function_jar"); + args.add(originalExtraFunctionFileName); + } } else if (instanceConfig.getFunctionDetails().getRuntime() == Function.FunctionDetails.Runtime.PYTHON) { args.add("python3"); if (!isEmpty(instanceConfig.getFunctionDetails().getRuntimeFlags())) { - for (String runtimeFlagArg : splitRuntimeArgs(instanceConfig.getFunctionDetails().getRuntimeFlags())) { - args.add(runtimeFlagArg); - } + Collections.addAll(args, splitRuntimeArgs(instanceConfig.getFunctionDetails().getRuntimeFlags())); } args.add(instanceFile); args.add("--py"); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index 50dc975075f6c..aea5efad90de7 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -22,6 +22,7 @@ import static java.net.HttpURLConnection.HTTP_CONFLICT; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.commons.lang3.StringUtils.isNotEmpty; import static org.apache.commons.lang3.StringUtils.left; import static org.apache.pulsar.functions.auth.FunctionAuthUtils.getFunctionAuthData; import static org.apache.pulsar.functions.utils.FunctionCommon.roundDecimal; @@ -139,6 +140,7 @@ public class KubernetesRuntime implements Runtime { private final String configAdminCLI; private final String userCodePkgUrl; private final String originalCodeFileName; + private final String originalExtraFunctionFileName; private final String pulsarAdminUrl; private final SecretsProviderConfigurator secretsProviderConfigurator; private int percentMemoryPadding; @@ -173,6 +175,7 @@ public class KubernetesRuntime implements Runtime { String configAdminCLI, String userCodePkgUrl, String originalCodeFileName, + String originalExtraFunctionFileName, String pulsarServiceUrl, String pulsarAdminUrl, String stateStorageServiceUrl, @@ -205,6 +208,7 @@ public class KubernetesRuntime implements Runtime { this.downloadDirectory = StringUtils.isNotEmpty(downloadDirectory) ? downloadDirectory : this.pulsarRootDir; // for backward comp this.originalCodeFileName = this.downloadDirectory + "/" + originalCodeFileName; + this.originalExtraFunctionFileName = this.downloadDirectory + "/" + originalExtraFunctionFileName; this.pulsarAdminUrl = pulsarAdminUrl; this.secretsProviderConfigurator = secretsProviderConfigurator; this.percentMemoryPadding = percentMemoryPadding; @@ -263,6 +267,7 @@ public class KubernetesRuntime implements Runtime { extraDependenciesDir, logDirectory, this.originalCodeFileName, + this.originalExtraFunctionFileName, pulsarServiceUrl, stateStorageServiceUrl, authConfig, @@ -844,13 +849,22 @@ public void deleteService() throws InterruptedException { } protected List getExecutorCommand() { - return Arrays.asList( + List cmds = Arrays.asList( "sh", "-c", - String.join(" ", getDownloadCommand(instanceConfig.getFunctionDetails(), originalCodeFileName)) - + " && " + setShardIdEnvironmentVariableCommand() - + " && " + String.join(" ", processArgs) - ); + String.join(" ", getDownloadCommand(instanceConfig.getFunctionDetails(), originalCodeFileName))); + if (isNotEmpty(originalExtraFunctionFileName)) { + cmds.add("&&"); + cmds.add("sh"); + cmds.add("-c"); + cmds.add(String.join(" ", + getDownloadCommand(instanceConfig.getFunctionDetails(), originalExtraFunctionFileName))); + } + cmds.add("&&"); + cmds.add(setShardIdEnvironmentVariableCommand()); + cmds.add("&&"); + cmds.add(String.join(" ", processArgs)); + return cmds; } private List getDownloadCommand(Function.FunctionDetails functionDetails, String userCodeFilePath) { diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java index 229dee095a237..082b67e828513 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java @@ -329,6 +329,7 @@ public KubernetesRuntime createContainer(InstanceConfig instanceConfig, String c configAdminCLI, codePkgUrl, originalCodeFileName, + originalExtraFunctionFileName, pulsarServiceUrl, pulsarAdminUrl, stateStorageServiceUri, diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java index 3c17633ea613b..c434fe0623962 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java @@ -82,6 +82,7 @@ class ProcessRuntime implements Runtime { String narExtractionDirectory, String logDirectory, String codeFile, + String extraFunctionFile, String pulsarServiceUrl, String stateStorageServiceUrl, AuthenticationConfig authConfig, @@ -133,6 +134,7 @@ class ProcessRuntime implements Runtime { ? extraDependenciesDir : null, logDirectory, codeFile, + extraFunctionFile, pulsarServiceUrl, stateStorageServiceUrl, authConfig, diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeFactory.java index c5c5b664d695e..c74201b732afb 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeFactory.java @@ -225,6 +225,7 @@ public ProcessRuntime createContainer(InstanceConfig instanceConfig, String code narExtractionDirectory, logDirectory, codeFile, + extraFunctionFile, pulsarServiceUrl, stateStorageServiceUrl, authConfig, diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java index 1651a7ba1861e..f7cdf707d3219 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/RuntimeUtilsTest.java @@ -191,6 +191,7 @@ public void getAdditionalJavaRuntimeArguments(boolean k8sRuntime) throws Excepti "extraDependenciesDir", /* extra dependencies for running instances */ "logDirectory", "originalCodeFileName", + "originalExtraFileName", "pulsarServiceUrl", "stateStorageServiceUrl", AuthenticationConfig.builder().build(), From 2288586603cc7482d0d9e2e722a7d3d1ea2a5bd5 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Wed, 6 Jul 2022 17:07:43 +0200 Subject: [PATCH 07/26] Support Function+Sink in KubernetesRuntime --- .../broker/admin/impl/FunctionsBase.java | 7 +- .../apache/pulsar/client/admin/Functions.java | 36 +++++++++ .../client/admin/internal/FunctionsImpl.java | 16 ++++ .../apache/pulsar/admin/cli/CmdFunctions.java | 7 +- .../runtime/kubernetes/KubernetesRuntime.java | 80 +++++++++---------- .../worker/rest/api/ComponentImpl.java | 10 ++- .../rest/api/v3/FunctionsApiV3Resource.java | 7 +- .../worker/service/api/Component.java | 15 +++- 8 files changed, 124 insertions(+), 54 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java index d5380407491ab..6c408522e0e84 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java @@ -711,9 +711,12 @@ public StreamingOutput downloadFunction( @ApiParam(value = "The namespace of a Pulsar Function") final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Function") - final @PathParam("functionName") String functionName) { + final @PathParam("functionName") String functionName, + @ApiParam(value = "Whether to download the extra-function") + final @QueryParam("extra-function") boolean extraFunction) { - return functions().downloadFunction(tenant, namespace, functionName, clientAppId(), clientAuthData()); + return functions() + .downloadFunction(tenant, namespace, functionName, clientAppId(), clientAuthData(), extraFunction); } @GET diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Functions.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Functions.java index 5bf289f9b6571..bce7d420aebf9 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Functions.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Functions.java @@ -757,6 +757,42 @@ void downloadFunction(String destinationFile, String tenant, String namespace, S CompletableFuture downloadFunctionAsync( String destinationFile, String tenant, String namespace, String function); + /** + * Download Function Code. + * + * @param destinationFile + * file where data should be downloaded to + * @param tenant + * Tenant name + * @param namespace + * Namespace name + * @param function + * Function name + * @param extraFunction + * Whether to download the extra-function + * @throws PulsarAdminException + */ + void downloadFunction(String destinationFile, String tenant, String namespace, String function, + boolean extraFunction) throws PulsarAdminException; + + /** + * Download Function Code asynchronously. + * + * @param destinationFile + * file where data should be downloaded to + * @param tenant + * Tenant name + * @param namespace + * Namespace name + * @param function + * Function name + * @param extraFunction + * Whether to download the extra-function + */ + CompletableFuture downloadFunctionAsync( + String destinationFile, String tenant, String namespace, String function, boolean extraFunction); + + /** * Deprecated in favor of getting sources and sinks for their own APIs. *

diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java index ff2c22daae539..4348eadd7ff9b 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java @@ -606,6 +606,22 @@ public CompletableFuture downloadFunctionAsync( functions.path(tenant).path(namespace).path(functionName).path("download")); } + @Override + public void downloadFunction(String destinationPath, String tenant, String namespace, String functionName, + boolean extraFunction) throws PulsarAdminException { + downloadFile(destinationPath, functions.path(tenant).path(namespace).path(functionName).path("download") + .queryParam("extra-function", extraFunction)); + } + + @Override + public CompletableFuture downloadFunctionAsync( + String destinationPath, String tenant, String namespace, String functionName, boolean extraFunction) { + return downloadFileAsync(destinationPath, + functions.path(tenant).path(namespace).path(functionName).path("download") + .queryParam("extra-function", extraFunction)); + } + + @Override public void downloadFunction(String destinationPath, String path) throws PulsarAdminException { downloadFile(destinationPath, functions.path("download").queryParam("path", path)); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index a37197ccc281f..6cc58e3c16852 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -1171,6 +1171,10 @@ class DownloadFunction extends FunctionCommand { description = "Path or functionPkgUrl to store the content", listConverter = StringConverter.class, required = false, hidden = true) protected String path; + @Parameter( + names = "--extra-function", + description = "Download the extra Function of the connector") + protected Boolean extraFunction = false; private void mergeArgs() { if (isBlank(destinationFile) && !isBlank(deprecatedDestinationFile)) { @@ -1195,7 +1199,8 @@ void runCmd() throws Exception { if (path != null) { getAdmin().functions().downloadFunction(destinationFile, path); } else { - getAdmin().functions().downloadFunction(destinationFile, tenant, namespace, functionName); + getAdmin().functions() + .downloadFunction(destinationFile, tenant, namespace, functionName, extraFunction); } print("Downloaded successfully"); } diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index aea5efad90de7..44c21321784bd 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -21,6 +21,7 @@ import static java.net.HttpURLConnection.HTTP_CONFLICT; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; +import static org.apache.commons.lang3.StringUtils.isEmpty; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.isNotEmpty; import static org.apache.commons.lang3.StringUtils.left; @@ -206,9 +207,11 @@ public class KubernetesRuntime implements Runtime { this.configAdminCLI = configAdminCLI; this.userCodePkgUrl = userCodePkgUrl; this.downloadDirectory = - StringUtils.isNotEmpty(downloadDirectory) ? downloadDirectory : this.pulsarRootDir; // for backward comp + isNotEmpty(downloadDirectory) ? downloadDirectory : this.pulsarRootDir; // for backward comp this.originalCodeFileName = this.downloadDirectory + "/" + originalCodeFileName; - this.originalExtraFunctionFileName = this.downloadDirectory + "/" + originalExtraFunctionFileName; + this.originalExtraFunctionFileName = isNotEmpty(originalExtraFunctionFileName) + ? this.downloadDirectory + "/" + originalExtraFunctionFileName + : originalExtraFunctionFileName; this.pulsarAdminUrl = pulsarAdminUrl; this.secretsProviderConfigurator = secretsProviderConfigurator; this.percentMemoryPadding = percentMemoryPadding; @@ -849,58 +852,28 @@ public void deleteService() throws InterruptedException { } protected List getExecutorCommand() { - List cmds = Arrays.asList( - "sh", - "-c", - String.join(" ", getDownloadCommand(instanceConfig.getFunctionDetails(), originalCodeFileName))); + List cmds = + new ArrayList<>(getDownloadCommand(instanceConfig.getFunctionDetails(), originalCodeFileName, false)); if (isNotEmpty(originalExtraFunctionFileName)) { cmds.add("&&"); - cmds.add("sh"); - cmds.add("-c"); - cmds.add(String.join(" ", - getDownloadCommand(instanceConfig.getFunctionDetails(), originalExtraFunctionFileName))); + cmds.addAll(getDownloadCommand(instanceConfig.getFunctionDetails(), originalExtraFunctionFileName, true)); } cmds.add("&&"); cmds.add(setShardIdEnvironmentVariableCommand()); cmds.add("&&"); - cmds.add(String.join(" ", processArgs)); - return cmds; + cmds.addAll(processArgs); + return Arrays.asList("sh", "-c", String.join(" ", cmds)); } - private List getDownloadCommand(Function.FunctionDetails functionDetails, String userCodeFilePath) { + private List getDownloadCommand(Function.FunctionDetails functionDetails, String userCodeFilePath, + boolean extraFunction) { return getDownloadCommand(functionDetails.getTenant(), functionDetails.getNamespace(), - functionDetails.getName(), userCodeFilePath); + functionDetails.getName(), userCodeFilePath, extraFunction); } - private List getDownloadCommand(String tenant, String namespace, String name, String userCodeFilePath) { - - // add auth plugin and parameters if necessary - if (authenticationEnabled && authConfig != null) { - if (isNotBlank(authConfig.getClientAuthenticationPlugin()) - && isNotBlank(authConfig.getClientAuthenticationParameters()) - && instanceConfig.getFunctionAuthenticationSpec() != null) { - return Arrays.asList( - pulsarRootDir + configAdminCLI, - "--auth-plugin", - authConfig.getClientAuthenticationPlugin(), - "--auth-params", - authConfig.getClientAuthenticationParameters(), - "--admin-url", - pulsarAdminUrl, - "functions", - "download", - "--tenant", - tenant, - "--namespace", - namespace, - "--name", - name, - "--destination-file", - userCodeFilePath); - } - } - - return Arrays.asList( + private List getDownloadCommand(String tenant, String namespace, String name, String userCodeFilePath, + boolean extraFunction) { + ArrayList cmd = new ArrayList<>(Arrays.asList( pulsarRootDir + configAdminCLI, "--admin-url", pulsarAdminUrl, @@ -913,7 +886,26 @@ && isNotBlank(authConfig.getClientAuthenticationParameters()) "--name", name, "--destination-file", - userCodeFilePath); + userCodeFilePath)); + + // add auth plugin and parameters if necessary + if (authenticationEnabled && authConfig != null) { + if (isNotBlank(authConfig.getClientAuthenticationPlugin()) + && isNotBlank(authConfig.getClientAuthenticationParameters()) + && instanceConfig.getFunctionAuthenticationSpec() != null) { + cmd.addAll(Arrays.asList( + "--auth-plugin", + authConfig.getClientAuthenticationPlugin(), + "--auth-params", + authConfig.getClientAuthenticationParameters())); + } + } + + if (extraFunction) { + cmd.add("--extra-function"); + } + + return cmd; } private static String setShardIdEnvironmentVariableCommand() { diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index 5d6e0a7973c5f..bc035cf380bde 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -1433,7 +1433,8 @@ public void uploadFunction(final InputStream uploadedInputStream, final String p @Override public StreamingOutput downloadFunction(String tenant, String namespace, String componentName, - String clientRole, AuthenticationDataSource clientAuthenticationDataHttps) { + String clientRole, AuthenticationDataSource clientAuthenticationDataHttps, + boolean extraFunction) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } @@ -1458,8 +1459,11 @@ public StreamingOutput downloadFunction(String tenant, String namespace, String String.format("%s %s doesn't exist", ComponentTypeUtils.toString(componentType), componentName)); } - String pkgPath = functionMetaDataManager.getFunctionMetaData(tenant, namespace, componentName) - .getPackageLocation().getPackagePath(); + FunctionMetaData functionMetaData = + functionMetaDataManager.getFunctionMetaData(tenant, namespace, componentName); + String pkgPath = extraFunction + ? functionMetaData.getExtraFunctionPackageLocation().getPackagePath() + : functionMetaData.getPackageLocation().getPackagePath(); return getStreamingOutput(pkgPath); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java index 3efad1da1cc31..21197e7cc0d97 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java @@ -339,9 +339,12 @@ public StreamingOutput downloadFunction( @ApiParam(value = "The namespace of functions") final @PathParam("namespace") String namespace, @ApiParam(value = "The name of functions") - final @PathParam("functionName") String functionName) { + final @PathParam("functionName") String functionName, + @ApiParam(value = "Whether to download the extra-function") + final @QueryParam("extra-function") boolean extraFunction) { - return functions().downloadFunction(tenant, namespace, functionName, clientAppId(), clientAuthData()); + return functions() + .downloadFunction(tenant, namespace, functionName, clientAppId(), clientAuthData(), extraFunction); } @GET diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java index c305d64b9f3cf..738a92e00d77e 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java @@ -172,7 +172,17 @@ StreamingOutput downloadFunction(String tenant, String namespace, String componentName, String clientRole, - AuthenticationDataSource clientAuthenticationDataHttps); + AuthenticationDataSource clientAuthenticationDataHttps, + boolean extraFunction); + + @Deprecated + default StreamingOutput downloadFunction(String tenant, + String namespace, + String componentName, + String clientRole, + AuthenticationDataSource clientAuthenticationDataHttps) { + return downloadFunction(tenant, namespace, componentName, clientRole, clientAuthenticationDataHttps, false); + } @Deprecated default StreamingOutput downloadFunction(String tenant, @@ -185,7 +195,8 @@ default StreamingOutput downloadFunction(String tenant, namespace, componentName, clientRole, - (AuthenticationDataSource) clientAuthenticationDataHttps); + (AuthenticationDataSource) clientAuthenticationDataHttps, + false); } List getListOfConnectors(); From 44b4aef28c184a610e9635dfe13e991e617b980e Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Tue, 19 Jul 2022 18:25:39 +0200 Subject: [PATCH 08/26] Fix usage of classloader and component type detection --- .../pulsar/functions/instance/JavaInstanceRunnable.java | 6 +++--- .../apache/pulsar/functions/worker/FunctionActioner.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index edc1d9b711f4a..6340908311428 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -760,11 +760,11 @@ private void setupInput(ContextImpl contextImpl) throws Exception { if (pulsarSourceConfig instanceof SingleConsumerPulsarSourceConfig) { object = new SingleConsumerPulsarSource(this.client, (SingleConsumerPulsarSourceConfig) pulsarSourceConfig, this.properties, - this.componentClassLoader); + this.functionClassLoader); } else { object = new MultiConsumerPulsarSource(this.client, (MultiConsumerPulsarSourceConfig) pulsarSourceConfig, - this.properties, this.componentClassLoader); + this.properties, this.functionClassLoader); } } else { @@ -849,7 +849,7 @@ private void setupOutput(ContextImpl contextImpl) throws Exception { } object = new PulsarSink(this.client, pulsarSinkConfig, this.properties, this.stats, - this.componentClassLoader); + this.functionClassLoader); } } else { object = Reflections.createInstance( diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java index 15d0398253b46..41f1c81ab0c34 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java @@ -118,7 +118,7 @@ public void startFunction(FunctionRuntimeInfo functionRuntimeInfo) { extraFunctionPackageFile = extraFunctionPkgLocation.getPackagePath(); } else { packageFile = getPackageFile(functionMetaData, functionDetails, instanceId, pkgLocation, - functionDetails.getComponentType()); + InstanceUtils.calculateSubjectType(functionDetails)); if (!isEmpty(extraFunctionPkgLocation.getPackagePath())) { extraFunctionPackageFile = getPackageFile(functionMetaData, functionDetails, instanceId, extraFunctionPkgLocation, From 38d230773260f94e282aa96c4498b7b240065550 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Wed, 20 Jul 2022 12:13:58 +0200 Subject: [PATCH 09/26] Support updating the Sink preprocessing function --- .../functions/worker/rest/api/SinksImpl.java | 65 +++++++++++-------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index 0e93eabd56bb0..86f12d7e081bf 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -228,35 +228,11 @@ public void registerSink(final String tenant, functionMetaDataBuilder.setPackageLocation(packageLocationMetaDataBuilder); - File functionPackageFile = null; String preprocessFunction = sinkConfig.getPreprocessFunction(); if (isNotBlank(preprocessFunction)) { - try { - String builtin = null; - if (!preprocessFunction.startsWith(Utils.BUILTIN)) { - functionPackageFile = getPackageFile(preprocessFunction); - } else { - builtin = preprocessFunction.replaceFirst("^builtin://", ""); - functionMetaDataBuilder.setFunctionDetails(functionDetails.toBuilder().setBuiltin(builtin)); - } - Function.PackageLocationMetaData.Builder functionPackageLocation = - getFunctionPackageLocation(functionMetaDataBuilder.build(), - preprocessFunction, null, functionPackageFile, - sinkName + "__sink-function", - Function.FunctionDetails.ComponentType.FUNCTION, builtin); - functionMetaDataBuilder.setExtraFunctionPackageLocation(functionPackageLocation); - } catch (Exception e) { - log.error("Failed process {} {}/{}/{} preprocess function package: ", - ComponentTypeUtils.toString(componentType), tenant, namespace, sinkName, e); - throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } finally { - if (functionPackageFile != null && functionPackageFile.exists()) { - if (!preprocessFunction.startsWith(Utils.FILE)) { - functionPackageFile.delete(); - } - } - } + setExtraFunctionPackageLocation(functionMetaDataBuilder, functionDetails, preprocessFunction); } + updateRequest(null, functionMetaDataBuilder.build()); } finally { if (componentPackageFile != null && componentPackageFile.exists()) { @@ -437,6 +413,12 @@ public void updateSink(final String tenant, functionMetaDataBuilder.setPackageLocation(packageLocationMetaDataBuilder); + String preprocessFunction = mergedConfig.getPreprocessFunction(); + if (isNotBlank(preprocessFunction) + && !preprocessFunction.equals(existingSinkConfig.getPreprocessFunction())) { + setExtraFunctionPackageLocation(functionMetaDataBuilder, functionDetails, preprocessFunction); + } + updateRequest(existingComponent, functionMetaDataBuilder.build()); } finally { if (componentPackageFile != null && componentPackageFile.exists()) { @@ -447,6 +429,37 @@ public void updateSink(final String tenant, } } + private void setExtraFunctionPackageLocation(Function.FunctionMetaData.Builder functionMetaDataBuilder, + Function.FunctionDetails functionDetails, String extraFunction) { + File functionPackageFile = null; + try { + String builtin = null; + if (!extraFunction.startsWith(Utils.BUILTIN)) { + functionPackageFile = getPackageFile(extraFunction); + } else { + builtin = extraFunction.replaceFirst("^builtin://", ""); + functionMetaDataBuilder.setFunctionDetails(functionDetails.toBuilder().setBuiltin(builtin)); + } + Function.PackageLocationMetaData.Builder functionPackageLocation = + getFunctionPackageLocation(functionMetaDataBuilder.build(), + extraFunction, null, functionPackageFile, + functionDetails.getName() + "__sink-function", + Function.FunctionDetails.ComponentType.FUNCTION, builtin); + functionMetaDataBuilder.setExtraFunctionPackageLocation(functionPackageLocation); + } catch (Exception e) { + log.error("Failed process {} {}/{}/{} extra function package: ", + ComponentTypeUtils.toString(componentType), functionDetails.getTenant(), + functionDetails.getNamespace(), functionDetails.getName(), e); + throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } finally { + if (functionPackageFile != null && functionPackageFile.exists()) { + if (!extraFunction.startsWith(Utils.FILE)) { + functionPackageFile.delete(); + } + } + } + } + private class GetSinkStatus extends GetStatus { @Override From 61275f2ed726f489a22b202b536c68ddd178dfd3 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Wed, 20 Jul 2022 16:59:27 +0200 Subject: [PATCH 10/26] Fix update extra function --- .../functions/utils/SinkConfigUtils.java | 25 +++++++++++++++++++ .../functions/worker/rest/api/SinksImpl.java | 3 --- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java index b1ccc95002e3f..4ea48433ad05f 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java @@ -234,6 +234,11 @@ public static FunctionDetails convert(SinkConfig sinkConfig, ExtractedSinkDetail sinkSpecBuilder.setBuiltin(builtin); } + if (!isEmpty(sinkConfig.getPreprocessFunction()) + && sinkConfig.getPreprocessFunction().startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN)) { + functionDetailsBuilder.setBuiltin(sinkConfig.getPreprocessFunction().replaceFirst("^builtin://", "")); + } + if (sinkConfig.getConfigs() != null) { sinkSpecBuilder.setConfigs(new Gson().toJson(sinkConfig.getConfigs())); } @@ -379,6 +384,17 @@ public static SinkConfig convertFromDetails(FunctionDetails functionDetails) { } } + if (!isEmpty(functionDetails.getBuiltin())) { + sinkConfig.setPreprocessFunction("builtin://" + functionDetails.getBuiltin()); + } + if (!functionDetails.getClassName().equals(IdentityFunction.class.getName())) { + sinkConfig.setPreprocessFunctionClassName(functionDetails.getClassName()); + } + if (!isEmpty(functionDetails.getUserConfig())) { + sinkConfig.setPreprocessFunctionConfig(functionDetails.getUserConfig()); + } + + return sinkConfig; } @@ -652,6 +668,15 @@ public static SinkConfig validateUpdate(SinkConfig existingConfig, SinkConfig ne if (newConfig.getCleanupSubscription() != null) { mergedConfig.setCleanupSubscription(newConfig.getCleanupSubscription()); } + if (newConfig.getPreprocessFunction() != null) { + mergedConfig.setPreprocessFunction(newConfig.getPreprocessFunction()); + } + if (newConfig.getPreprocessFunctionClassName() != null) { + mergedConfig.setPreprocessFunctionClassName(newConfig.getPreprocessFunctionClassName()); + } + if (newConfig.getPreprocessFunctionConfig() != null) { + mergedConfig.setPreprocessFunctionConfig(newConfig.getPreprocessFunctionConfig()); + } return mergedConfig; } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index 86f12d7e081bf..71a9286ca1b88 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -436,9 +436,6 @@ private void setExtraFunctionPackageLocation(Function.FunctionMetaData.Builder f String builtin = null; if (!extraFunction.startsWith(Utils.BUILTIN)) { functionPackageFile = getPackageFile(extraFunction); - } else { - builtin = extraFunction.replaceFirst("^builtin://", ""); - functionMetaDataBuilder.setFunctionDetails(functionDetails.toBuilder().setBuiltin(builtin)); } Function.PackageLocationMetaData.Builder functionPackageLocation = getFunctionPackageLocation(functionMetaDataBuilder.build(), From b54d9fb2579cfeb2efcf4342bf2d4c6c3055e0a9 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Thu, 21 Jul 2022 15:15:28 +0200 Subject: [PATCH 11/26] Add tests for Sink + Function --- .../apache/pulsar/admin/cli/TestCmdSinks.java | 62 +++++++++ .../functions/utils/SinkConfigUtils.java | 2 +- .../rest/api/v3/SinkApiV3ResourceTest.java | 119 ++++++++++++++---- 3 files changed, 160 insertions(+), 23 deletions(-) diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java index 7a00f0429fcb5..b15c51e602469 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java @@ -83,6 +83,9 @@ public class TestCmdSinks { private static final Long RAM = 1024L * 1024L; private static final Long DISK = 1024L * 1024L * 1024L; private static final String SINK_CONFIG_STRING = "{\"created_at\":\"Mon Jul 02 00:33:15 +0000 2018\",\"int\":1000,\"int_string\":\"1000\",\"float\":1000.0,\"float_string\":\"1000.0\"}"; + private static final String PREPROCESS_FUNCTION = "preprocess"; + private static final String PREPROCESS_FUNCTION_CLASSNAME = "PreProcessFunction"; + private static final String PREPROCESS_FUNCTION_CONFIG = "{\"test_function_config\": \"\"}"; private PulsarAdmin pulsarAdmin; private Sinks sink; @@ -146,6 +149,11 @@ public SinkConfig getSinkConfig() throws JsonProcessingException { sinkConfig.setArchive(JAR_FILE_PATH); sinkConfig.setResources(new Resources(CPU, RAM, DISK)); sinkConfig.setConfigs(createSink.parseConfigs(SINK_CONFIG_STRING)); + + sinkConfig.setPreprocessFunction(PREPROCESS_FUNCTION); + sinkConfig.setPreprocessFunctionClassName(PREPROCESS_FUNCTION_CLASSNAME); + sinkConfig.setPreprocessFunctionConfig(PREPROCESS_FUNCTION_CONFIG); + return sinkConfig; } @@ -166,6 +174,9 @@ public void testCliCorrect() throws Exception { RAM, DISK, SINK_CONFIG_STRING, + PREPROCESS_FUNCTION, + PREPROCESS_FUNCTION_CLASSNAME, + PREPROCESS_FUNCTION_CONFIG, sinkConfig ); } @@ -188,6 +199,9 @@ public void testMissingInput() throws Exception { RAM, DISK, SINK_CONFIG_STRING, + PREPROCESS_FUNCTION, + PREPROCESS_FUNCTION_CLASSNAME, + PREPROCESS_FUNCTION_CONFIG, sinkConfig ); } @@ -211,6 +225,9 @@ public void testMissingCustomSerdeInput() throws Exception { RAM, DISK, SINK_CONFIG_STRING, + PREPROCESS_FUNCTION, + PREPROCESS_FUNCTION_CLASSNAME, + PREPROCESS_FUNCTION_CONFIG, sinkConfig ); } @@ -233,6 +250,9 @@ public void testMissingTopicPattern() throws Exception { RAM, DISK, SINK_CONFIG_STRING, + PREPROCESS_FUNCTION, + PREPROCESS_FUNCTION_CLASSNAME, + PREPROCESS_FUNCTION_CONFIG, sinkConfig ); } @@ -255,6 +275,9 @@ public void testMissingProcessingGuarantees() throws Exception { RAM, DISK, SINK_CONFIG_STRING, + PREPROCESS_FUNCTION, + PREPROCESS_FUNCTION_CLASSNAME, + PREPROCESS_FUNCTION_CONFIG, sinkConfig ); } @@ -277,6 +300,9 @@ public void testMissingArchive() throws Exception { RAM, DISK, SINK_CONFIG_STRING, + PREPROCESS_FUNCTION, + PREPROCESS_FUNCTION_CLASSNAME, + PREPROCESS_FUNCTION_CONFIG, sinkConfig ); } @@ -301,6 +327,9 @@ public void testInvalidJar() throws Exception { RAM, DISK, SINK_CONFIG_STRING, + PREPROCESS_FUNCTION, + PREPROCESS_FUNCTION_CLASSNAME, + PREPROCESS_FUNCTION_CONFIG, sinkConfig ); } @@ -323,6 +352,9 @@ public void testMissingConfig() throws Exception { RAM, DISK, null, + PREPROCESS_FUNCTION, + PREPROCESS_FUNCTION_CLASSNAME, + PREPROCESS_FUNCTION_CONFIG, sinkConfig ); } @@ -341,6 +373,9 @@ private void testCmdSinkCliMissingArgs( Long ram, Long disk, String sinkConfigString, + String preprocessFunction, + String preprocessFunctionClassName, + String preprocessFunctionConfig, SinkConfig sinkConfig) throws Exception { // test create sink @@ -357,6 +392,9 @@ private void testCmdSinkCliMissingArgs( createSink.ram = ram; createSink.disk = disk; createSink.sinkConfigString = sinkConfigString; + createSink.preprocessFunction = preprocessFunction; + createSink.preprocessFunctionClassName = preprocessFunctionClassName; + createSink.preprocessFunctionConfig = preprocessFunctionConfig; createSink.processArguments(); @@ -376,6 +414,9 @@ private void testCmdSinkCliMissingArgs( updateSink.ram = ram; updateSink.disk = disk; updateSink.sinkConfigString = sinkConfigString; + updateSink.preprocessFunction = preprocessFunction; + updateSink.preprocessFunctionClassName = preprocessFunctionClassName; + updateSink.preprocessFunctionConfig = preprocessFunctionConfig; updateSink.processArguments(); @@ -395,6 +436,9 @@ private void testCmdSinkCliMissingArgs( localSinkRunner.ram = ram; localSinkRunner.disk = disk; localSinkRunner.sinkConfigString = sinkConfigString; + localSinkRunner.preprocessFunction = preprocessFunction; + localSinkRunner.preprocessFunctionClassName = preprocessFunctionClassName; + localSinkRunner.preprocessFunctionConfig = preprocessFunctionConfig; localSinkRunner.processArguments(); @@ -539,6 +583,9 @@ public void testCliOverwriteConfigFile() throws Exception { testSinkConfig.setResources(new Resources(CPU + 1, RAM + 1, DISK + 1)); testSinkConfig.setConfigs(createSink.parseConfigs("{\"created_at-prime\":\"Mon Jul 02 00:33:15 +0000 2018\", \"otherConfigProperties\":{\"property1.value\":\"value1\",\"property2.value\":\"value2\"}}")); + testSinkConfig.setPreprocessFunction(PREPROCESS_FUNCTION + "-prime"); + testSinkConfig.setPreprocessFunction(PREPROCESS_FUNCTION_CLASSNAME + "-prime"); + testSinkConfig.setPreprocessFunction("{\"test_function_config\": \"prime\"}"); SinkConfig expectedSinkConfig = getSinkConfig(); @@ -563,6 +610,9 @@ public void testCliOverwriteConfigFile() throws Exception { RAM, DISK, SINK_CONFIG_STRING, + PREPROCESS_FUNCTION, + PREPROCESS_FUNCTION_CLASSNAME, + PREPROCESS_FUNCTION_CONFIG, file.getAbsolutePath(), expectedSinkConfig ); @@ -583,6 +633,9 @@ private void testMixCliAndConfigFile( Long ram, Long disk, String sinkConfigString, + String preprocessFunction, + String preprocessFunctionClassName, + String preprocessFunctionConfig, String sinkConfigFile, SinkConfig sinkConfig ) throws Exception { @@ -602,6 +655,9 @@ private void testMixCliAndConfigFile( createSink.ram = ram; createSink.disk = disk; createSink.sinkConfigString = sinkConfigString; + createSink.preprocessFunction = preprocessFunction; + createSink.preprocessFunctionClassName = preprocessFunctionClassName; + createSink.preprocessFunctionConfig = preprocessFunctionConfig; createSink.sinkConfigFile = sinkConfigFile; createSink.processArguments(); @@ -622,6 +678,9 @@ private void testMixCliAndConfigFile( updateSink.ram = ram; updateSink.disk = disk; updateSink.sinkConfigString = sinkConfigString; + updateSink.preprocessFunction = preprocessFunction; + updateSink.preprocessFunctionClassName = preprocessFunctionClassName; + updateSink.preprocessFunctionConfig = preprocessFunctionConfig; updateSink.sinkConfigFile = sinkConfigFile; updateSink.processArguments(); @@ -642,6 +701,9 @@ private void testMixCliAndConfigFile( localSinkRunner.ram = ram; localSinkRunner.disk = disk; localSinkRunner.sinkConfigString = sinkConfigString; + localSinkRunner.preprocessFunction = preprocessFunction; + localSinkRunner.preprocessFunctionClassName = preprocessFunctionClassName; + localSinkRunner.preprocessFunctionConfig = preprocessFunctionConfig; localSinkRunner.sinkConfigFile = sinkConfigFile; diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java index 4ea48433ad05f..ba3607097a309 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java @@ -466,7 +466,7 @@ public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConf try { functionClassName = FunctionUtils.getFunctionClass(functionClassLoader); } catch (IOException e) { - throw new IllegalArgumentException("Failed to extract sink class from archive", e); + throw new IllegalArgumentException("Failed to extract function class from archive", e); } } Class functionClass; diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java index 54063f1e4e063..7ab82283e9dce 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java @@ -20,6 +20,7 @@ import static org.apache.pulsar.functions.proto.Function.ProcessingGuarantees.ATLEAST_ONCE; import static org.apache.pulsar.functions.source.TopicSchema.DEFAULT_SERDE; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.doAnswer; @@ -71,11 +72,14 @@ import org.apache.pulsar.functions.runtime.RuntimeFactory; import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.SinkConfigUtils; +import org.apache.pulsar.functions.utils.functions.FunctionArchive; +import org.apache.pulsar.functions.utils.functions.FunctionUtils; import org.apache.pulsar.functions.utils.io.Connector; import org.apache.pulsar.functions.utils.io.ConnectorUtils; import org.apache.pulsar.functions.worker.ConnectorsManager; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; import org.apache.pulsar.functions.worker.FunctionRuntimeManager; +import org.apache.pulsar.functions.worker.FunctionsManager; import org.apache.pulsar.functions.worker.LeaderService; import org.apache.pulsar.functions.worker.PulsarWorkerService; import org.apache.pulsar.functions.worker.WorkerConfig; @@ -635,13 +639,7 @@ public void testRegisterSinkConflictingFields() throws Exception { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false); - SinkConfig sinkConfig = new SinkConfig(); - sinkConfig.setTenant(tenant); - sinkConfig.setNamespace(namespace); - sinkConfig.setName(sink); - sinkConfig.setClassName(CASSANDRA_STRING_SINK); - sinkConfig.setParallelism(parallelism); - sinkConfig.setTopicToSerdeClassName(topicsToSerDeClassName); + SinkConfig sinkConfig = createDefaultSinkConfig(); try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { resource.registerSink( actualTenant, @@ -710,6 +708,50 @@ public void testRegisterSinkFailedWithWrongPackageName() throws PulsarAdminExcep } } + @Test + public void testRegisterSinkSuccessWithPreprocessFunction() throws Exception { + mockInstanceUtils(); + mockWorkerUtils(); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false); + + NarClassLoader mockedClassLoader = mock(NarClassLoader.class); + mockStatic(FunctionCommon.class, ctx -> { + ctx.when(FunctionCommon::createPkgTempFile).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getFunctionTypes(any(), anyBoolean())).thenReturn(new Class[]{String.class, String.class}); + ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); + }); + + mockStatic(FunctionUtils.class, ctx -> { + ctx.when(() -> FunctionUtils.getFunctionClass(any())).thenReturn("DummyFunction"); + }); + + FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); + FunctionArchive functionArchive = FunctionArchive.builder() + .classLoader(mockedClassLoader) + .build(); + when(mockedFunctionsManager.getFunction("preprocess")).thenReturn(functionArchive); + + when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); + + SinkConfig sinkConfig = createDefaultSinkConfig(); + sinkConfig.setPreprocessFunction("builtin://preprocess"); + sinkConfig.setPreprocessFunctionConfig("{\"dummy\": \"dummy\"}"); + + try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { + resource.registerSink( + tenant, + namespace, + sink, + inputStream, + mockedFormData, + null, + sinkConfig, + null, null); + } + } + // // Update Functions // @@ -882,7 +924,7 @@ private void testUpdateSinkMissingArguments( this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); - when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetaData); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))).thenReturn(mockedFunctionMetaData); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); @@ -928,13 +970,7 @@ private void updateDefaultSink() throws Exception { } private void updateDefaultSinkWithPackageUrl(String packageUrl) throws Exception { - SinkConfig sinkConfig = new SinkConfig(); - sinkConfig.setTenant(tenant); - sinkConfig.setNamespace(namespace); - sinkConfig.setName(sink); - sinkConfig.setClassName(CASSANDRA_STRING_SINK); - sinkConfig.setParallelism(parallelism); - sinkConfig.setTopicToSerdeClassName(topicsToSerDeClassName); + SinkConfig sinkConfig = createDefaultSinkConfig(); mockStatic(ConnectorUtils.class, ctx -> { ctx.when(() -> ConnectorUtils.getIOSinkClass(any(NarClassLoader.class))) @@ -1019,13 +1055,7 @@ public void testUpdateSinkWithUrl() throws Exception { String filePackageUrl = getPulsarIOCassandraNar().toURI().toString(); - SinkConfig sinkConfig = new SinkConfig(); - sinkConfig.setTopicToSerdeClassName(topicsToSerDeClassName); - sinkConfig.setTenant(tenant); - sinkConfig.setNamespace(namespace); - sinkConfig.setName(sink); - sinkConfig.setClassName(CASSANDRA_STRING_SINK); - sinkConfig.setParallelism(parallelism); + SinkConfig sinkConfig = createDefaultSinkConfig(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); @@ -1115,6 +1145,51 @@ public void testUpdateSinkInterrupted() throws Exception { } } + @Test + public void testUpdateSinkDifferentPreprocessFunction() throws Exception { + mockWorkerUtils(); + + SinkConfig sinkConfig = createDefaultSinkConfig(); + sinkConfig.setPreprocessFunction("builtin://preprocess"); + sinkConfig.setPreprocessFunctionClassName("DummyFunction"); + sinkConfig.setPreprocessFunctionConfig("{\"dummy\": \"dummy\"}"); + + NarClassLoader mockedClassLoader = mock(NarClassLoader.class); + mockStatic(FunctionCommon.class, ctx -> { + ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getFunctionTypes(any(), anyBoolean())).thenReturn(new Class[]{String.class, String.class}); + ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); + }); + + this.mockedFunctionMetaData = + FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))).thenReturn(mockedFunctionMetaData); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); + + FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); + FunctionArchive functionArchive = FunctionArchive.builder() + .classLoader(mockedClassLoader) + .build(); + when(mockedFunctionsManager.getFunction("preprocess")).thenReturn(functionArchive); + + when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); + + + try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { + resource.updateSink( + tenant, + namespace, + sink, + inputStream, + mockedFormData, + null, + sinkConfig, + null, null, null); + } + } + // // deregister sink // From b201b679c392d08d3202cb74798528662983436b Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Fri, 22 Jul 2022 16:12:41 +0200 Subject: [PATCH 12/26] Cleanup stored preprocessing function when unregistering sink --- .../apache/pulsar/client/admin/Functions.java | 4 +- .../worker/rest/api/ComponentImpl.java | 26 +++++--- .../rest/api/v3/SinkApiV3ResourceTest.java | 61 ++++++++++++------- 3 files changed, 57 insertions(+), 34 deletions(-) diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Functions.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Functions.java index bce7d420aebf9..4c6fa4fa3ee4e 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Functions.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Functions.java @@ -769,7 +769,7 @@ CompletableFuture downloadFunctionAsync( * @param function * Function name * @param extraFunction - * Whether to download the extra-function + * Whether to download the extra-function (for sources and sinks) * @throws PulsarAdminException */ void downloadFunction(String destinationFile, String tenant, String namespace, String function, @@ -787,7 +787,7 @@ void downloadFunction(String destinationFile, String tenant, String namespace, S * @param function * Function name * @param extraFunction - * Whether to download the extra-function + * Whether to download the extra-function (for sources and sinks) */ CompletableFuture downloadFunctionAsync( String destinationFile, String tenant, String namespace, String function, boolean extraFunction); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index bc035cf380bde..f97e07fb381e3 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -514,8 +514,19 @@ public void deregisterFunction(final String tenant, String.format("Error deleting %s @ /%s/%s/%s", ComponentTypeUtils.toString(componentType), tenant, namespace, componentName)); - // clean up component files stored in BK - String functionPackagePath = functionMetaData.getPackageLocation().getPackagePath(); + deleteComponentFromStorage(tenant, namespace, componentName, + functionMetaData.getPackageLocation().getPackagePath()); + + if (!isEmpty(functionMetaData.getExtraFunctionPackageLocation().getPackagePath())) { + deleteComponentFromStorage(tenant, namespace, componentName, + functionMetaData.getExtraFunctionPackageLocation().getPackagePath()); + } + + deleteStatestoreTableAsync(getStateNamespace(tenant, namespace), componentName); + } + + private void deleteComponentFromStorage(String tenant, String namespace, String componentName, + String functionPackagePath) { if (!functionPackagePath.startsWith(Utils.HTTP) && !functionPackagePath.startsWith(Utils.FILE) && !functionPackagePath.startsWith(Utils.BUILTIN)) { @@ -523,22 +534,19 @@ public void deregisterFunction(final String tenant, try { worker().getBrokerAdmin().packages().delete(functionPackagePath); } catch (PulsarAdminException e) { - log.error("{}/{}/{} Failed to cleanup package in package managemanet with url {}", tenant, - namespace, componentName, functionMetaData.getPackageLocation().getPackagePath(), e); + log.error("{}/{}/{} Failed to cleanup package in package management with url {}", tenant, + namespace, componentName, functionPackagePath, e); } } else { try { - WorkerUtils.deleteFromBookkeeper(worker().getDlogNamespace(), - functionMetaData.getPackageLocation().getPackagePath()); + WorkerUtils.deleteFromBookkeeper(worker().getDlogNamespace(), functionPackagePath); } catch (IOException e) { log.error("{}/{}/{} Failed to cleanup package in BK with path {}", tenant, namespace, componentName, - functionMetaData.getPackageLocation().getPackagePath(), e); + functionPackagePath, e); } } } - - deleteStatestoreTableAsync(getStateNamespace(tenant, namespace), componentName); } @Override diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java index 7ab82283e9dce..9f6fa61a77563 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java @@ -1313,7 +1313,7 @@ public void testDeregisterSinkInterrupted() throws Exception { } @Test - public void testDeregisterSinkBKPackageCleanup() throws IOException { + public void testDeregisterSinkBKPackageCleanup() { mockInstanceUtils(); try (final MockedStatic ctx = Mockito.mockStatic(WorkerUtils.class)) { @@ -1321,15 +1321,20 @@ public void testDeregisterSinkBKPackageCleanup() throws IOException { String packagePath = "public/default/test/591541f0-c7c5-40c0-983b-610c722f90b0-pulsar-io-batch-data-generator-2.7.0.nar"; + String extraFunctionPackagePath = + "public/default/test/591541f0-c7c5-40c0-983b-610c722f90b0-test-function.nar"; + FunctionMetaData functionMetaData = FunctionMetaData.newBuilder() + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath)) + .setExtraFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() + .setPackagePath(extraFunctionPackagePath)) + .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) - .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( - Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + .thenReturn(functionMetaData); deregisterDefaultSink(); - ctx.verify(() -> { - WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); - }, times(1)); + ctx.verify(() -> WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)), times(1)); + ctx.verify(() -> WorkerUtils.deleteFromBookkeeper(any(), eq(extraFunctionPackagePath)), times(1)); } } @@ -1341,16 +1346,19 @@ public void testDeregisterBuiltinSinkBKPackageCleanup() { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); String packagePath = String.format("%s://data-generator", Utils.BUILTIN); + String extraFunctionPackagePath = String.format("%s://exclamation", Utils.BUILTIN); + FunctionMetaData functionMetaData = FunctionMetaData.newBuilder() + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath)) + .setExtraFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() + .setPackagePath(extraFunctionPackagePath)) + .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) - .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( - Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + .thenReturn(functionMetaData); deregisterDefaultSink(); // if the sink is a builtin sink we shouldn't try to clean it up - ctx.verify(() -> { - WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); - }, times(0)); + ctx.verify(() -> WorkerUtils.deleteFromBookkeeper(any(), anyString()), times(0)); } } @@ -1363,22 +1371,25 @@ public void testDeregisterHTTPSinkBKPackageCleanup() { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); String packagePath = "http://foo.com/connector.jar"; + String extraFunctionPackagePath = "http://foo.com/function.jar"; + FunctionMetaData functionMetaData = FunctionMetaData.newBuilder() + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath)) + .setExtraFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() + .setPackagePath(extraFunctionPackagePath)) + .build(); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) - .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( - Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + .thenReturn(functionMetaData); deregisterDefaultSink(); // if the sink is a is download from a http url, we shouldn't try to clean it up - ctx.verify(() -> { - WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); - }, times(0)); - + ctx.verify(() -> WorkerUtils.deleteFromBookkeeper(any(), anyString()), times(0)); } } @Test - public void testDeregisterFileSinkBKPackageCleanup() throws IOException { + public void testDeregisterFileSinkBKPackageCleanup() { mockInstanceUtils(); try (final MockedStatic ctx = Mockito.mockStatic(WorkerUtils.class)) { @@ -1386,16 +1397,20 @@ public void testDeregisterFileSinkBKPackageCleanup() throws IOException { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); String packagePath = "file://foo/connector.jar"; + String extraFunctionPackagePath = "file://foo/function.jar"; + FunctionMetaData functionMetaData = FunctionMetaData.newBuilder() + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath)) + .setExtraFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() + .setPackagePath(extraFunctionPackagePath)) + .build(); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) - .thenReturn(FunctionMetaData.newBuilder().setPackageLocation( - Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath).build()).build()); + .thenReturn(functionMetaData); deregisterDefaultSink(); // if the sink package has a file url, we shouldn't try to clean it up - ctx.verify(() -> { - WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)); - }, times(0)); + ctx.verify(() -> WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)), times(0)); } } From 923a4309258795d7d80a436b3ffc3f95bee45e3e Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Fri, 22 Jul 2022 17:08:20 +0200 Subject: [PATCH 13/26] Add more tests --- .../worker/FunctionActionerTest.java | 67 +++++++++---------- .../worker/FunctionRuntimeManagerTest.java | 4 +- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java index 926b4fcf5f7e4..149542208e197 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java @@ -78,7 +78,6 @@ public void testStartFunctionWithDLNamespace() throws Exception { @SuppressWarnings("resource") FunctionActioner actioner = new FunctionActioner(workerConfig, factory, dlogNamespace, new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), mock(PulsarAdmin.class)); - Runtime runtime = mock(Runtime.class); Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() .setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant") .setNamespace("test-namespace").setName("func-1")) @@ -128,39 +127,49 @@ public void testStartFunctionWithPkgUrl() throws Exception { // (1) test with file url. functionActioner should be able to consider file-url and it should be able to call // RuntimeSpawner String pkgPathLocation = FILE + ":/user/my-file.jar"; - Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() - .setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant") - .setNamespace("test-namespace").setName("func-1")) - .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath(pkgPathLocation).build()) - .build(); - Function.Instance instance = Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0) - .build(); - FunctionRuntimeInfo functionRuntimeInfo = mock(FunctionRuntimeInfo.class); - doReturn(instance).when(functionRuntimeInfo).getFunctionInstance(); - - actioner.startFunction(functionRuntimeInfo); + startFunction(actioner, pkgPathLocation, pkgPathLocation); verify(runtime, times(1)).start(); // (2) test with http-url, downloading file from http should fail with UnknownHostException due to invalid url - pkgPathLocation = "http://invalid/my-file.jar"; - function1 = Function.FunctionMetaData.newBuilder() - .setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant") - .setNamespace("test-namespace").setName("func-1")) - .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath(pkgPathLocation).build()) - .build(); - instance = Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build(); - functionRuntimeInfo = mock(FunctionRuntimeInfo.class); - doReturn(instance).when(functionRuntimeInfo).getFunctionInstance(); - doThrow(new IllegalStateException("StartupException")).when(functionRuntimeInfo).setStartupException(any()); + String invalidPkgPathLocation = "http://invalid/my-file.jar"; try { - actioner.startFunction(functionRuntimeInfo); + startFunction(actioner, invalidPkgPathLocation, pkgPathLocation); + fail(); + } catch (IllegalStateException ex) { + assertEquals(ex.getMessage(), "StartupException"); + } + + try { + startFunction(actioner, pkgPathLocation, invalidPkgPathLocation); fail(); } catch (IllegalStateException ex) { assertEquals(ex.getMessage(), "StartupException"); } } + private void startFunction(FunctionActioner actioner, String pkgPathLocation, String extraPkgPathLocation) { + PackageLocationMetaData packageLocation = PackageLocationMetaData.newBuilder() + .setPackagePath(pkgPathLocation) + .build(); + PackageLocationMetaData extraPackageLocation = PackageLocationMetaData.newBuilder() + .setPackagePath(extraPkgPathLocation) + .build(); + Function.FunctionMetaData function = Function.FunctionMetaData.newBuilder() + .setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant") + .setNamespace("test-namespace").setName("func-1")) + .setPackageLocation(packageLocation) + .setExtraFunctionPackageLocation(extraPackageLocation) + .build(); + Function.Instance instance = Function.Instance.newBuilder().setFunctionMetaData(function).setInstanceId(0) + .build(); + FunctionRuntimeInfo functionRuntimeInfo = mock(FunctionRuntimeInfo.class); + doReturn(instance).when(functionRuntimeInfo).getFunctionInstance(); + doThrow(new IllegalStateException("StartupException")).when(functionRuntimeInfo).setStartupException(any()); + + actioner.startFunction(functionRuntimeInfo); + } + @Test public void testFunctionAuthDisabled() throws Exception { WorkerConfig workerConfig = new WorkerConfig(); @@ -253,17 +262,7 @@ public void testStartFunctionWithPackageUrl() throws Exception { // (1) test with file url. functionActioner should be able to consider file-url and it should be able to call // RuntimeSpawner String pkgPathLocation = "function://public/default/test-function@latest"; - Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() - .setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant") - .setNamespace("test-namespace").setName("func-1")) - .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath(pkgPathLocation).build()) - .build(); - Function.Instance instance = Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0) - .build(); - FunctionRuntimeInfo functionRuntimeInfo = mock(FunctionRuntimeInfo.class); - doReturn(instance).when(functionRuntimeInfo).getFunctionInstance(); - - actioner.startFunction(functionRuntimeInfo); + startFunction(actioner, pkgPathLocation, pkgPathLocation); verify(runtime, times(1)).start(); } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java index cc23c86c3590c..47537997315b2 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java @@ -742,7 +742,9 @@ public void testExternallyManagedRuntimeUpdate() throws Exception { functionRuntimeManager.setFunctionActioner(functionActioner); Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() - .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("path").build()) + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("path")) + .setExtraFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() + .setPackagePath("function-path")) .setFunctionDetails( Function.FunctionDetails.newBuilder() .setTenant("test-tenant").setNamespace("test-namespace").setName("func-1")).build(); From 299023e41576bb144122601e39734a8cc7af555e Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Fri, 22 Jul 2022 18:10:43 +0200 Subject: [PATCH 14/26] Add tests to KubernetesRuntimeTest and ProcessRuntimeTest --- .../functions/runtime/RuntimeUtils.java | 6 ++--- .../kubernetes/KubernetesRuntimeTest.java | 22 +++++++++++-------- .../runtime/process/ProcessRuntimeTest.java | 20 ++++++++++------- .../functions/utils/SinkConfigUtilsTest.java | 2 +- 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java index 60296e8368bfe..367eea3ed2b62 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java @@ -351,6 +351,8 @@ public static List getCmd(InstanceConfig instanceConfig, if (isNotEmpty(originalExtraFunctionFileName)) { args.add("--extra_function_jar"); args.add(originalExtraFunctionFileName); + args.add("--extra_function_id"); + args.add(instanceConfig.getExtraFunctionId()); } } else if (instanceConfig.getFunctionDetails().getRuntime() == Function.FunctionDetails.Runtime.PYTHON) { args.add("python3"); @@ -386,10 +388,6 @@ public static List getCmd(InstanceConfig instanceConfig, args.add(shardId); args.add("--function_id"); args.add(instanceConfig.getFunctionId()); - if (instanceConfig.getExtraFunctionId() != null) { - args.add("--extra_function_id"); - args.add(instanceConfig.getExtraFunctionId()); - } args.add("--function_version"); args.add(instanceConfig.getFunctionVersion()); args.add("--function_details"); diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java index cdec1e96abd1d..cc277b6f18942 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java @@ -284,6 +284,7 @@ InstanceConfig createJavaInstanceConfig(FunctionDetails.Runtime runtime, boolean config.setFunctionDetails(createFunctionDetails(runtime, addSecrets)); config.setFunctionId(java.util.UUID.randomUUID().toString()); + config.setExtraFunctionId(java.util.UUID.randomUUID().toString()); config.setFunctionVersion("1.0"); config.setInstanceId(0); config.setMaxBufferedTuples(1024); @@ -387,7 +388,7 @@ private void verifyJavaInstance(InstanceConfig config, String depsDir, boolean s List args; try (MockedStatic systemUtils = Mockito.mockStatic(SystemUtils.class, Mockito.CALLS_REAL_METHODS)) { systemUtils.when(() -> SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)).thenReturn(true); - container = factory.createContainer(config, userJarFile, userJarFile, null, null,30L); + container = factory.createContainer(config, userJarFile, userJarFile, userJarFile, userJarFile,30L); args = container.getProcessArgs(); } @@ -400,14 +401,14 @@ private void verifyJavaInstance(InstanceConfig config, String depsDir, boolean s if (null != depsDir) { extraDepsEnv = " -Dpulsar.functions.extra.dependencies.dir=" + depsDir; classpath = classpath + ":" + depsDir + "/*"; - totalArgs = 42; - portArg = 29; - metricsPortArg = 31; + totalArgs = 46; + portArg = 33; + metricsPortArg = 35; } else { extraDepsEnv = ""; - portArg = 28; - metricsPortArg = 30; - totalArgs = 41; + portArg = 32; + metricsPortArg = 34; + totalArgs = 45; } if (secretsAttached) { totalArgs += 4; @@ -441,8 +442,11 @@ private void verifyJavaInstance(InstanceConfig config, String depsDir, boolean s + " --add-opens java.base/sun.net=ALL-UNNAMED" + " -Xmx" + RESOURCES.getRam() + " org.apache.pulsar.functions.instance.JavaInstanceMain" - + " --jar " + jarLocation + " --instance_id " - + "$SHARD_ID" + " --function_id " + config.getFunctionId() + + " --jar " + jarLocation + + " --extra_function_jar " + jarLocation + + " --extra_function_id " + config.getExtraFunctionId() + + " --instance_id " + "$SHARD_ID" + + " --function_id " + config.getFunctionId() + " --function_version " + config.getFunctionVersion() + " --function_details '" + JsonFormat.printer().omittingInsignificantWhitespace().print(config.getFunctionDetails()) + "' --pulsar_serviceurl " + pulsarServiceUrl diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java index 59ea8c9c54f1f..1456330b2f8b4 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java @@ -289,7 +289,8 @@ private void verifyJavaInstance(InstanceConfig config, Path depsDir, String webS List args; try (MockedStatic systemUtils = Mockito.mockStatic(SystemUtils.class, Mockito.CALLS_REAL_METHODS)) { systemUtils.when(() -> SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)).thenReturn(true); - ProcessRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null,30L); + ProcessRuntime container = factory.createContainer(config, userJarFile, userJarFile, + userJarFile, userJarFile,30L); args = container.getProcessArgs(); } @@ -297,7 +298,7 @@ private void verifyJavaInstance(InstanceConfig config, Path depsDir, String webS String extraDepsEnv; int portArg; int metricsPortArg; - int totalArgCount = 44; + int totalArgCount = 48; if (webServiceUrl != null && config.isExposePulsarAdminClientEnabled()) { totalArgCount += 3; } @@ -305,13 +306,13 @@ private void verifyJavaInstance(InstanceConfig config, Path depsDir, String webS assertEquals(args.size(), totalArgCount); extraDepsEnv = " -Dpulsar.functions.extra.dependencies.dir=" + depsDir; classpath = classpath + ":" + depsDir + "/*"; - portArg = 27; - metricsPortArg = 29; + portArg = 31; + metricsPortArg = 33; } else { assertEquals(args.size(), totalArgCount-1); extraDepsEnv = ""; - portArg = 26; - metricsPortArg = 28; + portArg = 30; + metricsPortArg = 32; } if (webServiceUrl != null && config.isExposePulsarAdminClientEnabled()) { portArg += 3; @@ -330,8 +331,11 @@ private void verifyJavaInstance(InstanceConfig config, Path depsDir, String webS + " -Dio.netty.tryReflectionSetAccessible=true" + " --add-opens java.base/sun.net=ALL-UNNAMED" + " org.apache.pulsar.functions.instance.JavaInstanceMain" - + " --jar " + userJarFile + " --instance_id " - + config.getInstanceId() + " --function_id " + config.getFunctionId() + + " --jar " + userJarFile + + " --extra_function_jar " + userJarFile + + " --extra_function_id " + config.getExtraFunctionId() + + " --instance_id " + config.getInstanceId() + + " --function_id " + config.getFunctionId() + " --function_version " + config.getFunctionVersion() + " --function_details '" + JsonFormat.printer().omittingInsignificantWhitespace().print(config.getFunctionDetails()) + "' --pulsar_serviceurl " + pulsarServiceUrl diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java index b670b9b583e3c..ae9d10c9b7527 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java @@ -70,7 +70,7 @@ public void testAutoAckConvertFailed() throws IOException { assertThrows(IllegalArgumentException.class, () -> { SinkConfigUtils.convert(sinkConfig, - new SinkConfigUtils.ExtractedSinkDetails(null, null)); + new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); }); } From fac5f4adfe8cae3ad99c4dfc762a337a67fe0599 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Fri, 22 Jul 2022 18:38:49 +0200 Subject: [PATCH 15/26] Add tests to SinkConfigUtilsTest --- .../functions/utils/SinkConfigUtilsTest.java | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java index ae9d10c9b7527..114a5c13547c2 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java @@ -111,6 +111,9 @@ public void testConvertBackFidelity() throws IOException { sinkConfig.setResources(Resources.getDefaultResources()); + sinkConfig.setPreprocessFunction("builtin://preprocess"); + sinkConfig.setPreprocessFunctionConfig("{\"key\": \"value\"}"); + Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); assertEquals(Function.SubscriptionType.SHARED, functionDetails.getSource().getSubscriptionType()); SinkConfig convertedConfig = SinkConfigUtils.convertFromDetails(functionDetails); @@ -425,6 +428,57 @@ public void testMergeRuntimeFlags() { ); } + @Test + public void testMergeDifferentPreprocessFunction() { + SinkConfig sinkConfig = createSinkConfig(); + String newFunction = "builtin://new"; + SinkConfig newSinkConfig = createUpdatedSinkConfig("preprocessFunction", newFunction); + SinkConfig mergedConfig = SinkConfigUtils.validateUpdate(sinkConfig, newSinkConfig); + assertEquals( + mergedConfig.getPreprocessFunction(), + newFunction + ); + mergedConfig.setPreprocessFunction(sinkConfig.getPreprocessFunction()); + assertEquals( + new Gson().toJson(sinkConfig), + new Gson().toJson(mergedConfig) + ); + } + + @Test + public void testMergeDifferentPreprocessFunctionClassName() { + SinkConfig sinkConfig = createSinkConfig(); + String newFunctionClassName = "NewPreprocessFunction"; + SinkConfig newSinkConfig = createUpdatedSinkConfig("preprocessFunctionClassName", newFunctionClassName); + SinkConfig mergedConfig = SinkConfigUtils.validateUpdate(sinkConfig, newSinkConfig); + assertEquals( + mergedConfig.getPreprocessFunctionClassName(), + newFunctionClassName + ); + mergedConfig.setPreprocessFunctionClassName(sinkConfig.getPreprocessFunctionClassName()); + assertEquals( + new Gson().toJson(sinkConfig), + new Gson().toJson(mergedConfig) + ); + } + + @Test + public void testMergeDifferentPreprocessFunctionConfig() { + SinkConfig sinkConfig = createSinkConfig(); + String newFunctionConfig = "{\"new-key\": \"new-value\"}"; + SinkConfig newSinkConfig = createUpdatedSinkConfig("preprocessFunctionConfig", newFunctionConfig); + SinkConfig mergedConfig = SinkConfigUtils.validateUpdate(sinkConfig, newSinkConfig); + assertEquals( + mergedConfig.getPreprocessFunctionConfig(), + newFunctionConfig + ); + mergedConfig.setPreprocessFunctionConfig(sinkConfig.getPreprocessFunctionConfig()); + assertEquals( + new Gson().toJson(sinkConfig), + new Gson().toJson(mergedConfig) + ); + } + @Test public void testValidateConfig() { SinkConfig sinkConfig = createSinkConfig(); @@ -459,6 +513,9 @@ private SinkConfig createSinkConfig() { sinkConfig.setCleanupSubscription(true); sinkConfig.setArchive("DummyArchive.nar"); sinkConfig.setCleanupSubscription(true); + sinkConfig.setPreprocessFunction("builtin://preprocess"); + sinkConfig.setPreprocessFunctionClassName("Preprocess"); + sinkConfig.setPreprocessFunctionConfig("{\"key\": \"value\"}"); return sinkConfig; } From 8986be1a419da957e9f769c9aa1ed3f112920191 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Mon, 25 Jul 2022 11:47:06 +0200 Subject: [PATCH 16/26] Add HTTP donload function tests --- .../api/v3/FunctionApiV3ResourceTest.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java index a868d6325064e..81ab922ed7c6e 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java @@ -1688,6 +1688,66 @@ public void testDownloadFunctionBuiltin() throws Exception { } } + @Test + public void testDownloadFunctionByName() throws Exception { + URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); + File file = Paths.get(fileUrl.toURI()).toFile(); + String fileLocation = file.getAbsolutePath().replace('\\', '/'); + String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + doReturn(true).when(mockedWorkerService).isInitialized(); + WorkerConfig config = mock(WorkerConfig.class); + when(config.isAuthorizationEnabled()).thenReturn(false); + when(mockedWorkerService.getWorkerConfig()).thenReturn(config); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + FunctionMetaData metaData = FunctionMetaData.newBuilder() + .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("file:///" + fileLocation)) + .setExtraFunctionPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) + .build(); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); + + StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, null, null, false); + File pkgFile = new File(testDir, UUID.randomUUID().toString()); + OutputStream output = new FileOutputStream(pkgFile); + streamOutput.write(output); + Assert.assertTrue(pkgFile.exists()); + if (pkgFile.exists()) { + pkgFile.delete(); + } + } + + @Test + public void testDownloadExtraFunctionByName() throws Exception { + URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); + File file = Paths.get(fileUrl.toURI()).toFile(); + String fileLocation = file.getAbsolutePath().replace('\\', '/'); + String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + doReturn(true).when(mockedWorkerService).isInitialized(); + WorkerConfig config = mock(WorkerConfig.class); + when(config.isAuthorizationEnabled()).thenReturn(false); + when(mockedWorkerService.getWorkerConfig()).thenReturn(config); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + FunctionMetaData metaData = FunctionMetaData.newBuilder() + .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) + .setExtraFunctionPackageLocation(PackageLocationMetaData.newBuilder() + .setPackagePath("file:///" + fileLocation)) + .build(); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); + + StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, null, null, true); + File pkgFile = new File(testDir, UUID.randomUUID().toString()); + OutputStream output = new FileOutputStream(pkgFile); + streamOutput.write(output); + Assert.assertTrue(pkgFile.exists()); + if (pkgFile.exists()) { + pkgFile.delete(); + } + } + + @Test public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception { Configurator.setRootLevel(Level.DEBUG); From df9ef4e97367a65c69979894247f37bb87dd79d2 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Mon, 25 Jul 2022 12:17:51 +0200 Subject: [PATCH 17/26] Add admin CLI function download tests --- .../pulsar/admin/cli/CmdFunctionsTest.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java index e98894e9c6528..f47f7127d5396 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java @@ -859,6 +859,44 @@ public void testUpdateAuthData() throws Exception { verify(functions, times(1)).updateFunctionWithUrl(any(FunctionConfig.class), anyString(), eq(updateOptions)); } + @Test + public void testDownloadFunction() throws Exception { + cmd.run(new String[] { + "download", + "--destination-file", JAR_NAME, + "--name", FN_NAME, + "--tenant", TENANT, + "--namespace", NAMESPACE + }); + verify(functions, times(1)) + .downloadFunction(JAR_NAME, TENANT, NAMESPACE, FN_NAME, false); + } + + @Test + public void testDownloadFunctionByPath() throws Exception { + cmd.run(new String[] { + "download", + "--destination-file", JAR_NAME, + "--path", PACKAGE_URL + }); + verify(functions, times(1)) + .downloadFunction(JAR_NAME, PACKAGE_URL); + } + + @Test + public void testDownloadExtraFunction() throws Exception { + cmd.run(new String[] { + "download", + "--destination-file", JAR_NAME, + "--name", FN_NAME, + "--tenant", TENANT, + "--namespace", NAMESPACE, + "--extra-function" + }); + verify(functions, times(1)) + .downloadFunction(JAR_NAME, TENANT, NAMESPACE, FN_NAME, true); + } + public static class ConsoleOutputCapturer { private ByteArrayOutputStream stdout; From e404a9e97d9fadbc486e38f9ad1cf81f3d7a9676 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Tue, 26 Jul 2022 13:04:53 +0200 Subject: [PATCH 18/26] Add integration test for Sink with preprocessing function --- .../runtime/kubernetes/KubernetesRuntime.java | 2 - .../functions/utils/FunctionConfigUtils.java | 5 - .../tests/integration/io/TestLoggingSink.java | 56 +++++ .../io/SinkWithPreprocessingFunctionTest.java | 224 ++++++++++++++++++ 4 files changed, 280 insertions(+), 7 deletions(-) create mode 100644 tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestLoggingSink.java create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkWithPreprocessingFunctionTest.java diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index 44c21321784bd..d2b4e69fadcbd 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -21,7 +21,6 @@ import static java.net.HttpURLConnection.HTTP_CONFLICT; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; -import static org.apache.commons.lang3.StringUtils.isEmpty; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.isNotEmpty; import static org.apache.commons.lang3.StringUtils.left; @@ -76,7 +75,6 @@ import lombok.extern.slf4j.Slf4j; import okhttp3.Response; import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.functions.auth.KubernetesFunctionAuthProvider; import org.apache.pulsar.functions.instance.AuthenticationConfig; import org.apache.pulsar.functions.instance.InstanceConfig; diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java index cd86fedafbf5b..bcd56b1d25f38 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java @@ -33,9 +33,6 @@ import java.io.IOException; import java.lang.reflect.Type; import java.net.MalformedURLException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Collection; import java.util.HashMap; import java.util.LinkedList; @@ -46,13 +43,11 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; -import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.common.functions.ConsumerConfig; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.ProducerConfig; import org.apache.pulsar.common.functions.Resources; -import org.apache.pulsar.common.functions.Utils; import org.apache.pulsar.common.functions.WindowConfig; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.ObjectMapperFactory; diff --git a/tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestLoggingSink.java b/tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestLoggingSink.java new file mode 100644 index 0000000000000..193897b78b39c --- /dev/null +++ b/tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestLoggingSink.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.io; + +import java.util.Map; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.io.core.Sink; +import org.apache.pulsar.io.core.SinkContext; +import org.slf4j.Logger; + +public class TestLoggingSink implements Sink { + + private Logger logger; + private Producer producer; + + @Override + public void open(Map config, SinkContext sinkContext) throws Exception { + logger = sinkContext.getLogger(); + String topic = (String) sinkContext.getSinkConfig().getConfigs().get("log-topic"); + producer = sinkContext.getPulsarClient().newProducer(Schema.STRING) + .topic(topic) + .create(); + } + + @Override + public void write(Record record) throws Exception { + logger.info("Got message: " + record.getValue()); + producer.newMessage() + .properties(record.getProperties()) + .value(record.getValue()) + .send(); + } + + @Override + public void close() throws Exception { + + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkWithPreprocessingFunctionTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkWithPreprocessingFunctionTest.java new file mode 100644 index 0000000000000..1fe04b55db3f0 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkWithPreprocessingFunctionTest.java @@ -0,0 +1,224 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.io; + +import static org.apache.pulsar.tests.integration.functions.utils.CommandGenerator.JAVAJAR; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.tests.integration.containers.PulsarContainer; +import org.apache.pulsar.tests.integration.containers.StandaloneContainer; +import org.apache.pulsar.tests.integration.docker.ContainerExecException; +import org.apache.pulsar.tests.integration.docker.ContainerExecResult; +import org.apache.pulsar.tests.integration.suites.PulsarStandaloneTestSuite; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterTestBase; +import org.testcontainers.containers.Network; +import org.testng.annotations.Test; + +/** + * Test behaviour of sinks with a pre-processing function + */ +@Slf4j +public class SinkWithPreprocessingFunctionTest extends PulsarStandaloneTestSuite { + + //Use PIP-117 new defaults so that the package management service is enabled. + @Override + public void setUpCluster() throws Exception { + incrementSetupNumber(); + network = Network.newNetwork(); + String clusterName = PulsarClusterTestBase.randomName(8); + container = new StandaloneContainer(clusterName, PulsarContainer.DEFAULT_IMAGE_NAME) + .withNetwork(network) + .withNetworkAliases(StandaloneContainer.NAME + "-" + clusterName) + .withEnv("PF_stateStorageServiceUrl", "bk://localhost:4181"); + container.start(); + log.info("Pulsar cluster {} is up running:", clusterName); + log.info("\tBinary Service Url : {}", container.getPlainTextServiceUrl()); + log.info("\tHttp Service Url : {}", container.getHttpServiceUrl()); + + // add cluster to public tenant + ContainerExecResult result = container.execCmd( + "/pulsar/bin/pulsar-admin", "namespaces", "policies", "public/default"); + assertEquals(0, result.getExitCode()); + log.info("public/default namespace policies are {}", result.getStdout()); + } + + @Test(groups = {"sink"}) + public void testSinkWithPreprocessingFunction() throws Exception { + + @Cleanup PulsarClient client = PulsarClient.builder() + .serviceUrl(container.getPlainTextServiceUrl()) + .build(); + + final int numRecords = 10; + + String sinkName = "sink-with-preprocessing-function"; + String topicName = "test-sink-with-preprocessing-function"; + String packageName = "function://public/default/test-function@1.0"; + + submitPackage(packageName, "package-function", JAVAJAR); + + submitSinkConnector( + sinkName, + topicName, + "org.apache.pulsar.tests.integration.io.TestLoggingSink", + JAVAJAR, + "{\"log-topic\": \"log\"}", + packageName, + "org.apache.pulsar.functions.api.examples.RecordFunction"); + + getSinkInfoSuccess(sinkName); + getSinkStatus(sinkName); + + @Cleanup Producer producer = client.newProducer(Schema.STRING) + .topic(topicName) + .create(); + + @Cleanup Consumer consumer = client.newConsumer(Schema.STRING) + .topic("log") + .subscriptionName("sub") + .subscribe(); + + for (int i = 0; i < numRecords; i++) { + producer.send(i + "-test"); + } + + try { + log.info("waiting for sink {}", sinkName); + + for (int i = 0; i < numRecords; i++) { + Message receive = consumer.receive(5, TimeUnit.SECONDS); + assertNotNull(receive); + assertEquals(receive.getValue(), i + "-test!"); + } + } finally { + dumpFunctionLogs(sinkName); + } + + deleteSink(sinkName); + getSinkInfoNotFound(sinkName); + } + + private void submitPackage(String packageName, String description, String packagePath) throws Exception { + String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "packages", "upload", + packageName, + "--description", description, + "--path", packagePath + + }; + log.info("Run command : {}", StringUtils.join(commands, ' ')); + ContainerExecResult result = container.execCmd(commands); + assertTrue( + result.getStdout().contains("successfully"), + result.getStdout()); + } + + private void submitSinkConnector(String sinkName, + String inputTopicName, + String className, + String archive, + String configs, + String preprocessFunction, + String preprocessFunctionClassName) throws Exception { + String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "sinks", "create", + "--name", sinkName, + "-i", inputTopicName, + "--archive", archive, + "--classname", className, + "--sink-config", configs, + "--preprocess-function", preprocessFunction, + "--preprocess-function-classname", preprocessFunctionClassName + }; + log.info("Run command : {}", StringUtils.join(commands, ' ')); + ContainerExecResult result = container.execCmd(commands); + assertTrue( + result.getStdout().contains("Created successfully"), + result.getStdout()); + } + + private void getSinkInfoSuccess(String sinkName) throws Exception { + ContainerExecResult result = container.execCmd( + PulsarCluster.ADMIN_SCRIPT, + "sinks", + "get", + "--tenant", "public", + "--namespace", "default", + "--name", sinkName + ); + assertTrue(result.getStdout().contains("\"name\": \"" + sinkName + "\"")); + } + + private void getSinkStatus(String sinkName) throws Exception { + ContainerExecResult result = container.execCmd( + PulsarCluster.ADMIN_SCRIPT, + "sinks", + "status", + "--tenant", "public", + "--namespace", "default", + "--name", sinkName + ); + log.info(result.getStdout()); + log.info(result.getStderr()); + assertTrue(result.getStdout().contains("\"running\" : true")); + } + + private void deleteSink(String sinkName) throws Exception { + ContainerExecResult result = container.execCmd( + PulsarCluster.ADMIN_SCRIPT, + "sinks", + "delete", + "--tenant", "public", + "--namespace", "default", + "--name", sinkName + ); + assertTrue(result.getStdout().contains("successfully")); + result.assertNoStderr(); + } + + private void getSinkInfoNotFound(String sinkName) throws Exception { + try { + container.execCmd( + PulsarCluster.ADMIN_SCRIPT, + "sinks", + "get", + "--tenant", "public", + "--namespace", "default", + "--name", sinkName); + fail("Command should have exited with non-zero"); + } catch (ContainerExecException e) { + assertTrue(e.getResult().getStderr().contains(sinkName + " doesn't exist")); + } + } +} + From e0a3ecdebd2921c8a030e13e026ac61127e60839 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Tue, 26 Jul 2022 23:11:01 +0200 Subject: [PATCH 19/26] Check that Sink preprocess function output type is a Record --- .../org/apache/pulsar/admin/cli/CmdSinks.java | 3 +- .../apache/pulsar/functions/LocalRunner.java | 16 ++--- .../functions/utils/FunctionCommon.java | 54 ++++++++------ .../functions/utils/SinkConfigUtils.java | 5 ++ .../rest/api/v3/SinkApiV3ResourceTest.java | 71 +++++++++++++++++-- 5 files changed, 111 insertions(+), 38 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java index 221998d779890..2846b74e82076 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java @@ -402,7 +402,8 @@ abstract class SinkDetailsCommand extends BaseCommand { protected String preprocessFunction; @Parameter(names = "--preprocess-function-classname", description = "The preprocess function class name") protected String preprocessFunctionClassName; - @Parameter(names = "--preprocess-function-config", description = "Configuration of the preprocess function applied before the Sink") + @Parameter(names = "--preprocess-function-config", description = "Configuration of the preprocess function " + + "applied before the Sink") protected String preprocessFunctionConfig; protected SinkConfig sinkConfig; diff --git a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java index 3a50faa1ea9fa..6c7551f8ac321 100644 --- a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java +++ b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java @@ -439,19 +439,18 @@ public void start(boolean blocking) throws Exception { ClassLoader builtInSinkClassLoader = userCodeFile != null ? isBuiltInSink(userCodeFile) : null; if (builtInSinkClassLoader != null) { - //TODO: handle preprocess function functionDetails = SinkConfigUtils.convert( sinkConfig, SinkConfigUtils.validateAndExtractDetails( - sinkConfig, builtInSinkClassLoader, null,true)); + sinkConfig, builtInSinkClassLoader, null, true)); userCodeClassLoader = builtInSinkClassLoader; } else if (Utils.isFunctionPackageUrlSupported(userCodeFile)) { File file = FunctionCommon.extractFileFromPkgURL(userCodeFile); ClassLoader sinkClassLoader = FunctionCommon.getClassLoaderFromPackage( Function.FunctionDetails.ComponentType.SINK, sinkConfig.getClassName(), file, narExtractionDirectory); - //TODO: handle preprocess function functionDetails = SinkConfigUtils.convert( - sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, null,true)); + sinkConfig, + SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, null, true)); userCodeClassLoader = sinkClassLoader; userCodeClassLoaderCreated = true; } else if (userCodeFile != null) { @@ -462,19 +461,18 @@ public void start(boolean blocking) throws Exception { ClassLoader sinkClassLoader = FunctionCommon.getClassLoaderFromPackage( Function.FunctionDetails.ComponentType.SINK, sinkConfig.getClassName(), file, narExtractionDirectory); - //TODO: handle preprocess function functionDetails = SinkConfigUtils.convert( - sinkConfig, SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, null,true)); + sinkConfig, + SinkConfigUtils.validateAndExtractDetails(sinkConfig, sinkClassLoader, null, true)); userCodeClassLoader = sinkClassLoader; userCodeClassLoaderCreated = true; } else { if (!(runtimeEnv == null || runtimeEnv == RuntimeEnv.THREAD)) { throw new IllegalStateException("The archive property must be specified in SourceConfig."); } - //TODO: handle preprocess function functionDetails = SinkConfigUtils.convert( sinkConfig, SinkConfigUtils.validateAndExtractDetails( - sinkConfig, Thread.currentThread().getContextClassLoader(), null,true)); + sinkConfig, Thread.currentThread().getContextClassLoader(), null, true)); } } else { throw new IllegalArgumentException("Must specify Function, Source or Sink config"); @@ -576,7 +574,6 @@ private void startProcessMode(org.apache.pulsar.functions.proto.Function.Functio } } - // TODO: support preprocessing function in LocalRunner RuntimeSpawner runtimeSpawner = new RuntimeSpawner( instanceConfig, userCodeFile, @@ -683,7 +680,6 @@ private void startThreadedMode(org.apache.pulsar.functions.proto.Function.Functi } } - // TODO: support preprocessing Function in LocalRunner RuntimeSpawner runtimeSpawner = new RuntimeSpawner( instanceConfig, userCodeFile, diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java index 2bd24f0b7b0d0..54b23d5200beb 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java @@ -104,42 +104,50 @@ public static Class[] getFunctionTypes(FunctionConfig functionConfig, Class f return getFunctionTypes(functionClass, isWindowConfigPresent); } - public static Class[] getFunctionTypes(Class userClass, boolean isWindowConfigPresent) { - Class[] typeArgs; + public static Class[] getFunctionTypes(Class userClass, boolean isWindowConfigPresent) { + Class classParent = getFunctionClassParent(userClass, isWindowConfigPresent); + Class[] typeArgs = TypeResolver.resolveRawArguments(classParent, userClass); // if window function if (isWindowConfigPresent) { - if (WindowFunction.class.isAssignableFrom(userClass)) { - typeArgs = getFunctionTypesUnwrappingRecordIfNeeded(WindowFunction.class, userClass); - } else { - typeArgs = getFunctionTypesUnwrappingRecordIfNeeded(java.util.function.Function.class, userClass); + if (classParent.equals(java.util.function.Function.class)) { if (!typeArgs[0].equals(Collection.class)) { throw new IllegalArgumentException("Window function must take a collection as input"); } - Type type = TypeResolver.resolveGenericType(java.util.function.Function.class, userClass); - Type collectionType = ((ParameterizedType) type).getActualTypeArguments()[0]; - Type actualInputType = ((ParameterizedType) collectionType).getActualTypeArguments()[0]; - typeArgs[0] = (Class) actualInputType; + typeArgs[0] = (Class) unwrapType(classParent, userClass, 0); + } + } + if (typeArgs[1].equals(Record.class)) { + typeArgs[1] = (Class) unwrapType(classParent, userClass, 1); + } + + return typeArgs; + } + + public static Class[] getRawFunctionTypes(Class userClass, boolean isWindowConfigPresent) { + Class classParent = getFunctionClassParent(userClass, isWindowConfigPresent); + return TypeResolver.resolveRawArguments(classParent, userClass); + } + + public static Class getFunctionClassParent(Class userClass, boolean isWindowConfigPresent) { + if (isWindowConfigPresent) { + if (WindowFunction.class.isAssignableFrom(userClass)) { + return WindowFunction.class; + } else { + return java.util.function.Function.class; } } else { if (Function.class.isAssignableFrom(userClass)) { - typeArgs = getFunctionTypesUnwrappingRecordIfNeeded(Function.class, userClass); + return Function.class; } else { - typeArgs = getFunctionTypesUnwrappingRecordIfNeeded(java.util.function.Function.class, userClass); + return java.util.function.Function.class; } } - - return typeArgs; } - private static Class[] getFunctionTypesUnwrappingRecordIfNeeded(Class type, Class subType) { - Class[] typeArgs = TypeResolver.resolveRawArguments(type, subType); - if (typeArgs[1].equals(Record.class)) { - Type genericType = TypeResolver.resolveGenericType(type, subType); - Type recordType = ((ParameterizedType) genericType).getActualTypeArguments()[1]; - Type actualInputType = ((ParameterizedType) recordType).getActualTypeArguments()[0]; - typeArgs[1] = (Class) actualInputType; - } - return typeArgs; + private static Type unwrapType(Class type, Class subType, int position) { + Type genericType = TypeResolver.resolveGenericType(type, subType); + Type argType = ((ParameterizedType) genericType).getActualTypeArguments()[position]; + return ((ParameterizedType) argType).getActualTypeArguments()[0]; } public static Object createInstance(String userClassName, ClassLoader classLoader) { diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java index ba3607097a309..21bdffe8ed7fd 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java @@ -23,6 +23,7 @@ import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsSubscriptionPosition; import static org.apache.pulsar.functions.utils.FunctionCommon.convertProcessingGuarantee; import static org.apache.pulsar.functions.utils.FunctionCommon.getFunctionTypes; +import static org.apache.pulsar.functions.utils.FunctionCommon.getRawFunctionTypes; import static org.apache.pulsar.functions.utils.FunctionCommon.getSinkType; import com.fasterxml.jackson.core.type.TypeReference; import com.google.gson.Gson; @@ -51,6 +52,7 @@ import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.config.validation.ConfigValidation; +import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.Function.FunctionDetails; @@ -477,6 +479,9 @@ public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConf String.format("Function class %s not found in class loader", functionClassName), e); } // extract type from preprocess function class + if (!getRawFunctionTypes(functionClass, false)[1].equals(Record.class)) { + throw new IllegalArgumentException("Sink preprocess function output must be of type Record"); + } typeArg = getFunctionTypes(functionClass, false)[0]; inputClassLoader = functionClassLoader; } else { diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java index 9f6fa61a77563..f9608b2e7b26b 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java @@ -24,6 +24,7 @@ import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; @@ -64,6 +65,8 @@ import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.RestException; +import org.apache.pulsar.functions.api.examples.ExclamationFunction; +import org.apache.pulsar.functions.api.examples.RecordFunction; import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.apache.pulsar.functions.instance.InstanceUtils; import org.apache.pulsar.functions.proto.Function; @@ -716,15 +719,18 @@ public void testRegisterSinkSuccessWithPreprocessFunction() throws Exception { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false); NarClassLoader mockedClassLoader = mock(NarClassLoader.class); + doReturn(RecordFunction.class).when(mockedClassLoader).loadClass("RecordFunction"); mockStatic(FunctionCommon.class, ctx -> { ctx.when(FunctionCommon::createPkgTempFile).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getFunctionTypes(any(), anyBoolean())).thenReturn(new Class[]{String.class, String.class}); + ctx.when(() -> FunctionCommon.getRawFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getFunctionClassParent(any(), anyBoolean())).thenCallRealMethod(); ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); }); mockStatic(FunctionUtils.class, ctx -> { - ctx.when(() -> FunctionUtils.getFunctionClass(any())).thenReturn("DummyFunction"); + ctx.when(() -> FunctionUtils.getFunctionClass(any())).thenReturn("RecordFunction"); }); FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); @@ -752,6 +758,60 @@ public void testRegisterSinkSuccessWithPreprocessFunction() throws Exception { } } + @Test(expectedExceptions = RestException.class, + expectedExceptionsMessageRegExp = "Sink preprocess function output must be of type Record") + public void testRegisterSinkFailureWithInvalidPreprocessFunction() throws Exception { + mockInstanceUtils(); + mockWorkerUtils(); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false); + + NarClassLoader mockedClassLoader = mock(NarClassLoader.class); + doReturn(ExclamationFunction.class).when(mockedClassLoader).loadClass("ExclamationFunction"); + mockStatic(FunctionCommon.class, ctx -> { + ctx.when(FunctionCommon::createPkgTempFile).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getRawFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getFunctionClassParent(any(), anyBoolean())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); + }); + + mockStatic(FunctionUtils.class, ctx -> { + ctx.when(() -> FunctionUtils.getFunctionClass(any())).thenReturn("ExclamationFunction"); + }); + + FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); + FunctionArchive functionArchive = FunctionArchive.builder() + .classLoader(mockedClassLoader) + .build(); + when(mockedFunctionsManager.getFunction("preprocess")).thenReturn(functionArchive); + + when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); + + SinkConfig sinkConfig = createDefaultSinkConfig(); + sinkConfig.setPreprocessFunction("builtin://preprocess"); + sinkConfig.setPreprocessFunctionConfig("{\"dummy\": \"dummy\"}"); + + try { + try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { + resource.registerSink( + tenant, + namespace, + sink, + inputStream, + mockedFormData, + null, + sinkConfig, + null, null); + } + } catch (RestException e) { + // expected exception + assertEquals(e.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw e; + } + } + // // Update Functions // @@ -1155,10 +1215,13 @@ public void testUpdateSinkDifferentPreprocessFunction() throws Exception { sinkConfig.setPreprocessFunctionConfig("{\"dummy\": \"dummy\"}"); NarClassLoader mockedClassLoader = mock(NarClassLoader.class); + doReturn(RecordFunction.class).when(mockedClassLoader).loadClass("DummyFunction"); mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); + ctx.when(FunctionCommon::createPkgTempFile).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getRawFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getFunctionClassParent(any(), anyBoolean())).thenCallRealMethod(); ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getFunctionTypes(any(), anyBoolean())).thenReturn(new Class[]{String.class, String.class}); ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); }); From 662daa4a878a92ccb4a27c1075cd8ffad457dbbd Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Mon, 22 Aug 2022 18:24:02 +0200 Subject: [PATCH 20/26] Add encoding/decding step between the Function and the Sink If we detect that a Function is coupled with the Sink, we encode the record value with the Function schema and decode with the Sink schema. The Sink schema is created from the first schema type record received. At each record, a new schema version is created if it is the first time we get the hash of the record schema. --- .../instance/JavaInstanceRunnable.java | 151 +++++++++++++++++- .../instance/SinkSchemaInfoProvider.java | 50 ++++++ 2 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkSchemaInfoProvider.java diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index 6340908311428..bdb1bf91a3bff 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -22,11 +22,14 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.annotations.VisibleForTesting; import com.scurrilous.circe.checksum.Crc32cIntChecksum; +import io.netty.buffer.ByteBuf; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Consumer; @@ -42,16 +45,28 @@ import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.schema.GenericObject; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; +import org.apache.pulsar.client.impl.schema.AvroSchema; +import org.apache.pulsar.client.impl.schema.JSONSchema; +import org.apache.pulsar.client.impl.schema.ProtobufNativeSchema; +import org.apache.pulsar.client.impl.schema.ProtobufSchema; import org.apache.pulsar.common.functions.ConsumerConfig; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.ProducerConfig; +import org.apache.pulsar.common.protocol.schema.SchemaVersion; +import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.common.util.Reflections; import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.api.StateStore; import org.apache.pulsar.functions.api.StateStoreContext; +import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.apache.pulsar.functions.instance.state.BKStateStoreProviderImpl; import org.apache.pulsar.functions.instance.state.InstanceStateManager; import org.apache.pulsar.functions.instance.state.StateManager; @@ -136,6 +151,10 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable { // a read write lock for stats operations private final ReadWriteLock statsLock = new ReentrantReadWriteLock(); + private Class sinkTypeArg; + private final AtomicReference> sinkSchema = new AtomicReference<>(); + private SinkSchemaInfoProvider sinkSchemaInfoProvider = null; + public JavaInstanceRunnable(InstanceConfig instanceConfig, ClientBuilder clientBuilder, PulsarClient pulsarClient, @@ -230,6 +249,10 @@ private synchronized void setup() throws Exception { // start any log topic handler setupLogHandler(); + if (!(object instanceof IdentityFunction) && !(sink instanceof PulsarSink)) { + sinkSchemaInfoProvider = new SinkSchemaInfoProvider(); + } + javaInstance = new JavaInstance(contextImpl, object, instanceConfig); try { Thread.currentThread().setContextClassLoader(functionClassLoader); @@ -389,7 +412,38 @@ private void sendOutputMessage(Record srcRecord, Object output) throws Exception } AbstractSinkRecord sinkRecord; if (output instanceof Record) { - sinkRecord = new OutputRecordSinkRecord<>(srcRecord, (Record) output); + Record record = (Record) output; + if (sinkSchemaInfoProvider != null) { + // Function and Sink coupled together so we need to encode with the Function Schema + // and decode with the Sink schema + byte[] encoded = record.getSchema().encode(record.getValue()); + + if (sinkSchema.get() == null) { + Schema schema = getSinkSchema(record, sinkTypeArg); + schema.setSchemaInfoProvider(sinkSchemaInfoProvider); + sinkSchema.compareAndSet(null, schema); + } + Schema schema = sinkSchema.get(); + SchemaVersion schemaVersion = sinkSchemaInfoProvider.getSchemaVersion(record); + final byte[] schemaVersionBytes = schemaVersion.bytes(); + Object decoded = schema.decode(encoded, schemaVersionBytes); + + sinkRecord = new OutputRecordSinkRecord<>(srcRecord, record) { + @Override + public Object getValue() { + return decoded; + } + + @Override + public Schema getSchema() { + return schema instanceof AutoConsumeSchema + ? ((AutoConsumeSchema) schema).getInternalSchema(schemaVersionBytes) + : schema; + } + }; + } else { + sinkRecord = new OutputRecordSinkRecord<>(srcRecord, record); + } } else { sinkRecord = new SinkRecord<>(srcRecord, output); } @@ -859,6 +913,7 @@ private void setupOutput(ContextImpl contextImpl) throws Exception { if (object instanceof Sink) { this.sink = (Sink) object; + this.sinkTypeArg = TypeResolver.resolveRawArguments(Sink.class, object.getClass())[0]; } else { throw new RuntimeException("Sink does not implement correct interface"); } @@ -888,4 +943,98 @@ private void setupOutput(ContextImpl contextImpl) throws Exception { Thread.currentThread().setContextClassLoader(this.instanceClassLoader); } } + + private static Schema getSinkSchema(Record record, Class clazz) { + SchemaType type = getSchemaTypeOrDefault(record, clazz); + switch (type) { + case NONE: + if (ByteBuffer.class.isAssignableFrom(clazz)) { + return (Schema) Schema.BYTEBUFFER; + } else { + return (Schema) Schema.BYTES; + } + + case AUTO_CONSUME: + case AUTO: + return (Schema) Schema.AUTO_CONSUME(); + + case STRING: + return (Schema) Schema.STRING; + + case AVRO: + return AvroSchema.of(SchemaDefinition.builder() + .withPojo(clazz).build()); + + case JSON: + return JSONSchema.of(SchemaDefinition.builder().withPojo(clazz).build()); + + case KEY_VALUE: + return (Schema) Schema.KV_BYTES(); + + case PROTOBUF: + return ProtobufSchema.ofGenericClass(clazz, new HashMap<>()); + + case PROTOBUF_NATIVE: + return ProtobufNativeSchema.ofGenericClass(clazz, new HashMap<>()); + + case AUTO_PUBLISH: + return (Schema) Schema.AUTO_PRODUCE_BYTES(); + + default: + throw new RuntimeException("Unsupported schema type" + type); + } + } + + private static SchemaType getSchemaTypeOrDefault(Record record, Class clazz) { + if (GenericObject.class.isAssignableFrom(clazz)) { + return SchemaType.AUTO_CONSUME; + } else if (byte[].class.equals(clazz) + || ByteBuf.class.equals(clazz) + || ByteBuffer.class.equals(clazz)) { + // if sink uses bytes, we should ignore + return SchemaType.NONE; + } else { + Schema schema = record.getSchema(); + if (schema != null) { + if (schema.getSchemaInfo().getType() == SchemaType.NONE) { + return getDefaultSchemaType(clazz); + } else { + return schema.getSchemaInfo().getType(); + } + } else { + return getDefaultSchemaType(clazz); + } + } + } + + private static SchemaType getDefaultSchemaType(Class clazz) { + if (byte[].class.equals(clazz) + || ByteBuf.class.equals(clazz) + || ByteBuffer.class.equals(clazz)) { + return SchemaType.NONE; + } else if (GenericObject.class.isAssignableFrom(clazz)) { + // the sink is taking generic record/object, so we do auto schema detection + return SchemaType.AUTO_CONSUME; + } else if (String.class.equals(clazz)) { + // If type is String, then we use schema type string, otherwise we fallback on default schema + return SchemaType.STRING; + } else if (isProtobufClass(clazz)) { + return SchemaType.PROTOBUF; + } else if (KeyValue.class.equals(clazz)) { + return SchemaType.KEY_VALUE; + } else { + return SchemaType.JSON; + } + } + + private static boolean isProtobufClass(Class pojoClazz) { + try { + Class protobufBaseClass = Class.forName("com.google.protobuf.GeneratedMessageV3"); + return protobufBaseClass.isAssignableFrom(pojoClazz); + } catch (ClassNotFoundException | NoClassDefFoundError e) { + // If sink does not have protobuf in classpath then it cannot be protobuf + return false; + } + } + } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkSchemaInfoProvider.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkSchemaInfoProvider.java new file mode 100644 index 0000000000000..b35434e454de4 --- /dev/null +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkSchemaInfoProvider.java @@ -0,0 +1,50 @@ +package org.apache.pulsar.functions.instance; + +import java.nio.ByteBuffer; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.pulsar.client.api.schema.SchemaInfoProvider; +import org.apache.pulsar.common.protocol.schema.BytesSchemaVersion; +import org.apache.pulsar.common.protocol.schema.SchemaHash; +import org.apache.pulsar.common.protocol.schema.SchemaVersion; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.functions.api.Record; + +class SinkSchemaInfoProvider implements SchemaInfoProvider { + + AtomicLong latestVersion = new AtomicLong(0); + ConcurrentHashMap schemaInfos = new ConcurrentHashMap<>(); + ConcurrentHashMap schemaVersions = new ConcurrentHashMap<>(); + + public SchemaVersion getSchemaVersion(Record record) { + SchemaHash schemaHash = SchemaHash.of(record.getSchema()); + return schemaVersions.computeIfAbsent(schemaHash, s -> { + long l = latestVersion.incrementAndGet(); + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(l); + SchemaVersion schemaVersion = BytesSchemaVersion.of(buffer.array()); + schemaInfos.put(schemaVersion, record.getSchema().getSchemaInfo()); + return schemaVersion; + }); + } + + @Override + public CompletableFuture getSchemaByVersion(byte[] schemaVersion) { + return CompletableFuture.completedFuture(schemaInfos.get(BytesSchemaVersion.of(schemaVersion))); + } + + @Override + public CompletableFuture getLatestSchema() { + long l = latestVersion.get(); + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(l); + SchemaVersion schemaVersion = BytesSchemaVersion.of(buffer.array()); + return CompletableFuture.completedFuture(schemaInfos.get(schemaVersion)); + } + + @Override + public String getTopicName() { + return "__INTERNAL__"; + } +} From 9af64c6b11aed4c0d3dab4281184b1006b265e6f Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Mon, 22 Aug 2022 19:20:25 +0200 Subject: [PATCH 21/26] Use a Sink with a type different than the Function in integration tests --- .../instance/JavaInstanceRunnable.java | 7 +------ .../tests/integration/io/TestLoggingSink.java | 17 +++++++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index bdb1bf91a3bff..1309cd282144f 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -450,12 +450,7 @@ public Schema getSchema() { try { this.sink.write(sinkRecord); } catch (Exception e) { - if (e instanceof ClassCastException && functionClassLoader != componentClassLoader) { - log.info("Encountered ClassCastException in sink write. " - + "Is the Function return type compatible with the Sink type ? ", e); - } else { - log.info("Encountered exception in sink write: ", e); - } + log.info("Encountered exception in sink write: ", e); stats.incrSinkExceptions(e); // fail the source record srcRecord.fail(); diff --git a/tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestLoggingSink.java b/tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestLoggingSink.java index 193897b78b39c..1447b67ed3d99 100644 --- a/tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestLoggingSink.java +++ b/tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestLoggingSink.java @@ -21,12 +21,13 @@ import java.util.Map; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.schema.GenericObject; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.core.Sink; import org.apache.pulsar.io.core.SinkContext; import org.slf4j.Logger; -public class TestLoggingSink implements Sink { +public class TestLoggingSink implements Sink { private Logger logger; private Producer producer; @@ -41,16 +42,20 @@ public void open(Map config, SinkContext sinkContext) throws Exc } @Override - public void write(Record record) throws Exception { - logger.info("Got message: " + record.getValue()); - producer.newMessage() + public void write(Record record) throws Exception { + Object nativeObject = record.getValue().getNativeObject(); + logger.info("Got message: " + nativeObject); + if (nativeObject instanceof String) { + logger.info("Got message: " + nativeObject); + producer.newMessage() .properties(record.getProperties()) - .value(record.getValue()) + .value((String) nativeObject) .send(); + } } @Override - public void close() throws Exception { + public void close() { } } From 93d74d77a9bf3d1ff536978a79c52d71f57a130a Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Wed, 24 Aug 2022 01:13:07 +0200 Subject: [PATCH 22/26] Fix encoding/decoding when the record value is KeyValue with SEPARATED encoding --- .../instance/JavaInstanceRunnable.java | 35 +++- .../instance/SinkSchemaInfoProvider.java | 26 ++- .../tests/integration/io/TestLoggingSink.java | 27 ++- .../io/SinkWithPreprocessingFunctionTest.java | 158 +++++++++++++++++- 4 files changed, 224 insertions(+), 22 deletions(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index 1309cd282144f..bd4c2179dcdb0 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -47,11 +47,13 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.schema.GenericObject; +import org.apache.pulsar.client.api.schema.KeyValueSchema; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.client.impl.schema.JSONSchema; +import org.apache.pulsar.client.impl.schema.KeyValueSchemaImpl; import org.apache.pulsar.client.impl.schema.ProtobufNativeSchema; import org.apache.pulsar.client.impl.schema.ProtobufSchema; import org.apache.pulsar.common.functions.ConsumerConfig; @@ -59,6 +61,7 @@ import org.apache.pulsar.common.functions.ProducerConfig; import org.apache.pulsar.common.protocol.schema.SchemaVersion; import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.common.util.Reflections; @@ -416,7 +419,18 @@ private void sendOutputMessage(Record srcRecord, Object output) throws Exception if (sinkSchemaInfoProvider != null) { // Function and Sink coupled together so we need to encode with the Function Schema // and decode with the Sink schema - byte[] encoded = record.getSchema().encode(record.getValue()); + Schema encodingSchema = record.getSchema(); + boolean isKeyValueSeparated = false; + if (encodingSchema instanceof KeyValueSchema) { + KeyValueSchema kvSchema = (KeyValueSchema) encodingSchema; + // If the encoding is SEPARATED, it's easier to encode/decode with INLINE + // and rebuild the SEPARATED KeyValueSchema after decoding + if (kvSchema.getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED) { + encodingSchema = KeyValueSchemaImpl.of(kvSchema.getKeySchema(), kvSchema.getValueSchema()); + isKeyValueSeparated = true; + } + } + byte[] encoded = encodingSchema.encode(record.getValue()); if (sinkSchema.get() == null) { Schema schema = getSinkSchema(record, sinkTypeArg); @@ -424,10 +438,23 @@ private void sendOutputMessage(Record srcRecord, Object output) throws Exception sinkSchema.compareAndSet(null, schema); } Schema schema = sinkSchema.get(); - SchemaVersion schemaVersion = sinkSchemaInfoProvider.getSchemaVersion(record); + SchemaVersion schemaVersion = sinkSchemaInfoProvider.getSchemaVersion(encodingSchema); final byte[] schemaVersionBytes = schemaVersion.bytes(); Object decoded = schema.decode(encoded, schemaVersionBytes); + if (schema instanceof AutoConsumeSchema) { + schema = ((AutoConsumeSchema) schema).getInternalSchema(schemaVersionBytes); + } + + final Schema finalSchema; + if (isKeyValueSeparated && schema instanceof KeyValueSchema) { + KeyValueSchema kvSchema = (KeyValueSchema) schema; + finalSchema = KeyValueSchemaImpl.of(kvSchema.getKeySchema(), kvSchema.getValueSchema(), + KeyValueEncodingType.SEPARATED); + } else { + finalSchema = schema; + } + sinkRecord = new OutputRecordSinkRecord<>(srcRecord, record) { @Override public Object getValue() { @@ -436,9 +463,7 @@ public Object getValue() { @Override public Schema getSchema() { - return schema instanceof AutoConsumeSchema - ? ((AutoConsumeSchema) schema).getInternalSchema(schemaVersionBytes) - : schema; + return finalSchema; } }; } else { diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkSchemaInfoProvider.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkSchemaInfoProvider.java index b35434e454de4..73fb4246f17fb 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkSchemaInfoProvider.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkSchemaInfoProvider.java @@ -1,15 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.pulsar.functions.instance; import java.nio.ByteBuffer; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.schema.SchemaInfoProvider; import org.apache.pulsar.common.protocol.schema.BytesSchemaVersion; import org.apache.pulsar.common.protocol.schema.SchemaHash; import org.apache.pulsar.common.protocol.schema.SchemaVersion; import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.functions.api.Record; class SinkSchemaInfoProvider implements SchemaInfoProvider { @@ -17,14 +35,14 @@ class SinkSchemaInfoProvider implements SchemaInfoProvider { ConcurrentHashMap schemaInfos = new ConcurrentHashMap<>(); ConcurrentHashMap schemaVersions = new ConcurrentHashMap<>(); - public SchemaVersion getSchemaVersion(Record record) { - SchemaHash schemaHash = SchemaHash.of(record.getSchema()); + public SchemaVersion getSchemaVersion(Schema schema) { + SchemaHash schemaHash = SchemaHash.of(schema); return schemaVersions.computeIfAbsent(schemaHash, s -> { long l = latestVersion.incrementAndGet(); ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); buffer.putLong(l); SchemaVersion schemaVersion = BytesSchemaVersion.of(buffer.array()); - schemaInfos.put(schemaVersion, record.getSchema().getSchemaInfo()); + schemaInfos.put(schemaVersion, schema.getSchemaInfo()); return schemaVersion; }); } diff --git a/tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestLoggingSink.java b/tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestLoggingSink.java index 1447b67ed3d99..571eb6a9d1ac4 100644 --- a/tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestLoggingSink.java +++ b/tests/docker-images/java-test-functions/src/main/java/org/apache/pulsar/tests/integration/io/TestLoggingSink.java @@ -22,6 +22,7 @@ import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.schema.GenericObject; +import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.core.Sink; import org.apache.pulsar.io.core.SinkContext; @@ -44,14 +45,26 @@ public void open(Map config, SinkContext sinkContext) throws Exc @Override public void write(Record record) throws Exception { Object nativeObject = record.getValue().getNativeObject(); - logger.info("Got message: " + nativeObject); - if (nativeObject instanceof String) { - logger.info("Got message: " + nativeObject); - producer.newMessage() - .properties(record.getProperties()) - .value((String) nativeObject) - .send(); + logger.info("Got message: " + nativeObject + " with schema" + record.getSchema()); + String payload = nativeObject.toString(); + if (nativeObject instanceof KeyValue) { + KeyValue kv = (KeyValue) nativeObject; + String key = kv.getKey().toString(); + String value = kv.getValue().toString(); + + if (kv.getKey() instanceof GenericObject) { + key = ((GenericObject) kv.getKey()).getNativeObject().toString(); + } + if (kv.getValue() instanceof GenericObject) { + value = ((GenericObject) kv.getValue()).getNativeObject().toString(); + } + payload = "(key = " + key + ", value = " + value + ")"; } + producer.newMessage() + .properties(record.getProperties()) + .value(record.getSchema().getSchemaInfo().getType().name() + " - " + payload) + .send(); + record.ack(); } @Override diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkWithPreprocessingFunctionTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkWithPreprocessingFunctionTest.java index 1fe04b55db3f0..b7a5ecab9cbc2 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkWithPreprocessingFunctionTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkWithPreprocessingFunctionTest.java @@ -32,6 +32,9 @@ import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.schema.KeyValueEncodingType; +import org.apache.pulsar.functions.api.examples.pojo.Users; import org.apache.pulsar.tests.integration.containers.PulsarContainer; import org.apache.pulsar.tests.integration.containers.StandaloneContainer; import org.apache.pulsar.tests.integration.docker.ContainerExecException; @@ -79,9 +82,10 @@ public void testSinkWithPreprocessingFunction() throws Exception { final int numRecords = 10; - String sinkName = "sink-with-preprocessing-function"; - String topicName = "test-sink-with-preprocessing-function"; - String packageName = "function://public/default/test-function@1.0"; + String sinkName = "sink-with-function"; + String topicName = "sink-with-function"; + String logTopicName = "log-sink-with-function"; + String packageName = "function://public/default/sink-with-function-function@1.0"; submitPackage(packageName, "package-function", JAVAJAR); @@ -90,7 +94,7 @@ public void testSinkWithPreprocessingFunction() throws Exception { topicName, "org.apache.pulsar.tests.integration.io.TestLoggingSink", JAVAJAR, - "{\"log-topic\": \"log\"}", + "{\"log-topic\": \"" + logTopicName + "\"}", packageName, "org.apache.pulsar.functions.api.examples.RecordFunction"); @@ -102,7 +106,7 @@ public void testSinkWithPreprocessingFunction() throws Exception { .create(); @Cleanup Consumer consumer = client.newConsumer(Schema.STRING) - .topic("log") + .topic(logTopicName) .subscriptionName("sub") .subscribe(); @@ -116,7 +120,7 @@ public void testSinkWithPreprocessingFunction() throws Exception { for (int i = 0; i < numRecords; i++) { Message receive = consumer.receive(5, TimeUnit.SECONDS); assertNotNull(receive); - assertEquals(receive.getValue(), i + "-test!"); + assertEquals(receive.getValue(), "STRING - " + i + "-test!"); } } finally { dumpFunctionLogs(sinkName); @@ -126,6 +130,148 @@ public void testSinkWithPreprocessingFunction() throws Exception { getSinkInfoNotFound(sinkName); } + @Test(groups = {"sink"}) + public void testGenericObjectSinkWithPreprocessingFunction() throws Exception { + + @Cleanup PulsarClient client = PulsarClient.builder() + .serviceUrl(container.getPlainTextServiceUrl()) + .build(); + + final int numRecords = 10; + + String sinkName = "sink-with-genericobject-function"; + String topicName = "sink-with-genericobject-function"; + String logTopicName = "log-sink-with-genericobject-function"; + String packageName = "function://public/default/sink-with-genericobject-function-function@1.0"; + + submitPackage(packageName, "package-function", JAVAJAR); + + submitSinkConnector( + sinkName, + topicName, + "org.apache.pulsar.tests.integration.io.TestLoggingSink", + JAVAJAR, + "{\"log-topic\": \"" + logTopicName + "\"}", + packageName, + "org.apache.pulsar.tests.integration.functions.RemoveAvroFieldRecordFunction"); + + getSinkInfoSuccess(sinkName); + getSinkStatus(sinkName); + + try { + @Cleanup Consumer consumer = client.newConsumer(Schema.STRING) + .topic(logTopicName) + .subscriptionName("sub") + .subscribe(); + + @Cleanup Producer producer1 = client.newProducer(Schema.AVRO(Users.UserV1.class)) + .topic(topicName) + .create(); + + for (int i = 0; i < numRecords; i++) { + producer1.send(new Users.UserV1("foo" + i, i)); + } + + for (int i = 0; i < numRecords; i++) { + Message receive = consumer.receive(5, TimeUnit.SECONDS); + assertNotNull(receive); + assertEquals(receive.getValue(), "AVRO - {\"name\": \"foo" + i + "\"}"); + } + + // Test with schema evolution + @Cleanup Producer producer2 = client.newProducer(Schema.AVRO(Users.UserV2.class)) + .topic(topicName) + .create(); + + for (int i = 0; i < numRecords; i++) { + producer2.send(new Users.UserV2("foo" + i, i, "bar" + i)); + } + + for (int i = 0; i < numRecords; i++) { + Message receive = consumer.receive(5, TimeUnit.SECONDS); + assertNotNull(receive); + assertEquals(receive.getValue(), "AVRO - {\"name\": \"foo" + i + "\", \"phone\": \"bar"+ i + "\"}"); + } + + for (int i = 0; i < numRecords; i++) { + producer1.send(new Users.UserV1("foo" + i, i)); + } + + for (int i = 0; i < numRecords; i++) { + Message receive = consumer.receive(5, TimeUnit.SECONDS); + assertNotNull(receive); + assertEquals(receive.getValue(), "AVRO - {\"name\": \"foo" + i + "\"}"); + } + } finally { + dumpFunctionLogs(sinkName); + } + + deleteSink(sinkName); + getSinkInfoNotFound(sinkName); + } + + @Test(groups = {"sink"}) + public void testKeyValueSinkWithPreprocessingFunction() throws Exception { + + @Cleanup PulsarClient client = PulsarClient.builder() + .serviceUrl(container.getPlainTextServiceUrl()) + .build(); + + final int numRecords = 10; + + String sinkName = "sink-with-kv-function"; + String topicName = "sink-with-kv-function"; + String logTopicName = "log-sink-with-kv-function"; + String packageName = "function://public/default/sink-with-kv-function-function@1.0"; + + submitPackage(packageName, "package-function", JAVAJAR); + + submitSinkConnector( + sinkName, + topicName, + "org.apache.pulsar.tests.integration.io.TestLoggingSink", + JAVAJAR, + "{\"log-topic\": \"" + logTopicName + "\"}", + packageName, + "org.apache.pulsar.tests.integration.functions.RemoveAvroFieldRecordFunction"); + + getSinkInfoSuccess(sinkName); + getSinkStatus(sinkName); + + try { + @Cleanup Consumer consumer = client.newConsumer(Schema.STRING) + .topic(logTopicName) + .subscriptionName("sub") + .subscribe(); + + @Cleanup Producer> producer = client + .newProducer(Schema.KeyValue( + Schema.AVRO(Users.UserV1.class), + Schema.AVRO(Users.UserV1.class), KeyValueEncodingType.SEPARATED)) + .topic(topicName) + .create(); + + for (int i = 0; i < numRecords; i++) { + producer.send(new KeyValue<>(new Users.UserV1("foo" + i, i), + new Users.UserV1("bar" + i, i + 100))); + } + + for (int i = 0; i < numRecords; i++) { + Message receive = consumer.receive(5, TimeUnit.SECONDS); + assertNotNull(receive); + assertEquals(receive.getValue(), "KEY_VALUE - (key = {\"age\": " + i + + ", \"name\": \"foo" + i + "\"}, value = " + + "{\"name\": \"bar" + i + "\"})"); + } + } finally { + dumpFunctionLogs(sinkName); + } + + deleteSink(sinkName); + getSinkInfoNotFound(sinkName); + } + + private void submitPackage(String packageName, String description, String packagePath) throws Exception { String[] commands = { PulsarCluster.ADMIN_SCRIPT, From 1df5425ab2768b39fddf28e4dd5ac6bf1d2ea841 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Thu, 25 Aug 2022 10:56:30 +0200 Subject: [PATCH 23/26] Minor improvements --- .../instance/JavaInstanceRunnable.java | 88 +++++++++---------- .../instance/OutputRecordSinkRecord.java | 11 +++ .../instance/SinkSchemaInfoProvider.java | 29 ++++-- 3 files changed, 72 insertions(+), 56 deletions(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index bd4c2179dcdb0..b22c2ea3118cd 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -419,53 +419,7 @@ private void sendOutputMessage(Record srcRecord, Object output) throws Exception if (sinkSchemaInfoProvider != null) { // Function and Sink coupled together so we need to encode with the Function Schema // and decode with the Sink schema - Schema encodingSchema = record.getSchema(); - boolean isKeyValueSeparated = false; - if (encodingSchema instanceof KeyValueSchema) { - KeyValueSchema kvSchema = (KeyValueSchema) encodingSchema; - // If the encoding is SEPARATED, it's easier to encode/decode with INLINE - // and rebuild the SEPARATED KeyValueSchema after decoding - if (kvSchema.getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED) { - encodingSchema = KeyValueSchemaImpl.of(kvSchema.getKeySchema(), kvSchema.getValueSchema()); - isKeyValueSeparated = true; - } - } - byte[] encoded = encodingSchema.encode(record.getValue()); - - if (sinkSchema.get() == null) { - Schema schema = getSinkSchema(record, sinkTypeArg); - schema.setSchemaInfoProvider(sinkSchemaInfoProvider); - sinkSchema.compareAndSet(null, schema); - } - Schema schema = sinkSchema.get(); - SchemaVersion schemaVersion = sinkSchemaInfoProvider.getSchemaVersion(encodingSchema); - final byte[] schemaVersionBytes = schemaVersion.bytes(); - Object decoded = schema.decode(encoded, schemaVersionBytes); - - if (schema instanceof AutoConsumeSchema) { - schema = ((AutoConsumeSchema) schema).getInternalSchema(schemaVersionBytes); - } - - final Schema finalSchema; - if (isKeyValueSeparated && schema instanceof KeyValueSchema) { - KeyValueSchema kvSchema = (KeyValueSchema) schema; - finalSchema = KeyValueSchemaImpl.of(kvSchema.getKeySchema(), kvSchema.getValueSchema(), - KeyValueEncodingType.SEPARATED); - } else { - finalSchema = schema; - } - - sinkRecord = new OutputRecordSinkRecord<>(srcRecord, record) { - @Override - public Object getValue() { - return decoded; - } - - @Override - public Schema getSchema() { - return finalSchema; - } - }; + sinkRecord = encodeWithRecordSchemaAndDecodeWithSinkSchema(srcRecord, record); } else { sinkRecord = new OutputRecordSinkRecord<>(srcRecord, record); } @@ -485,6 +439,46 @@ public Schema getSchema() { } } + private OutputRecordSinkRecord encodeWithRecordSchemaAndDecodeWithSinkSchema(Record srcRecord, Record record) { + AbstractSinkRecord sinkRecord; + Schema encodingSchema = record.getSchema(); + boolean isKeyValueSeparated = false; + if (encodingSchema instanceof KeyValueSchema) { + KeyValueSchema kvSchema = (KeyValueSchema) encodingSchema; + // If the encoding is SEPARATED, it's easier to encode/decode with INLINE + // and rebuild the SEPARATED KeyValueSchema after decoding + if (kvSchema.getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED) { + encodingSchema = KeyValueSchemaImpl.of(kvSchema.getKeySchema(), kvSchema.getValueSchema()); + isKeyValueSeparated = true; + } + } + byte[] encoded = encodingSchema.encode(record.getValue()); + + if (sinkSchema.get() == null) { + Schema schema = getSinkSchema(record, sinkTypeArg); + schema.setSchemaInfoProvider(sinkSchemaInfoProvider); + sinkSchema.compareAndSet(null, schema); + } + Schema schema = sinkSchema.get(); + SchemaVersion schemaVersion = sinkSchemaInfoProvider.addSchemaIfNeeded(encodingSchema); + final byte[] schemaVersionBytes = schemaVersion.bytes(); + Object decoded = schema.decode(encoded, schemaVersionBytes); + + if (schema instanceof AutoConsumeSchema) { + schema = ((AutoConsumeSchema) schema).getInternalSchema(schemaVersionBytes); + } + + final Schema finalSchema; + if (isKeyValueSeparated && schema instanceof KeyValueSchema) { + KeyValueSchema kvSchema = (KeyValueSchema) schema; + finalSchema = KeyValueSchemaImpl.of(kvSchema.getKeySchema(), kvSchema.getValueSchema(), + KeyValueEncodingType.SEPARATED); + } else { + finalSchema = schema; + } + return new OutputRecordSinkRecord(srcRecord, record, decoded, finalSchema); + } + private Record readInput() throws Exception { Record record; if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SOURCE) { diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/OutputRecordSinkRecord.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/OutputRecordSinkRecord.java index 5b63c6e8896f5..233100cd2ee8a 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/OutputRecordSinkRecord.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/OutputRecordSinkRecord.java @@ -31,10 +31,21 @@ public class OutputRecordSinkRecord extends AbstractSinkRecord { private final Record sinkRecord; + private final T value; + private final Schema schema; OutputRecordSinkRecord(Record sourceRecord, Record sinkRecord) { super(sourceRecord); this.sinkRecord = sinkRecord; + this.value = sinkRecord.getValue(); + this.schema = getRecordSchema(sinkRecord); + } + + OutputRecordSinkRecord(Record sourceRecord, Record sinkRecord, T value, Schema schema) { + super(sourceRecord); + this.sinkRecord = sinkRecord; + this.value = value; + this.schema = schema; } @Override diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkSchemaInfoProvider.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkSchemaInfoProvider.java index 73fb4246f17fb..1bd41f226d1a8 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkSchemaInfoProvider.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkSchemaInfoProvider.java @@ -29,22 +29,33 @@ import org.apache.pulsar.common.protocol.schema.SchemaVersion; import org.apache.pulsar.common.schema.SchemaInfo; +/** + * SchemaInfo provider that creates a new schema version for each new schema hash. + */ class SinkSchemaInfoProvider implements SchemaInfoProvider { AtomicLong latestVersion = new AtomicLong(0); ConcurrentHashMap schemaInfos = new ConcurrentHashMap<>(); ConcurrentHashMap schemaVersions = new ConcurrentHashMap<>(); - public SchemaVersion getSchemaVersion(Schema schema) { + /** + * Creates a new schema version with the info of the provided schema if the hash of the schema is a new one. + * + * @param schema schema for which we create a version + * @return the version of the schema + */ + public SchemaVersion addSchemaIfNeeded(Schema schema) { SchemaHash schemaHash = SchemaHash.of(schema); - return schemaVersions.computeIfAbsent(schemaHash, s -> { - long l = latestVersion.incrementAndGet(); - ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); - buffer.putLong(l); - SchemaVersion schemaVersion = BytesSchemaVersion.of(buffer.array()); - schemaInfos.put(schemaVersion, schema.getSchemaInfo()); - return schemaVersion; - }); + return schemaVersions.computeIfAbsent(schemaHash, s -> createNewSchemaInfo(schema.getSchemaInfo())); + } + + private SchemaVersion createNewSchemaInfo(SchemaInfo schemaInfo) { + long l = latestVersion.incrementAndGet(); + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(l); + BytesSchemaVersion schemaVersion = BytesSchemaVersion.of(buffer.array()); + schemaInfos.put(schemaVersion, schemaInfo); + return schemaVersion; } @Override From 483044dfda9f938d6e13b62b3156658534ff9979 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Sat, 27 Aug 2022 19:48:22 +0200 Subject: [PATCH 24/26] Rename preprocess function to transform function --- .../apache/pulsar/common/io/SinkConfig.java | 6 +- .../org/apache/pulsar/admin/cli/CmdSinks.java | 24 ++-- .../apache/pulsar/admin/cli/TestCmdSinks.java | 120 +++++++++--------- .../functions/utils/SinkConfigUtils.java | 34 ++--- .../functions/utils/SinkConfigUtilsTest.java | 36 +++--- .../functions/worker/rest/api/SinksImpl.java | 24 ++-- .../rest/api/v3/SinkApiV3ResourceTest.java | 28 ++-- ...ava => SinkWithTransformFunctionTest.java} | 18 +-- 8 files changed, 145 insertions(+), 145 deletions(-) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{SinkWithPreprocessingFunctionTest.java => SinkWithTransformFunctionTest.java} (95%) diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SinkConfig.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SinkConfig.java index ebb2adf2e825c..316cdd69e073b 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SinkConfig.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/io/SinkConfig.java @@ -91,7 +91,7 @@ public class SinkConfig { // to change behavior at runtime. Currently, this primarily used by the KubernetesManifestCustomizer // interface private String customRuntimeOptions; - private String preprocessFunction; - private String preprocessFunctionClassName; - private String preprocessFunctionConfig; + private String transformFunction; + private String transformFunctionClassName; + private String transformFunctionConfig; } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java index 2846b74e82076..7aae6ece7990f 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java @@ -398,13 +398,13 @@ abstract class SinkDetailsCommand extends BaseCommand { @Parameter(names = "--secrets", description = "The map of secretName to an object that encapsulates " + "how the secret is fetched by the underlying secrets provider") protected String secretsString; - @Parameter(names = "--preprocess-function", description = "Preprocess function applied before the Sink") - protected String preprocessFunction; - @Parameter(names = "--preprocess-function-classname", description = "The preprocess function class name") - protected String preprocessFunctionClassName; - @Parameter(names = "--preprocess-function-config", description = "Configuration of the preprocess function " + @Parameter(names = "--transform-function", description = "Transform function applied before the Sink") + protected String transformFunction; + @Parameter(names = "--transform-function-classname", description = "The transform function class name") + protected String transformFunctionClassName; + @Parameter(names = "--transform-function-config", description = "Configuration of the transform function " + "applied before the Sink") - protected String preprocessFunctionConfig; + protected String transformFunctionConfig; protected SinkConfig sinkConfig; @@ -585,16 +585,16 @@ void processArguments() throws Exception { sinkConfig.setSecrets(secretsMap); } - if (preprocessFunction != null) { - sinkConfig.setPreprocessFunction(preprocessFunction); + if (transformFunction != null) { + sinkConfig.setTransformFunction(transformFunction); } - if (preprocessFunctionClassName != null) { - sinkConfig.setPreprocessFunctionClassName(preprocessFunctionClassName); + if (transformFunctionClassName != null) { + sinkConfig.setTransformFunctionClassName(transformFunctionClassName); } - if (preprocessFunctionConfig != null) { - sinkConfig.setPreprocessFunctionConfig(preprocessFunctionConfig); + if (transformFunctionConfig != null) { + sinkConfig.setTransformFunctionConfig(transformFunctionConfig); } // check if configs are valid diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java index b15c51e602469..1cb86102802e0 100644 --- a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java +++ b/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/TestCmdSinks.java @@ -83,9 +83,9 @@ public class TestCmdSinks { private static final Long RAM = 1024L * 1024L; private static final Long DISK = 1024L * 1024L * 1024L; private static final String SINK_CONFIG_STRING = "{\"created_at\":\"Mon Jul 02 00:33:15 +0000 2018\",\"int\":1000,\"int_string\":\"1000\",\"float\":1000.0,\"float_string\":\"1000.0\"}"; - private static final String PREPROCESS_FUNCTION = "preprocess"; - private static final String PREPROCESS_FUNCTION_CLASSNAME = "PreProcessFunction"; - private static final String PREPROCESS_FUNCTION_CONFIG = "{\"test_function_config\": \"\"}"; + private static final String TRANSFORM_FUNCTION = "transform"; + private static final String TRANSFORM_FUNCTION_CLASSNAME = "TransformFunction"; + private static final String TRANSFORM_FUNCTION_CONFIG = "{\"test_function_config\": \"\"}"; private PulsarAdmin pulsarAdmin; private Sinks sink; @@ -150,9 +150,9 @@ public SinkConfig getSinkConfig() throws JsonProcessingException { sinkConfig.setResources(new Resources(CPU, RAM, DISK)); sinkConfig.setConfigs(createSink.parseConfigs(SINK_CONFIG_STRING)); - sinkConfig.setPreprocessFunction(PREPROCESS_FUNCTION); - sinkConfig.setPreprocessFunctionClassName(PREPROCESS_FUNCTION_CLASSNAME); - sinkConfig.setPreprocessFunctionConfig(PREPROCESS_FUNCTION_CONFIG); + sinkConfig.setTransformFunction(TRANSFORM_FUNCTION); + sinkConfig.setTransformFunctionClassName(TRANSFORM_FUNCTION_CLASSNAME); + sinkConfig.setTransformFunctionConfig(TRANSFORM_FUNCTION_CONFIG); return sinkConfig; } @@ -174,9 +174,9 @@ public void testCliCorrect() throws Exception { RAM, DISK, SINK_CONFIG_STRING, - PREPROCESS_FUNCTION, - PREPROCESS_FUNCTION_CLASSNAME, - PREPROCESS_FUNCTION_CONFIG, + TRANSFORM_FUNCTION, + TRANSFORM_FUNCTION_CLASSNAME, + TRANSFORM_FUNCTION_CONFIG, sinkConfig ); } @@ -199,9 +199,9 @@ public void testMissingInput() throws Exception { RAM, DISK, SINK_CONFIG_STRING, - PREPROCESS_FUNCTION, - PREPROCESS_FUNCTION_CLASSNAME, - PREPROCESS_FUNCTION_CONFIG, + TRANSFORM_FUNCTION, + TRANSFORM_FUNCTION_CLASSNAME, + TRANSFORM_FUNCTION_CONFIG, sinkConfig ); } @@ -225,9 +225,9 @@ public void testMissingCustomSerdeInput() throws Exception { RAM, DISK, SINK_CONFIG_STRING, - PREPROCESS_FUNCTION, - PREPROCESS_FUNCTION_CLASSNAME, - PREPROCESS_FUNCTION_CONFIG, + TRANSFORM_FUNCTION, + TRANSFORM_FUNCTION_CLASSNAME, + TRANSFORM_FUNCTION_CONFIG, sinkConfig ); } @@ -250,9 +250,9 @@ public void testMissingTopicPattern() throws Exception { RAM, DISK, SINK_CONFIG_STRING, - PREPROCESS_FUNCTION, - PREPROCESS_FUNCTION_CLASSNAME, - PREPROCESS_FUNCTION_CONFIG, + TRANSFORM_FUNCTION, + TRANSFORM_FUNCTION_CLASSNAME, + TRANSFORM_FUNCTION_CONFIG, sinkConfig ); } @@ -275,9 +275,9 @@ public void testMissingProcessingGuarantees() throws Exception { RAM, DISK, SINK_CONFIG_STRING, - PREPROCESS_FUNCTION, - PREPROCESS_FUNCTION_CLASSNAME, - PREPROCESS_FUNCTION_CONFIG, + TRANSFORM_FUNCTION, + TRANSFORM_FUNCTION_CLASSNAME, + TRANSFORM_FUNCTION_CONFIG, sinkConfig ); } @@ -300,9 +300,9 @@ public void testMissingArchive() throws Exception { RAM, DISK, SINK_CONFIG_STRING, - PREPROCESS_FUNCTION, - PREPROCESS_FUNCTION_CLASSNAME, - PREPROCESS_FUNCTION_CONFIG, + TRANSFORM_FUNCTION, + TRANSFORM_FUNCTION_CLASSNAME, + TRANSFORM_FUNCTION_CONFIG, sinkConfig ); } @@ -327,9 +327,9 @@ public void testInvalidJar() throws Exception { RAM, DISK, SINK_CONFIG_STRING, - PREPROCESS_FUNCTION, - PREPROCESS_FUNCTION_CLASSNAME, - PREPROCESS_FUNCTION_CONFIG, + TRANSFORM_FUNCTION, + TRANSFORM_FUNCTION_CLASSNAME, + TRANSFORM_FUNCTION_CONFIG, sinkConfig ); } @@ -352,9 +352,9 @@ public void testMissingConfig() throws Exception { RAM, DISK, null, - PREPROCESS_FUNCTION, - PREPROCESS_FUNCTION_CLASSNAME, - PREPROCESS_FUNCTION_CONFIG, + TRANSFORM_FUNCTION, + TRANSFORM_FUNCTION_CLASSNAME, + TRANSFORM_FUNCTION_CONFIG, sinkConfig ); } @@ -373,9 +373,9 @@ private void testCmdSinkCliMissingArgs( Long ram, Long disk, String sinkConfigString, - String preprocessFunction, - String preprocessFunctionClassName, - String preprocessFunctionConfig, + String transformFunction, + String transformFunctionClassName, + String transformFunctionConfig, SinkConfig sinkConfig) throws Exception { // test create sink @@ -392,9 +392,9 @@ private void testCmdSinkCliMissingArgs( createSink.ram = ram; createSink.disk = disk; createSink.sinkConfigString = sinkConfigString; - createSink.preprocessFunction = preprocessFunction; - createSink.preprocessFunctionClassName = preprocessFunctionClassName; - createSink.preprocessFunctionConfig = preprocessFunctionConfig; + createSink.transformFunction = transformFunction; + createSink.transformFunctionClassName = transformFunctionClassName; + createSink.transformFunctionConfig = transformFunctionConfig; createSink.processArguments(); @@ -414,9 +414,9 @@ private void testCmdSinkCliMissingArgs( updateSink.ram = ram; updateSink.disk = disk; updateSink.sinkConfigString = sinkConfigString; - updateSink.preprocessFunction = preprocessFunction; - updateSink.preprocessFunctionClassName = preprocessFunctionClassName; - updateSink.preprocessFunctionConfig = preprocessFunctionConfig; + updateSink.transformFunction = transformFunction; + updateSink.transformFunctionClassName = transformFunctionClassName; + updateSink.transformFunctionConfig = transformFunctionConfig; updateSink.processArguments(); @@ -436,9 +436,9 @@ private void testCmdSinkCliMissingArgs( localSinkRunner.ram = ram; localSinkRunner.disk = disk; localSinkRunner.sinkConfigString = sinkConfigString; - localSinkRunner.preprocessFunction = preprocessFunction; - localSinkRunner.preprocessFunctionClassName = preprocessFunctionClassName; - localSinkRunner.preprocessFunctionConfig = preprocessFunctionConfig; + localSinkRunner.transformFunction = transformFunction; + localSinkRunner.transformFunctionClassName = transformFunctionClassName; + localSinkRunner.transformFunctionConfig = transformFunctionConfig; localSinkRunner.processArguments(); @@ -583,9 +583,9 @@ public void testCliOverwriteConfigFile() throws Exception { testSinkConfig.setResources(new Resources(CPU + 1, RAM + 1, DISK + 1)); testSinkConfig.setConfigs(createSink.parseConfigs("{\"created_at-prime\":\"Mon Jul 02 00:33:15 +0000 2018\", \"otherConfigProperties\":{\"property1.value\":\"value1\",\"property2.value\":\"value2\"}}")); - testSinkConfig.setPreprocessFunction(PREPROCESS_FUNCTION + "-prime"); - testSinkConfig.setPreprocessFunction(PREPROCESS_FUNCTION_CLASSNAME + "-prime"); - testSinkConfig.setPreprocessFunction("{\"test_function_config\": \"prime\"}"); + testSinkConfig.setTransformFunction(TRANSFORM_FUNCTION + "-prime"); + testSinkConfig.setTransformFunction(TRANSFORM_FUNCTION_CLASSNAME + "-prime"); + testSinkConfig.setTransformFunction("{\"test_function_config\": \"prime\"}"); SinkConfig expectedSinkConfig = getSinkConfig(); @@ -610,9 +610,9 @@ public void testCliOverwriteConfigFile() throws Exception { RAM, DISK, SINK_CONFIG_STRING, - PREPROCESS_FUNCTION, - PREPROCESS_FUNCTION_CLASSNAME, - PREPROCESS_FUNCTION_CONFIG, + TRANSFORM_FUNCTION, + TRANSFORM_FUNCTION_CLASSNAME, + TRANSFORM_FUNCTION_CONFIG, file.getAbsolutePath(), expectedSinkConfig ); @@ -633,9 +633,9 @@ private void testMixCliAndConfigFile( Long ram, Long disk, String sinkConfigString, - String preprocessFunction, - String preprocessFunctionClassName, - String preprocessFunctionConfig, + String transformFunction, + String transformFunctionClassName, + String transformFunctionConfig, String sinkConfigFile, SinkConfig sinkConfig ) throws Exception { @@ -655,9 +655,9 @@ private void testMixCliAndConfigFile( createSink.ram = ram; createSink.disk = disk; createSink.sinkConfigString = sinkConfigString; - createSink.preprocessFunction = preprocessFunction; - createSink.preprocessFunctionClassName = preprocessFunctionClassName; - createSink.preprocessFunctionConfig = preprocessFunctionConfig; + createSink.transformFunction = transformFunction; + createSink.transformFunctionClassName = transformFunctionClassName; + createSink.transformFunctionConfig = transformFunctionConfig; createSink.sinkConfigFile = sinkConfigFile; createSink.processArguments(); @@ -678,9 +678,9 @@ private void testMixCliAndConfigFile( updateSink.ram = ram; updateSink.disk = disk; updateSink.sinkConfigString = sinkConfigString; - updateSink.preprocessFunction = preprocessFunction; - updateSink.preprocessFunctionClassName = preprocessFunctionClassName; - updateSink.preprocessFunctionConfig = preprocessFunctionConfig; + updateSink.transformFunction = transformFunction; + updateSink.transformFunctionClassName = transformFunctionClassName; + updateSink.transformFunctionConfig = transformFunctionConfig; updateSink.sinkConfigFile = sinkConfigFile; updateSink.processArguments(); @@ -701,9 +701,9 @@ private void testMixCliAndConfigFile( localSinkRunner.ram = ram; localSinkRunner.disk = disk; localSinkRunner.sinkConfigString = sinkConfigString; - localSinkRunner.preprocessFunction = preprocessFunction; - localSinkRunner.preprocessFunctionClassName = preprocessFunctionClassName; - localSinkRunner.preprocessFunctionConfig = preprocessFunctionConfig; + localSinkRunner.transformFunction = transformFunction; + localSinkRunner.transformFunctionClassName = transformFunctionClassName; + localSinkRunner.transformFunctionConfig = transformFunctionConfig; localSinkRunner.sinkConfigFile = sinkConfigFile; diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java index 21bdffe8ed7fd..2a06339d00a15 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java @@ -98,8 +98,8 @@ public static FunctionDetails convert(SinkConfig sinkConfig, ExtractedSinkDetail } else { functionDetailsBuilder.setClassName(IdentityFunction.class.getName()); } - if (sinkConfig.getPreprocessFunctionConfig() != null) { - functionDetailsBuilder.setUserConfig(sinkConfig.getPreprocessFunctionConfig()); + if (sinkConfig.getTransformFunctionConfig() != null) { + functionDetailsBuilder.setUserConfig(sinkConfig.getTransformFunctionConfig()); } if (sinkConfig.getProcessingGuarantees() != null) { functionDetailsBuilder.setProcessingGuarantees( @@ -236,9 +236,9 @@ public static FunctionDetails convert(SinkConfig sinkConfig, ExtractedSinkDetail sinkSpecBuilder.setBuiltin(builtin); } - if (!isEmpty(sinkConfig.getPreprocessFunction()) - && sinkConfig.getPreprocessFunction().startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN)) { - functionDetailsBuilder.setBuiltin(sinkConfig.getPreprocessFunction().replaceFirst("^builtin://", "")); + if (!isEmpty(sinkConfig.getTransformFunction()) + && sinkConfig.getTransformFunction().startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN)) { + functionDetailsBuilder.setBuiltin(sinkConfig.getTransformFunction().replaceFirst("^builtin://", "")); } if (sinkConfig.getConfigs() != null) { @@ -387,13 +387,13 @@ public static SinkConfig convertFromDetails(FunctionDetails functionDetails) { } if (!isEmpty(functionDetails.getBuiltin())) { - sinkConfig.setPreprocessFunction("builtin://" + functionDetails.getBuiltin()); + sinkConfig.setTransformFunction("builtin://" + functionDetails.getBuiltin()); } if (!functionDetails.getClassName().equals(IdentityFunction.class.getName())) { - sinkConfig.setPreprocessFunctionClassName(functionDetails.getClassName()); + sinkConfig.setTransformFunctionClassName(functionDetails.getClassName()); } if (!isEmpty(functionDetails.getUserConfig())) { - sinkConfig.setPreprocessFunctionConfig(functionDetails.getUserConfig()); + sinkConfig.setTransformFunctionConfig(functionDetails.getUserConfig()); } @@ -458,7 +458,7 @@ public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConf String.format("Sink class %s not found in class loader", sinkClassName), e); } - String functionClassName = sinkConfig.getPreprocessFunctionClassName(); + String functionClassName = sinkConfig.getTransformFunctionClassName(); Class typeArg; ClassLoader inputClassLoader; if (functionClassLoader != null) { @@ -478,9 +478,9 @@ public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConf throw new IllegalArgumentException( String.format("Function class %s not found in class loader", functionClassName), e); } - // extract type from preprocess function class + // extract type from transform function class if (!getRawFunctionTypes(functionClass, false)[1].equals(Record.class)) { - throw new IllegalArgumentException("Sink preprocess function output must be of type Record"); + throw new IllegalArgumentException("Sink transform function output must be of type Record"); } typeArg = getFunctionTypes(functionClass, false)[0]; inputClassLoader = functionClassLoader; @@ -673,14 +673,14 @@ public static SinkConfig validateUpdate(SinkConfig existingConfig, SinkConfig ne if (newConfig.getCleanupSubscription() != null) { mergedConfig.setCleanupSubscription(newConfig.getCleanupSubscription()); } - if (newConfig.getPreprocessFunction() != null) { - mergedConfig.setPreprocessFunction(newConfig.getPreprocessFunction()); + if (newConfig.getTransformFunction() != null) { + mergedConfig.setTransformFunction(newConfig.getTransformFunction()); } - if (newConfig.getPreprocessFunctionClassName() != null) { - mergedConfig.setPreprocessFunctionClassName(newConfig.getPreprocessFunctionClassName()); + if (newConfig.getTransformFunctionClassName() != null) { + mergedConfig.setTransformFunctionClassName(newConfig.getTransformFunctionClassName()); } - if (newConfig.getPreprocessFunctionConfig() != null) { - mergedConfig.setPreprocessFunctionConfig(newConfig.getPreprocessFunctionConfig()); + if (newConfig.getTransformFunctionConfig() != null) { + mergedConfig.setTransformFunctionConfig(newConfig.getTransformFunctionConfig()); } return mergedConfig; diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java index 114a5c13547c2..4f316cac7c7d6 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java @@ -111,8 +111,8 @@ public void testConvertBackFidelity() throws IOException { sinkConfig.setResources(Resources.getDefaultResources()); - sinkConfig.setPreprocessFunction("builtin://preprocess"); - sinkConfig.setPreprocessFunctionConfig("{\"key\": \"value\"}"); + sinkConfig.setTransformFunction("builtin://transform"); + sinkConfig.setTransformFunctionConfig("{\"key\": \"value\"}"); Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); assertEquals(Function.SubscriptionType.SHARED, functionDetails.getSource().getSubscriptionType()); @@ -429,16 +429,16 @@ public void testMergeRuntimeFlags() { } @Test - public void testMergeDifferentPreprocessFunction() { + public void testMergeDifferentTransformFunction() { SinkConfig sinkConfig = createSinkConfig(); String newFunction = "builtin://new"; - SinkConfig newSinkConfig = createUpdatedSinkConfig("preprocessFunction", newFunction); + SinkConfig newSinkConfig = createUpdatedSinkConfig("transformFunction", newFunction); SinkConfig mergedConfig = SinkConfigUtils.validateUpdate(sinkConfig, newSinkConfig); assertEquals( - mergedConfig.getPreprocessFunction(), + mergedConfig.getTransformFunction(), newFunction ); - mergedConfig.setPreprocessFunction(sinkConfig.getPreprocessFunction()); + mergedConfig.setTransformFunction(sinkConfig.getTransformFunction()); assertEquals( new Gson().toJson(sinkConfig), new Gson().toJson(mergedConfig) @@ -446,16 +446,16 @@ public void testMergeDifferentPreprocessFunction() { } @Test - public void testMergeDifferentPreprocessFunctionClassName() { + public void testMergeDifferentTransformFunctionClassName() { SinkConfig sinkConfig = createSinkConfig(); - String newFunctionClassName = "NewPreprocessFunction"; - SinkConfig newSinkConfig = createUpdatedSinkConfig("preprocessFunctionClassName", newFunctionClassName); + String newFunctionClassName = "NewTransformFunction"; + SinkConfig newSinkConfig = createUpdatedSinkConfig("transformFunctionClassName", newFunctionClassName); SinkConfig mergedConfig = SinkConfigUtils.validateUpdate(sinkConfig, newSinkConfig); assertEquals( - mergedConfig.getPreprocessFunctionClassName(), + mergedConfig.getTransformFunctionClassName(), newFunctionClassName ); - mergedConfig.setPreprocessFunctionClassName(sinkConfig.getPreprocessFunctionClassName()); + mergedConfig.setTransformFunctionClassName(sinkConfig.getTransformFunctionClassName()); assertEquals( new Gson().toJson(sinkConfig), new Gson().toJson(mergedConfig) @@ -463,16 +463,16 @@ public void testMergeDifferentPreprocessFunctionClassName() { } @Test - public void testMergeDifferentPreprocessFunctionConfig() { + public void testMergeDifferentTransformFunctionConfig() { SinkConfig sinkConfig = createSinkConfig(); String newFunctionConfig = "{\"new-key\": \"new-value\"}"; - SinkConfig newSinkConfig = createUpdatedSinkConfig("preprocessFunctionConfig", newFunctionConfig); + SinkConfig newSinkConfig = createUpdatedSinkConfig("transformFunctionConfig", newFunctionConfig); SinkConfig mergedConfig = SinkConfigUtils.validateUpdate(sinkConfig, newSinkConfig); assertEquals( - mergedConfig.getPreprocessFunctionConfig(), + mergedConfig.getTransformFunctionConfig(), newFunctionConfig ); - mergedConfig.setPreprocessFunctionConfig(sinkConfig.getPreprocessFunctionConfig()); + mergedConfig.setTransformFunctionConfig(sinkConfig.getTransformFunctionConfig()); assertEquals( new Gson().toJson(sinkConfig), new Gson().toJson(mergedConfig) @@ -513,9 +513,9 @@ private SinkConfig createSinkConfig() { sinkConfig.setCleanupSubscription(true); sinkConfig.setArchive("DummyArchive.nar"); sinkConfig.setCleanupSubscription(true); - sinkConfig.setPreprocessFunction("builtin://preprocess"); - sinkConfig.setPreprocessFunctionClassName("Preprocess"); - sinkConfig.setPreprocessFunctionConfig("{\"key\": \"value\"}"); + sinkConfig.setTransformFunction("builtin://transform"); + sinkConfig.setTransformFunctionClassName("Transform"); + sinkConfig.setTransformFunctionConfig("{\"key\": \"value\"}"); return sinkConfig; } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index 71a9286ca1b88..3f1526f442d54 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -228,9 +228,9 @@ public void registerSink(final String tenant, functionMetaDataBuilder.setPackageLocation(packageLocationMetaDataBuilder); - String preprocessFunction = sinkConfig.getPreprocessFunction(); - if (isNotBlank(preprocessFunction)) { - setExtraFunctionPackageLocation(functionMetaDataBuilder, functionDetails, preprocessFunction); + String transformFunction = sinkConfig.getTransformFunction(); + if (isNotBlank(transformFunction)) { + setExtraFunctionPackageLocation(functionMetaDataBuilder, functionDetails, transformFunction); } updateRequest(null, functionMetaDataBuilder.build()); @@ -413,10 +413,10 @@ public void updateSink(final String tenant, functionMetaDataBuilder.setPackageLocation(packageLocationMetaDataBuilder); - String preprocessFunction = mergedConfig.getPreprocessFunction(); - if (isNotBlank(preprocessFunction) - && !preprocessFunction.equals(existingSinkConfig.getPreprocessFunction())) { - setExtraFunctionPackageLocation(functionMetaDataBuilder, functionDetails, preprocessFunction); + String transformFunction = mergedConfig.getTransformFunction(); + if (isNotBlank(transformFunction) + && !transformFunction.equals(existingSinkConfig.getTransformFunction())) { + setExtraFunctionPackageLocation(functionMetaDataBuilder, functionDetails, transformFunction); } updateRequest(existingComponent, functionMetaDataBuilder.build()); @@ -779,16 +779,16 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant } ClassLoader functionClassLoader = null; - if (isNotBlank(sinkConfig.getPreprocessFunction())) { + if (isNotBlank(sinkConfig.getTransformFunction())) { functionClassLoader = - getBuiltinFunctionClassLoader(sinkConfig.getPreprocessFunction()); + getBuiltinFunctionClassLoader(sinkConfig.getTransformFunction()); if (functionClassLoader == null) { - File functionPackageFile = getPackageFile(sinkConfig.getPreprocessFunction()); - functionClassLoader = getClassLoaderFromPackage(sinkConfig.getPreprocessFunctionClassName(), + File functionPackageFile = getPackageFile(sinkConfig.getTransformFunction()); + functionClassLoader = getClassLoaderFromPackage(sinkConfig.getTransformFunctionClassName(), functionPackageFile, worker().getWorkerConfig().getNarExtractionDirectory()); } if (functionClassLoader == null) { - throw new IllegalArgumentException("Pre-process Function package not found"); + throw new IllegalArgumentException("Transform Function package not found"); } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java index f9608b2e7b26b..aea6565e05bb5 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java @@ -712,7 +712,7 @@ public void testRegisterSinkFailedWithWrongPackageName() throws PulsarAdminExcep } @Test - public void testRegisterSinkSuccessWithPreprocessFunction() throws Exception { + public void testRegisterSinkSuccessWithTransformFunction() throws Exception { mockInstanceUtils(); mockWorkerUtils(); @@ -737,13 +737,13 @@ public void testRegisterSinkSuccessWithPreprocessFunction() throws Exception { FunctionArchive functionArchive = FunctionArchive.builder() .classLoader(mockedClassLoader) .build(); - when(mockedFunctionsManager.getFunction("preprocess")).thenReturn(functionArchive); + when(mockedFunctionsManager.getFunction("transform")).thenReturn(functionArchive); when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); SinkConfig sinkConfig = createDefaultSinkConfig(); - sinkConfig.setPreprocessFunction("builtin://preprocess"); - sinkConfig.setPreprocessFunctionConfig("{\"dummy\": \"dummy\"}"); + sinkConfig.setTransformFunction("builtin://transform"); + sinkConfig.setTransformFunctionConfig("{\"dummy\": \"dummy\"}"); try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { resource.registerSink( @@ -759,8 +759,8 @@ public void testRegisterSinkSuccessWithPreprocessFunction() throws Exception { } @Test(expectedExceptions = RestException.class, - expectedExceptionsMessageRegExp = "Sink preprocess function output must be of type Record") - public void testRegisterSinkFailureWithInvalidPreprocessFunction() throws Exception { + expectedExceptionsMessageRegExp = "Sink transform function output must be of type Record") + public void testRegisterSinkFailureWithInvalidTransformFunction() throws Exception { mockInstanceUtils(); mockWorkerUtils(); @@ -785,13 +785,13 @@ public void testRegisterSinkFailureWithInvalidPreprocessFunction() throws Except FunctionArchive functionArchive = FunctionArchive.builder() .classLoader(mockedClassLoader) .build(); - when(mockedFunctionsManager.getFunction("preprocess")).thenReturn(functionArchive); + when(mockedFunctionsManager.getFunction("transform")).thenReturn(functionArchive); when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); SinkConfig sinkConfig = createDefaultSinkConfig(); - sinkConfig.setPreprocessFunction("builtin://preprocess"); - sinkConfig.setPreprocessFunctionConfig("{\"dummy\": \"dummy\"}"); + sinkConfig.setTransformFunction("builtin://transform"); + sinkConfig.setTransformFunctionConfig("{\"dummy\": \"dummy\"}"); try { try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { @@ -1206,13 +1206,13 @@ public void testUpdateSinkInterrupted() throws Exception { } @Test - public void testUpdateSinkDifferentPreprocessFunction() throws Exception { + public void testUpdateSinkDifferentTransformFunction() throws Exception { mockWorkerUtils(); SinkConfig sinkConfig = createDefaultSinkConfig(); - sinkConfig.setPreprocessFunction("builtin://preprocess"); - sinkConfig.setPreprocessFunctionClassName("DummyFunction"); - sinkConfig.setPreprocessFunctionConfig("{\"dummy\": \"dummy\"}"); + sinkConfig.setTransformFunction("builtin://transform"); + sinkConfig.setTransformFunctionClassName("DummyFunction"); + sinkConfig.setTransformFunctionConfig("{\"dummy\": \"dummy\"}"); NarClassLoader mockedClassLoader = mock(NarClassLoader.class); doReturn(RecordFunction.class).when(mockedClassLoader).loadClass("DummyFunction"); @@ -1235,7 +1235,7 @@ public void testUpdateSinkDifferentPreprocessFunction() throws Exception { FunctionArchive functionArchive = FunctionArchive.builder() .classLoader(mockedClassLoader) .build(); - when(mockedFunctionsManager.getFunction("preprocess")).thenReturn(functionArchive); + when(mockedFunctionsManager.getFunction("transform")).thenReturn(functionArchive); when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkWithPreprocessingFunctionTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkWithTransformFunctionTest.java similarity index 95% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkWithPreprocessingFunctionTest.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkWithTransformFunctionTest.java index b7a5ecab9cbc2..b5faf59d25a2d 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkWithPreprocessingFunctionTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkWithTransformFunctionTest.java @@ -46,10 +46,10 @@ import org.testng.annotations.Test; /** - * Test behaviour of sinks with a pre-processing function + * Test behaviour of sinks with a transform function */ @Slf4j -public class SinkWithPreprocessingFunctionTest extends PulsarStandaloneTestSuite { +public class SinkWithTransformFunctionTest extends PulsarStandaloneTestSuite { //Use PIP-117 new defaults so that the package management service is enabled. @Override @@ -74,7 +74,7 @@ public void setUpCluster() throws Exception { } @Test(groups = {"sink"}) - public void testSinkWithPreprocessingFunction() throws Exception { + public void testSinkWithTransformFunction() throws Exception { @Cleanup PulsarClient client = PulsarClient.builder() .serviceUrl(container.getPlainTextServiceUrl()) @@ -131,7 +131,7 @@ public void testSinkWithPreprocessingFunction() throws Exception { } @Test(groups = {"sink"}) - public void testGenericObjectSinkWithPreprocessingFunction() throws Exception { + public void testGenericObjectSinkWithTransformFunction() throws Exception { @Cleanup PulsarClient client = PulsarClient.builder() .serviceUrl(container.getPlainTextServiceUrl()) @@ -211,7 +211,7 @@ public void testGenericObjectSinkWithPreprocessingFunction() throws Exception { } @Test(groups = {"sink"}) - public void testKeyValueSinkWithPreprocessingFunction() throws Exception { + public void testKeyValueSinkWithTransformFunction() throws Exception { @Cleanup PulsarClient client = PulsarClient.builder() .serviceUrl(container.getPlainTextServiceUrl()) @@ -293,8 +293,8 @@ private void submitSinkConnector(String sinkName, String className, String archive, String configs, - String preprocessFunction, - String preprocessFunctionClassName) throws Exception { + String transformFunction, + String transformFunctionClassName) throws Exception { String[] commands = { PulsarCluster.ADMIN_SCRIPT, "sinks", "create", @@ -303,8 +303,8 @@ private void submitSinkConnector(String sinkName, "--archive", archive, "--classname", className, "--sink-config", configs, - "--preprocess-function", preprocessFunction, - "--preprocess-function-classname", preprocessFunctionClassName + "--transform-function", transformFunction, + "--transform-function-classname", transformFunctionClassName }; log.info("Run command : {}", StringUtils.join(commands, ' ')); ContainerExecResult result = container.execCmd(commands); From da3ed442ee8a19e41ee3bc934f64ad09352c36bb Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Mon, 29 Aug 2022 10:35:44 +0200 Subject: [PATCH 25/26] Rename extra function to transform function --- .../broker/admin/impl/FunctionsBase.java | 6 ++--- .../apache/pulsar/client/admin/Functions.java | 12 ++++----- .../client/admin/internal/FunctionsImpl.java | 8 +++--- .../pulsar/admin/cli/CmdFunctionsTest.java | 4 +-- .../apache/pulsar/admin/cli/CmdFunctions.java | 8 +++--- .../functions/instance/InstanceConfig.java | 2 +- .../instance/JavaInstanceRunnable.java | 6 +++-- .../proto/src/main/proto/Function.proto | 2 +- .../runtime/JavaInstanceStarter.java | 14 +++++----- .../functions/runtime/RuntimeFactory.java | 2 +- .../functions/runtime/RuntimeSpawner.java | 6 ++--- .../functions/runtime/RuntimeUtils.java | 16 +++++------ .../runtime/kubernetes/KubernetesRuntime.java | 27 ++++++++++--------- .../kubernetes/KubernetesRuntimeFactory.java | 6 ++--- .../runtime/process/ProcessRuntime.java | 4 +-- .../process/ProcessRuntimeFactory.java | 6 ++--- .../runtime/thread/ThreadRuntime.java | 14 +++++----- .../runtime/thread/ThreadRuntimeFactory.java | 6 ++--- .../kubernetes/KubernetesRuntimeTest.java | 6 ++--- .../runtime/process/ProcessRuntimeTest.java | 4 +-- .../functions/worker/FunctionActioner.java | 25 ++++++++--------- .../worker/FunctionRuntimeManager.java | 2 +- .../worker/rest/api/ComponentImpl.java | 10 +++---- .../functions/worker/rest/api/SinksImpl.java | 18 ++++++------- .../rest/api/v3/FunctionsApiV3Resource.java | 6 ++--- .../worker/service/api/Component.java | 2 +- .../worker/FunctionActionerTest.java | 2 +- .../worker/FunctionRuntimeManagerTest.java | 4 +-- .../api/v3/FunctionApiV3ResourceTest.java | 6 ++--- .../rest/api/v3/SinkApiV3ResourceTest.java | 26 +++++++++--------- 30 files changed, 132 insertions(+), 128 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java index 6c408522e0e84..be9dc8c7b714c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java @@ -712,11 +712,11 @@ public StreamingOutput downloadFunction( final @PathParam("namespace") String namespace, @ApiParam(value = "The name of a Pulsar Function") final @PathParam("functionName") String functionName, - @ApiParam(value = "Whether to download the extra-function") - final @QueryParam("extra-function") boolean extraFunction) { + @ApiParam(value = "Whether to download the transform-function") + final @QueryParam("transform-function") boolean transformFunction) { return functions() - .downloadFunction(tenant, namespace, functionName, clientAppId(), clientAuthData(), extraFunction); + .downloadFunction(tenant, namespace, functionName, clientAppId(), clientAuthData(), transformFunction); } @GET diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Functions.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Functions.java index 4c6fa4fa3ee4e..55e0855ca5523 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Functions.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Functions.java @@ -768,12 +768,12 @@ CompletableFuture downloadFunctionAsync( * Namespace name * @param function * Function name - * @param extraFunction - * Whether to download the extra-function (for sources and sinks) + * @param transformFunction + * Whether to download the transform function (for sources and sinks) * @throws PulsarAdminException */ void downloadFunction(String destinationFile, String tenant, String namespace, String function, - boolean extraFunction) throws PulsarAdminException; + boolean transformFunction) throws PulsarAdminException; /** * Download Function Code asynchronously. @@ -786,11 +786,11 @@ void downloadFunction(String destinationFile, String tenant, String namespace, S * Namespace name * @param function * Function name - * @param extraFunction - * Whether to download the extra-function (for sources and sinks) + * @param transformFunction + * Whether to download the transform function (for sources and sinks) */ CompletableFuture downloadFunctionAsync( - String destinationFile, String tenant, String namespace, String function, boolean extraFunction); + String destinationFile, String tenant, String namespace, String function, boolean transformFunction); /** diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java index 4348eadd7ff9b..482846ad8ac1e 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java @@ -608,17 +608,17 @@ public CompletableFuture downloadFunctionAsync( @Override public void downloadFunction(String destinationPath, String tenant, String namespace, String functionName, - boolean extraFunction) throws PulsarAdminException { + boolean transformFunction) throws PulsarAdminException { downloadFile(destinationPath, functions.path(tenant).path(namespace).path(functionName).path("download") - .queryParam("extra-function", extraFunction)); + .queryParam("transform-function", transformFunction)); } @Override public CompletableFuture downloadFunctionAsync( - String destinationPath, String tenant, String namespace, String functionName, boolean extraFunction) { + String destinationPath, String tenant, String namespace, String functionName, boolean transformFunction) { return downloadFileAsync(destinationPath, functions.path(tenant).path(namespace).path(functionName).path("download") - .queryParam("extra-function", extraFunction)); + .queryParam("transform-function", transformFunction)); } diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java index f47f7127d5396..4fc723e5f4c87 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java @@ -884,14 +884,14 @@ public void testDownloadFunctionByPath() throws Exception { } @Test - public void testDownloadExtraFunction() throws Exception { + public void testDownloadTransformFunction() throws Exception { cmd.run(new String[] { "download", "--destination-file", JAR_NAME, "--name", FN_NAME, "--tenant", TENANT, "--namespace", NAMESPACE, - "--extra-function" + "--transform-function" }); verify(functions, times(1)) .downloadFunction(JAR_NAME, TENANT, NAMESPACE, FN_NAME, true); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index 6cc58e3c16852..43d13bc97dd45 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -1172,9 +1172,9 @@ class DownloadFunction extends FunctionCommand { listConverter = StringConverter.class, required = false, hidden = true) protected String path; @Parameter( - names = "--extra-function", - description = "Download the extra Function of the connector") - protected Boolean extraFunction = false; + names = "--transform-function", + description = "Download the transform Function of the connector") + protected Boolean transformFunction = false; private void mergeArgs() { if (isBlank(destinationFile) && !isBlank(deprecatedDestinationFile)) { @@ -1200,7 +1200,7 @@ void runCmd() throws Exception { getAdmin().functions().downloadFunction(destinationFile, path); } else { getAdmin().functions() - .downloadFunction(destinationFile, tenant, namespace, functionName, extraFunction); + .downloadFunction(destinationFile, tenant, namespace, functionName, transformFunction); } print("Downloaded successfully"); } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java index b66dbdda4f104..3aefa6b2dca50 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/InstanceConfig.java @@ -33,7 +33,7 @@ public class InstanceConfig { private int instanceId; private String functionId; - private String extraFunctionId; + private String transformFunctionId; private String functionVersion; private FunctionDetails functionDetails; private int maxBufferedTuples; diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index b22c2ea3118cd..bd779e0fccc73 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -167,7 +167,7 @@ public JavaInstanceRunnable(InstanceConfig instanceConfig, SecretsProvider secretsProvider, FunctionCollectorRegistry collectorRegistry, ClassLoader componentClassLoader, - ClassLoader extraFunctionClassLoader) throws PulsarClientException { + ClassLoader transformFunctionClassLoader) throws PulsarClientException { this.instanceConfig = instanceConfig; this.clientBuilder = clientBuilder; this.client = (PulsarClientImpl) pulsarClient; @@ -176,7 +176,9 @@ public JavaInstanceRunnable(InstanceConfig instanceConfig, this.stateStorageServiceUrl = stateStorageServiceUrl; this.secretsProvider = secretsProvider; this.componentClassLoader = componentClassLoader; - this.functionClassLoader = extraFunctionClassLoader != null ? extraFunctionClassLoader : componentClassLoader; + this.functionClassLoader = transformFunctionClassLoader != null + ? transformFunctionClassLoader + : componentClassLoader; this.metricsLabels = new String[]{ instanceConfig.getFunctionDetails().getTenant(), String.format("%s/%s", instanceConfig.getFunctionDetails().getTenant(), diff --git a/pulsar-functions/proto/src/main/proto/Function.proto b/pulsar-functions/proto/src/main/proto/Function.proto index 65b5e0f0fc076..e67899abd223e 100644 --- a/pulsar-functions/proto/src/main/proto/Function.proto +++ b/pulsar-functions/proto/src/main/proto/Function.proto @@ -211,7 +211,7 @@ message FunctionMetaData { uint64 createTime = 4; map instanceStates = 5; FunctionAuthenticationSpec functionAuthSpec = 6; - PackageLocationMetaData extraFunctionPackageLocation = 7; + PackageLocationMetaData transformFunctionPackageLocation = 7; } message FunctionAuthenticationSpec { diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java index 3cd26c7ffb520..5e74d65621ee0 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java @@ -62,10 +62,10 @@ public class JavaInstanceStarter implements AutoCloseable { public String jarFile; @Parameter( - names = "--extra_function_jar", - description = "Path to Extra Function Jar\n", + names = "--transform_function_jar", + description = "Path to Transform Function Jar\n", listConverter = StringConverter.class) - public String extraFunctionJarFile; + public String transformFunctionJarFile; @Parameter(names = "--instance_id", description = "Instance Id\n", required = true) public int instanceId; @@ -73,8 +73,8 @@ public class JavaInstanceStarter implements AutoCloseable { @Parameter(names = "--function_id", description = "Function Id\n", required = true) public String functionId; - @Parameter(names = "--extra_function_id", description = "Extra Function Id\n", required = true) - public String extraFunctionId; + @Parameter(names = "--transform_function_id", description = "Transform Function Id\n", required = true) + public String transformFunctionId; @Parameter(names = "--function_version", description = "Function Version\n", required = true) public String functionVersion; @@ -166,7 +166,7 @@ public void start(String[] args, ClassLoader functionInstanceClassLoader, ClassL InstanceConfig instanceConfig = new InstanceConfig(); instanceConfig.setFunctionId(functionId); - instanceConfig.setExtraFunctionId(extraFunctionId); + instanceConfig.setTransformFunctionId(transformFunctionId); instanceConfig.setFunctionVersion(functionVersion); instanceConfig.setInstanceId(instanceId); instanceConfig.setMaxBufferedTuples(maxBufferedTuples); @@ -230,7 +230,7 @@ public void start(String[] args, ClassLoader functionInstanceClassLoader, ClassL instanceConfig, jarFile, null, // we really dont use this in thread container - extraFunctionJarFile, + transformFunctionJarFile, null, // we really dont use this in thread container containerFactory, expectedHealthCheckInterval * 1000); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeFactory.java index 04d750b02e567..0ee2c430a72b1 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeFactory.java @@ -52,7 +52,7 @@ void initialize(WorkerConfig workerConfig, */ Runtime createContainer( InstanceConfig instanceConfig, String codeFile, String originalCodeFileName, - String extraFunctionFile, String originalExtraFunctionFileName, + String transformFunctionFile, String originalTransformFunctionFileName, Long expectedHealthCheckInterval) throws Exception; default boolean externallyManaged() { diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java index 02be49ee512b0..4f9e9b672cb55 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java @@ -57,8 +57,8 @@ public class RuntimeSpawner implements AutoCloseable { public RuntimeSpawner(InstanceConfig instanceConfig, String codeFile, String originalCodeFileName, - String extraFunctionFile, - String originalExtraFunctionFileName, + String transformFunctionFile, + String originalTransformFunctionFileName, RuntimeFactory containerFactory, long instanceLivenessCheckFreqMs) { this.instanceConfig = instanceConfig; this.runtimeFactory = containerFactory; @@ -67,7 +67,7 @@ public RuntimeSpawner(InstanceConfig instanceConfig, this.instanceLivenessCheckFreqMs = instanceLivenessCheckFreqMs; try { this.runtime = runtimeFactory.createContainer(this.instanceConfig, codeFile, originalCodeFileName, - extraFunctionFile, originalExtraFunctionFileName, + transformFunctionFile, originalTransformFunctionFileName, instanceLivenessCheckFreqMs / 1000); } catch (Exception e) { throw new RuntimeException(e); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java index 367eea3ed2b62..e5baa7ae9671a 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java @@ -68,7 +68,7 @@ public static List composeCmd(InstanceConfig instanceConfig, String extraDependenciesDir, /* extra dependencies for running instances */ String logDirectory, String originalCodeFileName, - String originalExtraFunctionFileName, + String originalTransformFunctionFileName, String pulsarServiceUrl, String stateStorageServiceUrl, AuthenticationConfig authConfig, @@ -88,7 +88,7 @@ public static List composeCmd(InstanceConfig instanceConfig, final List cmd = getArgsBeforeCmd(instanceConfig, extraDependenciesDir); cmd.addAll(getCmd(instanceConfig, instanceFile, extraDependenciesDir, logDirectory, - originalCodeFileName, originalExtraFunctionFileName, pulsarServiceUrl, stateStorageServiceUrl, + originalCodeFileName, originalTransformFunctionFileName, pulsarServiceUrl, stateStorageServiceUrl, authConfig, shardId, grpcPort, expectedHealthCheckInterval, logConfigFile, secretsProviderClassName, secretsProviderConfig, installUserCodeDependencies, pythonDependencyRepository, @@ -265,7 +265,7 @@ public static List getCmd(InstanceConfig instanceConfig, String extraDependenciesDir, /* extra dependencies for running instances */ String logDirectory, String originalCodeFileName, - String originalExtraFunctionFileName, + String originalTransformFunctionFileName, String pulsarServiceUrl, String stateStorageServiceUrl, AuthenticationConfig authConfig, @@ -348,11 +348,11 @@ public static List getCmd(InstanceConfig instanceConfig, args.add("--jar"); args.add(originalCodeFileName); - if (isNotEmpty(originalExtraFunctionFileName)) { - args.add("--extra_function_jar"); - args.add(originalExtraFunctionFileName); - args.add("--extra_function_id"); - args.add(instanceConfig.getExtraFunctionId()); + if (isNotEmpty(originalTransformFunctionFileName)) { + args.add("--transform_function_jar"); + args.add(originalTransformFunctionFileName); + args.add("--transform_function_id"); + args.add(instanceConfig.getTransformFunctionId()); } } else if (instanceConfig.getFunctionDetails().getRuntime() == Function.FunctionDetails.Runtime.PYTHON) { args.add("python3"); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index d2b4e69fadcbd..f7791e716d96b 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -139,7 +139,7 @@ public class KubernetesRuntime implements Runtime { private final String configAdminCLI; private final String userCodePkgUrl; private final String originalCodeFileName; - private final String originalExtraFunctionFileName; + private final String originalTransformFunctionFileName; private final String pulsarAdminUrl; private final SecretsProviderConfigurator secretsProviderConfigurator; private int percentMemoryPadding; @@ -174,7 +174,7 @@ public class KubernetesRuntime implements Runtime { String configAdminCLI, String userCodePkgUrl, String originalCodeFileName, - String originalExtraFunctionFileName, + String originalTransformFunctionFileName, String pulsarServiceUrl, String pulsarAdminUrl, String stateStorageServiceUrl, @@ -207,9 +207,9 @@ public class KubernetesRuntime implements Runtime { this.downloadDirectory = isNotEmpty(downloadDirectory) ? downloadDirectory : this.pulsarRootDir; // for backward comp this.originalCodeFileName = this.downloadDirectory + "/" + originalCodeFileName; - this.originalExtraFunctionFileName = isNotEmpty(originalExtraFunctionFileName) - ? this.downloadDirectory + "/" + originalExtraFunctionFileName - : originalExtraFunctionFileName; + this.originalTransformFunctionFileName = isNotEmpty(originalTransformFunctionFileName) + ? this.downloadDirectory + "/" + originalTransformFunctionFileName + : originalTransformFunctionFileName; this.pulsarAdminUrl = pulsarAdminUrl; this.secretsProviderConfigurator = secretsProviderConfigurator; this.percentMemoryPadding = percentMemoryPadding; @@ -268,7 +268,7 @@ public class KubernetesRuntime implements Runtime { extraDependenciesDir, logDirectory, this.originalCodeFileName, - this.originalExtraFunctionFileName, + this.originalTransformFunctionFileName, pulsarServiceUrl, stateStorageServiceUrl, authConfig, @@ -852,9 +852,10 @@ public void deleteService() throws InterruptedException { protected List getExecutorCommand() { List cmds = new ArrayList<>(getDownloadCommand(instanceConfig.getFunctionDetails(), originalCodeFileName, false)); - if (isNotEmpty(originalExtraFunctionFileName)) { + if (isNotEmpty(originalTransformFunctionFileName)) { cmds.add("&&"); - cmds.addAll(getDownloadCommand(instanceConfig.getFunctionDetails(), originalExtraFunctionFileName, true)); + cmds.addAll(getDownloadCommand(instanceConfig.getFunctionDetails(), + originalTransformFunctionFileName, true)); } cmds.add("&&"); cmds.add(setShardIdEnvironmentVariableCommand()); @@ -864,13 +865,13 @@ protected List getExecutorCommand() { } private List getDownloadCommand(Function.FunctionDetails functionDetails, String userCodeFilePath, - boolean extraFunction) { + boolean transformFunction) { return getDownloadCommand(functionDetails.getTenant(), functionDetails.getNamespace(), - functionDetails.getName(), userCodeFilePath, extraFunction); + functionDetails.getName(), userCodeFilePath, transformFunction); } private List getDownloadCommand(String tenant, String namespace, String name, String userCodeFilePath, - boolean extraFunction) { + boolean transformFunction) { ArrayList cmd = new ArrayList<>(Arrays.asList( pulsarRootDir + configAdminCLI, "--admin-url", @@ -899,8 +900,8 @@ && isNotBlank(authConfig.getClientAuthenticationParameters()) } } - if (extraFunction) { - cmd.add("--extra-function"); + if (transformFunction) { + cmd.add("--transform-function"); } return cmd; diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java index 082b67e828513..740d6cc0fdc4b 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java @@ -269,8 +269,8 @@ public void initialize(WorkerConfig workerConfig, AuthenticationConfig authentic @Override public KubernetesRuntime createContainer(InstanceConfig instanceConfig, String codePkgUrl, String originalCodeFileName, - String extraFunctionPkgUrl, - String originalExtraFunctionFileName, + String transformFunctionPkgUrl, + String originalTransformFunctionFileName, Long expectedHealthCheckInterval) throws Exception { String instanceFile = null; switch (instanceConfig.getFunctionDetails().getRuntime()) { @@ -329,7 +329,7 @@ public KubernetesRuntime createContainer(InstanceConfig instanceConfig, String c configAdminCLI, codePkgUrl, originalCodeFileName, - originalExtraFunctionFileName, + originalTransformFunctionFileName, pulsarServiceUrl, pulsarAdminUrl, stateStorageServiceUri, diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java index c434fe0623962..678296eeec05c 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntime.java @@ -82,7 +82,7 @@ class ProcessRuntime implements Runtime { String narExtractionDirectory, String logDirectory, String codeFile, - String extraFunctionFile, + String transformFunctionFile, String pulsarServiceUrl, String stateStorageServiceUrl, AuthenticationConfig authConfig, @@ -134,7 +134,7 @@ class ProcessRuntime implements Runtime { ? extraDependenciesDir : null, logDirectory, codeFile, - extraFunctionFile, + transformFunctionFile, pulsarServiceUrl, stateStorageServiceUrl, authConfig, diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeFactory.java index c74201b732afb..2324d0ee441f6 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeFactory.java @@ -193,8 +193,8 @@ private void initialize(String pulsarServiceUrl, @Override public ProcessRuntime createContainer(InstanceConfig instanceConfig, String codeFile, String originalCodeFileName, - String extraFunctionFile, - String originalExtraFunctionFileName, + String transformFunctionFile, + String originalTransformFunctionFileName, Long expectedHealthCheckInterval) throws Exception { String instanceFile = null; switch (instanceConfig.getFunctionDetails().getRuntime()) { @@ -225,7 +225,7 @@ public ProcessRuntime createContainer(InstanceConfig instanceConfig, String code narExtractionDirectory, logDirectory, codeFile, - extraFunctionFile, + transformFunctionFile, pulsarServiceUrl, stateStorageServiceUrl, authConfig, diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java index 25a0488c105a7..d1b536732090a 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java @@ -64,7 +64,7 @@ public class ThreadRuntime implements Runtime { private final ThreadGroup threadGroup; private final FunctionCacheManager fnCache; private final String jarFile; - private final String extraFunctionFile; + private final String transformFunctionFile; private final ClientBuilder clientBuilder; private final PulsarClient pulsarClient; private final PulsarAdmin pulsarAdmin; @@ -80,7 +80,7 @@ public class ThreadRuntime implements Runtime { FunctionCacheManager fnCache, ThreadGroup threadGroup, String jarFile, - String extraFunctionFile, + String transformFunctionFile, PulsarClient client, ClientBuilder clientBuilder, PulsarAdmin pulsarAdmin, @@ -99,7 +99,7 @@ public class ThreadRuntime implements Runtime { this.threadGroup = threadGroup; this.fnCache = fnCache; this.jarFile = jarFile; - this.extraFunctionFile = extraFunctionFile; + this.transformFunctionFile = transformFunctionFile; this.clientBuilder = clientBuilder; this.pulsarClient = client; this.pulsarAdmin = pulsarAdmin; @@ -199,9 +199,9 @@ public void start() throws Exception { fnCache, connectorsManager, functionsManager, InstanceUtils.calculateSubjectType(instanceConfig.getFunctionDetails())); - ClassLoader extraFunctionClassLoader = extraFunctionFile == null ? null : getFunctionClassLoader( - instanceConfig, instanceConfig.getExtraFunctionId(), extraFunctionFile, narExtractionDirectory, fnCache, - connectorsManager, functionsManager, Function.FunctionDetails.ComponentType.FUNCTION); + ClassLoader transformFunctionClassLoader = transformFunctionFile == null ? null : getFunctionClassLoader( + instanceConfig, instanceConfig.getTransformFunctionId(), transformFunctionFile, narExtractionDirectory, + fnCache, connectorsManager, functionsManager, Function.FunctionDetails.ComponentType.FUNCTION); // re-initialize JavaInstanceRunnable so that variables in constructor can be re-initialized this.javaInstanceRunnable = new JavaInstanceRunnable( @@ -214,7 +214,7 @@ public void start() throws Exception { secretsProvider, collectorRegistry, functionClassLoader, - extraFunctionClassLoader); + transformFunctionClassLoader); log.info("ThreadContainer starting function with instanceId {} functionId {} namespace {}", instanceConfig.getInstanceId(), diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java index f53454e587182..64ea2d0f17298 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java @@ -177,8 +177,8 @@ public void initialize(WorkerConfig workerConfig, AuthenticationConfig authentic @Override public ThreadRuntime createContainer(InstanceConfig instanceConfig, String jarFile, String originalCodeFileName, - String extraFunctionFile, - String originalExtraFunctionFileName, + String transformFunctionFile, + String originalTransformFunctionFileName, Long expectedHealthCheckInterval) { SecretsProvider secretsProvider = defaultSecretsProvider; if (secretsProvider == null) { @@ -198,7 +198,7 @@ public ThreadRuntime createContainer(InstanceConfig instanceConfig, String jarFi fnCache, threadGroup, jarFile, - extraFunctionFile, + transformFunctionFile, pulsarClient, clientBuilder, pulsarAdmin, diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java index cc277b6f18942..33451a316e4e1 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java @@ -284,7 +284,7 @@ InstanceConfig createJavaInstanceConfig(FunctionDetails.Runtime runtime, boolean config.setFunctionDetails(createFunctionDetails(runtime, addSecrets)); config.setFunctionId(java.util.UUID.randomUUID().toString()); - config.setExtraFunctionId(java.util.UUID.randomUUID().toString()); + config.setTransformFunctionId(java.util.UUID.randomUUID().toString()); config.setFunctionVersion("1.0"); config.setInstanceId(0); config.setMaxBufferedTuples(1024); @@ -443,8 +443,8 @@ private void verifyJavaInstance(InstanceConfig config, String depsDir, boolean s + " -Xmx" + RESOURCES.getRam() + " org.apache.pulsar.functions.instance.JavaInstanceMain" + " --jar " + jarLocation - + " --extra_function_jar " + jarLocation - + " --extra_function_id " + config.getExtraFunctionId() + + " --transform_function_jar " + jarLocation + + " --transform_function_id " + config.getTransformFunctionId() + " --instance_id " + "$SHARD_ID" + " --function_id " + config.getFunctionId() + " --function_version " + config.getFunctionVersion() diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java index 1456330b2f8b4..ea1d6bbc6293f 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/process/ProcessRuntimeTest.java @@ -332,8 +332,8 @@ private void verifyJavaInstance(InstanceConfig config, Path depsDir, String webS + " --add-opens java.base/sun.net=ALL-UNNAMED" + " org.apache.pulsar.functions.instance.JavaInstanceMain" + " --jar " + userJarFile - + " --extra_function_jar " + userJarFile - + " --extra_function_id " + config.getExtraFunctionId() + + " --transform_function_jar " + userJarFile + + " --transform_function_id " + config.getTransformFunctionId() + " --instance_id " + config.getInstanceId() + " --function_id " + config.getFunctionId() + " --function_version " + config.getFunctionVersion() diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java index 41f1c81ab0c34..a4f8532dbc60b 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java @@ -107,21 +107,21 @@ public void startFunction(FunctionRuntimeInfo functionRuntimeInfo) { functionDetails.getName(), instanceId); String packageFile; - String extraFunctionPackageFile = null; + String transformFunctionPackageFile = null; Function.PackageLocationMetaData pkgLocation = functionMetaData.getPackageLocation(); - Function.PackageLocationMetaData extraFunctionPkgLocation = - functionMetaData.getExtraFunctionPackageLocation(); + Function.PackageLocationMetaData transformFunctionPkgLocation = + functionMetaData.getTransformFunctionPackageLocation(); if (runtimeFactory.externallyManaged()) { packageFile = pkgLocation.getPackagePath(); - extraFunctionPackageFile = extraFunctionPkgLocation.getPackagePath(); + transformFunctionPackageFile = transformFunctionPkgLocation.getPackagePath(); } else { packageFile = getPackageFile(functionMetaData, functionDetails, instanceId, pkgLocation, InstanceUtils.calculateSubjectType(functionDetails)); - if (!isEmpty(extraFunctionPkgLocation.getPackagePath())) { - extraFunctionPackageFile = - getPackageFile(functionMetaData, functionDetails, instanceId, extraFunctionPkgLocation, + if (!isEmpty(transformFunctionPkgLocation.getPackagePath())) { + transformFunctionPackageFile = + getPackageFile(functionMetaData, functionDetails, instanceId, transformFunctionPkgLocation, FunctionDetails.ComponentType.FUNCTION); } } @@ -130,7 +130,7 @@ public void startFunction(FunctionRuntimeInfo functionRuntimeInfo) { setupBatchSource(functionDetails); RuntimeSpawner runtimeSpawner = getRuntimeSpawner(functionRuntimeInfo.getFunctionInstance(), - packageFile, extraFunctionPackageFile); + packageFile, transformFunctionPackageFile); functionRuntimeInfo.setRuntimeSpawner(runtimeSpawner); runtimeSpawner.start(); @@ -173,7 +173,8 @@ private String getPackageFile(FunctionMetaData functionMetaData, FunctionDetails return packageFile; } - RuntimeSpawner getRuntimeSpawner(Function.Instance instance, String packageFile, String extraFunctionPackageFile) { + RuntimeSpawner getRuntimeSpawner(Function.Instance instance, String packageFile, + String transformFunctionPackageFile) { FunctionMetaData functionMetaData = instance.getFunctionMetaData(); int instanceId = instance.getInstanceId(); @@ -194,8 +195,8 @@ RuntimeSpawner getRuntimeSpawner(Function.Instance instance, String packageFile, RuntimeSpawner runtimeSpawner = new RuntimeSpawner(instanceConfig, packageFile, functionMetaData.getPackageLocation().getOriginalFileName(), - extraFunctionPackageFile, - functionMetaData.getExtraFunctionPackageLocation().getOriginalFileName(), + transformFunctionPackageFile, + functionMetaData.getTransformFunctionPackageLocation().getOriginalFileName(), runtimeFactory, workerConfig.getInstanceLivenessCheckFreqMs()); return runtimeSpawner; @@ -207,7 +208,7 @@ InstanceConfig createInstanceConfig(FunctionDetails functionDetails, Function.Fu instanceConfig.setFunctionDetails(functionDetails); // TODO: set correct function id and version when features implemented instanceConfig.setFunctionId(UUID.randomUUID().toString()); - instanceConfig.setExtraFunctionId(UUID.randomUUID().toString()); + instanceConfig.setTransformFunctionId(UUID.randomUUID().toString()); instanceConfig.setFunctionVersion(UUID.randomUUID().toString()); instanceConfig.setInstanceId(instanceId); instanceConfig.setMaxBufferedTuples(1024); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java index 0f84e81844eb5..c05ffe9ca8ca1 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java @@ -762,7 +762,7 @@ private void updateAssignment(Assignment assignment) { assignment .getInstance() .getFunctionMetaData() - .getExtraFunctionPackageLocation() + .getTransformFunctionPackageLocation() .getPackagePath()); // re-initialize if necessary runtimeSpawner.getRuntime().reinitialize(); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index f97e07fb381e3..132641c8f01c7 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -517,9 +517,9 @@ public void deregisterFunction(final String tenant, deleteComponentFromStorage(tenant, namespace, componentName, functionMetaData.getPackageLocation().getPackagePath()); - if (!isEmpty(functionMetaData.getExtraFunctionPackageLocation().getPackagePath())) { + if (!isEmpty(functionMetaData.getTransformFunctionPackageLocation().getPackagePath())) { deleteComponentFromStorage(tenant, namespace, componentName, - functionMetaData.getExtraFunctionPackageLocation().getPackagePath()); + functionMetaData.getTransformFunctionPackageLocation().getPackagePath()); } deleteStatestoreTableAsync(getStateNamespace(tenant, namespace), componentName); @@ -1442,7 +1442,7 @@ public void uploadFunction(final InputStream uploadedInputStream, final String p @Override public StreamingOutput downloadFunction(String tenant, String namespace, String componentName, String clientRole, AuthenticationDataSource clientAuthenticationDataHttps, - boolean extraFunction) { + boolean transformFunction) { if (!isWorkerServiceAvailable()) { throwUnavailableException(); } @@ -1469,8 +1469,8 @@ public StreamingOutput downloadFunction(String tenant, String namespace, String FunctionMetaData functionMetaData = functionMetaDataManager.getFunctionMetaData(tenant, namespace, componentName); - String pkgPath = extraFunction - ? functionMetaData.getExtraFunctionPackageLocation().getPackagePath() + String pkgPath = transformFunction + ? functionMetaData.getTransformFunctionPackageLocation().getPackagePath() : functionMetaData.getPackageLocation().getPackagePath(); return getStreamingOutput(pkgPath); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index 3f1526f442d54..dede6307e76f0 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -230,7 +230,7 @@ public void registerSink(final String tenant, String transformFunction = sinkConfig.getTransformFunction(); if (isNotBlank(transformFunction)) { - setExtraFunctionPackageLocation(functionMetaDataBuilder, functionDetails, transformFunction); + setTransformFunctionPackageLocation(functionMetaDataBuilder, functionDetails, transformFunction); } updateRequest(null, functionMetaDataBuilder.build()); @@ -416,7 +416,7 @@ public void updateSink(final String tenant, String transformFunction = mergedConfig.getTransformFunction(); if (isNotBlank(transformFunction) && !transformFunction.equals(existingSinkConfig.getTransformFunction())) { - setExtraFunctionPackageLocation(functionMetaDataBuilder, functionDetails, transformFunction); + setTransformFunctionPackageLocation(functionMetaDataBuilder, functionDetails, transformFunction); } updateRequest(existingComponent, functionMetaDataBuilder.build()); @@ -429,20 +429,20 @@ public void updateSink(final String tenant, } } - private void setExtraFunctionPackageLocation(Function.FunctionMetaData.Builder functionMetaDataBuilder, - Function.FunctionDetails functionDetails, String extraFunction) { + private void setTransformFunctionPackageLocation(Function.FunctionMetaData.Builder functionMetaDataBuilder, + Function.FunctionDetails functionDetails, String transformFunction) { File functionPackageFile = null; try { String builtin = null; - if (!extraFunction.startsWith(Utils.BUILTIN)) { - functionPackageFile = getPackageFile(extraFunction); + if (!transformFunction.startsWith(Utils.BUILTIN)) { + functionPackageFile = getPackageFile(transformFunction); } Function.PackageLocationMetaData.Builder functionPackageLocation = getFunctionPackageLocation(functionMetaDataBuilder.build(), - extraFunction, null, functionPackageFile, + transformFunction, null, functionPackageFile, functionDetails.getName() + "__sink-function", Function.FunctionDetails.ComponentType.FUNCTION, builtin); - functionMetaDataBuilder.setExtraFunctionPackageLocation(functionPackageLocation); + functionMetaDataBuilder.setTransformFunctionPackageLocation(functionPackageLocation); } catch (Exception e) { log.error("Failed process {} {}/{}/{} extra function package: ", ComponentTypeUtils.toString(componentType), functionDetails.getTenant(), @@ -450,7 +450,7 @@ private void setExtraFunctionPackageLocation(Function.FunctionMetaData.Builder f throw new RestException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); } finally { if (functionPackageFile != null && functionPackageFile.exists()) { - if (!extraFunction.startsWith(Utils.FILE)) { + if (!transformFunction.startsWith(Utils.FILE)) { functionPackageFile.delete(); } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java index 21197e7cc0d97..092844db489dd 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionsApiV3Resource.java @@ -340,11 +340,11 @@ public StreamingOutput downloadFunction( final @PathParam("namespace") String namespace, @ApiParam(value = "The name of functions") final @PathParam("functionName") String functionName, - @ApiParam(value = "Whether to download the extra-function") - final @QueryParam("extra-function") boolean extraFunction) { + @ApiParam(value = "Whether to download the transform function") + final @QueryParam("transform-function") boolean transformFunction) { return functions() - .downloadFunction(tenant, namespace, functionName, clientAppId(), clientAuthData(), extraFunction); + .downloadFunction(tenant, namespace, functionName, clientAppId(), clientAuthData(), transformFunction); } @GET diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java index 738a92e00d77e..db315f1181e81 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/service/api/Component.java @@ -173,7 +173,7 @@ StreamingOutput downloadFunction(String tenant, String componentName, String clientRole, AuthenticationDataSource clientAuthenticationDataHttps, - boolean extraFunction); + boolean transformFunction); @Deprecated default StreamingOutput downloadFunction(String tenant, diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java index 149542208e197..2412aebc294b2 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java @@ -159,7 +159,7 @@ private void startFunction(FunctionActioner actioner, String pkgPathLocation, St .setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant") .setNamespace("test-namespace").setName("func-1")) .setPackageLocation(packageLocation) - .setExtraFunctionPackageLocation(extraPackageLocation) + .setTransformFunctionPackageLocation(extraPackageLocation) .build(); Function.Instance instance = Function.Instance.newBuilder().setFunctionMetaData(function).setInstanceId(0) .build(); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java index 47537997315b2..61237cdf481ea 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java @@ -743,7 +743,7 @@ public void testExternallyManagedRuntimeUpdate() throws Exception { Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("path")) - .setExtraFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() + .setTransformFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() .setPackagePath("function-path")) .setFunctionDetails( Function.FunctionDetails.newBuilder() @@ -774,7 +774,7 @@ public void testExternallyManagedRuntimeUpdate() throws Exception { .setFunctionInstance(instance) .setRuntimeSpawner(functionActioner .getRuntimeSpawner(instance, function1.getPackageLocation().getPackagePath(), - function1.getExtraFunctionPackageLocation().getPackagePath())); + function1.getTransformFunctionPackageLocation().getPackagePath())); functionRuntimeManager.functionRuntimeInfos.put( "test-tenant/test-namespace/func-1:0", functionRuntimeInfo); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java index 81ab922ed7c6e..8d19869b4738a 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java @@ -1703,7 +1703,7 @@ public void testDownloadFunctionByName() throws Exception { FunctionMetaData metaData = FunctionMetaData.newBuilder() .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("file:///" + fileLocation)) - .setExtraFunctionPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) + .setTransformFunctionPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); @@ -1718,7 +1718,7 @@ public void testDownloadFunctionByName() throws Exception { } @Test - public void testDownloadExtraFunctionByName() throws Exception { + public void testDownloadTransformFunctionByName() throws Exception { URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); File file = Paths.get(fileUrl.toURI()).toFile(); String fileLocation = file.getAbsolutePath().replace('\\', '/'); @@ -1732,7 +1732,7 @@ public void testDownloadExtraFunctionByName() throws Exception { FunctionMetaData metaData = FunctionMetaData.newBuilder() .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) - .setExtraFunctionPackageLocation(PackageLocationMetaData.newBuilder() + .setTransformFunctionPackageLocation(PackageLocationMetaData.newBuilder() .setPackagePath("file:///" + fileLocation)) .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java index aea6565e05bb5..7fd0fb1030fc9 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java @@ -1384,12 +1384,12 @@ public void testDeregisterSinkBKPackageCleanup() { String packagePath = "public/default/test/591541f0-c7c5-40c0-983b-610c722f90b0-pulsar-io-batch-data-generator-2.7.0.nar"; - String extraFunctionPackagePath = + String transformFunctionPackagePath = "public/default/test/591541f0-c7c5-40c0-983b-610c722f90b0-test-function.nar"; FunctionMetaData functionMetaData = FunctionMetaData.newBuilder() .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath)) - .setExtraFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() - .setPackagePath(extraFunctionPackagePath)) + .setTransformFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() + .setPackagePath(transformFunctionPackagePath)) .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) .thenReturn(functionMetaData); @@ -1397,7 +1397,7 @@ public void testDeregisterSinkBKPackageCleanup() { deregisterDefaultSink(); ctx.verify(() -> WorkerUtils.deleteFromBookkeeper(any(), eq(packagePath)), times(1)); - ctx.verify(() -> WorkerUtils.deleteFromBookkeeper(any(), eq(extraFunctionPackagePath)), times(1)); + ctx.verify(() -> WorkerUtils.deleteFromBookkeeper(any(), eq(transformFunctionPackagePath)), times(1)); } } @@ -1409,11 +1409,11 @@ public void testDeregisterBuiltinSinkBKPackageCleanup() { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); String packagePath = String.format("%s://data-generator", Utils.BUILTIN); - String extraFunctionPackagePath = String.format("%s://exclamation", Utils.BUILTIN); + String transformFunctionPackagePath = String.format("%s://exclamation", Utils.BUILTIN); FunctionMetaData functionMetaData = FunctionMetaData.newBuilder() .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath)) - .setExtraFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() - .setPackagePath(extraFunctionPackagePath)) + .setTransformFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() + .setPackagePath(transformFunctionPackagePath)) .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) .thenReturn(functionMetaData); @@ -1434,11 +1434,11 @@ public void testDeregisterHTTPSinkBKPackageCleanup() { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); String packagePath = "http://foo.com/connector.jar"; - String extraFunctionPackagePath = "http://foo.com/function.jar"; + String transformFunctionPackagePath = "http://foo.com/function.jar"; FunctionMetaData functionMetaData = FunctionMetaData.newBuilder() .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath)) - .setExtraFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() - .setPackagePath(extraFunctionPackagePath)) + .setTransformFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() + .setPackagePath(transformFunctionPackagePath)) .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) @@ -1460,11 +1460,11 @@ public void testDeregisterFileSinkBKPackageCleanup() { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); String packagePath = "file://foo/connector.jar"; - String extraFunctionPackagePath = "file://foo/function.jar"; + String transformFunctionPackagePath = "file://foo/function.jar"; FunctionMetaData functionMetaData = FunctionMetaData.newBuilder() .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath(packagePath)) - .setExtraFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() - .setPackagePath(extraFunctionPackagePath)) + .setTransformFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() + .setPackagePath(transformFunctionPackagePath)) .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))) From d260282a5f8a5e78aee4b2370c396ab8387e9bc9 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Wed, 31 Aug 2022 10:38:24 +0200 Subject: [PATCH 26/26] Fix: make --transform_function_id param non required --- .../pulsar/functions/runtime/JavaInstanceStarter.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java index 5e74d65621ee0..3978e0f34fd5d 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java @@ -73,15 +73,15 @@ public class JavaInstanceStarter implements AutoCloseable { @Parameter(names = "--function_id", description = "Function Id\n", required = true) public String functionId; - @Parameter(names = "--transform_function_id", description = "Transform Function Id\n", required = true) - public String transformFunctionId; - @Parameter(names = "--function_version", description = "Function Version\n", required = true) public String functionVersion; @Parameter(names = "--pulsar_serviceurl", description = "Pulsar Service Url\n", required = true) public String pulsarServiceUrl; + @Parameter(names = "--transform_function_id", description = "Transform Function Id\n") + public String transformFunctionId; + @Parameter(names = "--client_auth_plugin", description = "Client auth plugin name\n") public String clientAuthenticationPlugin;