[Testing] Improve LocalRunner to support .nar files that aren't on the classpath

pulsar-broker/pom.xml

@@ -111,7 +111,7 @@
       <artifactId>pulsar-transaction-common</artifactId>
       <version>${project.version}</version>
     </dependency>
-    
+
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>pulsar-io-batch-discovery-triggerers</artifactId>
@@ -145,7 +145,7 @@
        <groupId>io.dropwizard.metrics</groupId>
        <artifactId>metrics-core</artifactId>
     </dependency>
-    
+
     <!-- zookeeper server -->
     <dependency>
        <groupId>org.xerial.snappy</groupId>
@@ -158,7 +158,7 @@
       <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>
-    </dependency>   
+    </dependency>
 
     <!-- functions related dependencies (begin) -->
 
@@ -315,20 +315,23 @@
       <groupId>${project.groupId}</groupId>
       <artifactId>pulsar-functions-api-examples</artifactId>
       <version>${project.version}</version>
+      <type>pom</type>
       <scope>test</scope>
     </dependency>
 
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>pulsar-io-batch-data-generator</artifactId>
       <version>${project.version}</version>
+      <type>pom</type>
       <scope>test</scope>
     </dependency>
 
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>pulsar-io-data-generator</artifactId>
       <version>${project.version}</version>
+      <type>pom</type>
       <scope>test</scope>
     </dependency>
 
@@ -407,49 +410,79 @@
       </plugin>
 
       <plugin>
-        <groupId>org.xolstice.maven.plugins</groupId>
-        <artifactId>protobuf-maven-plugin</artifactId>
-        <version>${protobuf-maven-plugin.version}</version>
-        <configuration>
-          <protocArtifact>com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier}</protocArtifact>
-          <checkStaleness>true</checkStaleness>
-        </configuration>
+        <artifactId>maven-dependency-plugin</artifactId>
         <executions>
           <execution>
-            <phase>generate-sources</phase>
+            <id>copy-pulsar-io-connectors</id>
+            <phase>generate-test-resources</phase>
             <goals>
-              <goal>compile</goal>
-              <goal>test-compile</goal>
+              <goal>copy</goal>
             </goals>
+            <configuration>
+              <artifactItems>
+                <artifactItem>
+                  <groupId>${project.groupId}</groupId>
+                  <artifactId>pulsar-io-data-generator</artifactId>
+                  <version>${project.version}</version>
+                  <type>jar</type>
+                  <overWrite>true</overWrite>
+                  <outputDirectory>${project.build.directory}</outputDirectory>
+                  <destFileName>pulsar-io-data-generator.nar</destFileName>
+                </artifactItem>
+                <artifactItem>
+                  <groupId>${project.groupId}</groupId>
+                  <artifactId>pulsar-io-batch-data-generator</artifactId>
+                  <version>${project.version}</version>
+                  <type>jar</type>
+                  <overWrite>true</overWrite>
+                  <outputDirectory>${project.build.directory}</outputDirectory>
+                  <destFileName>pulsar-io-batch-data-generator.nar</destFileName>
+                </artifactItem>
+                <artifactItem>
+                  <groupId>${project.groupId}</groupId>
+                  <artifactId>pulsar-functions-api-examples</artifactId>
+                  <version>${project.version}</version>
+                  <type>jar</type>
+                  <overWrite>true</overWrite>
+                  <outputDirectory>${project.build.directory}</outputDirectory>
+                  <destFileName>pulsar-functions-api-examples.jar</destFileName>
+                </artifactItem>
+              </artifactItems>
+            </configuration>
           </execution>
         </executions>
       </plugin>
 
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-antrun-plugin</artifactId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <systemPropertyVariables>
+            <pulsar-io-data-generator.nar.path>${project.build.directory}/pulsar-io-data-generator.nar</pulsar-io-data-generator.nar.path>
+            <pulsar-functions-api-examples.jar.path>${project.build.directory}/pulsar-functions-api-examples.jar</pulsar-functions-api-examples.jar.path>
+            <pulsar-io-batch-data-generator.nar.path>${project.build.directory}/pulsar-io-batch-data-generator.nar</pulsar-io-batch-data-generator.nar.path>
+          </systemPropertyVariables>
+        </configuration>
+      </plugin>
+
+      <plugin>
+        <groupId>org.xolstice.maven.plugins</groupId>
+        <artifactId>protobuf-maven-plugin</artifactId>
+        <version>${protobuf-maven-plugin.version}</version>
+        <configuration>
+          <protocArtifact>com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier}</protocArtifact>
+          <checkStaleness>true</checkStaleness>
+        </configuration>
         <executions>
           <execution>
-            <phase>compile</phase>
+            <phase>generate-sources</phase>
             <goals>
-              <goal>run</goal>
+              <goal>compile</goal>
+              <goal>test-compile</goal>
             </goals>
-            <configuration>
-              <target>
-                <echo>copy test examples package</echo>
-                <mkdir dir="${basedir}/src/test/resources"/>
-                <copy file="${basedir}/../pulsar-functions/java-examples/target/pulsar-functions-api-examples.jar"
-                      tofile="${basedir}/src/test/resources/pulsar-functions-api-examples.jar"/>
-                <copy file="${basedir}/../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar"
-                      tofile="${basedir}/src/test/resources/pulsar-io-data-generator.nar"/>
-                <copy file="${basedir}/../pulsar-io/batch-data-generator/target/pulsar-io-batch-data-generator-${project.version}.nar"
-                      tofile="${basedir}/src/test/resources/pulsar-io-batch-data-generator.nar"/>
-              </target>
-            </configuration>
           </execution>
         </executions>
       </plugin>
-
     </plugins>
     <resources>
       <resource>

pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java

@@ -148,6 +148,7 @@ public void initNamespace() throws Exception {
     @Override
     @BeforeMethod
     public void setup() throws Exception {
+        resetConfig();
         conf.setClusterName(testLocalCluster);
         super.internalSetup();
 

pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/PackagesApiTest.java

@@ -63,8 +63,7 @@ public void testPackagesOperations() throws Exception {
         admin.packages().upload(originalMetadata, packageName, file.getPath());
 
         // testing download api
-        String directory = file.getParent();
-        String downloadPath = directory + "package-api-test-download.package";
+        String downloadPath = new File(file.getParentFile(), "package-api-test-download.package").getPath();
         admin.packages().download(packageName, downloadPath);
         File downloadFile = new File(downloadPath);
         assertTrue(downloadFile.exists());

pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/FileServer.java

@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.functions.worker;
+
+import com.google.api.Http;
+import com.sun.net.httpserver.Headers;
+import com.sun.net.httpserver.HttpServer;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Simple http server for serving files in Pulsar Function test cases
+ */
+@Slf4j
+public class FileServer implements AutoCloseable {
+    private final HttpServer httpServer;
+
+    public FileServer() throws IOException {
+        httpServer = HttpServer.create(new InetSocketAddress(0), 0);
+        // creates a default executor
+        httpServer.setExecutor(null);
+    }
+
+    public void serveFile(String path, File file) {
+        httpServer.createContext(path, he -> {
+            try {
+                Headers headers = he.getResponseHeaders();
+                headers.add("Content-Type", "application/octet-stream");
+
+                he.sendResponseHeaders(200, file.length());
+                try (OutputStream outputStream = he.getResponseBody()) {
+                    Files.copy(file.toPath(), outputStream);
+                }
+            } catch (Exception e) {
+                log.error("Error serving file {} for path {}", file, path, e);
+            }
+        });
+    }
+
+    public void start() {
+        httpServer.start();
+    }
+
+    public void stop() {
+        httpServer.stop(0);
+    }
+
+    public String getUrl(String path) {
+        return "http://127.0.0.1:" + httpServer.getAddress().getPort() + path;
+    }
+
+    @Override
+    public void close() throws Exception {
+        stop();
+    }
+}

pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java

@@ -21,6 +21,7 @@
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically;
 import static org.apache.pulsar.functions.utils.functioncache.FunctionCacheEntry.JAVA_INSTANCE_JAR_PROPERTY;
+import static org.apache.pulsar.functions.worker.PulsarFunctionLocalRunTest.getPulsarApiExamplesJar;
 import static org.mockito.Mockito.spy;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotEquals;
@@ -293,7 +294,7 @@ public void testAuthorizationWithAnonymousUser() throws Exception {
                 PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).build())
         ) {
 
-            String jarFilePathUrl = Utils.FILE + ":" + getClass().getClassLoader().getResource("pulsar-functions-api-examples.jar").getFile();
+            String jarFilePathUrl = getPulsarApiExamplesJar().toURI().toString();
 
             FunctionConfig functionConfig = createFunctionConfig(TENANT, NAMESPACE, functionName,
                     sourceTopic, sinkTopic, subscriptionName);
@@ -563,7 +564,7 @@ public void testAuthorization() throws Exception {
                     PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).authentication(authToken2).build())
         ) {
 
-            String jarFilePathUrl = Utils.FILE + ":" + getClass().getClassLoader().getResource("pulsar-functions-api-examples.jar").getFile();
+            String jarFilePathUrl = getPulsarApiExamplesJar().toURI().toString();
 
             FunctionConfig functionConfig = createFunctionConfig(TENANT, NAMESPACE, functionName,
                     sourceTopic, sinkTopic, subscriptionName);