From 3af7c783a31e5fad34397acb4ae7eb05b101940b Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Tue, 13 May 2025 15:02:47 +0200 Subject: [PATCH 01/14] [SYSTEMDS-3886] Federated Worker JVM Codecoverage --- .github/workflows/javaTests.yml | 4 ++++ pom.xml | 2 +- .../apache/sysds/test/AutomatedTestBase.java | 19 +++++++++++++++++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index 37c28d30dce..4d067f2e29a 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -103,6 +103,10 @@ jobs: ARTIFACT_NAME=${ARTIFACT_NAME//\*/x} # replace * with x echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> $GITHUB_ENV + - name: Merge Jacoco SingleTest Artifacts + run: mvn jacoco:merge + + - name: Save Java Test Coverage as Artifact uses: actions/upload-artifact@v4 with: diff --git a/pom.xml b/pom.xml index 8723b8e3f54..9f4613f6172 100644 --- a/pom.xml +++ b/pom.xml @@ -459,7 +459,7 @@ ${project.build.directory} - */jacoco.exec + *.exec diff --git a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java index a81dedd0975..e2624902293 100644 --- a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java @@ -28,6 +28,8 @@ import java.io.File; import java.io.IOException; import java.io.PrintStream; +import java.lang.management.ManagementFactory; +import java.lang.management.RuntimeMXBean; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.nio.charset.Charset; @@ -1663,9 +1665,22 @@ protected static Process startLocalFedWorker(int port, String[] addArgs, int sle "--add-opens=java.base/java.lang=ALL-UNNAMED" , "--add-opens=java.base/java.lang.ref=ALL-UNNAMED" , "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED" , - "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",}; + + RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean(); + List jvmArgs = runtimeMxBean.getInputArguments(); + + for(String arg : jvmArgs) { + // add code coverage report + System.out.println(arg); + if(arg.contains("org.jacoco.agent")) + args = ArrayUtils.addAll(args, + new String[] {arg.replace("target/jacoco.exec", String.format("target/jacoco-%d.exec", port))}); + } + + args = ArrayUtils.addAll(args, new String[]{ "-cp", classpath, - DMLScript.class.getName(), "-w", Integer.toString(port), "-stats"}; + DMLScript.class.getName(), "-w", Integer.toString(port), "-stats"}); if(addArgs != null) args = ArrayUtils.addAll(args, addArgs); From 8ce667769f06984f2e4bf2cc4a1bf9af3f9533bb Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Tue, 13 May 2025 15:29:41 +0200 Subject: [PATCH 02/14] chown --- .github/workflows/javaTests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index 4d067f2e29a..1a46a2cd3e7 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -104,7 +104,9 @@ jobs: echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> $GITHUB_ENV - name: Merge Jacoco SingleTest Artifacts - run: mvn jacoco:merge + run: | + chown $(whoami) target/jacoco.exec + mvn jacoco:merge - name: Save Java Test Coverage as Artifact From 13d3ea4c373bc3c3cec569796ddd288aba3c56ac Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Tue, 13 May 2025 15:58:55 +0200 Subject: [PATCH 03/14] try different way of merging via docker image --- .github/workflows/javaTests.yml | 6 ------ docker/entrypoint.sh | 1 + docker/sysds.Dockerfile | 2 +- docker/testsysds.Dockerfile | 2 +- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index 1a46a2cd3e7..37c28d30dce 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -103,12 +103,6 @@ jobs: ARTIFACT_NAME=${ARTIFACT_NAME//\*/x} # replace * with x echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> $GITHUB_ENV - - name: Merge Jacoco SingleTest Artifacts - run: | - chown $(whoami) target/jacoco.exec - mvn jacoco:merge - - - name: Save Java Test Coverage as Artifact uses: actions/upload-artifact@v4 with: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index d79c61b8928..4d26990ea33 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -31,6 +31,7 @@ export MAVEN_OPTS="-Xmx512m" log="/tmp/sysdstest.log" mvn -ntp -B test-compile 2>&1 | grep -E "BUILD|Total time:|---|Building SystemDS" mvn -ntp -B test -D maven.test.skip=false -D automatedtestbase.outputbuffering=true -D test=$1 2>&1 | grep -v "already exists in destination." | tee $log +mvn -ntp -B jacoco:merge grep_args="SUCCESS" grepvals="$( tail -n 100 $log | grep $grep_args)" diff --git a/docker/sysds.Dockerfile b/docker/sysds.Dockerfile index 359bb14bb9e..cc6ef605d53 100644 --- a/docker/sysds.Dockerfile +++ b/docker/sysds.Dockerfile @@ -19,7 +19,7 @@ # #------------------------------------------------------------- -FROM ubuntu:24.04 +FROM ubuntu:24.04@sha256:6015f66923d7afbc53558d7ccffd325d43b4e249f41a6e93eef074c9505d2233 WORKDIR /usr/src/ diff --git a/docker/testsysds.Dockerfile b/docker/testsysds.Dockerfile index 3b4781c16b4..68ad36e9233 100644 --- a/docker/testsysds.Dockerfile +++ b/docker/testsysds.Dockerfile @@ -19,7 +19,7 @@ # #------------------------------------------------------------- -FROM ubuntu:24.04 +FROM ubuntu:24.04@sha256:6015f66923d7afbc53558d7ccffd325d43b4e249f41a6e93eef074c9505d2233 WORKDIR /usr/src/ From 03e7876bb2b395498dc1216b778f1be3db5d364c Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Tue, 13 May 2025 16:23:58 +0200 Subject: [PATCH 04/14] do not forcefully terminate federated workers --- .../federated/FederatedWorker.java | 37 +++++++++-------- .../apache/sysds/test/AutomatedTestBase.java | 1 - .../java/org/apache/sysds/test/TestUtils.java | 41 +++++++++++++------ 3 files changed, 48 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java index b055a6848fc..55f2f17cd8a 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorker.java @@ -28,13 +28,6 @@ import javax.net.ssl.SSLException; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.ChannelOutboundHandlerAdapter; -import io.netty.channel.ChannelPipeline; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.log4j.Logger; import org.apache.sysds.api.DMLScript; @@ -50,25 +43,32 @@ import org.apache.sysds.runtime.lineage.LineageCache; import org.apache.sysds.runtime.lineage.LineageCacheConfig; import org.apache.sysds.runtime.lineage.LineageCacheConfig.ReuseCacheType; +import org.apache.sysds.runtime.lineage.LineageItem; import org.apache.sysds.utils.stats.InfrastructureAnalyzer; import org.apache.sysds.utils.stats.Timing; -import org.apache.sysds.runtime.lineage.LineageItem; import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.ChannelOutboundHandlerAdapter; +import io.netty.channel.ChannelPipeline; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.codec.serialization.ClassResolvers; +import io.netty.handler.codec.serialization.ObjectDecoder; import io.netty.handler.codec.serialization.ObjectEncoder; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.util.SelfSignedCertificate; -import io.netty.handler.codec.serialization.ObjectDecoder; -import io.netty.handler.codec.serialization.ClassResolvers; @SuppressWarnings("deprecation") public class FederatedWorker { - protected static Logger log = Logger.getLogger(FederatedWorker.class); + protected static Logger LOG = Logger.getLogger(FederatedWorker.class); private final int _port; private final FederatedLookupTable _flt; @@ -96,7 +96,7 @@ public FederatedWorker(int port, boolean debug) { } private void run() { - log.info("Setting up Federated Worker on port " + _port); + LOG.info("Setting up Federated Worker on port " + _port); int par_conn = ConfigurationManager.getDMLConfig().getIntValue(DMLConfig.FEDERATED_PAR_CONN); final int EVENT_LOOP_THREADS = (par_conn > 0) ? par_conn : InfrastructureAnalyzer.getLocalParallelism(); NioEventLoopGroup bossGroup = new NioEventLoopGroup(1); @@ -113,22 +113,23 @@ private void run() { b.option(ChannelOption.SO_BACKLOG, 128); b.childOption(ChannelOption.SO_KEEPALIVE, true); - log.info("Starting Federated Worker server at port: " + _port); + LOG.info("Starting Federated Worker server at port: " + _port); ChannelFuture f = b.bind(_port).sync(); - log.info("Started Federated Worker at port: " + _port); + LOG.info("Started Federated Worker at port: " + _port); f.channel().closeFuture().sync(); - } + } catch(Exception e) { - log.info("Federated worker interrupted"); + LOG.info("Federated worker interrupted"); if(_debug) { - log.error(e.getMessage()); + LOG.error(e.getMessage()); e.printStackTrace(); } } finally { - log.info("Federated Worker Shutting down."); + LOG.info("Federated Worker Shutting down."); workerGroup.shutdownGracefully(); bossGroup.shutdownGracefully(); + } } diff --git a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java index e2624902293..03ebb0fb4eb 100644 --- a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java @@ -1672,7 +1672,6 @@ protected static Process startLocalFedWorker(int port, String[] addArgs, int sle for(String arg : jvmArgs) { // add code coverage report - System.out.println(arg); if(arg.contains("org.jacoco.agent")) args = ArrayUtils.addAll(args, new String[] {arg.replace("target/jacoco.exec", String.format("target/jacoco-%d.exec", port))}); diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java index aa869a29e35..195e36d6065 100644 --- a/src/test/java/org/apache/sysds/test/TestUtils.java +++ b/src/test/java/org/apache/sysds/test/TestUtils.java @@ -19,6 +19,12 @@ package org.apache.sysds.test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.DataOutputStream; @@ -26,11 +32,11 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; -import java.io.RandomAccessFile; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.io.PrintWriter; +import java.io.RandomAccessFile; import java.text.NumberFormat; import java.util.ArrayList; import java.util.Collections; @@ -44,12 +50,8 @@ import java.util.Random; import java.util.Set; import java.util.StringTokenizer; +import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.NotImplementedException; @@ -3489,13 +3491,28 @@ public static void shutdownThread(Thread t) { public static void shutdownThread(Process t) { // kill the worker if( t != null ) { - Process d = t.destroyForcibly(); + sendSigInt(t);// Attempt graceful termination try { - d.waitFor(); - } - catch (InterruptedException e) { - e.printStackTrace(); - } + // Wait up to 1 second for the process to exit + if (!t.waitFor(10, TimeUnit.SECONDS)) { + // If still alive after 1 second, force kill + Process forciblyDestroyed = t.destroyForcibly(); + forciblyDestroyed.waitFor(); // Wait until it's definitely terminated + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + public static void sendSigInt(Process process) { + long pid = process.pid(); + ProcessBuilder pb = new ProcessBuilder("kill", "-SIGINT", Long.toString(pid)); + try { + pb.inheritIO().start().waitFor(); + } + catch(IOException | InterruptedException e) { + e.printStackTrace(); } } From fa7f80045c391dcebf0d7bd9ae8b18694adfb2a6 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Wed, 14 May 2025 10:51:44 +0200 Subject: [PATCH 05/14] fix merging (hopefully) --- docker/build.sh | 2 +- docker/entrypoint.sh | 3 +++ docker/testsysds.Dockerfile | 11 +++++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docker/build.sh b/docker/build.sh index 02e45675f10..2898effdc22 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -23,7 +23,7 @@ # Build the docker containers # The first build is for running systemds through docker. -docker image build -f docker/sysds.Dockerfile -t apache/systemds:latest . +# docker image build -f docker/sysds.Dockerfile -t apache/systemds:latest . # The second build is for testing systemds. This image installs the R dependencies needed to run the tests. docker image build -f docker/testsysds.Dockerfile -t apache/systemds:testing-latest . diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 4d26990ea33..190243254ff 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -31,6 +31,9 @@ export MAVEN_OPTS="-Xmx512m" log="/tmp/sysdstest.log" mvn -ntp -B test-compile 2>&1 | grep -E "BUILD|Total time:|---|Building SystemDS" mvn -ntp -B test -D maven.test.skip=false -D automatedtestbase.outputbuffering=true -D test=$1 2>&1 | grep -v "already exists in destination." | tee $log + +# Merge Federated test runs. +[ -f target/jacoco.exec ] && mv target/jacoco.exec target/jacoco_main.exec mvn -ntp -B jacoco:merge grep_args="SUCCESS" diff --git a/docker/testsysds.Dockerfile b/docker/testsysds.Dockerfile index 68ad36e9233..2f63dace7fa 100644 --- a/docker/testsysds.Dockerfile +++ b/docker/testsysds.Dockerfile @@ -21,7 +21,6 @@ FROM ubuntu:24.04@sha256:6015f66923d7afbc53558d7ccffd325d43b4e249f41a6e93eef074c9505d2233 - WORKDIR /usr/src/ ENV MAVEN_VERSION=3.9.9 ENV MAVEN_HOME=/usr/lib/mvn @@ -34,9 +33,6 @@ ENV LC_ALL=en_US.UTF-8 ENV LANG=en_US.UTF-8 ENV LD_LIBRARY_PATH=/usr/local/lib/ -COPY ./src/test/scripts/installDependencies.R installDependencies.R -COPY ./docker/entrypoint.sh /entrypoint.sh - RUN apt-get update -qq \ && apt-get upgrade -y \ && apt-get install -y --no-install-recommends \ @@ -74,7 +70,9 @@ RUN apt-get install -y --no-install-recommends \ r-base-dev \ r-base-core + # Install R packages +COPY ./src/test/scripts/installDependencies.R installDependencies.R RUN Rscript installDependencies.R \ && rm -rf installDependencies.R \ && rm -rf /var/lib/apt/lists/* @@ -86,4 +84,9 @@ RUN wget -qO- https://github.com/microsoft/SEAL/archive/refs/tags/v3.7.0.tar.gz && cmake --build build \ && cmake --install build +# Finally copy the entrypoint script +# This is last to enable quick updates to the script after initial local build. +COPY ./docker/entrypoint.sh /entrypoint.sh + + ENTRYPOINT ["/entrypoint.sh"] From 6bcf2ad59bffd51e3fef3d53daae1625921ee0c5 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Wed, 14 May 2025 14:48:43 +0200 Subject: [PATCH 06/14] jacoco merge fix --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9f4613f6172..34bdc1c7c5c 100644 --- a/pom.xml +++ b/pom.xml @@ -459,7 +459,7 @@ ${project.build.directory} - *.exec + *jacoco* From c74ee72db8239ddc1d83140c3ca9450d580e56a5 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Wed, 14 May 2025 16:02:48 +0200 Subject: [PATCH 07/14] cov test --- .github/workflows/javaTests.yml | 74 +++++++++++++++++---------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index 37c28d30dce..8a9587320c9 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -49,41 +49,41 @@ jobs: fail-fast: false matrix: tests: [ - "org.apache.sysds.test.applications.**", - "**.test.usertest.**", - "**.component.c**.** -Dtest-threadCount=1 -Dtest-forkCount=1", - "**.component.e**.**,**.component.f**.**,**.component.m**.**", - "**.component.p**.**,**.component.r**.**,**.component.s**.**,**.component.t**.**,**.component.u**.**", - "**.functions.a**.**,**.functions.binary.matrix.**,**.functions.binary.scalar.**,**.functions.binary.tensor.**", - "**.functions.blocks.**,**.functions.data.rand.**,", - "**.functions.countDistinct.**,**.functions.countDistinctApprox.**", - "**.functions.data.misc.**,**.functions.lineage.**", - "**.functions.compress.**,**.functions.data.tensor.**,**.functions.codegenalg.parttwo.**,**.functions.codegen.**,**.functions.caching.**", - "**.functions.binary.matrix_full_cellwise.**,**.functions.binary.matrix_full_other.**", - "**.functions.federated.algorithms.**,**.functions.federated.io.**,**.functions.federated.paramserv.**", - "**.functions.federated.transform.**,**.functions.federated.fedplanner.**", - "**.functions.federated.primitives.part1.** -Dtest-threadCount=1 -Dtest-forkCount=1", - "**.functions.federated.primitives.part2.** -Dtest-threadCount=1 -Dtest-forkCount=1", - "**.functions.federated.primitives.part3.** -Dtest-threadCount=1 -Dtest-forkCount=1", - "**.functions.federated.primitives.part4.** -Dtest-threadCount=1 -Dtest-forkCount=1", - "**.functions.federated.primitives.part5.** -Dtest-threadCount=1 -Dtest-forkCount=1", - "**.functions.federated.monitoring.**,**.functions.federated.multitenant.**", - "**.functions.federated.codegen.**,**.functions.federated.FederatedTestObjectConstructor", - "**.functions.codegenalg.partone.**", - "**.functions.builtin.part1.**", - "**.functions.builtin.part2.**", - "**.functions.frame.**,**.functions.indexing.**,**.functions.io.**,**.functions.iogen.**", - "**.functions.dnn.**", - "**.functions.paramserv.**", - "**.functions.recompile.**,**.functions.misc.**", - "**.functions.mlcontext.**", - "**.functions.nary.**,**.functions.quaternary.**", - "**.functions.parfor.**,**.functions.pipelines.**", + # "org.apache.sysds.test.applications.**", + # "**.test.usertest.**", + # "**.component.c**.** -Dtest-threadCount=1 -Dtest-forkCount=1", + # "**.component.e**.**,**.component.f**.**,**.component.m**.**", + # "**.component.p**.**,**.component.r**.**,**.component.s**.**,**.component.t**.**,**.component.u**.**", + # "**.functions.a**.**,**.functions.binary.matrix.**,**.functions.binary.scalar.**,**.functions.binary.tensor.**", + # "**.functions.blocks.**,**.functions.data.rand.**,", + # "**.functions.countDistinct.**,**.functions.countDistinctApprox.**", + # "**.functions.data.misc.**,**.functions.lineage.**", + # "**.functions.compress.**,**.functions.data.tensor.**,**.functions.codegenalg.parttwo.**,**.functions.codegen.**,**.functions.caching.**", + # "**.functions.binary.matrix_full_cellwise.**,**.functions.binary.matrix_full_other.**", + # "**.functions.federated.algorithms.**,**.functions.federated.io.**,**.functions.federated.paramserv.**", + # "**.functions.federated.transform.**,**.functions.federated.fedplanner.**", + # "**.functions.federated.primitives.part1.** -Dtest-threadCount=1 -Dtest-forkCount=1", + # "**.functions.federated.primitives.part2.** -Dtest-threadCount=1 -Dtest-forkCount=1", + # "**.functions.federated.primitives.part3.** -Dtest-threadCount=1 -Dtest-forkCount=1", + # "**.functions.federated.primitives.part4.** -Dtest-threadCount=1 -Dtest-forkCount=1", + # "**.functions.federated.primitives.part5.** -Dtest-threadCount=1 -Dtest-forkCount=1", + # "**.functions.federated.monitoring.**,**.functions.federated.multitenant.**", + # "**.functions.federated.codegen.**,**.functions.federated.FederatedTestObjectConstructor", + # "**.functions.codegenalg.partone.**", + # "**.functions.builtin.part1.**", + # "**.functions.builtin.part2.**", + # "**.functions.frame.**,**.functions.indexing.**,**.functions.io.**,**.functions.iogen.**", + # "**.functions.dnn.**", + # "**.functions.paramserv.**", + # "**.functions.recompile.**,**.functions.misc.**", + # "**.functions.mlcontext.**", + # "**.functions.nary.**,**.functions.quaternary.**", + # "**.functions.parfor.**,**.functions.pipelines.**", "**.functions.homomorphicEncryption.**", - "**.functions.unary.scalar.**,**.functions.updateinplace.**,**.functions.vect.**", - "**.functions.reorg.**,**.functions.rewrite.**,**.functions.ternary.**", - "**.functions.transform.**","**.functions.unique.**", - "**.functions.unary.matrix.**,**.functions.linearization.**,**.functions.jmlc.**" + # "**.functions.unary.scalar.**,**.functions.updateinplace.**,**.functions.vect.**", + # "**.functions.reorg.**,**.functions.rewrite.**,**.functions.ternary.**", + # "**.functions.transform.**","**.functions.unique.**", + # "**.functions.unary.matrix.**,**.functions.linearization.**,**.functions.jmlc.**" ] name: ${{ matrix.tests }} steps: @@ -101,6 +101,7 @@ jobs: ARTIFACT_NAME="transient_jacoco" ARTIFACT_NAME+="-${{ matrix.tests }}" ARTIFACT_NAME=${ARTIFACT_NAME//\*/x} # replace * with x + ARTIFACT_NAME+=".exec" echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> $GITHUB_ENV - name: Save Java Test Coverage as Artifact @@ -140,7 +141,10 @@ jobs: path: target - name: Merge Jacoco Artifacts - run: mvn jacoco:merge + run: | + ls + ls target + mvn jacoco:merge - name: Process Classes run: mvn process-classes From 6af6e08d786a80cce18169375914b60eab0a534c Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Wed, 14 May 2025 16:07:56 +0200 Subject: [PATCH 08/14] again --- .github/workflows/javaTests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index 8a9587320c9..de1d5ce0265 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -143,6 +143,7 @@ jobs: - name: Merge Jacoco Artifacts run: | ls + echo "And Now Target " ls target mvn jacoco:merge From 2003ef0ee9fc8332d4dcb72f205271b16b864084 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Thu, 15 May 2025 10:04:36 +0200 Subject: [PATCH 09/14] see lsla --- .github/workflows/javaTests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index de1d5ce0265..d9998b7545f 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -144,7 +144,7 @@ jobs: run: | ls echo "And Now Target " - ls target + ls -la target mvn jacoco:merge - name: Process Classes From ed3f687e9412e5e3a4d85b3e5bbfec36567b247d Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Thu, 15 May 2025 10:09:07 +0200 Subject: [PATCH 10/14] treated as a folder? --- .github/workflows/javaTests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index d9998b7545f..c730f4f0cf5 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -145,6 +145,7 @@ jobs: ls echo "And Now Target " ls -la target + ls -la target/transient_jacoco-xx.functions.homomorphicEncryption.xx.exec mvn jacoco:merge - name: Process Classes From 7909a9d0b8b1cd76e968c46025e0295a9762502d Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Thu, 15 May 2025 10:18:41 +0200 Subject: [PATCH 11/14] merge subfolders --- pom.xml | 14 +++++++++++++- .../org/apache/sysds/test/AutomatedTestBase.java | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 34bdc1c7c5c..a7d0f775fe0 100644 --- a/pom.xml +++ b/pom.xml @@ -459,7 +459,19 @@ ${project.build.directory} - *jacoco* + *.exec + + + + ${project.build.directory} + + trancient_jacoco**/*.exec + + + + ${project.build.directory} + + federated_jacoco/*.exec diff --git a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java index 03ebb0fb4eb..2c3dd11c6d0 100644 --- a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java @@ -1674,7 +1674,7 @@ protected static Process startLocalFedWorker(int port, String[] addArgs, int sle // add code coverage report if(arg.contains("org.jacoco.agent")) args = ArrayUtils.addAll(args, - new String[] {arg.replace("target/jacoco.exec", String.format("target/jacoco-%d.exec", port))}); + new String[] {arg.replace("target/jacoco.exec", String.format("target/federated_jacoco/jacoco-%d.exec", port))}); } args = ArrayUtils.addAll(args, new String[]{ From 1cac72063ec0c2c7ebe29045c52ac7a3ae41893b Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Thu, 15 May 2025 10:19:53 +0200 Subject: [PATCH 12/14] minor cleanup --- .github/workflows/javaTests.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index c730f4f0cf5..2baac8a19bb 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -101,7 +101,6 @@ jobs: ARTIFACT_NAME="transient_jacoco" ARTIFACT_NAME+="-${{ matrix.tests }}" ARTIFACT_NAME=${ARTIFACT_NAME//\*/x} # replace * with x - ARTIFACT_NAME+=".exec" echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> $GITHUB_ENV - name: Save Java Test Coverage as Artifact @@ -141,12 +140,7 @@ jobs: path: target - name: Merge Jacoco Artifacts - run: | - ls - echo "And Now Target " - ls -la target - ls -la target/transient_jacoco-xx.functions.homomorphicEncryption.xx.exec - mvn jacoco:merge + run: mvn jacoco:merge - name: Process Classes run: mvn process-classes From f910694609784b6c5a9caf99d46358ef4c363202 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Thu, 15 May 2025 10:28:24 +0200 Subject: [PATCH 13/14] fix spelling mistake in trancient --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a7d0f775fe0..b25d94cc7db 100644 --- a/pom.xml +++ b/pom.xml @@ -465,7 +465,7 @@ ${project.build.directory} - trancient_jacoco**/*.exec + transient_jacoco**/*.exec From 89ccaa5fa35afbe5c8bc63488c6e52bad123afc2 Mon Sep 17 00:00:00 2001 From: Sebastian Baunsgaard Date: Thu, 15 May 2025 10:36:25 +0200 Subject: [PATCH 14/14] reenable all tests --- .github/workflows/javaTests.yml | 68 ++++++++++++++++----------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/javaTests.yml b/.github/workflows/javaTests.yml index 2baac8a19bb..37c28d30dce 100644 --- a/.github/workflows/javaTests.yml +++ b/.github/workflows/javaTests.yml @@ -49,41 +49,41 @@ jobs: fail-fast: false matrix: tests: [ - # "org.apache.sysds.test.applications.**", - # "**.test.usertest.**", - # "**.component.c**.** -Dtest-threadCount=1 -Dtest-forkCount=1", - # "**.component.e**.**,**.component.f**.**,**.component.m**.**", - # "**.component.p**.**,**.component.r**.**,**.component.s**.**,**.component.t**.**,**.component.u**.**", - # "**.functions.a**.**,**.functions.binary.matrix.**,**.functions.binary.scalar.**,**.functions.binary.tensor.**", - # "**.functions.blocks.**,**.functions.data.rand.**,", - # "**.functions.countDistinct.**,**.functions.countDistinctApprox.**", - # "**.functions.data.misc.**,**.functions.lineage.**", - # "**.functions.compress.**,**.functions.data.tensor.**,**.functions.codegenalg.parttwo.**,**.functions.codegen.**,**.functions.caching.**", - # "**.functions.binary.matrix_full_cellwise.**,**.functions.binary.matrix_full_other.**", - # "**.functions.federated.algorithms.**,**.functions.federated.io.**,**.functions.federated.paramserv.**", - # "**.functions.federated.transform.**,**.functions.federated.fedplanner.**", - # "**.functions.federated.primitives.part1.** -Dtest-threadCount=1 -Dtest-forkCount=1", - # "**.functions.federated.primitives.part2.** -Dtest-threadCount=1 -Dtest-forkCount=1", - # "**.functions.federated.primitives.part3.** -Dtest-threadCount=1 -Dtest-forkCount=1", - # "**.functions.federated.primitives.part4.** -Dtest-threadCount=1 -Dtest-forkCount=1", - # "**.functions.federated.primitives.part5.** -Dtest-threadCount=1 -Dtest-forkCount=1", - # "**.functions.federated.monitoring.**,**.functions.federated.multitenant.**", - # "**.functions.federated.codegen.**,**.functions.federated.FederatedTestObjectConstructor", - # "**.functions.codegenalg.partone.**", - # "**.functions.builtin.part1.**", - # "**.functions.builtin.part2.**", - # "**.functions.frame.**,**.functions.indexing.**,**.functions.io.**,**.functions.iogen.**", - # "**.functions.dnn.**", - # "**.functions.paramserv.**", - # "**.functions.recompile.**,**.functions.misc.**", - # "**.functions.mlcontext.**", - # "**.functions.nary.**,**.functions.quaternary.**", - # "**.functions.parfor.**,**.functions.pipelines.**", + "org.apache.sysds.test.applications.**", + "**.test.usertest.**", + "**.component.c**.** -Dtest-threadCount=1 -Dtest-forkCount=1", + "**.component.e**.**,**.component.f**.**,**.component.m**.**", + "**.component.p**.**,**.component.r**.**,**.component.s**.**,**.component.t**.**,**.component.u**.**", + "**.functions.a**.**,**.functions.binary.matrix.**,**.functions.binary.scalar.**,**.functions.binary.tensor.**", + "**.functions.blocks.**,**.functions.data.rand.**,", + "**.functions.countDistinct.**,**.functions.countDistinctApprox.**", + "**.functions.data.misc.**,**.functions.lineage.**", + "**.functions.compress.**,**.functions.data.tensor.**,**.functions.codegenalg.parttwo.**,**.functions.codegen.**,**.functions.caching.**", + "**.functions.binary.matrix_full_cellwise.**,**.functions.binary.matrix_full_other.**", + "**.functions.federated.algorithms.**,**.functions.federated.io.**,**.functions.federated.paramserv.**", + "**.functions.federated.transform.**,**.functions.federated.fedplanner.**", + "**.functions.federated.primitives.part1.** -Dtest-threadCount=1 -Dtest-forkCount=1", + "**.functions.federated.primitives.part2.** -Dtest-threadCount=1 -Dtest-forkCount=1", + "**.functions.federated.primitives.part3.** -Dtest-threadCount=1 -Dtest-forkCount=1", + "**.functions.federated.primitives.part4.** -Dtest-threadCount=1 -Dtest-forkCount=1", + "**.functions.federated.primitives.part5.** -Dtest-threadCount=1 -Dtest-forkCount=1", + "**.functions.federated.monitoring.**,**.functions.federated.multitenant.**", + "**.functions.federated.codegen.**,**.functions.federated.FederatedTestObjectConstructor", + "**.functions.codegenalg.partone.**", + "**.functions.builtin.part1.**", + "**.functions.builtin.part2.**", + "**.functions.frame.**,**.functions.indexing.**,**.functions.io.**,**.functions.iogen.**", + "**.functions.dnn.**", + "**.functions.paramserv.**", + "**.functions.recompile.**,**.functions.misc.**", + "**.functions.mlcontext.**", + "**.functions.nary.**,**.functions.quaternary.**", + "**.functions.parfor.**,**.functions.pipelines.**", "**.functions.homomorphicEncryption.**", - # "**.functions.unary.scalar.**,**.functions.updateinplace.**,**.functions.vect.**", - # "**.functions.reorg.**,**.functions.rewrite.**,**.functions.ternary.**", - # "**.functions.transform.**","**.functions.unique.**", - # "**.functions.unary.matrix.**,**.functions.linearization.**,**.functions.jmlc.**" + "**.functions.unary.scalar.**,**.functions.updateinplace.**,**.functions.vect.**", + "**.functions.reorg.**,**.functions.rewrite.**,**.functions.ternary.**", + "**.functions.transform.**","**.functions.unique.**", + "**.functions.unary.matrix.**,**.functions.linearization.**,**.functions.jmlc.**" ] name: ${{ matrix.tests }} steps: