From 98617aedf2a26bbaf1b7a0527a83ec51b8648399 Mon Sep 17 00:00:00 2001 From: Rui Abreu Date: Sun, 19 Jul 2026 11:57:58 +0200 Subject: [PATCH 1/3] Fix supervisor-launched workers: add lib-common to the worker framework classpath Since the lib de-duplication (#8819) the distribution ships the worker jars in lib-common instead of lib-worker. bin/storm.py was updated, but BasicContainer.frameworkClasspath() still built the worker launch classpath from lib-worker only, so supervisor-launched workers crashed with ClassNotFoundException: org.apache.storm.LogWriter. Add lib-common to the framework classpath, keeping lib-worker as the drop-in location for worker-only jars. --- .../org/apache/storm/daemon/supervisor/BasicContainer.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java index 7583a96db79..35ce6c56f7e 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java @@ -372,6 +372,10 @@ protected String getWildcardDir(File dir) { } protected List frameworkClasspath(SimpleVersion topoVersion) { + // Jars shared by the daemon and worker classpaths are de-duplicated into lib-common + // (see storm-dist dedup-libs.py); storm-client and its dependencies live there, so the + // worker classpath needs lib-common in addition to lib-worker. + File stormCommonLibDir = new File(stormHome, "lib-common"); File stormWorkerLibDir = new File(stormHome, "lib-worker"); String topoConfDir = System.getenv("STORM_CONF_DIR") != null ? System.getenv("STORM_CONF_DIR") @@ -379,6 +383,7 @@ protected List frameworkClasspath(SimpleVersion topoVersion) { File stormExtlibDir = new File(stormHome, "extlib"); String extcp = System.getenv("STORM_EXT_CLASSPATH"); List pathElements = new LinkedList<>(); + pathElements.add(getWildcardDir(stormCommonLibDir)); pathElements.add(getWildcardDir(stormWorkerLibDir)); pathElements.add(getWildcardDir(stormExtlibDir)); pathElements.add(extcp); From c9bb9e852cca57238be3d1d40f0ba6e44e342b89 Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Sun, 19 Jul 2026 11:58:33 +0200 Subject: [PATCH 2/3] test: cover the real frameworkClasspath() in BasicContainerTest The existing launch tests mock frameworkClasspath() away (FRAMEWORK_CP), which is how the lib-common regression slipped through. Exercise the real implementation via a hook in MockBasicContainer and assert that both lib-common/* and lib-worker/* are on the worker classpath. --- .../daemon/supervisor/BasicContainerTest.java | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java index a4f6ec93aa1..4f941d42ef4 100644 --- a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java +++ b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java @@ -31,11 +31,13 @@ import org.apache.storm.utils.LocalState; import org.apache.storm.utils.SimpleVersion; import org.apache.storm.utils.Utils; +import org.apache.storm.utils.VersionInfo; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -402,6 +404,47 @@ public void testLaunch() throws Exception { "storm.log.dir", stormLogDir); } + @Test + public void testFrameworkClasspathIncludesSharedAndWorkerLibs() throws Exception { + final String topoId = "test_topology_classpath"; + final int supervisorPort = 6628; + final int port = 8080; + final String stormHome = ContainerTest.asAbsPath("tmp", "storm-home"); + final String stormLogDir = ContainerTest.asFile(".", "target").getCanonicalPath(); + final String stormLocal = ContainerTest.asAbsPath("tmp", "storm-local"); + + final Map superConf = new HashMap<>(); + superConf.put(Config.STORM_LOCAL_DIR, stormLocal); + superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal); + + LocalAssignment la = new LocalAssignment(); + la.set_topology_id(topoId); + + AdvancedFSOps ops = mock(AdvancedFSOps.class); + when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true); + + LocalState ls = mock(LocalState.class); + MockResourceIsolationManager iso = new MockResourceIsolationManager(); + + checkpoint(() -> { + MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, + "SUPERVISOR", supervisorPort, port, la, iso, ls, "worker-id", new StormMetricsRegistry(), + new HashMap<>(), ops, "profile"); + + List cp = mc.realFrameworkClasspath(VersionInfo.OUR_VERSION); + + // The distribution de-duplicates the jars shared by the daemon and worker + // classpaths into lib-common; storm-client (LogWriter, Worker) ships there, + // so a worker launched without lib-common on the classpath cannot start. + String libCommon = stormHome + File.separator + "lib-common" + File.separator + "*"; + String libWorker = stormHome + File.separator + "lib-worker" + File.separator + "*"; + assertTrue(cp.contains(libCommon), "worker classpath must include lib-common/*, got: " + cp); + assertTrue(cp.contains(libWorker), "worker classpath must include lib-worker/*, got: " + cp); + }, + ConfigUtils.STORM_HOME, stormHome, + "storm.log.dir", stormLogDir); + } + @Test public void testLaunchStorm1version() throws Exception { final String topoId = "test_topology_storm_1.x"; @@ -712,6 +755,10 @@ protected List frameworkClasspath(SimpleVersion version) { return Collections.singletonList("FRAMEWORK_CP"); } + public List realFrameworkClasspath(SimpleVersion version) { + return super.frameworkClasspath(version); + } + @Override protected String javaLibraryPath(String stormRoot, Map conf) { return "JLP"; From f7a0d52bc937d8735f85caa2dfa5be758991f239 Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Sun, 19 Jul 2026 11:50:58 +0200 Subject: [PATCH 3/3] build: ship an empty lib-worker drop-in directory in the distribution The lib de-duplication removed lib-worker entirely, but both bin/storm.py and BasicContainer keep lib-worker/* on the worker classpath, and operators use the directory as the drop-in point for worker-only jars. Restore it as an empty directory holding only a README that documents the new lib / lib-common / lib-worker layout, in both the full and lite distributions. --- .../final-package/src/main/assembly/common.xml | 6 ++++++ .../src/main/dist/lib-worker/README.md | 15 +++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 storm-dist/binary/final-package/src/main/dist/lib-worker/README.md diff --git a/storm-dist/binary/final-package/src/main/assembly/common.xml b/storm-dist/binary/final-package/src/main/assembly/common.xml index 6b50cc8ada8..402fd3837b2 100644 --- a/storm-dist/binary/final-package/src/main/assembly/common.xml +++ b/storm-dist/binary/final-package/src/main/assembly/common.xml @@ -38,6 +38,12 @@ *.jar + + + ${project.basedir}/src/main/dist/lib-worker + lib-worker + ${project.basedir}/../storm-webapp-bin/target/webapp/webapp/ . diff --git a/storm-dist/binary/final-package/src/main/dist/lib-worker/README.md b/storm-dist/binary/final-package/src/main/dist/lib-worker/README.md new file mode 100644 index 00000000000..65f2d6a5a9a --- /dev/null +++ b/storm-dist/binary/final-package/src/main/dist/lib-worker/README.md @@ -0,0 +1,15 @@ +# lib-worker + +Drop-in directory for worker-only jars. + +Jars placed here are added to the classpath of every worker JVM launched by +the supervisor, but not to the daemon (nimbus / supervisor / ui) classpath. + +The distribution itself ships no worker-only jars: the jars shared by the +daemons and the workers live in `lib-common/`, daemon-only jars in `lib/`. +The classpaths are composed as: + + daemon classpath = lib-common + lib + worker classpath = lib-common + lib-worker + +See also `extlib/`, which is added to both the daemon and worker classpaths.