Test and publish a multi-release jar

The previously published jars only worked on Java 21 because classes that need `--enable-preview` are not compatible between Java releases (even when the API is identical). We now still build the base contents of the jar on Java 21, but we add Java 22 overrides for all classes that have the preview flag set.

Author: szeiger

.github/workflows/build.yaml

@@ -57,3 +57,38 @@ jobs:
       if: matrix.java == 23
       run: |
         sbt core/doc
+
+  integration:
+    runs-on: ubuntu-latest
+    name: Multi-Release JAR Integration Test
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Java
+      uses: actions/setup-java@v4
+      with:
+        java-version: |
+          21
+          22
+        distribution: 'temurin'
+        cache: 'sbt'
+    - name: Set up sbt
+      uses: sbt/setup-sbt@v1
+    - name: Run tests
+      run: |
+        # Build multi-release override classes on Java 22
+        PATH=$JAVA_HOME_22_X64/bin:$PATH JAVA_HOME=$JAVA_HOME_22_X64 sbt \
+          core/compile
+        cp -r core/target/classes override-core-j22
+        # Build jar on Java 21 with --enable-preview and include Java 22 overrides
+        PATH=$JAVA_HOME_21_X64/bin:$PATH JAVA_HOME=$JAVA_HOME_21_X64 JAVA_OPTS=--enable-preview \
+          OVERRIDE_CORE_J22=`realpath override-core-j22` sbt \
+          core/clean \
+          core/packageBin
+        # Run tests with multi-release jar
+        cp `realpath core/target/perfio-*.jar` override-test.jar
+        PATH=$JAVA_HOME_21_X64/bin:$PATH JAVA_HOME=$JAVA_HOME_21_X64 JAVA_OPTS=--enable-preview \
+          OVERRIDE_TEST_JAR=`realpath override-test.jar` sbt \
+          test/test
+        PATH=$JAVA_HOME_22_X64/bin:$PATH JAVA_HOME=$JAVA_HOME_22_X64 \
+          OVERRIDE_TEST_JAR=`realpath override-test.jar` sbt \
+          test/test

.github/workflows/release.yaml

@@ -23,6 +23,7 @@ jobs:
       with:
         java-version: |
           21
+          22
           23
         distribution: 'temurin'
         cache: 'sbt'
@@ -42,9 +43,15 @@ jobs:
         SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
         SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
       run: |
-        # Publish binaries with Java 21 (because we can't use --enable-preview for 21 from 23)
-        PATH=$JAVA_HOME_21_X64/bin:$PATH JAVA_HOME=$JAVA_HOME_21_X64 JAVA_OPTS=--enable-preview sbt \
+        # Build multi-release override classes with Java 22
+        PATH=$JAVA_HOME_22_X64/bin:$PATH JAVA_HOME=$JAVA_HOME_22_X64 sbt \
+          core/compile
+        cp -r core/target/classes override-core-j22
+        # Publish binaries with Java 21 with --enable-preview and include Java 22 overrides
+        PATH=$JAVA_HOME_21_X64/bin:$PATH JAVA_HOME=$JAVA_HOME_21_X64 JAVA_OPTS=--enable-preview \
+          OVERRIDE_CORE_J22=`realpath override-core-j22` sbt \
           "set core/Compile/packageDoc/publishArtifact := false" \
+          core/clean \
           core/publishSigned
         ls -lR target/sonatype-staging
         # Publish javadoc with Java 23 (because we need Markdown support)

build.sbt

@@ -1,5 +1,6 @@
+import java.io.FileInputStream
 import java.nio.file.{Files, Path}
-import scala.jdk.CollectionConverters._
+import scala.jdk.CollectionConverters.*
 
 val PROTOBUF_HOME = sys.env.getOrElse("PROTOBUF_HOME", s"${sys.props("user.home")}/protobuf")
 val protobufVersion = "4.29.0-RC2"
@@ -24,6 +25,9 @@ val compileOpts = Seq(
   "--enable-preview",
 ))
 
+// External jar to use instead of `core` for tests and benchmarks
+val overrideTestJar = sys.env.get("OVERRIDE_TEST_JAR")
+
 javaOptions in Global ++= runtimeOpts
 javacOptions in Global ++= compileOpts
 scalacOptions in Global ++= Seq("-java-output-version", release.toString)
@@ -78,13 +82,47 @@ lazy val core = (project in file("core"))
     crossPaths := false,
     autoScalaLibrary := false,
     name := "perfio",
+    // When building the Java 21 version, add multi-release overrides for preview API classes
+    // by copying them from a previously built Java 22 output directory
+    Compile / packageBin / mappings := {
+      val orig = (Compile / packageBin / mappings).value
+      sys.env.get("OVERRIDE_CORE_J22") match {
+        case Some(j22classes) =>
+          println("Building multi-release jar")
+          val overrides = orig.collect {
+            case (f, s) if s.endsWith(".class") && isPreview(f) =>
+            val nf = new File(j22classes, s)
+            assert(nf.exists())
+            val ns = "META-INF/versions/22/" + s
+            println("  Adding override "+ns)
+            (nf, ns)
+          }
+          orig ++ overrides
+        case None => orig
+      }
+    },
+    (Compile / packageBin) / packageOptions +=
+      Package.ManifestAttributes("Multi-Release" -> "true"),
   )
 
 lazy val scalaApi = (project in file("scalaapi"))
-  .dependsOn(core)
+  .dependsOn(
+    (overrideTestJar match {
+      case Some(_) => Nil
+      case None => Seq(core: ClasspathDep[ProjectReference])
+    }): _*
+  )
   .settings(
     name := "perfio-scala",
     publish / skip := true,
+    (Compile / unmanagedJars) ++= {
+      overrideTestJar match {
+        case Some(j) =>
+          println("Using override test jar "+j)
+          Seq(file(j))
+        case None => Nil
+      }
+    },
   )
 
 lazy val benchUtil = (project in file("bench-util"))
@@ -215,3 +253,11 @@ test_ / Test / javap := {
   val pb = new ProcessBuilder(Seq("javap", "-cp", cp.iterator.map(_.data).mkString(sys.props.getOrElse("path.separator", ":"))) ++ args: _*).inheritIO()
   pb.start().waitFor()
 }
+
+def isPreview(classFile: File): Boolean = {
+  val in = new FileInputStream(classFile)
+  try {
+    val a = in.readNBytes(5)
+    a(4) != 0
+  } finally in.close()
+}