diff --git a/docs/_plugins/build_api_docs.rb b/docs/_plugins/build_api_docs.rb index 1ef80bfaf09a4..3072e2ddf85d0 100644 --- a/docs/_plugins/build_api_docs.rb +++ b/docs/_plugins/build_api_docs.rb @@ -129,7 +129,9 @@ def build_spark_scala_and_java_docs_if_necessary return end - command = "build/sbt -Pkinesis-asl unidoc" + # Drop stale genjavadoc trees (target/java) before unifying Javadoc; leftover stubs + # (e.g. from a partial or Test/doc run) can make JavaUnidoc fail (SPARK-56827). + command = "build/sbt -Pkinesis-asl cleanGenjavadocOutput unidoc" puts "Running '#{command}'..." # Two filter passes on the unidoc output: diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 66947a58ac590..f471b186e45b0 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1605,6 +1605,11 @@ object Unidoc { import sbtunidoc.JavaUnidocPlugin.autoImport._ import sbtunidoc.ScalaUnidocPlugin.autoImport._ + val cleanGenjavadocOutput = taskKey[Unit]( + "Delete each module's target/java tree (genjavadoc stubs) before unified API docs " + + "so JavaUnidoc cannot pick up stale files (SPARK-56827)." + ) + protected def ignoreUndocumentedPackages(packages: Seq[Seq[File]]): Seq[Seq[File]] = { packages .map(_.filterNot(_.getName.contains("$"))) @@ -1729,6 +1734,13 @@ object Unidoc { inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, kubernetes, yarn, tags, streamingKafka010, sqlKafka010, connectCommon, connect, connectJdbc, connectClient, connectShims, protobuf, profiler, udfWorkerProto, udfWorkerCore), + + cleanGenjavadocOutput := { + val dirs = target.all(ScopeFilter(inAnyProject)).value + .map(_ / "java") + .filter(_.isDirectory) + IO.delete(dirs) + } ) }