[SPARK-56666][INFRA] Reduce unidoc CI log noise with -Xdoclint:-missing and -verbose post-filter

docs/_plugins/build_api_docs.rb

@@ -134,20 +134,20 @@ def build_spark_scala_and_java_docs_if_necessary
   command = "build/sbt -Pkinesis-asl unidoc"
   puts "Running '#{command}'..."
 
-  # Suppress genjavadoc-stub diagnostic blocks from the visible log. javadoc
-  # emits ~3500 `[error]` lines per unidoc run on stubs under `target/java/`
-  # -- all benign because `--ignore-source-errors` is set, but they bury
-  # everything else. Each diagnostic is a header line followed by 3-5
-  # `[error|warn]`-prefixed continuation lines (snippet, caret,
-  # symbol/location); the state machine drops both.
+  # Two filter passes on the unidoc output:
   #
-  # Match by *message text*, not just by `target/java/` path. Otherwise
-  # legitimate doclint diagnostics on stub paths would be hidden too --
-  # those messages are real signal. The patterns below are the known-benign
-  # genjavadoc structural errors; anything else on a `target/java/` path is
-  # echoed. Diagnostic mirror lines from `SparkUnidocDoclet` use the
-  # `[unidoc-doclet]` prefix and don't match either regex, so they always
-  # pass through.
+  # 1. Genjavadoc-stub diagnostic blocks (~28 `[error]` lines on stubs under
+  #    `target/java/`, plus 3-5 continuation lines each). Inert because
+  #    `--ignore-source-errors` is set; matched by message text so legitimate
+  #    doclint diagnostics on stub paths still pass through.
+  #
+  # 2. `-verbose` progress lines (~13K total): `Loading source file ...`,
+  #    `[parsing started/completed ...]`, `[loading /path/X.class]`,
+  #    `Generating .../X.html`. These are dominant in the log when `-verbose`
+  #    is set (which it is in `JavaUnidoc / unidoc / javacOptions` to surface
+  #    per-file `error: reference not found` diagnostics) but carry no signal
+  #    of their own. Suppressing them brings the visible log from ~17K to ~5K
+  #    lines on a typical run while leaving every diagnostic untouched.
   ansi = /\e\[[0-9;]*[A-Za-z]/
   stub_header = %r{
     \[(?:error|warn)\]\s+
@@ -159,11 +159,24 @@ def build_spark_scala_and_java_docs_if_necessary
      |.*?\s+is\s+not\s+public\s+in\s+\S+;\s+cannot\s+be\s+accessed\s+from\s+outside\s+package)
   }x
   stub_cont = %r{\A\s*\[(?:error|warn)\]\s+(?!/\S+\.java:\d+(?::\d+)?:\s)}
+  verbose_line = %r{
+    \[(?:error|warn)\]\s+
+    (?:Loading\s+source\s+file\s
+     |\[parsing\s+(?:started|completed)\s
+     |\[loading\s
+     |\[checking\s
+     |\[wrote\s
+     |Generating\s+\S+\.html
+    )
+  }x
   in_stub = false
   IO.popen("#{command} 2>&1", 'r') do |pipe|
     pipe.each_line do |line|
       plain = line.gsub(ansi, '')
-      if plain =~ stub_header
+      if plain =~ verbose_line
+        in_stub = false
+        # suppress -verbose progress line
+      elsif plain =~ stub_header
         in_stub = true
       elsif in_stub && plain =~ stub_cont
         # continuation of a stub block; suppress

project/SparkBuild.scala

@@ -253,9 +253,7 @@ object SparkBuild extends PomBuild {
   // violations surface, with file/line annotations from `dev/scalastyle`.
   def enableScalaStyle: Seq[sbt.Def.Setting[_]] = Seq(
     scalaStyleOnCompile := cachedScalaStyle(Compile).value,
-    scalaStyleOnTest := cachedScalaStyle(Test).value,
-    (scalaStyleOnCompile / logLevel) := Level.Warn,
-    (scalaStyleOnTest / logLevel) := Level.Warn
+    scalaStyleOnTest := cachedScalaStyle(Test).value
   )
 
   lazy val compilerWarningSettings: Seq[sbt.Def.Setting[_]] = Seq(
@@ -1698,7 +1696,10 @@ object Unidoc {
         "-tag", "todo:X",
         "-tag", "groupname:X",
         "-tag", "inheritdoc",
-        "--ignore-source-errors", "-notree"
+        "--ignore-source-errors", "-notree",
+        "-Xmaxerrs", "0",
+        "-verbose",
+        "-Xdoclint:all", "-Xdoclint:-missing"
       )
     },