From 1c7f435f7c831855d10fb7821734c6498a2d1fd3 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 26 Aug 2024 09:33:47 -0700 Subject: [PATCH 1/8] Add namespaces (packages) to packageInfo --- eng/pipelines/templates/jobs/ci.yml | 10 ++ .../Save-Package-Namespace-Property.ps1 | 102 ++++++++++++++++++ eng/scripts/docs/Docs-ToC.ps1 | 40 ++++--- 3 files changed, 138 insertions(+), 14 deletions(-) create mode 100644 eng/scripts/Save-Package-Namespace-Property.ps1 diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 2822000b3ee2..c158ec62416c 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -197,6 +197,16 @@ jobs: -Artifacts ('${{ replace(convertToJson(parameters.ReleaseArtifacts), '''', '`''') }}' | ConvertFrom-Json | Where-Object -Not skipPublishPackage ) -InformationAction Continue + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/scripts/Save-Package-Namespaces-Property.ps1 + arguments: > + -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) + -Artifacts ('$(ArtifactsJson)' | ConvertFrom-Json | Select-Object name, groupId, skipPublishDocMs | Where-Object -Not "skipPublishDocMs") + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Update package properties with namespaces + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: $(Build.ArtifactStagingDirectory) diff --git a/eng/scripts/Save-Package-Namespace-Property.ps1 b/eng/scripts/Save-Package-Namespace-Property.ps1 new file mode 100644 index 000000000000..0b2ff28513d0 --- /dev/null +++ b/eng/scripts/Save-Package-Namespace-Property.ps1 @@ -0,0 +1,102 @@ +<# +.SYNOPSIS +Given an artifact staging directory, loop through all of the PackageInfo files +in the PackageInfo subdirectory and compute the namespaces if they don't already +exist. Yes, they're called packages in Java but namespaces are being used to keep +code that processes them common amongst the languages. + +.DESCRIPTION +Given an artifact staging directory, loop through all of the PackageInfo files +in the PackageInfo subdirectory. For each PackageInfo file, find its corresponding +javadoc jar file and use that to compute the namespace information. + +.PARAMETER ArtifactStagingDirectory +The root directory of the staged artifacts. The PackageInfo files will be in the +PackageInfo subdirectory. The artifacts root directories are GroupId based, meaning +any artifact with that GroupId will be in a subdirectory. For example: Most Spring +libraries are com.azure.spring and their javadoc jars will be under that subdirectory +but azure-spring-data-cosmos' GroupId is com.azure and its javadoc jar will be under +com.azure. + +.PARAMETER ArtifactsList +The list of artifacts to gather namespaces for, this is only done for libraries that are +producing docs. +-ArtifactsList ('$(ArtifactsJson)' | ConvertFrom-Json | Select-Object name, groupId, skipPublishDocMs | Where-Object -Not "skipPublishDocMs") +#> +[CmdletBinding()] +Param ( + [Parameter(Mandatory = $True)] + [string] $ArtifactStagingDirectory, + # -ArtifactsList ('$(ArtifactsJson)' | ConvertFrom-Json | Select-Object name, groupId, skipPublishDocMs | Where-Object -Not "skipPublishDocMs") + [Parameter(Mandatory=$true)] + [AllowNull()] + [array] $ArtifactsList +) + +. (Join-Path $PSScriptRoot ".." common scripts common.ps1) + +if (-not $ArtifactsList) { + Write-Host "ArtifactsList is empty, nothing to process. This can happen if skipPublishDocMs is set to true for all libraries being built." + exit 0 +} + +Write-Host "ArtifactStagingDirectory=$ArtifactStagingDirectory" +if (-not (Test-Path -Path $ArtifactStagingDirectory)) { + LogError "ArtifactStagingDirectory '$ArtifactStagingDirectory' does not exist." + exit 1 +} + +Write-Host "" +Write-Host "ArtifactsList:" +$ArtifactsList | Format-Table -Property GroupId, Name | Out-String | Write-Host + +$packageInfoDirectory = Join-Path $ArtifactStagingDirectory "PackageInfo" + +$foundError = $false +# At this point the packageInfo files should have been already been created. +# The only thing being done here is adding or updating namespaces for libraries +# that will be producing docs. This ArtifactsList is +foreach($artifact in $ArtifactsList) { + # Get the version from the packageInfo file + $packageInfoFile = Join-Path $packageInfoDirectory "$($artifact.Name).json" + Write-Host "processing $($packageInfoFile.FullName)" + $packageInfo = ConvertFrom-Json (Get-Content $packageInfoFile -Raw) + $version = $packageInfo.Version + # If the dev version is set, use that. This will be set for nightly builds + if ($packageInfo.DevVersion) { + $version = $packageInfo.DevVersion + } + # From the $packageInfo piece together the path to the javadoc jar file + $javadocJar = Join-Path $ArtifactStagingDirectory $packageInfo.Group $packageInfo.Name "$($packageInfo.Name)-$($version)-javadoc.jar" + if (!(Test-Path $javadocJar -PathType Leaf)) { + LogError "Javadoc Jar file, $javadocJar, was not found. Please ensure that a Javadoc jar is being created for the library." + $foundError = $true + continue + } + $namespaces = Fetch-Namespaces-From-Javadoc $packageInfo.Name $packageInfo.Group $version $javadocJar + if ($namespaces.Count -gt 0) { + Write-Host "Adding/Updating Namespaces property with the following namespaces:" + $namespaces | Write-Host + if ($packageInfo.PSobject.Properties.Name -contains "Namespaces") { + Write-Host "Contains Namespaces property, updating" + $packageInfo.Namespaces = $namespaces + } + else { + Write-Host "Adding Namespaces property" + $packageInfo = $packageInfo | Add-Member -MemberType NoteProperty -Name Namespaces -Value $namespaces -PassThru + } + $packageInfoJson = ConvertTo-Json -InputObject $packageInfo -Depth 100 + Write-Host "The updated packageInfo for $packageInfoFile is:" + Write-Host "$packageInfoJson" + Set-Content ` + -Path $packageInfoFile ` + -Value $packageInfoJson + } else { + Write-Host "Fetch-Namespaces-From-Javadoc::returned no namespaces to add" + } +} + +if ($foundError) { + exit 1 +} +exit 0 \ No newline at end of file diff --git a/eng/scripts/docs/Docs-ToC.ps1 b/eng/scripts/docs/Docs-ToC.ps1 index 153229377075..ae64aef5468c 100644 --- a/eng/scripts/docs/Docs-ToC.ps1 +++ b/eng/scripts/docs/Docs-ToC.ps1 @@ -144,7 +144,7 @@ function Get-Toc-Children($package, $docRepoLocation) { } # Given a library, groupId and version, return the list of namespaces -function Fetch-Namespaces-From-Javadoc($package, $groupId, $version) { +function Fetch-Namespaces-From-Javadoc($package, $groupId, $version, $javadocJarFile = $null) { $namespaces = @() # Create a temporary directory to drop the jar into @@ -152,19 +152,31 @@ function Fetch-Namespaces-From-Javadoc($package, $groupId, $version) { New-Item $tempDirectory -ItemType Directory | Out-Null $artifact = "${groupId}:${package}:${version}:jar:javadoc" try { - # Download the Jar file - Write-Host "mvn dependency:copy -Dartifact=""$artifact"" -DoutputDirectory=""$tempDirectory""" - $mvnResults = mvn ` - dependency:copy ` - -Dartifact="$artifact" ` - -DoutputDirectory="$tempDirectory" - - if ($LASTEXITCODE -ne 0) { - LogWarning "Could not download javadoc artifact: $artifact" - $mvnResults | Write-Host + # If the $javadocJarFile is passed in, copy it to the $tempDirectory, otherwise download it + # from the dev feed or maven + if ($javadocJarFile) { + if (Test-Path $javadocJar -PathType Leaf) { + Copy-Item $javadocJarFile -Destination $tempDirectory + } else { + LogWarning "javadocJar parameter's jar file, $javadocJar, does not exist." + } } else { + # Download the Jar file + Write-Host "mvn dependency:copy -Dartifact=""$artifact"" -DoutputDirectory=""$tempDirectory""" + $mvnResults = mvn ` + dependency:copy ` + -Dartifact="$artifact" ` + -DoutputDirectory="$tempDirectory" + if ($LASTEXITCODE -ne 0) { + LogWarning "Could not download javadoc artifact: $artifact" + $mvnResults | Write-Host + } + } + $javadocLocation = "$tempDirectory/$package-$version-javadoc.jar" + # If the Jar file doesn't exit the error has already been reported above and + # processing will return an empty namespaces list + if (Test-Path $javadocLocation -PathType Leaf) { # Unpack the Jar file - $javadocLocation = "$tempDirectory/$package-$version-javadoc.jar" $unpackDirectory = Join-Path $tempDirectory "unpackedJavadoc" New-Item $unpackDirectory -ItemType Directory | Out-Null Add-Type -AssemblyName System.IO.Compression.FileSystem @@ -205,12 +217,12 @@ function Fetch-Namespaces-From-Javadoc($package, $groupId, $version) { } } else { - LogWarning "Unable to determine namespaces from $artifact." + LogWarning "Unable to determine namespaces from $artifact. Please ensure that the a javadoc jar is being built for the artifact and that it's not empty (ex. START/END: Empty Java Doc & Sources isn't in the POM file)." } } } catch { - LogError "Exception while trying to download: $artifact" + LogError "Exception while trying to retrieve namespaces from: $artifact" LogError $_ LogError $_.ScriptStackTrace } From a338f36e081dd4a8ea2e9af7de608ee3f891cbb7 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 26 Aug 2024 10:08:07 -0700 Subject: [PATCH 2/8] Actually name the script correctly --- ...amespace-Property.ps1 => Save-Package-Namespaces-Property.ps1} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename eng/scripts/{Save-Package-Namespace-Property.ps1 => Save-Package-Namespaces-Property.ps1} (100%) diff --git a/eng/scripts/Save-Package-Namespace-Property.ps1 b/eng/scripts/Save-Package-Namespaces-Property.ps1 similarity index 100% rename from eng/scripts/Save-Package-Namespace-Property.ps1 rename to eng/scripts/Save-Package-Namespaces-Property.ps1 From 7539338ac37ce8836299a734b876443c33a062b6 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 26 Aug 2024 10:18:38 -0700 Subject: [PATCH 3/8] Fix incorrect parameter name --- eng/pipelines/templates/jobs/ci.yml | 2 +- .../templates/stages/archetype-sdk-client-patch.yml | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index c158ec62416c..4f3b13ad511c 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -202,7 +202,7 @@ jobs: filePath: $(Build.SourcesDirectory)/eng/scripts/Save-Package-Namespaces-Property.ps1 arguments: > -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) - -Artifacts ('$(ArtifactsJson)' | ConvertFrom-Json | Select-Object name, groupId, skipPublishDocMs | Where-Object -Not "skipPublishDocMs") + -ArtifactsList ('$(ArtifactsJson)' | ConvertFrom-Json | Select-Object name, groupId, skipPublishDocMs | Where-Object -Not "skipPublishDocMs") pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Update package properties with namespaces diff --git a/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml b/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml index daf985ee0e00..2bcfd622d4c5 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml @@ -160,6 +160,16 @@ extends: -Artifacts ('$(ArtifactsJsonEscaped)' | ConvertFrom-Json | Where-Object -Not skipPublishPackage) -InformationAction Continue + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/scripts/Save-Package-Namespaces-Property.ps1 + arguments: > + -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) + -ArtifactsList ('$(ArtifactsJson)' | ConvertFrom-Json | Select-Object name, groupId, skipPublishDocMs | Where-Object -Not "skipPublishDocMs") + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Update package properties with namespaces + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: $(Build.ArtifactStagingDirectory) From 07ba502e2a93e937cbd08bee38135cfae7d34d01 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 26 Aug 2024 11:11:50 -0700 Subject: [PATCH 4/8] correct fail the step if no namespaces can be determined for one of the libraries being processed --- eng/scripts/Save-Package-Namespaces-Property.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/scripts/Save-Package-Namespaces-Property.ps1 b/eng/scripts/Save-Package-Namespaces-Property.ps1 index 0b2ff28513d0..16b7068049e4 100644 --- a/eng/scripts/Save-Package-Namespaces-Property.ps1 +++ b/eng/scripts/Save-Package-Namespaces-Property.ps1 @@ -92,7 +92,8 @@ foreach($artifact in $ArtifactsList) { -Path $packageInfoFile ` -Value $packageInfoJson } else { - Write-Host "Fetch-Namespaces-From-Javadoc::returned no namespaces to add" + LogError "Unable to determine namespaces for $($packageInfo.Group):$($packageInfo.Name). Please ensure that skipPublishDocMs isn't incorrectly set to true or that the library isn't producing an empty java doc jar." + $foundError = $true } } From 5b7f9f85630da8cf5928d5341ec181ab1b521692 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 26 Aug 2024 11:20:33 -0700 Subject: [PATCH 5/8] Fix the pom file to remove the duplicate org.apache.maven.plugins:maven-jar-plugin definition --- .../pom.xml | 26 +++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/sdk/spring/spring-cloud-azure-starter-appconfiguration-config/pom.xml b/sdk/spring/spring-cloud-azure-starter-appconfiguration-config/pom.xml index 73aa8d0224a4..d8d9831b3e14 100644 --- a/sdk/spring/spring-cloud-azure-starter-appconfiguration-config/pom.xml +++ b/sdk/spring/spring-cloud-azure-starter-appconfiguration-config/pom.xml @@ -145,21 +145,15 @@ - - - - - org.apache.maven.plugins - maven-jar-plugin - 3.4.2 + + empty-javadoc-jar-with-readme @@ -223,8 +217,8 @@ - + From 515f45df8fdda9ae600bf378505b28633b78fa25 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 27 Aug 2024 08:15:26 -0700 Subject: [PATCH 6/8] turn off docs for spring-cloud-azure-appconfiguration-config-web until it actually generates docs --- sdk/spring/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/spring/ci.yml b/sdk/spring/ci.yml index d68609fb6b36..f42a7fb39940 100644 --- a/sdk/spring/ci.yml +++ b/sdk/spring/ci.yml @@ -578,8 +578,8 @@ extends: - name: spring-cloud-azure-appconfiguration-config-web groupId: com.azure.spring safeName: springcloudazureappconfigurationconfigweb - skipPublishDocGithubIo: false - skipPublishDocMs: false + skipPublishDocGithubIo: true + skipPublishDocMs: true skipUpdatePackageJson: true skipVerifyChangelog: true releaseInBatch: ${{ parameters.release_springcloudazureappconfigurationconfigweb }} From 8c10388ea34ce285fe855d0103af111e90cc58cb Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 27 Aug 2024 10:18:04 -0700 Subject: [PATCH 7/8] Updates for feedback --- eng/pipelines/templates/jobs/ci.yml | 2 +- .../templates/stages/archetype-sdk-client-patch.yml | 2 +- eng/scripts/Save-Package-Namespaces-Property.ps1 | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 4f3b13ad511c..d1bad5e988eb 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -202,7 +202,7 @@ jobs: filePath: $(Build.SourcesDirectory)/eng/scripts/Save-Package-Namespaces-Property.ps1 arguments: > -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) - -ArtifactsList ('$(ArtifactsJson)' | ConvertFrom-Json | Select-Object name, groupId, skipPublishDocMs | Where-Object -Not "skipPublishDocMs") + -ArtifactsList ('$(ArtifactsJson)' | ConvertFrom-Json) pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Update package properties with namespaces diff --git a/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml b/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml index 2bcfd622d4c5..f3672569eebc 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml @@ -165,7 +165,7 @@ extends: filePath: $(Build.SourcesDirectory)/eng/scripts/Save-Package-Namespaces-Property.ps1 arguments: > -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) - -ArtifactsList ('$(ArtifactsJson)' | ConvertFrom-Json | Select-Object name, groupId, skipPublishDocMs | Where-Object -Not "skipPublishDocMs") + -ArtifactsList ('$(ArtifactsJson)' | ConvertFrom-Json) pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Update package properties with namespaces diff --git a/eng/scripts/Save-Package-Namespaces-Property.ps1 b/eng/scripts/Save-Package-Namespaces-Property.ps1 index 16b7068049e4..1b4fac330c5b 100644 --- a/eng/scripts/Save-Package-Namespaces-Property.ps1 +++ b/eng/scripts/Save-Package-Namespaces-Property.ps1 @@ -21,18 +21,18 @@ com.azure. .PARAMETER ArtifactsList The list of artifacts to gather namespaces for, this is only done for libraries that are producing docs. --ArtifactsList ('$(ArtifactsJson)' | ConvertFrom-Json | Select-Object name, groupId, skipPublishDocMs | Where-Object -Not "skipPublishDocMs") +-ArtifactsList ('$(ArtifactsJson)' | ConvertFrom-Json) #> [CmdletBinding()] Param ( [Parameter(Mandatory = $True)] [string] $ArtifactStagingDirectory, - # -ArtifactsList ('$(ArtifactsJson)' | ConvertFrom-Json | Select-Object name, groupId, skipPublishDocMs | Where-Object -Not "skipPublishDocMs") [Parameter(Mandatory=$true)] - [AllowNull()] [array] $ArtifactsList ) +$ArtifactsList = $ArtifactsList | Where-Object -Not "skipPublishDocMs" + . (Join-Path $PSScriptRoot ".." common scripts common.ps1) if (-not $ArtifactsList) { From d49329061b85b241922a2d5ca9dc881f23080134 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 27 Aug 2024 10:45:51 -0700 Subject: [PATCH 8/8] Fix dependency version issue --- sdk/spring/spring-cloud-azure-appconfiguration-config/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/spring/spring-cloud-azure-appconfiguration-config/pom.xml b/sdk/spring/spring-cloud-azure-appconfiguration-config/pom.xml index b68abc0933e3..63d5c90c1882 100644 --- a/sdk/spring/spring-cloud-azure-appconfiguration-config/pom.xml +++ b/sdk/spring/spring-cloud-azure-appconfiguration-config/pom.xml @@ -26,7 +26,7 @@ org.springframework.boot spring-boot-configuration-processor - 3.3.2 + 3.3.3 true