diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 0cb3f6205908..c128f858955d 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -13,7 +13,6 @@ jobs: DailyDocRepoLocation: $(Pipeline.Workspace)/daily DocRepoOwner: Azure DocRepoName: azure-docs-sdk-java - DocValidationImageId: azuresdkimages.azurecr.io/javarefautocr:latest steps: # Sync docs repo onboarding files/folders - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -29,10 +28,6 @@ jobs: WorkingDirectory: $(DocRepoLocation) - Name: azure-sdk/$(DocRepoName) WorkingDirectory: $(DailyDocRepoLocation) - # Pull and build the docker image. - - template: /eng/common/pipelines/templates/steps/docker-pull-image.yml - parameters: - ImageId: "$(DocValidationImageId)" - task: Powershell@2 inputs: @@ -46,7 +41,7 @@ jobs: inputs: pwsh: true filePath: eng/common/scripts/Update-DocsMsPackages.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) -ImageId '$(DocValidationImageId)' + arguments: -DocRepoLocation $(DocRepoLocation) displayName: Update Docs Onboarding for main branch condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) @@ -85,7 +80,7 @@ jobs: parameters: BaseRepoBranch: $(DefaultBranch) BaseRepoOwner: $(DocRepoOwner) - CommitMsg: "Update docs CI configuration" + CommitMsg: "Update docs CI configuration Build: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" TargetRepoName: $(DocRepoName) TargetRepoOwner: $(DocRepoOwner) WorkingDirectory: $(DocRepoLocation) @@ -95,20 +90,7 @@ jobs: parameters: DailyBranchVariableName: DailyDocsBranchName - # Docs daily updates is supposed to download packages from public feed repository, so we have to specify additional repositories in a POM or the profile. - # Here is maven documentation: https://maven.apache.org/guides/mini/guide-multiple-repositories.html - - powershell: | - # Linux mvn `setting.xml` is sitting under path `~/.m2/setting.xml` - Get-Command mvn - if (!(Test-Path '~/.m2/')) { - mkdir ~/.m2/ - } - if (Test-Path '~/.m2/setting.xml') { - Write-Host "'setting.xml' exists. Overwriting the file to support multiple repositories." - } - Copy-Item "./eng/repo-docs/docms/daily.update.setting.xml" -Destination "~/.m2/settings.xml" - displayName: 'Configure mvn' - workingDirectory: $(Build.SourcesDirectory) + - template: /eng/pipelines/templates/steps/mvn-linux-settings-for-docs.yml - task: Powershell@2 inputs: diff --git a/eng/pipelines/templates/stages/archetype-java-release-batch.yml b/eng/pipelines/templates/stages/archetype-java-release-batch.yml index 24a2d7c7e723..7b0cf0945427 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-batch.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-batch.yml @@ -319,10 +319,6 @@ stages: image: azsdk-pool-mms-ubuntu-2004-1espt os: linux - variables: - - name: DocValidationImageId - value: azuresdkimages.azurecr.io/javarefautocr:latest - strategy: runOnce: deploy: @@ -341,6 +337,8 @@ stages: - template: /eng/pipelines/templates/steps/mvn-linux-repository-settings.yml + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: PackageInfoLocations: @@ -354,7 +352,6 @@ stages: SparseCheckoutPaths: - docs-ref-services/ - metadata/ - DocValidationImageId: "$(DocValidationImageId)" - deployment: PublishDocs displayName: Publish Docs to GitHubIO Blob Storage @@ -464,9 +461,6 @@ stages: name: azsdk-pool-mms-ubuntu-2004-general image: azsdk-pool-mms-ubuntu-2004-1espt os: linux - variables: - - name: DocValidationImageId - value: azuresdkimages.azurecr.io/javarefautocr:latest steps: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: @@ -485,6 +479,8 @@ stages: - template: /eng/pipelines/templates/steps/mvn-linux-repository-settings.yml + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: PackageInfoLocations: @@ -499,6 +495,5 @@ stages: SparseCheckoutPaths: - docs-ref-services/ - metadata/ - DocValidationImageId: "$(DocValidationImageId)" - template: /eng/common/pipelines/templates/steps/docsms-ensure-validation.yml diff --git a/eng/pipelines/templates/stages/archetype-java-release-patch.yml b/eng/pipelines/templates/stages/archetype-java-release-patch.yml index 50e817b937fd..c22cff36f32c 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-patch.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-patch.yml @@ -239,8 +239,6 @@ stages: variables: - template: /eng/pipelines/templates/variables/globals.yml - - name: DocValidationImageId - value: azuresdkimages.azurecr.io/javarefautocr:latest strategy: runOnce: @@ -261,6 +259,8 @@ stages: - template: /eng/pipelines/templates/steps/mvn-linux-repository-settings.yml + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: PackageInfoLocations: @@ -274,7 +274,6 @@ stages: SparseCheckoutPaths: - docs-ref-services/ - metadata/ - DocValidationImageId: "$(DocValidationImageId)" - deployment: PublishDocs displayName: Publish Docs to GitHubIO Blob Storage diff --git a/eng/pipelines/templates/stages/archetype-java-release.yml b/eng/pipelines/templates/stages/archetype-java-release.yml index bcb7e5c37892..6d689be35605 100644 --- a/eng/pipelines/templates/stages/archetype-java-release.yml +++ b/eng/pipelines/templates/stages/archetype-java-release.yml @@ -257,8 +257,6 @@ stages: variables: - template: /eng/pipelines/templates/variables/globals.yml - - name: DocValidationImageId - value: azuresdkimages.azurecr.io/javarefautocr:latest strategy: runOnce: @@ -275,10 +273,11 @@ stages: - download: current displayName: 'Download Artifact: ${{parameters.ArtifactName}}' artifact: ${{parameters.ArtifactName}} - # Pull and build the docker image. - template: /eng/pipelines/templates/steps/mvn-linux-repository-settings.yml + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: PackageInfoLocations: @@ -290,7 +289,6 @@ stages: SparseCheckoutPaths: - docs-ref-services/ - metadata/ - DocValidationImageId: "$(DocValidationImageId)" - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: - deployment: PublishDocs @@ -446,8 +444,6 @@ stages: os: linux variables: - template: /eng/pipelines/templates/variables/globals.yml - - name: DocValidationImageId - value: azuresdkimages.azurecr.io/javarefautocr:latest steps: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: @@ -466,6 +462,8 @@ stages: - template: /eng/pipelines/templates/steps/mvn-linux-repository-settings.yml + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: PackageInfoLocations: @@ -480,6 +478,5 @@ stages: SparseCheckoutPaths: - docs-ref-services/ - metadata/ - DocValidationImageId: "$(DocValidationImageId)" - template: /eng/common/pipelines/templates/steps/docsms-ensure-validation.yml diff --git a/eng/pipelines/templates/steps/install-rex-validation-tool.yml b/eng/pipelines/templates/steps/install-rex-validation-tool.yml new file mode 100644 index 000000000000..904e04402123 --- /dev/null +++ b/eng/pipelines/templates/steps/install-rex-validation-tool.yml @@ -0,0 +1,22 @@ +steps: + # Create a java2docfx subdirectory in the $(Build.BinariesDirectory) and install the java2docfx there + # This way, the jar file is in its own subdirectory and isolated. + - pwsh: | + $java2docfxVer = Get-Content eng/scripts/docs/java2docfx.version.txt + $java2docfxDir = Join-Path $(Build.BinariesDirectory) "java2docfx" + New-Item $java2docfxDir -ItemType Directory | Out-Null + $originLocation = Get-Location + try { + # Set the location into the created java2docfx directory . Running mvn from the repository root + # picks up the root POM and processes that while doing the dependency:copy in spite of + # the fact it has nothing to do with this download. + Set-Location $java2docfxDir + Write-Host "mvn dependency:copy -Dartifact=""com.microsoft:java2docfx:$java2docfxVer"" -DoutputDirectory=""$java2docfxDir""" + mvn dependency:copy -Dartifact="com.microsoft:java2docfx:$java2docfxVer" -DoutputDirectory="$java2docfxDir" + $java2docfxJarLoc = Join-Path $java2docfxDir -ChildPath "java2docfx-$java2docfxVer.jar" + Write-Host "Testing the install, running java -jar $java2docfxJarLoc -h" + java -jar $java2docfxJarLoc -h + } finally { + Set-Location $originLocation + } + displayName: Install java2docfx for package validation \ No newline at end of file diff --git a/eng/repo-docs/docms/daily.update.setting.xml b/eng/repo-docs/docms/daily.update.setting.xml index fc1b359a3a62..f30128b6d146 100644 --- a/eng/repo-docs/docms/daily.update.setting.xml +++ b/eng/repo-docs/docms/daily.update.setting.xml @@ -18,6 +18,16 @@ true + + docs-public-packages + https://docfx.pkgs.visualstudio.com/docfx/_packaging/docs-public-packages/maven/v1 + + true + + + true + + diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index ac6cf7eb809c..1d70e68094b8 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -163,7 +163,11 @@ function Get-java-DocsMsDevLanguageSpecificPackageInfo($packageInfo, $packageSou # through the javadoc, like track 1 libraries whose javadoc.jar files don't contain anything, in # the metadata json files. if ($namespaces.Count -gt 0) { + Write-Host "Get-java-DocsMsDevLanguageSpecificPackageInfo:adding namespaces property with the following namespaces:" + $namespaces | Write-Host $packageInfo | Add-Member -Type NoteProperty -Name "Namespaces" -Value $namespaces + } else { + Write-Host "Get-java-DocsMsDevLanguageSpecificPackageInfo: no namespaces to add" } } return $packageInfo @@ -262,412 +266,6 @@ function Get-java-GithubIoDocIndex() GenerateDocfxTocContent -tocContent $tocContent -lang "Java" -campaignId "UA-62780441-42" } -# a "package.json configures target packages for all the monikers in a Repository, it also has a slightly different -# schema than the moniker-specific json config that is seen in python and js -function Update-java-CIConfig($pkgs, $ciRepo, $locationInDocRepo, $monikerId=$null) -{ - $pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo) - - if (-not (Test-Path $pkgJsonLoc)) { - Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting." - exit(1) - } - - $allJsonData = Get-Content $pkgJsonLoc | ConvertFrom-Json - - $visibleInCI = @{} - - for ($i=0; $i -lt $allJsonData[$monikerId].packages.Length; $i++) { - $pkgDef = $allJsonData[$monikerId].packages[$i] - $visibleInCI[$pkgDef.packageArtifactId] = $i - } - - foreach ($releasingPkg in $pkgs) { - if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) { - $packagesIndex = $visibleInCI[$releasingPkg.PackageId] - $existingPackageDef = $allJsonData[$monikerId].packages[$packagesIndex] - $existingPackageDef.packageVersion = $releasingPkg.PackageVersion - } - else { - $newItem = New-Object PSObject -Property @{ - packageDownloadUrl = $PackageRepositoryUri - packageGroupId = $releasingPkg.GroupId - packageArtifactId = $releasingPkg.PackageId - packageVersion = $releasingPkg.PackageVersion - inputPath = @() - excludePath = @() - } - - $allJsonData[$monikerId].packages += $newItem - } - } - - $jsonContent = $allJsonData | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " } - - Set-Content -Path $pkgJsonLoc -Value $jsonContent -} - -$PackageExclusions = @{ - "azure-core-experimental" = "Don't want to include an experimental package."; - "azure-core-test" = "Don't want to include the test framework package."; - "azure-sdk-bom" = "Don't want to include the sdk bom."; - "azure-storage-internal-avro" = "No external APIs."; - "azure-cosmos-spark_3-1_2-12" = "Javadoc dependency issue."; - "azure-cosmos-spark_3-2_2-12" = "Javadoc dependency issue."; - "azure-cosmos-spark_3-3_2-12" = "Javadoc dependency issue."; - "azure-cosmos-spark_3-4_2-12" = "Javadoc dependency issue."; - "azure-cosmos-spark_3-5_2-12" = "Javadoc dependency issue."; - "azure-cosmos-test" = "Don't want to include the test framework package."; - "azure-aot-graalvm-support-netty" = "No Javadocs for the package."; - "azure-aot-graalvm-support" = "No Javadocs for the package."; - "azure-sdk-template" = "Depends on unreleased core."; - "azure-sdk-template-two" = "Depends on unreleased core."; - "azure-sdk-template-three" = "Depends on unreleased core."; - "azure-ai-personalizer" = "No java docs in this package."; - "azure-sdk-build-tool" = "Do not release docs for this package."; - "azure-resourcemanager-voiceservices" = "Doc build attempts to download a package that does not have published sources."; - "azure-resourcemanager-storagemover" = "Attempts to azure-sdk-build-tool and fails"; - "azure-security-keyvault-jca" = "Consistently hangs docs build, might be a spring package https://github.com/Azure/azure-sdk-for-java/issues/35389"; -} - -# Validates if the package will succeed in the CI build by validating the -# existence of a com folder in the unzipped source package -function SourcePackageHasComFolder($artifactNamePrefix, $packageDirectory) { - try - { - $packageArtifact = "${artifactNamePrefix}:jar:sources" - $mvnResults = mvn ` - dependency:copy ` - -Dartifact="$packageArtifact" ` - -DoutputDirectory="$packageDirectory" - - if ($LASTEXITCODE) { - LogWarning "Could not download source artifact: $packageArtifact" - $mvnResults | Write-Host - return $false - } - - $sourcesJarPath = (Get-ChildItem -File -Path $packageDirectory -Filter "*-sources.jar")[0] - $sourcesExtractPath = Join-Path $packageDirectory "sources" - - # Ensure that the sources folder is empty before extracting the jar - # otherwise there could be file collisions from a previous extraction run on - # the same system. - Remove-Item $sourcesExtractPath/* -Force -Recurse -ErrorAction Ignore - - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($sourcesJarPath, $sourcesExtractPath) - - if (!(Test-Path "$sourcesExtractPath\com")) { - LogWarning "Could not locate 'com' folder extracting $packageArtifact" - return $false - } - } - catch - { - LogError "Exception while updating checking if package can be documented: $($package.packageGroupId):$($package.packageArtifactId)" - LogError $_ - LogError $_.ScriptStackTrace - return $false - } - - return $true -} - -function PackageDependenciesResolve($artifactNamePrefix, $packageDirectory) { - - $pomArtifactName = "${artifactNamePrefix}:pom" - $artifactDownloadOutput = mvn ` - dependency:copy ` - -Dartifact="$pomArtifactName" ` - -DoutputDirectory="$packageDirectory" - - if ($LASTEXITCODE) { - LogWarning "Could not download pom artifact: $pomArtifactName" - $artifactDownloadOutput | Write-Host - return $false - } - - $downloadedPomPath = (Get-ChildItem -File -Path $packageDirectory -Filter '*.pom')[0] - - # -P '!azure-mgmt-sdk-test-jar' excludes the unpublished test jar from - # dependencies - $copyDependencyOutput = mvn ` - -f $downloadedPomPath ` - dependency:copy-dependencies ` - -P '!azure-mgmt-sdk-test-jar' ` - -DoutputDirectory="$packageDirectory" - - if ($LASTEXITCODE) { - LogWarning "Could not resolve dependencies for: $pomArtifactName" - $copyDependencyOutput | Write-Host - return $false - } - - return $true -} - -function ValidatePackage($groupId, $artifactId, $version, $DocValidationImageId) { - return ValidatePackages @{ Group = $groupId; Name = $artifactId; Version = $version; } $DocValidationImageId -} - -function ValidatePackages([array]$packageInfos, $DocValidationImageId) { - $workingDirectory = Join-Path ([System.IO.Path]::GetTempPath()) "validation" - if (!(Test-Path $workingDirectory)) { - New-Item -ItemType Directory -Force -Path $workingDirectory | Out-Null - } - - # Add more validation by replicating as much of the docs CI process as - # possible - # https://github.com/Azure/azure-sdk-for-python/issues/20109 - if (!$DocValidationImageId) - { - return FallbackValidation -packageInfos $packageInfos -workingDirectory $workingDirectory - } - else - { - return DockerValidation -packageInfos $packageInfos -DocValidationImageId $DocValidationImageId -workingDirectory $workingDirectory - } -} - -function FallbackValidation ($packageInfos, $workingDirectory) { - $results = @() - - foreach ($packageInfo in $packageInfos) { - $groupId = $packageInfo.Group - $artifactId = $packageInfo.Name - $version = $packageInfo.Version - - Write-Host "Validating using mvn command directly on $artifactId." - - $artifactNamePrefix = "${groupId}:${artifactId}:${version}" - - $packageDirectory = Join-Path $workingDirectory "${groupId}__${artifactId}__${version}" - New-Item -ItemType Directory -Path $packageDirectory -Force | Out-Null - - $isValid = (SourcePackageHasComFolder $artifactNamePrefix $packageDirectory) ` - -and (PackageDependenciesResolve $artifactNamePrefix $packageDirectory) - - if (!$isValid) { - LogWarning "Package $artifactNamePrefix ref docs validation failed." - } - - $results += $isValid - } - - $allValid = $results.Where({ $_ -eq $false }).Count -eq 0 - - return $allValid -} - -function DockerValidation ($packageInfos, $DocValidationImageId, $workingDirectory) { - Write-Host "Validating $($packageInfos.Length) package(s) using $DocValidationImageId." - - $containerWorkingDirectory = '/workdir/out' - $configurationFileName = 'configuration.json' - - $hostConfigurationPath = Join-Path $workingDirectory $configurationFileName - - # Cannot use Join-Path because the container and host path separators may differ - $containerConfigurationPath = "$containerWorkingDirectory/$configurationFileName" - - $configuration = [ordered]@{ - "output_path" = "docs-ref-autogen"; - "packages" = @($packageInfos | ForEach-Object { [ordered]@{ - packageGroupId = $_.Group; - packageArtifactId = $_.Name; - packageVersion = $_.Version; - packageDownloadUrl = $PackageRepositoryUri; - } }); - } - - Set-Content -Path $hostConfigurationPath -Value ($configuration | ConvertTo-Json) | Out-Null - - docker run -v "${workingDirectory}:${containerWorkingDirectory}" ` - -e TARGET_CONFIGURATION_PATH=$containerConfigurationPath $DocValidationImageId 2>&1 ` - | Where-Object { -not ($_ -match '^Progress .*B\s*$') } ` # Remove progress messages - | Out-Host - - if ($LASTEXITCODE -ne 0) { - LogWarning "The `docker` command failed with exit code $LASTEXITCODE." - - # The docker exit codes: https://docs.docker.com/engine/reference/run/#exit-status - # If the docker validation failed because of docker itself instead of the application, or if we don't know which - # package failed, fall back to mvn validation - if ($LASTEXITCODE -in 125..127 -Or $packageInfos.Length -gt 1) { - return FallbackValidation -packageInfos $packageInfos -workingDirectory $workingDirectory - } - - return $false - } - - return $true -} - -function Update-java-DocsMsPackages($DocsRepoLocation, $DocsMetadata, $DocValidationImageId) { - Write-Host "Excluded packages:" - foreach ($excludedPackage in $PackageExclusions.Keys) { - Write-Host " $excludedPackage - $($PackageExclusions[$excludedPackage])" - } - - # Also exclude 'spring' packages - # https://github.com/Azure/azure-sdk-for-java/issues/23087 - $FilteredMetadata = $DocsMetadata.Where({ !($PackageExclusions.ContainsKey($_.Package) -or $_.Type -eq 'spring') }) - - UpdateDocsMsPackages ` - (Join-Path $DocsRepoLocation 'package.json') ` - 'preview' ` - $FilteredMetadata ` - $DocValidationImageId - - UpdateDocsMsPackages ` - (Join-Path $DocsRepoLocation 'package.json') ` - 'latest' ` - $FilteredMetadata ` - $DocValidationImageId -} - -function UpdateDocsMsPackages($DocConfigFile, $Mode, $DocsMetadata, $DocValidationImageId) { - $packageConfig = Get-Content $DocConfigFile -Raw | ConvertFrom-Json - - $packageOutputPath = 'docs-ref-autogen' - if ($Mode -eq 'preview') { - $packageOutputPath = 'preview/docs-ref-autogen' - } - $targetPackageList = $packageConfig.Where({ $_.output_path -eq $packageOutputPath}) - if ($targetPackageList.Length -eq 0) { - LogError "Unable to find package config for $packageOutputPath in $DocConfigFile" - exit 1 - } elseif ($targetPackageList.Length -gt 1) { - LogError "Found multiple package configs for $packageOutputPath in $DocConfigFile" - exit 1 - } - - $targetPackageList = $targetPackageList[0] - - $outputPackages = @() - foreach ($package in $targetPackageList.packages) { - $packageGroupId = $package.packageGroupId - $packageName = $package.packageArtifactId - - $matchingPublishedPackageArray = $DocsMetadata.Where({ - $_.Package -eq $packageName -and $_.GroupId -eq $packageGroupId - }) - - # If this package does not match any published packages keep it in the list. - # This handles packages which are not tracked in metadata but still need to - # be built in Docs CI. - if ($matchingPublishedPackageArray.Count -eq 0) { - Write-Host "Keep non-tracked package: $packageName" - $outputPackages += $package - continue - } - - if ($matchingPublishedPackageArray.Count -gt 1) { - LogWarning "Found more than one matching published package in metadata for $packageName; only updating first entry" - } - $matchingPublishedPackage = $matchingPublishedPackageArray[0] - - if ($Mode -eq 'preview' -and !$matchingPublishedPackage.VersionPreview.Trim()) { - # If we are in preview mode and the package does not have a superseding - # preview version, remove the package from the list. - Write-Host "Remove superseded preview package: $packageName" - continue - } - - if ($Mode -eq 'latest' -and !$matchingPublishedPackage.VersionGA.Trim()) { - LogWarning "Metadata is missing GA version for GA package $packageName. Keeping existing package." - $outputPackages += $package - continue - } - - $packageVersion = $($matchingPublishedPackage.VersionGA) - if ($Mode -eq 'preview') { - if (!$matchingPublishedPackage.VersionPreview.Trim()) { - LogWarning "Metadata is missing preview version for preview package $packageName. Keeping existing package." - $outputPackages += $package - continue - } - $packageVersion = $matchingPublishedPackage.VersionPreview - } - - # If upgrading the package, run basic sanity checks against the package - if ($package.packageVersion -ne $packageVersion) { - Write-Host "Validating new version detected for $packageName ($packageVersion)" - $validatePackageResult = ValidatePackage $package.packageGroupId $package.packageArtifactId $packageVersion $DocValidationImageId - - if (!$validatePackageResult) { - LogWarning "Package is not valid: $packageName. Keeping old version." - $outputPackages += $package - continue - } - - $package.packageVersion = $packageVersion - } - - Write-Host "Keeping tracked package: $packageName." - $outputPackages += $package - } - - $outputPackagesHash = @{} - foreach ($package in $outputPackages) { - $outputPackagesHash["$($package.packageGroupId):$($package.packageArtifactId)"] = $true - } - - $remainingPackages = @() - if ($Mode -eq 'preview') { - $remainingPackages = $DocsMetadata.Where({ - ![string]::IsNullOrWhiteSpace($_.VersionPreview) -and !$outputPackagesHash.ContainsKey("$($_.GroupId):$($_.Package)") - }) - } else { - $remainingPackages = $DocsMetadata.Where({ - ![string]::IsNullOrWhiteSpace($_.VersionGA) -and !$outputPackagesHash.ContainsKey("$($_.GroupId):$($_.Package)") - }) - } - - # Add packages that exist in the metadata but are not onboarded in docs config - foreach ($package in $remainingPackages) { - $packageName = $package.Package - $packageGroupId = $package.GroupId - $packageVersion = $package.VersionGA - if ($Mode -eq 'preview') { - $packageVersion = $package.VersionPreview - } - - Write-Host "Validating new package $($packageGroupId):$($packageName):$($packageVersion)" - $validatePackageResult = ValidatePackage $packageGroupId $packageName $packageVersion $DocValidationImageId - if (!$validatePackageResult) { - LogWarning "Package is not valid: ${packageGroupId}:$packageName. Cannot onboard." - continue - } - - Write-Host "Add new package from metadata: ${packageGroupId}:$packageName" - $package = [ordered]@{ - packageArtifactId = $packageName - packageGroupId = $packageGroupId - packageVersion = $packageVersion - packageDownloadUrl = $PackageRepositoryUri - } - - $outputPackages += $package - } - - $targetPackageList.packages = $outputPackages - - # It is assumed that there is a matching config from above when the number of - # matching $targetPackageList is 1 - foreach ($config in $packageConfig) { - if ($config.output_path -eq $packageOutputPath) { - $config = $targetPackageList - break - } - } - - $outputJson = ConvertTo-Json $packageConfig -Depth 100 - Set-Content -Path $DocConfigFile -Value $outputJson - Write-Host "Onboarding configuration $Mode written to: $DocConfigFile" -} - # function is used to filter packages to submit to API view tool function Find-java-Artifacts-For-Apireview($artifactDir, $pkgName) { @@ -754,6 +352,8 @@ function GetExistingPackageVersions ($PackageName, $GroupId=$null) } } +# Defined in common.ps1 +# $GetDocsMsMetadataForPackageFn = "Get-${Language}-DocsMsMetadataForPackage" function Get-java-DocsMsMetadataForPackage($PackageInfo) { $readmeName = $PackageInfo.Name.ToLower() Write-Host "Docs.ms Readme name: $($readmeName)" @@ -787,12 +387,81 @@ function Validate-java-DocMsPackages ($PackageInfo, $PackageInfos, $DocValidatio $PackageInfos = @($PackageInfo) } - if (!(ValidatePackages $PackageInfos $DocValidationImageId)) { - Write-Error "Package validation failed" -ErrorAction Continue + # The install-rex-validation-tool.yml will install the java2docfx jar file into the Build.BinariesDirectory + # which is a DevOps variable for the directory. In PS that variable is BUILD_BINARIESDIRECTORY. + # The reason why this is necessary is that the command for java2docfx is in the following format: + # java –jar java2docfx-1.0.0.jar.jar --packagesJsonFile "C\temp\package.json" + # or + # java –jar java2docfx-1.0.0.jar --package "::" + # which means we need to know where, exactly, because the java command requires the full path + # to the jar file as an argument + $java2docfxJar = $null + if (!$Env:BUILD_BINARIESDIRECTORY) { + LogError "Env:BUILD_BINARIESDIRECTORY is not set and this is where the java2docfx jar file should be installed." + return $false + } + $java2docfxDir = Join-Path $Env:BUILD_BINARIESDIRECTORY "java2docfx" + if (!(Test-Path $java2docfxDir)) { + LogError "There should be a java2docfx directory under Env:BUILD_BINARIESDIRECTORY. Ensure that the /eng/pipelines/templates/steps/install-rex-validation-tool.yml template was run prior to whatever step is running this." + return $false + } + $java2docfxJarLoc = @(Get-ChildItem -Path $java2docfxDir -File -Filter "java2docfx*.jar") + if (!$java2docfxJarLoc) { + LogError "The java2docfx jar file should be installed in $java2docfxDir and is not there." return $false + } else { + # In theory, this shouldn't happen as the install-rex-validation-tool.yml is the only thing + # that'll ever install the jar + if ($java2docfxJarLoc.Count -gt 1) { + Write-Host "There were $($java2docfxJarLoc.Count) java2docfx jar files found in $Build_BinariesDirectory, using the first one" + } + $java2docfxJar = $java2docfxJarLoc[0] + Write-Host "java2docfx jar location=$java2docfxJar" + } + + $allSuccess = $true + $originLocation = Get-Location + foreach ($packageInfo in $PackageInfos) { + $artifact = "$($packageInfo.Group):$($packageInfo.Name):$($packageInfo.Version)" + $tempDirectory = Join-Path ([System.IO.Path]::GetTempPath()) "$($packageInfo.Group)-$($packageInfo.Name)-$($packageInfo.Version)" + New-Item $tempDirectory -ItemType Directory | Out-Null + # Set the location to the temp directory. The reason being is that it'll effectively be empty, no + # other jars, no POM files aka nothing Java related to pick up. + Set-Location $tempDirectory + try { + Write-Host "Calling java2docfx for $artifact" + Write-Host "java -jar ""$java2docfxJar"" -p ""$artifact""" + $java2docfxResults = java ` + -jar "$java2docfxJar"` + -p "$artifact" + # JRS-TODO: The -o option is something I'm currently questioning the behavior of but + # I can do some initial testing without that option being set + # -p "$artifact" ` + # -o "$tempDirectory" + + if ($LASTEXITCODE -ne 0) { + LogWarning "java2docfx failed for $artifact" + $java2docfxResults | Write-Host + $allSuccess = $false + } + } + catch { + LogError "Exception while trying to download: $artifact" + LogError $_ + LogError $_.ScriptStackTrace + $allSuccess = $false + } + finally { + # Ensure that the origianl location is restored + Set-Location $originLocation + # everything is contained within the temp directory, clean it up every time + if (Test-Path $tempDirectory) { + Remove-Item $tempDirectory -Recurse -Force + } + } } - return $true + return $allSuccess } function Get-java-EmitterName() { diff --git a/eng/scripts/docs/java2docfx.version.txt b/eng/scripts/docs/java2docfx.version.txt new file mode 100644 index 000000000000..a6a3a43c3a04 --- /dev/null +++ b/eng/scripts/docs/java2docfx.version.txt @@ -0,0 +1 @@ +1.0.4 \ No newline at end of file