Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions eng/pipelines/aggregate-reports.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,11 @@ jobs:
pool:
vmImage: 'ubuntu-18.04'
steps:
- template: ../steps/cache-maven-repository.yml

# We `install` the code quality reports tooling into our local m2 cache separately from building the Maven project
# reports. This means it is available as part of that, but also so that this is not documented in the project report.
- template: templates/steps/install-reporting-tools.yml
parameters:
Options: --batch-mode
MavenOptions: $(MemoryOptions) $(LoggingOptions)

- task: Maven@3
displayName: 'Build All Libraries'
Expand Down Expand Up @@ -88,6 +87,9 @@ jobs:
condition: succeededOrFailed()
displayName: 'Publish Report Artifacts'
artifact: reports

- template: ../steps/upload-repository-on-failure.yml
Comment thread
alzimmermsft marked this conversation as resolved.
Outdated

- job: Compliance
displayName: Analyze (no build)
timeoutInMinutes: 120
Expand Down Expand Up @@ -142,7 +144,7 @@ jobs:
endColumn = $_.locations.physicalLocation.region.endColumn
level = $_.level
}
}
}
$csvRows | Export-Csv -Path "$(Pipeline.workspace)\.gdn\.r\CredScan\001\CredScan-matches.csv" -NoTypeInformation
displayName: Add converted csv file for CredScan-matches.sarif
- task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3
Expand Down
8 changes: 4 additions & 4 deletions eng/pipelines/templates/jobs/build-validate-pom.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,13 @@ jobs:
- template: ../steps/cache-maven-repository.yml

# HACK: Ideally I wouldn't need to do this but I need to figure out the right
# approach here around getting this dependency installed whilst targetting
# the parent POMs via the root aggregeate (given they aren't listed)
# anywhere in the depedency graph.
# approach here around getting this dependency installed whilst targeting
# the parent POMs via the root aggregate (given they aren't listed)
# anywhere in the dependency graph.
#
# The answer might be to finally strip out the <module /> references from
# the parent POMs (something we want to do anyway) and then add them to
# the graph so that we can build off the root POM targetting the parent
# the graph so that we can build off the root POM targeting the parent
# projects and then it would be able to find sdk-build-tools. But for now
# I am more interested getting the end-to-end scenario working with this
# hack rather than getting bogged down.
Expand Down
13 changes: 4 additions & 9 deletions eng/pipelines/templates/jobs/ci.tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ parameters:
default: '$(DefaultOptions) -T 2C -DskipTests -Dgpg.skip -DtrimStackTrace=false -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -Djacoco.skip=true'
- name: TestOptions
type: string
default: '$(DefaultOptions) -DtrimStackTrace=false -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -Dparallel-test-playback'
default: '$(DefaultTestOptions) -Dparallel-test-playback -DAZURE_TEST_DEBUG=$(IsDebug)'
- name: Matrix
type: object
- name: CloudConfig
Expand Down Expand Up @@ -62,12 +62,7 @@ jobs:
ServiceDirectory: ${{ parameters.ServiceDirectory }}
CheckoutRecordings: true

- template: ../steps/cache-maven-repository.yml

- template: ../steps/install-reporting-tools.yml
parameters:
Options: --batch-mode
MavenOptions: $(MemoryOptions) $(LoggingOptions)

- ${{ parameters.PreTestSteps }}

Expand All @@ -88,7 +83,7 @@ jobs:
displayName: 'Build for non-From Source run'
inputs:
mavenPomFile: pom.xml
options: ${{ parameters.BuildOptions }} -pl $(PLSkipOptions)$(ProjectList) -am
options: ${{ parameters.BuildOptions }} -pl $(ProjectList) -am
mavenOptions: '$(MemoryOptions) $(LoggingOptions)'
javaHomeOption: 'JDKVersion'
jdkVersionOption: $(JavaTestVersion)
Expand All @@ -102,7 +97,7 @@ jobs:
displayName: 'Run tests'
inputs:
mavenPomFile: pom.xml
options: ${{ parameters.TestOptions }} $(TestOptions) -pl $(PLSkipOptions)$(ProjectList) $(AdditionalOptions) -DAZURE_TEST_DEBUG=$(IsDebug)
options: ${{ parameters.TestOptions }} $(TestOptions) $(AdditionalOptions)
mavenOptions: '$(MemoryOptions) $(LoggingOptions)'
javaHomeOption: 'JDKVersion'
jdkVersionOption: $(JavaTestVersion)
Expand Down Expand Up @@ -152,7 +147,7 @@ jobs:
mavenPomFile: ClientFromSourcePom.xml
# For the From Source runs we don't want the -am switch as we don't care about running tests for our dependencies
# but we do want the -amd switch because we want to run tests on things that depend on us.
options: ${{ parameters.TestOptions }} $(TestOptions) -pl $(PLSkipOptions)$(ProjectList) -amd -T 1C -DAZURE_TEST_DEBUG=$(IsDebug)
options: ${{ parameters.TestOptions }} $(TestOptions) -amd -T 1C
mavenOptions: '$(MemoryOptions) $(LoggingOptions)'
javaHomeOption: 'JDKVersion'
jdkVersionOption: $(JavaTestVersion)
Expand Down
31 changes: 12 additions & 19 deletions eng/pipelines/templates/jobs/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,31 +92,27 @@ jobs:
parameters:
ServiceDirectory: ${{parameters.ServiceDirectory}}

- ${{ each artifact in parameters.Artifacts }}:
- script: |
python3 --version
python3 eng/versioning/set_versions.py --build-type ${{parameters.SDKType}} --build-qualifier alpha.$(Build.BuildNumber) --artifact-id ${{artifact.name}} --group-id ${{artifact.groupId}}
condition: and(succeeded(), eq(variables['SetDevVersion'],'true'))
displayName: Append dev package version suffix for ${{artifact.name}}

- script: |
- pwsh: |
$artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json
python3 --version

# Append dev package version suffix for each artifact
foreach ($artifact in $artifacts) {
python3 eng/versioning/set_versions.py --build-type ${{parameters.SDKType}} --build-qualifier alpha.$(Build.BuildNumber) --artifact-id $artifact.name --group-id $artifact.groupId
}

# Set zero-dev-version for packages
python3 eng/versioning/set_versions.py --set-dev-zero-version --build-type ${{parameters.SDKType}} --build-qualifier alpha.$(Build.BuildNumber)
condition: and(succeeded(), eq(variables['SetDevVersion'],'true'))
Comment thread
weshaggard marked this conversation as resolved.
Outdated
displayName: Set zero-dev-version for packages

- script: |
python3 --version
# Apply version settings to repository
python3 eng/versioning/update_versions.py --update-type library --build-type ${{parameters.SDKType}} --sr
condition: and(succeeded(), eq(variables['SetDevVersion'],'true'))
displayName: Apply version settings to repository
displayName: Setup Dev versioning

- script: |
pip install markdown2 BeautifulSoup4
displayName: 'pip install markdown2 and BeautifulSoup4'

- template: ../steps/cache-maven-repository.yml

- task: Maven@3
displayName: 'Build and Package, JDK Version: $(JavaBuildVersion)'
inputs:
Expand Down Expand Up @@ -231,7 +227,7 @@ jobs:
parameters:
ServiceDirectory: ${{ parameters.ServiceDirectory }}

# Use BasePathLength of 38 instead of the default 49 as some GA'd files fail when the number is higher.
# Use BasePathLength of 38 instead of the default 49 as some released files fail when the number is higher.
- template: /eng/common/pipelines/templates/steps/verify-path-length.yml
parameters:
SourceDirectory: $(Build.SourcesDirectory)
Expand All @@ -248,9 +244,6 @@ jobs:
# We `install` the code quality reports tooling into our local m2 cache separately from building the Maven project
# reports. This means it is available as part of that, but also so that this is not documented in the project report.
- template: ../steps/install-reporting-tools.yml
parameters:
Options: --batch-mode
MavenOptions: $(MemoryOptions) $(LoggingOptions)

# maven dependency:tree needs to be able to resolve dependencies, so these should be installed.
- task: Maven@3
Expand Down
4 changes: 2 additions & 2 deletions eng/pipelines/templates/stages/archetype-sdk-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,8 @@ parameters:
- name: TestStepMavenInputs
type: object
default:
options: '$(DefaultOptions) -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -DtrimStackTrace=false -pl $(ProjectList)'
mavenOptions: '$(MemoryOptions) $(LoggingOptions)'
options: '$(DefaultTestOptions)'
mavenOptions: '$(DefaultTestMavenOptions)'
javaHomeOption: 'JDKVersion'
jdkVersionOption: '$(JavaTestVersion)'
jdkArchitectureOption: 'x64'
Expand Down
9 changes: 2 additions & 7 deletions eng/pipelines/templates/stages/cosmos-sdk-client.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,6 @@ stages:
Artifacts: ${{ parameters.Artifacts }}
AdditionalModules: ${{ parameters.AdditionalModules }}
SDKType: ${{parameters.SDKType}}
PreTestSteps:
- template: ../steps/install-reporting-tools.yml
MatrixConfigs:
- Name: Cosmos_ci_test
Path: eng/pipelines/templates/stages/platform-matrix.json
Expand Down Expand Up @@ -73,7 +71,8 @@ stages:
TimeoutInMinutes: 90
TestStepMavenInputs:
goals: clean verify
options: '$(ProfileFlag) -Dgpg.skip -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -DtrimStackTrace=false $(AdditionalArgs) -pl $(ProjectList)'
options: '$(DefaultTestOptions) $(ProfileFlag) $(AdditionalArgs)'
mavenOptions: '$(DefaultTestMavenOptions)'
mavenAuthenticateFeed: true
jdkVersionOption: $(JavaTestVersion)

Expand Down Expand Up @@ -117,10 +116,6 @@ stages:
Write-Host "New Path environment variable: $updatedPath"
displayName: 'Download and Extract Hadoop winutils and update env variables for Hadoop usage'

# We `install` separately from running `site:site site:stage` so that the `install` brings in the non-shipping-modules,
# but we don't include them in the Maven site commands (so that we don't generate reports for the non-shipping modules).
- template: ../steps/install-reporting-tools.yml

# The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch.
- ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}:
- template: archetype-java-release.yml
Expand Down
17 changes: 4 additions & 13 deletions eng/pipelines/templates/steps/cache-maven-repository.yml
Original file line number Diff line number Diff line change
@@ -1,15 +1,6 @@
steps:
- script: |
echo "##vso[task.setvariable variable=Maven.RepositoryPath;]%USERPROFILE%\.m2\repository"
condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
displayName: Detecting Maven repository on Windows
- script: |
echo "##vso[task.setvariable variable=Maven.RepositoryPath;]$HOME/.m2/repository"
condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'))
displayName: Detecting Maven repository on Linux and macOS
- task: CacheBeta@0
- task: Cache@2
inputs:
key: $(Agent.JobName)|$(CacheSalt)|$(Build.SourcesDirectory)/sdk/**/pom.xml|$(Build.SourcesDirectory)/pom*.xml
path: $(Maven.RepositoryPath)
displayName: 'Download/upload cache'
condition: and(succeeded(), ne(variables['EnableCaching'], 'false'))
key: 'maven | $(CacheSalt) | $(Agent.OS) | $(Build.SourcesDirectory)/eng/versioning/external_dependencies.txt'
path: $(MAVEN_CACHE_FOLDER)
displayName: Cache Maven local repo
35 changes: 16 additions & 19 deletions eng/pipelines/templates/steps/generate-project-list.yml
Original file line number Diff line number Diff line change
@@ -1,31 +1,28 @@
# This job template uses the artifact list in the ci.yml file
# to generate a variable at build time that can be passed into
# the Maven -pl command-line switch to constrain what subset of
# projects are targetted based on the artifacts this pipeline
# projects are targeted based on the artifacts this pipeline
# produces. The -am and -amd switches are then applied as
# appropriate to constrain the full graph that is built depending
# on whether we are doing a recorded test play back or a full live
# test.

steps:
- template: ../steps/cache-maven-repository.yml

- pwsh: |
Write-Host "##vso[task.setvariable variable=ProjectList;]"
displayName: Initialize project list variable
$artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json
Comment thread
weshaggard marked this conversation as resolved.
Outdated
$additionalModules = '${{ convertToJson(parameters.AdditionalModules) }}' | ConvertFrom-Json

# These artifacts are the principal outputs of the build. We source
# them from the artifact list.
- ${{each artifact in parameters.Artifacts }}:
- pwsh: |
Write-Host "##vso[task.setvariable variable=ProjectList;]$(ProjectList),${{artifact.groupId}}:${{artifact.name}}"
displayName: Append ${{artifact.groupId}}:${{artifact.name}} to project list
$projectList = @()
foreach ($artifact in $artifacts) {
$projectList += "$($artifact.groupId):$($artifact.name)"
}
foreach ($artifact in $additionalModules) {
$projectList += "$($artifact.groupId):$($artifact.name)"
}
$projects = $projectList -join ','

# From time to time we might need to also build another module which
# we don't actually want to ship, so we list that here. We can't rely
# on the -amd switch to do this since is doesn't resolve dependencies
# AFTER dependents have been resolved, which means some projects might
# be missing from the resulting build graph. This works around that
# limitation in Maven.
- ${{each artifact in parameters.AdditionalModules }}:
- pwsh: |
Write-Host "##vso[task.setvariable variable=ProjectList;]$(ProjectList),${{artifact.groupId}}:${{artifact.name}}"
displayName: Append ${{artifact.groupId}}:${{artifact.name}} to project list
Write-Host "ProjectList = $projects"
Write-Host "##vso[task.setvariable variable=ProjectList;]$projects"
displayName: Initialize project list variable
4 changes: 2 additions & 2 deletions eng/pipelines/templates/steps/install-reporting-tools.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@

parameters:
Options: '--batch-mode -Dmaven.wagon.http.pool=false --settings eng/settings.xml'
MavenOptions: '-Dorg.slf4j.simpleLogger.defaultLogLevel=error -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn'
Options: '$(DefaultOptions)'
MavenOptions: '$(MemoryOptions) $(LoggingOptions)'

steps:
- task: Maven@3
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,12 @@ steps:
condition: or(eq(variables['CaptureRepositoryOnFailure'], 'true'), and(failed(), eq(variables['Build.Reason'],'Schedule')))
displayName: 'Capture repo state for analysis'
artifact: repository-$(System.JobName)

- pwsh: |
# Remove any installed packages that we build in the repo so we don't cache temporary builds
Remove-Item "$(MAVEN_CACHE_FOLDER)/com/azure" -Recurse -ErrorAction Ignore
Comment thread
weshaggard marked this conversation as resolved.
Outdated
Remove-Item "$(MAVEN_CACHE_FOLDER)/com/microsoft/azure" -Recurse -ErrorAction Ignore

Write-Host "POM files left in the cache folder"
Get-ChildItem $env:MAVEN_CACHE_FOLDER -Recurse -Include *.pom | Select Name
displayName: Clean up maven local cache
22 changes: 13 additions & 9 deletions eng/pipelines/templates/variables/globals.yml
Original file line number Diff line number Diff line change
@@ -1,16 +1,20 @@
variables:
DocWardenVersion: '0.7.1'
WagonOptions: '-Dmaven.wagon.rto=300000 -Dmaven.wagon.http.retryHandler.count=3'
DefaultOptions: '--batch-mode --fail-at-end --settings eng/settings.xml $(WagonOptions)'
LoggingOptions: '-Dorg.slf4j.simpleLogger.defaultLogLevel=error -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn'
MemoryOptions: '-Xmx3072m'
#Agent.Source.Git.ShallowFetchDepth: 1
skipComponentGovernanceDetection: true
AdditionalOptions: ''
# format !<group1Id>:<artifact1Id>, !<group2Id>:<artifact2Id>,... no trailing comma is necessary
PLSkipOptions: ''
# This is the default Java build version. It's the version used to build the shipping libraries, Spotbugs etc.
JavaBuildVersion: '1.11'
# This is the default Java test version. It's the version used when running tests.
JavaTestVersion: '1.11'

# Maven build/test options
MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
# See https://github.com/actions/virtual-environments/issues/1499 for more info about the wagon options
WagonOptions: '-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120'
DefaultOptions: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) --batch-mode --fail-at-end --settings eng/settings.xml $(WagonOptions)'
LoggingOptions: '-Dorg.slf4j.simpleLogger.defaultLogLevel=error -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn'
MemoryOptions: '-Xmx3072m'
DefaultTestOptions: '$(DefaultOptions) -Dgpg.skip -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -DtrimStackTrace=false -pl $(ProjectList)'
DefaultTestMavenOptions: '$(MemoryOptions) $(LoggingOptions)'
AdditionalOptions: ''

skipComponentGovernanceDetection: true
IsDebug: $[coalesce(variables['System.Debug'], 'false')]
6 changes: 4 additions & 2 deletions sdk/cosmos/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ stages:
- template: /eng/pipelines/templates/steps/install-reporting-tools.yml
TestStepMavenInputs:
goals: verify
options: '$(ProfileFlag) -Dgpg.skip $(AdditionalArgs) -pl $(ProjectList)'
options: '$(DefaultTestOptions) $(ProfileFlag) $(AdditionalArgs)'
mavenOptions: '$(DefaultTestMavenOptions)'
TestResultsFiles: '**/junitreports/TEST-*.xml'

- template: /eng/pipelines/templates/stages/archetype-sdk-tests.yml
Expand All @@ -59,7 +60,8 @@ stages:
- template: /eng/pipelines/templates/steps/install-reporting-tools.yml
TestStepMavenInputs:
goals: clean verify
options: '$(ProfileFlag) -Dgpg.skip -Dmaven.wagon.http.pool=false $(DefaultOptions) -Dmaven.javadoc.skip=true -Drevapi.skip=true -pl $(ProjectList)'
options: '$(DefaultTestOptions) $(ProfileFlag)'
mavenOptions: '$(DefaultTestMavenOptions)'

- ${{ if or(eq(variables['Build.Reason'], 'Manual'), eq(variables['Build.Reason'], 'PullRequest')) }}:
- template: /sdk/cosmos/ci.spark.databricks.yml
Expand Down
7 changes: 5 additions & 2 deletions sdk/spring/spring-test-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,8 @@ stages:
TestResourceDirectories: ${{ parameters.TestResourceDirectories }}
Artifacts: ${{ parameters.Artifacts }}
TestStepMavenInputs:
options: '-Dmaven.wagon.http.pool=false $(DefaultOptions) -Dmaven.javadoc.skip=true -Drevapi.skip=true -DskipSpringITs=false -pl $(ProjectList)'
options: '$(DefaultTestOptions) -DskipSpringITs=false'
mavenOptions: '$(DefaultTestMavenOptions)'
goals: "verify"
PreSteps:
- script: |
Expand Down Expand Up @@ -151,7 +152,9 @@ stages:
inputs:
mavenPomFile: pom.xml
goals: verify
options: '-Dmaven.wagon.http.pool=false $(DefaultOptions) -Dmaven.javadoc.skip=true -Drevapi.skip=true -DskipSpringITs=false -pl $(ProjectList)'
options: '$(DefaultTestOptions) -DskipSpringITs=false'
Comment thread
alzimmermsft marked this conversation as resolved.
Outdated
mavenOptions: '$(DefaultTestMavenOptions)'

env:
${{ parameters.EnvVars }}

Expand Down
3 changes: 2 additions & 1 deletion sdk/storage/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ stages:
Location: canadacentral
Clouds: 'Preview'
TestStepMavenInputs:
options: '$(DefaultOptions) -T 1C -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -DtrimStackTrace=false -pl $(ProjectList)'
options: '$(DefaultTestOptions) -T 1C'
Comment thread
weshaggard marked this conversation as resolved.
Outdated
mavenOptions: '$(DefaultTestMavenOptions)'
MatrixReplace:
# Use dedicated storage pool in canadacentral with higher memory capacity
- Pool=(.*)-general/$1-storage
Expand Down