Add PoliCheck

eng/guardian-tools/policheck/PolicheckExclusions.xml

@@ -0,0 +1,11 @@
+<PoliCheckExclusions>
+<!-- Each of these exclusions is a folder name - if \[name]\ exists in the file path, it will be skipped -->
+<!--<Exclusion Type="FolderPathFull">ABC|XYZ</Exclusion>-->
+<!-- Each of these exclusions is a folder name - if any folder or file starts with "\[name]", it will be 
+skipped -->
+<!--<Exclusion Type="FolderPathStart">ABC|XYZ</Exclusion>-->
+<!-- Each of these file types will be completely skipped for the entire scan -->
+<!--<Exclusion Type="FileType">.ABC|.XYZ</Exclusion>-->
+<!-- The specified file names will be skipped during the scan regardless which folder they are in -->
+<!--<Exclusion Type="FileName">ABC.TXT|XYZ.CS</Exclusion>-->
+</PoliCheckExclusions>

eng/pipelines/aggregate-reports.yml

@@ -8,60 +8,105 @@ pr:
     include:
       - eng/pipelines/aggregate-reports.yml
 
-jobs:
-- job: 'ValidateDependencies'
-  variables:
+pool:
+  name: azsdk-pool-mms-win-2019-general
+  vmImage: MMS2019
+
+variables:
   - template: ./templates/variables/globals.yml
 
-  pool:
-    name: azsdk-pool-mms-win-2019-general
-    vmImage: MMS2019
-
-  steps:
-  - template: /eng/pipelines/templates/steps/analyze_dependency.yml
-
-  - task: AzureFileCopy@2
-    displayName: 'Upload dependency report'
-    condition: and(succeededOrFailed(), eq(variables['System.TeamProject'], 'internal'))
-    inputs:
-      sourcePath: '$(Build.ArtifactStagingDirectory)/reports'
-      azureSubscription: 'Azure SDK Artifacts'
-      destination: AzureBlob
-      storage: azuresdkartifacts
-      containerName: 'azure-sdk-for-python'
-      blobPrefix: dependencies
-
-  - task: PowerShell@2
-    displayName: "Verify Repository Resource Refs"
-    inputs:
-      pwsh: true
-      workingDirectory: $(Build.SourcesDirectory)
-      filePath: eng/common/scripts/Verify-Resource-Ref.ps1
-
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3
-    displayName: 'Run CredScan'
-    condition: succeededOrFailed()
-    inputs:
-      suppressionsFile: 'eng\CredScanSuppression.json'
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2
-    displayName: 'Post Analysis'
-    condition: succeededOrFailed()
-    inputs:
-      GdnBreakAllTools: false
-      GdnBreakGdnToolCredScan: true
-      GdnBreakGdnToolCredScanSeverity: Error
-      GdnBreakBaselineFiles: $(Build.SourcesDirectory)\eng\python.gdnbaselines
-      GdnBreakBaselines: baseline
-      # Used for generating baseline file.
-      # GdnBreakOutputBaselineFile: python
-      # GdnBreakOutputBaseline: baseline
-    continueOnError: true
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3
-    displayName: 'Publish Security Analysis Logs'
-    continueOnError: true
-    condition: succeededOrFailed()
-  - template: ../common/pipelines/templates/steps/verify-links.yml
-    parameters:
-      Directory: ""
-      CheckLinkGuidance: $true
-      Condition: succeededOrFailed()
+stages:
+  - stage: ValidateDependencies
+    displayName: Validate Dependencies
+
+    jobs:
+      - job: ValidateDependencies
+        timeoutInMinutes: 120
+        steps:
+
+          - template: /eng/pipelines/templates/steps/analyze_dependency.yml
+
+          - task: AzureFileCopy@2
+            displayName: 'Upload dependency report'
+            condition: and(succeededOrFailed(), eq(variables['System.TeamProject'], 'internal'))
+            inputs:
+              sourcePath: '$(Build.ArtifactStagingDirectory)/reports'
+              azureSubscription: 'Azure SDK Artifacts'
+              destination: AzureBlob
+              storage: azuresdkartifacts
+              containerName: 'azure-sdk-for-python'
+              blobPrefix: dependencies
+
+          - task: PowerShell@2
+            displayName: "Verify Repository Resource Refs"
+            inputs:
+              pwsh: true
+              workingDirectory: $(Build.SourcesDirectory)
+              filePath: eng/common/scripts/Verify-Resource-Ref.ps1
+
+          - template: ../common/pipelines/templates/steps/verify-links.yml
+            parameters:
+              Directory: ""
+              CheckLinkGuidance: $true
+              Condition: succeededOrFailed()
+
+  - stage: ComplianceTools
+    displayName: Compliance Tools
+    dependsOn: []
+
+    jobs:
+      - job: ComplianceTools
+        timeoutInMinutes: 120
+        steps:
+          - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3
+            displayName: 'Run CredScan'
+            condition: succeededOrFailed()
+            inputs:
+              suppressionsFile: 'eng\CredScanSuppression.json'
+
+          - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2
+            displayName: 'Post Analysis'
+            condition: succeededOrFailed()
+            inputs:
+              GdnBreakAllTools: false
+              GdnBreakGdnToolCredScan: true
+              GdnBreakGdnToolCredScanSeverity: Error
+              GdnBreakBaselineFiles: $(Build.SourcesDirectory)\eng\python.gdnbaselines
+              GdnBreakBaselines: baseline
+              # Used for generating baseline file.
+              # GdnBreakOutputBaselineFile: python
+              # GdnBreakOutputBaseline: baseline
+            continueOnError: true
+
+          - pwsh: |
+              azcopy copy "https://azuresdkartifacts.blob.core.windows.net/policheck/PythonPoliCheckExclusion.mdb?$(azuresdk-policheck-blob-SAS)" `
+              "$(Build.BinariesDirectory)"
+            displayName: 'Download PoliCheck Exclusion Database'
+            condition: succeededOrFailed()
+
+          - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@2
+            displayName: 'Run PoliCheck'
+            inputs:
+              targetType: F
+              targetArgument: '$(Build.SourcesDirectory)'
+              result: PoliCheck.sarif
+              optionsFC: 0
+              optionsXS: 1
+              optionsPE: 1|2|3|4
+              optionsRulesDBPath: "$(Build.BinariesDirectory)/PythonPoliCheckExclusion.mdb"
+              optionsUEPATH: "$(Build.SourcesDirectory)/eng/guardian-tools/policheck/PolicheckExclusions.xml"
+            condition: succeededOrFailed()
+
+          - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2
+            displayName: 'Post Analysis (PoliCheck)'
+            inputs:
+              GdnBreakAllTools: false
+              GdnBreakGdnToolPoliCheck: true
+              GdnBreakGdnToolPoliCheckSeverity: Warning
+            condition: succeededOrFailed()
+            continueOnError: true
+
+          - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3
+            displayName: 'Publish Security Analysis Logs'
+            continueOnError: true
+            condition: succeededOrFailed()

sdk/cognitiveservices/azure-cognitiveservices-vision-contentmoderator/README.md

@@ -1,23 +1,23 @@
-## Microsoft Azure SDK for Python
+# Microsoft Azure SDK for Python
 
-This is the Microsoft Azure Cognitive Services Content Moderator Client
-Library.
+This is the Microsoft Azure Cognitive Services Content Moderator Client Library.
+This package has been tested with Python 2.7, 3.6+.
+For a more complete view of Azure libraries, see the [azure sdk python release](https://aka.ms/azsdk/python/all).
 
-This package has been tested with Python 2.7, 3.4, 3.5, 3.6 and 3.7.
 
-For a more complete set of Azure libraries, see the
-[azure sdk python release](https://aka.ms/azsdk/python/all).
+# Usage
 
-## Usage
 
-For code examples, see [Cognitive Services Content
-Moderator](https://docs.microsoft.com/python/api/overview/azure/cognitive-services)
-on docs.microsoft.com.
 
-## Provide Feedback
 
-If you encounter any bugs or have suggestions, please file an issue in
-the [Issues](https://github.com/Azure/azure-sdk-for-python/issues)
-section of the project.
+For code examples, see [Cognitive Services Content Moderator](https://docs.microsoft.com/python/api/overview/azure/cognitive-services) on docs.microsoft.com.
 
-![image](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-python%2Fazure-cognitiveservices-vision-contentmoderator%2FREADME.png)
+
+# Provide Feedback
+
+If you encounter any bugs or have suggestions, please file an issue in the
+[Issues](https://github.com/Azure/azure-sdk-for-python/issues)
+section of the project. 
+
+
+![Impressions](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-python%2Fazure-cognitiveservices-vision-contentmoderator%2FREADME.png)

sdk/cognitiveservices/azure-cognitiveservices-vision-contentmoderator/azure/__init__.py

@@ -1 +1 @@
-__path__ = __import__('pkgutil').extend_path(__path__, __name__)
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

sdk/cognitiveservices/azure-cognitiveservices-vision-contentmoderator/azure/cognitiveservices/__init__.py

@@ -1 +1 @@
-__path__ = __import__('pkgutil').extend_path(__path__, __name__)
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

sdk/cognitiveservices/azure-cognitiveservices-vision-contentmoderator/azure/cognitiveservices/vision/__init__.py

@@ -1 +1 @@
-__path__ = __import__('pkgutil').extend_path(__path__, __name__)
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

sdk/cognitiveservices/azure-cognitiveservices-vision-contentmoderator/azure/cognitiveservices/vision/contentmoderator/operations/text_moderation_operations.py

@@ -35,10 +35,10 @@ def __init__(self, client, config, serializer, deserializer):
 
     def screen_text(
             self, text_content_type, text_content, language=None, autocorrect=False, pii=False, list_id=None, classify=False, custom_headers=None, raw=False, callback=None, **operation_config):
-        """Detect profanity and match against custom and shared blacklists.
+        """Detect profanity and match against custom and shared blocklists.
 
         Detects profanity in more than 100 languages and match against custom
-        and shared blacklists.
+        and shared blocklists.
 
         :param text_content_type: The content type. Possible values include:
          'text/plain', 'text/html', 'text/xml', 'text/markdown'

sdk/cognitiveservices/azure-cognitiveservices-vision-contentmoderator/setup.py

@@ -36,7 +36,9 @@
     pass
 
 # Version extraction inspired from 'requests'
-with open(os.path.join(package_folder_path, 'version.py'), 'r') as fd:
+with open(os.path.join(package_folder_path, 'version.py')
+          if os.path.exists(os.path.join(package_folder_path, 'version.py'))
+          else os.path.join(package_folder_path, '_version.py'), 'r') as fd:
     version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]',
                         fd.read(), re.MULTILINE).group(1)
 
@@ -64,10 +66,10 @@
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'License :: OSI Approved :: MIT License',
     ],
     zip_safe=False,
@@ -79,7 +81,7 @@
         'azure.cognitiveservices.vision',
     ]),
     install_requires=[
-        'msrest>=0.5.0',
+        'msrest>=0.6.21',
         'azure-common~=1.1',
     ],
     extras_require={

sdk/core/azure-core/azure/core/pipeline/policies/_redirect.py

@@ -45,14 +45,14 @@ class RedirectPolicyBase(object):
 
     REDIRECT_STATUSES = frozenset([300, 301, 302, 303, 307, 308])
 
-    REDIRECT_HEADERS_BLACKLIST = frozenset(['Authorization'])
+    REDIRECT_HEADERS_BLOCKLIST = frozenset(['Authorization'])
 
     def __init__(self, **kwargs):
         self.allow = kwargs.get('permit_redirects', True)
         self.max_redirects = kwargs.get('redirect_max', 30)
 
         remove_headers = set(kwargs.get('redirect_remove_headers', []))
-        self._remove_headers_on_redirect = remove_headers.union(self.REDIRECT_HEADERS_BLACKLIST)
+        self._remove_headers_on_redirect = remove_headers.union(self.REDIRECT_HEADERS_BLOCKLIST)
         redirect_status = set(kwargs.get('redirect_on_status_codes', []))
         self._redirect_on_status_codes = redirect_status.union(self.REDIRECT_STATUSES)
         super(RedirectPolicyBase, self).__init__()

sdk/core/azure-core/azure/core/pipeline/policies/_retry.py

@@ -73,7 +73,7 @@ def __init__(self, **kwargs):
         retry_codes = self._RETRY_CODES
         status_codes = kwargs.pop('retry_on_status_codes', [])
         self._retry_on_status_codes = set(status_codes) | retry_codes
-        self._method_whitelist = frozenset(['HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE'])
+        self._method_allowlist = frozenset(['HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE'])
         self._respect_retry_after_header = True
         super(RetryPolicyBase, self).__init__()
 
@@ -97,7 +97,7 @@ def configure_retries(self, options):
             'status': options.pop("retry_status", self.status_retries),
             'backoff': options.pop("retry_backoff_factor", self.backoff_factor),
             'max_backoff': options.pop("retry_backoff_max", self.BACKOFF_MAX),
-            'methods': options.pop("retry_on_methods", self._method_whitelist),
+            'methods': options.pop("retry_on_methods", self._method_allowlist),
             'timeout': options.pop("timeout", self.timeout),
             'history': []
         }
@@ -152,14 +152,14 @@ def _is_read_error(self, err):
 
     def _is_method_retryable(self, settings, request, response=None):
         """Checks if a given HTTP method should be retried upon, depending if
-        it is included on the method whitelist.
+        it is included on the method allowlist.
 
         :param dict settings: The retry settings.
         :param request: The PipelineRequest object.
         :type request: ~azure.core.pipeline.PipelineRequest
         :param response: The PipelineResponse object.
         :type response: ~azure.core.pipeline.PipelineResponse
-        :return: True if method should be retried upon. False if not in method whitelist.
+        :return: True if method should be retried upon. False if not in method allowlist.
         :rtype: bool
         """
         if response and request.method.upper() in ['POST', 'PATCH'] and \
@@ -173,7 +173,7 @@ def _is_method_retryable(self, settings, request, response=None):
     def is_retry(self, settings, response):
         """Checks if method/status code is retryable.
 
-        Based on whitelists and control variables such as the number of
+        Based on allowlists and control variables such as the number of
         total retries to allow, whether to respect the Retry-After header,
         whether this header is present, and whether the returned status
         code is on the list of status codes to be retried upon on the
@@ -244,7 +244,7 @@ def increment(self, settings, response=None, error=None):
 
         else:
             # Incrementing because of a server error like a 500 in
-            # status_forcelist and a the given method is in the whitelist
+            # status_forcelist and a the given method is in the allowlist
             if response:
                 settings['status'] -= 1
                 if hasattr(response, 'http_request') and hasattr(response, 'http_response'):

sdk/core/azure-core/azure/core/pipeline/policies/_universal.py

@@ -350,7 +350,7 @@ class HttpLoggingPolicy(SansIOHTTPPolicy):
     """The Pipeline policy that handles logging of HTTP requests and responses.
     """
 
-    DEFAULT_HEADERS_WHITELIST = set([
+    DEFAULT_HEADERS_ALLOWLIST = set([
         "x-ms-request-id",
         "x-ms-client-request-id",
         "x-ms-return-client-request-id",
@@ -382,7 +382,7 @@ def __init__(self, logger=None, **kwargs):  # pylint: disable=unused-argument
             "azure.core.pipeline.policies.http_logging_policy"
         )
         self.allowed_query_params = set()
-        self.allowed_header_names = set(self.__class__.DEFAULT_HEADERS_WHITELIST)
+        self.allowed_header_names = set(self.__class__.DEFAULT_HEADERS_ALLOWLIST)
 
     def _redact_query_param(self, key, value):
         lower_case_allowed_query_params = [

sdk/eventhub/azure-eventhub-checkpointstoreblob-aio/azure/eventhub/extensions/checkpointstoreblobaio/_vendor/storage/blob/_shared/policies.py

@@ -75,7 +75,7 @@ def retry_hook(settings, **kwargs):
 
 
 def is_retry(response, mode):
-    """Is this method/status code retryable? (Based on whitelists and control
+    """Is this method/status code retryable? (Based on allowlists and control
     variables such as the number of total retries to allow, whether to
     respect the Retry-After header, whether this header is present, and
     whether the returned status code is on the list of status codes to
@@ -456,7 +456,7 @@ def increment(self, settings, request, response=None, error=None):
 
         else:
             # Incrementing because of a server error like a 500 in
-            # status_forcelist and a the given method is in the whitelist
+            # status_forcelist and a the given method is in the allowlist
             if response:
                 settings['status'] -= 1
                 settings['history'].append(RequestHistory(request, http_response=response))

sdk/eventhub/azure-eventhub-checkpointstoreblob/azure/eventhub/extensions/checkpointstoreblob/_vendor/storage/blob/_shared/policies.py

@@ -75,7 +75,7 @@ def retry_hook(settings, **kwargs):
 
 
 def is_retry(response, mode):
-    """Is this method/status code retryable? (Based on whitelists and control
+    """Is this method/status code retryable? (Based on allowlists and control
     variables such as the number of total retries to allow, whether to
     respect the Retry-After header, whether this header is present, and
     whether the returned status code is on the list of status codes to
@@ -456,7 +456,7 @@ def increment(self, settings, request, response=None, error=None):
 
         else:
             # Incrementing because of a server error like a 500 in
-            # status_forcelist and a the given method is in the whitelist
+            # status_forcelist and a the given method is in the allowlist
             if response:
                 settings['status'] -= 1
                 settings['history'].append(RequestHistory(request, http_response=response))