From 9b018c7dd45d8f2ed0713a0fd4ad399db8e743cd Mon Sep 17 00:00:00 2001 From: Krista Pratico Date: Fri, 11 Jun 2021 09:39:40 -0700 Subject: [PATCH 1/5] add categories_filter to RecognizePiiEntitiesAction --- .../azure/ai/textanalytics/_models.py | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_models.py b/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_models.py index d2bfdd7dc4cc..af4762d45ca4 100644 --- a/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_models.py +++ b/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_models.py @@ -11,7 +11,7 @@ ) from ._generated.v3_0 import models as _v3_0_models -from ._generated.v3_1 import models as _latest_preview_models +from ._generated.v3_1 import models as _v3_1_models def _get_indices(relation): return [int(s) for s in re.findall(r"\d+", relation)] @@ -1466,8 +1466,8 @@ def __repr__(self, **kwargs): .format(self.model_version, self.string_index_type, self.disable_service_logs)[:1024] def to_generated(self): - return _latest_preview_models.EntitiesTask( - parameters=_latest_preview_models.EntitiesTaskParameters( + return _v3_1_models.EntitiesTask( + parameters=_v3_1_models.EntitiesTaskParameters( model_version=self.model_version, string_index_type=self.string_index_type, logging_opt_out=self.disable_service_logs, @@ -1537,8 +1537,8 @@ def __repr__(self, **kwargs): )[:1024] def to_generated(self): - return _latest_preview_models.SentimentAnalysisTask( - parameters=_latest_preview_models.SentimentAnalysisTaskParameters( + return _v3_1_models.SentimentAnalysisTask( + parameters=_v3_1_models.SentimentAnalysisTaskParameters( model_version=self.model_version, opinion_mining=self.show_opinion_mining, string_index_type=self.string_index_type, @@ -1558,6 +1558,11 @@ class RecognizePiiEntitiesAction(DictMixin): :keyword str model_version: The model version to use for the analysis. :keyword str domain_filter: An optional string to set the PII domain to include only a subset of the PII entity categories. Possible values include 'phi' or None. + :keyword categories_filter: Instead of filtering over all PII entity categories, you can pass in a list of + the specific PII entity categories you want to filter out. For example, if you only want to filter out + U.S. social security numbers in a document, you can pass in + `[PiiEntityCategoryType.US_SOCIAL_SECURITY_NUMBER]` for this kwarg. + :paramtype categories_filter: list[~azure.ai.textanalytics.PiiEntityCategoryType] :keyword str string_index_type: Specifies the method used to interpret string offsets. `UnicodeCodePoint`, the Python encoding, is the default. To override the Python default, you can also pass in `Utf16CodePoint` or TextElement_v8`. For additional information @@ -1590,23 +1595,26 @@ class RecognizePiiEntitiesAction(DictMixin): def __init__(self, **kwargs): self.model_version = kwargs.get("model_version", "latest") self.domain_filter = kwargs.get("domain_filter", None) + self.categories_filter = kwargs.get("categories_filter", None) self.string_index_type = kwargs.get("string_index_type", "UnicodeCodePoint") self.disable_service_logs = kwargs.get("disable_service_logs", False) def __repr__(self, **kwargs): - return "RecognizePiiEntitiesAction(model_version={}, domain_filter={}, string_index_type={}, "\ - "disable_service_logs={}".format( + return "RecognizePiiEntitiesAction(model_version={}, domain_filter={}, categories_filter={}, "\ + "string_index_type={}, disable_service_logs={}".format( self.model_version, self.domain_filter, + self.categories_filter, self.string_index_type, self.disable_service_logs, )[:1024] def to_generated(self): - return _latest_preview_models.PiiTask( - parameters=_latest_preview_models.PiiTaskParameters( + return _v3_1_models.PiiTask( + parameters=_v3_1_models.PiiTaskParameters( model_version=self.model_version, domain=self.domain_filter, + pii_categories=self.categories_filter, string_index_type=self.string_index_type, logging_opt_out=self.disable_service_logs ) @@ -1650,8 +1658,8 @@ def __repr__(self, **kwargs): .format(self.model_version, self.disable_service_logs)[:1024] def to_generated(self): - return _latest_preview_models.KeyPhrasesTask( - parameters=_latest_preview_models.KeyPhrasesTaskParameters( + return _v3_1_models.KeyPhrasesTask( + parameters=_v3_1_models.KeyPhrasesTaskParameters( model_version=self.model_version, logging_opt_out=self.disable_service_logs, ) @@ -1706,8 +1714,8 @@ def __repr__(self, **kwargs): )[:1024] def to_generated(self): - return _latest_preview_models.EntityLinkingTask( - parameters=_latest_preview_models.EntityLinkingTaskParameters( + return _v3_1_models.EntityLinkingTask( + parameters=_v3_1_models.EntityLinkingTaskParameters( model_version=self.model_version, string_index_type=self.string_index_type, logging_opt_out=self.disable_service_logs, From 76e0c1edc35a837ab267bdf1d3024a3115a7c88b Mon Sep 17 00:00:00 2001 From: Krista Pratico Date: Fri, 11 Jun 2021 10:13:44 -0700 Subject: [PATCH 2/5] add tests for categories_filter --- .../tests/test_analyze.py | 34 ++++++++++++++++ .../tests/test_analyze_async.py | 39 ++++++++++++++++++- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze.py b/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze.py index 96e7419257fc..231f2f87e5f6 100644 --- a/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze.py +++ b/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze.py @@ -27,6 +27,7 @@ VERSION, TextAnalyticsApiVersion, AnalyzeActionsType, + PiiEntityCategoryType ) # pre-apply the client_cls positional argument so it needn't be explicitly passed below @@ -698,3 +699,36 @@ def callback(resp): polling_interval=self._interval(), raw_response_hook=callback, ).result() + + @GlobalTextAnalyticsAccountPreparer() + @TextAnalyticsClientPreparer() + def test_pii_action_categories_filter(self, client): + + docs = [{"id": "1", "text": "My SSN is 859-98-0987."}, + {"id": "2", + "text": "Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check."}, + {"id": "3", "text": "Is 998.214.865-68 your Brazilian CPF number?"}] + + actions = [ + RecognizePiiEntitiesAction( + categories_filter=[ + PiiEntityCategoryType.US_SOCIAL_SECURITY_NUMBER, + PiiEntityCategoryType.ABA_ROUTING_NUMBER, + PiiEntityCategoryType.BRCPF_NUMBER + ] + ), + ] + + result = client.begin_analyze_actions(documents=docs, actions=actions, polling_interval=self._interval()).result() + action_results = list(result) + assert len(action_results) == 1 + action_result = action_results[0] + assert action_result.action_type == AnalyzeActionsType.RECOGNIZE_PII_ENTITIES + assert len(action_result.document_results) == len(docs) + + assert action_result.document_results[0].entities[0].text == "859-98-0987" + assert action_result.document_results[0].entities[0].category == PiiEntityCategoryType.US_SOCIAL_SECURITY_NUMBER + assert action_result.document_results[1].entities[0].text == "111000025" + assert action_result.document_results[1].entities[0].category == PiiEntityCategoryType.ABA_ROUTING_NUMBER + assert action_result.document_results[2].entities[0].text == "998.214.865-68" + assert action_result.document_results[2].entities[0].category == PiiEntityCategoryType.BRCPF_NUMBER diff --git a/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze_async.py b/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze_async.py index 6230eb51c9ce..877b5a6c7f36 100644 --- a/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze_async.py +++ b/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze_async.py @@ -29,7 +29,8 @@ RecognizePiiEntitiesAction, ExtractKeyPhrasesAction, AnalyzeSentimentAction, - AnalyzeActionsType + AnalyzeActionsType, + PiiEntityCategoryType ) # pre-apply the client_cls positional argument so it needn't be explicitly passed below @@ -739,3 +740,39 @@ async def test_disable_service_logs(self, client): actions=actions, polling_interval=self._interval(), )).result() + + @GlobalTextAnalyticsAccountPreparer() + @TextAnalyticsClientPreparer() + async def test_pii_action_categories_filter(self, client): + + docs = [{"id": "1", "text": "My SSN is 859-98-0987."}, + {"id": "2", + "text": "Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check."}, + {"id": "3", "text": "Is 998.214.865-68 your Brazilian CPF number?"}] + + actions = [ + RecognizePiiEntitiesAction( + categories_filter=[ + PiiEntityCategoryType.US_SOCIAL_SECURITY_NUMBER, + PiiEntityCategoryType.ABA_ROUTING_NUMBER, + PiiEntityCategoryType.BRCPF_NUMBER + ] + ), + ] + async with client: + result = await (await client.begin_analyze_actions(documents=docs, actions=actions, polling_interval=self._interval())).result() + action_results = [] + async for p in result: + action_results.append(p) + + assert len(action_results) == 1 + action_result = action_results[0] + assert action_result.action_type == AnalyzeActionsType.RECOGNIZE_PII_ENTITIES + assert len(action_result.document_results) == len(docs) + + assert action_result.document_results[0].entities[0].text == "859-98-0987" + assert action_result.document_results[0].entities[0].category == PiiEntityCategoryType.US_SOCIAL_SECURITY_NUMBER + assert action_result.document_results[1].entities[0].text == "111000025" + assert action_result.document_results[1].entities[0].category == PiiEntityCategoryType.ABA_ROUTING_NUMBER + assert action_result.document_results[2].entities[0].text == "998.214.865-68" + assert action_result.document_results[2].entities[0].category == PiiEntityCategoryType.BRCPF_NUMBER From 7fa2a1c919539aed8d6cbc86dae9db8bf6a27ab8 Mon Sep 17 00:00:00 2001 From: Krista Pratico Date: Fri, 11 Jun 2021 10:21:07 -0700 Subject: [PATCH 3/5] fix tests and docstring --- .../azure-ai-textanalytics/azure/ai/textanalytics/_models.py | 5 +++++ .../azure-ai-textanalytics/tests/test_analyze.py | 4 +--- .../azure-ai-textanalytics/tests/test_analyze_async.py | 3 +-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_models.py b/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_models.py index af4762d45ca4..3d6cf31eaf7f 100644 --- a/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_models.py +++ b/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_models.py @@ -1578,6 +1578,11 @@ class RecognizePiiEntitiesAction(DictMixin): :ivar str model_version: The model version to use for the analysis. :ivar str domain_filter: An optional string to set the PII domain to include only a subset of the PII entity categories. Possible values include 'phi' or None. + :ivar categories_filter: Instead of filtering over all PII entity categories, you can pass in a list of + the specific PII entity categories you want to filter out. For example, if you only want to filter out + U.S. social security numbers in a document, you can pass in + `[PiiEntityCategoryType.US_SOCIAL_SECURITY_NUMBER]` for this kwarg. + :vartype categories_filter: list[~azure.ai.textanalytics.PiiEntityCategoryType] :ivar str string_index_type: Specifies the method used to interpret string offsets. `UnicodeCodePoint`, the Python encoding, is the default. To override the Python default, you can also pass in `Utf16CodePoint` or TextElement_v8`. For additional information diff --git a/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze.py b/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze.py index 231f2f87e5f6..a74c7b9026fa 100644 --- a/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze.py +++ b/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze.py @@ -714,7 +714,6 @@ def test_pii_action_categories_filter(self, client): categories_filter=[ PiiEntityCategoryType.US_SOCIAL_SECURITY_NUMBER, PiiEntityCategoryType.ABA_ROUTING_NUMBER, - PiiEntityCategoryType.BRCPF_NUMBER ] ), ] @@ -730,5 +729,4 @@ def test_pii_action_categories_filter(self, client): assert action_result.document_results[0].entities[0].category == PiiEntityCategoryType.US_SOCIAL_SECURITY_NUMBER assert action_result.document_results[1].entities[0].text == "111000025" assert action_result.document_results[1].entities[0].category == PiiEntityCategoryType.ABA_ROUTING_NUMBER - assert action_result.document_results[2].entities[0].text == "998.214.865-68" - assert action_result.document_results[2].entities[0].category == PiiEntityCategoryType.BRCPF_NUMBER + assert action_result.document_results[2].entities[0].entities == [] # No Brazilian CPF since not in categories_filter diff --git a/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze_async.py b/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze_async.py index 877b5a6c7f36..1baeb08126a5 100644 --- a/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze_async.py +++ b/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze_async.py @@ -774,5 +774,4 @@ async def test_pii_action_categories_filter(self, client): assert action_result.document_results[0].entities[0].category == PiiEntityCategoryType.US_SOCIAL_SECURITY_NUMBER assert action_result.document_results[1].entities[0].text == "111000025" assert action_result.document_results[1].entities[0].category == PiiEntityCategoryType.ABA_ROUTING_NUMBER - assert action_result.document_results[2].entities[0].text == "998.214.865-68" - assert action_result.document_results[2].entities[0].category == PiiEntityCategoryType.BRCPF_NUMBER + assert action_result.document_results[2].entities[0].entities == [] # No Brazilian CPF since not in categories_filter From 3da157ac5909d07658f53fc6082902f2d6272194 Mon Sep 17 00:00:00 2001 From: Krista Pratico Date: Fri, 11 Jun 2021 10:21:20 -0700 Subject: [PATCH 4/5] update changelog --- sdk/textanalytics/azure-ai-textanalytics/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/textanalytics/azure-ai-textanalytics/CHANGELOG.md b/sdk/textanalytics/azure-ai-textanalytics/CHANGELOG.md index 953e4956eeaa..95e15caa5494 100644 --- a/sdk/textanalytics/azure-ai-textanalytics/CHANGELOG.md +++ b/sdk/textanalytics/azure-ai-textanalytics/CHANGELOG.md @@ -2,6 +2,8 @@ ## 5.1.0b8 (Unreleased) +**New Features** +- Added `catagories_filter` to `RecognizePiiEntitiesAction` ## 5.1.0b7 (2021-05-18) From 64eca22e17602f6a9da30f236967a2c2ceb0bb7f Mon Sep 17 00:00:00 2001 From: Krista Pratico Date: Tue, 15 Jun 2021 09:38:31 -0700 Subject: [PATCH 5/5] updating categories filter tests to new analyze design --- .../tests/test_analyze.py | 17 +++++++---------- .../tests/test_analyze_async.py | 18 +++++++----------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze.py b/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze.py index adefa18b3ba4..b6f0a0071cf8 100644 --- a/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze.py +++ b/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze.py @@ -701,13 +701,10 @@ def test_pii_action_categories_filter(self, client): result = client.begin_analyze_actions(documents=docs, actions=actions, polling_interval=self._interval()).result() action_results = list(result) - assert len(action_results) == 1 - action_result = action_results[0] - assert action_result.action_type == _AnalyzeActionsType.RECOGNIZE_PII_ENTITIES - assert len(action_result.document_results) == len(docs) - - assert action_result.document_results[0].entities[0].text == "859-98-0987" - assert action_result.document_results[0].entities[0].category == PiiEntityCategoryType.US_SOCIAL_SECURITY_NUMBER - assert action_result.document_results[1].entities[0].text == "111000025" - assert action_result.document_results[1].entities[0].category == PiiEntityCategoryType.ABA_ROUTING_NUMBER - assert action_result.document_results[2].entities[0].entities == [] # No Brazilian CPF since not in categories_filter + assert len(action_results) == 3 + + assert action_results[0][0].entities[0].text == "859-98-0987" + assert action_results[0][0].entities[0].category == PiiEntityCategoryType.US_SOCIAL_SECURITY_NUMBER + assert action_results[1][0].entities[0].text == "111000025" + assert action_results[1][0].entities[0].category == PiiEntityCategoryType.ABA_ROUTING_NUMBER + assert action_results[2][0].entities == [] # No Brazilian CPF since not in categories_filter diff --git a/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze_async.py b/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze_async.py index 3eb41a4a7e41..eba9d69ea505 100644 --- a/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze_async.py +++ b/sdk/textanalytics/azure-ai-textanalytics/tests/test_analyze_async.py @@ -742,8 +742,7 @@ async def test_pii_action_categories_filter(self, client): RecognizePiiEntitiesAction( categories_filter=[ PiiEntityCategoryType.US_SOCIAL_SECURITY_NUMBER, - PiiEntityCategoryType.ABA_ROUTING_NUMBER, - PiiEntityCategoryType.BRCPF_NUMBER + PiiEntityCategoryType.ABA_ROUTING_NUMBER ] ), ] @@ -753,13 +752,10 @@ async def test_pii_action_categories_filter(self, client): async for p in result: action_results.append(p) - assert len(action_results) == 1 - action_result = action_results[0] - assert action_result.action_type == _AnalyzeActionsType.RECOGNIZE_PII_ENTITIES - assert len(action_result.document_results) == len(docs) + assert len(action_results) == 3 - assert action_result.document_results[0].entities[0].text == "859-98-0987" - assert action_result.document_results[0].entities[0].category == PiiEntityCategoryType.US_SOCIAL_SECURITY_NUMBER - assert action_result.document_results[1].entities[0].text == "111000025" - assert action_result.document_results[1].entities[0].category == PiiEntityCategoryType.ABA_ROUTING_NUMBER - assert action_result.document_results[2].entities[0].entities == [] # No Brazilian CPF since not in categories_filter + assert action_results[0][0].entities[0].text == "859-98-0987" + assert action_results[0][0].entities[0].category == PiiEntityCategoryType.US_SOCIAL_SECURITY_NUMBER + assert action_results[1][0].entities[0].text == "111000025" + assert action_results[1][0].entities[0].category == PiiEntityCategoryType.ABA_ROUTING_NUMBER + assert action_results[2][0].entities == [] # No Brazilian CPF since not in categories_filter