From b1ca1b7e45178702e604bd18bb879be9ba84c9e8 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 28 Jan 2022 11:16:59 +0800 Subject: [PATCH 001/120] fix auto-ask-check bug --- scripts/release_issue_status/main.py | 22 +++++++------------ .../release_issue_status.yml | 6 +++++ .../release_issue_status/reply_generator.py | 3 ++- scripts/release_issue_status/requirement.txt | 3 +-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/scripts/release_issue_status/main.py b/scripts/release_issue_status/main.py index eff130e04ba5..f892a08eae46 100644 --- a/scripts/release_issue_status/main.py +++ b/scripts/release_issue_status/main.py @@ -7,9 +7,8 @@ import logging from github import Github -from azure.storage.blob import BlobClient +from reply_generator import AUTO_ASK_FOR_CHECK, begin_reply_generate -import reply_generator as rg from utils import update_issue_body, get_readme_and_output_folder, \ get_python_pipelines, get_pipeline_url, auto_close_issue @@ -144,7 +143,7 @@ def _latest_comment_time(comments, delay_from_create_date): return delay_from_create_date if not q else int((time.time() - q[-1][0]) / 3600 / 24) -def auto_reply(item, request_repo, rest_repo, sdk_repo, duplicated_issue, python_piplines, assigner_repoes): +def auto_reply(item, request_repo, rest_repo, duplicated_issue, python_piplines, assigner_repoes): logging.info("new issue number: {}".format(item.issue_object.number)) assigner_repo = assigner_repoes[item.assignee] if 'auto-link' not in item.labels: @@ -171,8 +170,7 @@ def auto_reply(item, request_repo, rest_repo, sdk_repo, duplicated_issue, python raise try: pipeline_url = get_pipeline_url(python_piplines, output_folder) - rg.begin_reply_generate(item=item, rest_repo=rest_repo, readme_link=readme_link, - pipeline_url=pipeline_url) + begin_reply_generate(item=item, rest_repo=rest_repo, readme_link=readme_link, pipeline_url=pipeline_url) if 'Configured' in item.labels: item.issue_object.remove_from_labels('Configured') except Exception as e: @@ -253,10 +251,11 @@ def main(): item.issue_object.add_to_assignees(_PYTHON_SDK_ASSIGNEES[assign_count]) item.assignee = item.issue_object.assignee.login item.issue_object.add_to_labels('assigned') - try: - auto_reply(item, request_repo, rest_repo, sdk_repo, duplicated_issue, python_piplines, assigner_repoes) - except Exception as e: - continue + if AUTO_ASK_FOR_CHECK not in item.labels: + try: + auto_reply(item, request_repo, rest_repo, duplicated_issue, python_piplines, assigner_repoes) + except Exception as e: + continue elif not item.author_latest_comment in _PYTHON_SDK_ADMINISTRATORS: item.bot_advice = 'new comment.
' if item.comment_num > 1 and item.language == 'Python': @@ -295,11 +294,6 @@ def main(): print_check('git commit -m \"update excel\"') print_check('git push -f origin HEAD') -# upload to storage account(it is created in advance) -# blob = BlobClient.from_connection_string(conn_str=os.getenv('CONN_STR'), container_name=os.getenv('FILE'), -# blob_name=_FILE_OUT) -# with open(_FILE_OUT, 'rb') as data: -# blob.upload_blob(data, overwrite=True) if __name__ == '__main__': main() diff --git a/scripts/release_issue_status/release_issue_status.yml b/scripts/release_issue_status/release_issue_status.yml index d79e7dd19b2d..2e384ec47b36 100644 --- a/scripts/release_issue_status/release_issue_status.yml +++ b/scripts/release_issue_status/release_issue_status.yml @@ -7,6 +7,12 @@ trigger: exclude: - '*' +# avoid being triggered as part of CI check +pr: + branches: + exclude: + - '*' + schedules: - cron: "0,30 1-9 * * *" displayName: Daily release diff --git a/scripts/release_issue_status/reply_generator.py b/scripts/release_issue_status/reply_generator.py index 3659f19bd33d..5a3ebc6b206c 100644 --- a/scripts/release_issue_status/reply_generator.py +++ b/scripts/release_issue_status/reply_generator.py @@ -6,6 +6,7 @@ logging.basicConfig(level=logging.INFO, format='[auto-reply log] - %(funcName)s[line:%(lineno)d] - %(levelname)s: %(message)s') +AUTO_ASK_FOR_CHECK = 'auto-ask-check' def readme_comparison(rest_repo, link_dict, labels): # to see whether need change readme @@ -68,6 +69,6 @@ def begin_reply_generate(item, rest_repo, readme_link, pipeline_url): logging.info(f'{issue_object_rg.number} run pipeline successfully') else: logging.info(f'{issue_object_rg.number} run pipeline fail') - issue_object_rg.add_to_labels('auto-ask-check') + issue_object_rg.add_to_labels(AUTO_ASK_FOR_CHECK) else: logging.info('issue {} need config readme'.format(issue_object_rg.number)) diff --git a/scripts/release_issue_status/requirement.txt b/scripts/release_issue_status/requirement.txt index 512d9702666a..95f800db46db 100644 --- a/scripts/release_issue_status/requirement.txt +++ b/scripts/release_issue_status/requirement.txt @@ -2,6 +2,5 @@ PyGithub datetime requests bs4 -azure.storage.blob==12.8.1 -azure-devops +azure-devops msrest From 5ae58ab80ac68b7f25e74ad99abe0b4e5894fc2a Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Wed, 16 Feb 2022 17:07:08 +0800 Subject: [PATCH 002/120] update comment --- scripts/auto_release/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 34e56022cd2e..eb2f1a746e27 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -410,6 +410,11 @@ def check_file(self): def sdk_code_path(self) -> str: return str(Path(f'sdk/{self.sdk_folder}/azure-mgmt-{self.package_name}')) + @property + def whether_single_path(self) -> bool: + path = sum([os.path.isdir(listx) for listx in os.listdir(str(Path(f'sdk/{self.sdk_folder}'))]) + return True if path==1 else False + @return_origin_path def install_package_locally(self): os.chdir(self.sdk_code_path()) @@ -454,6 +459,8 @@ def create_pr_proc(self): pr_head = "{}:{}".format(os.getenv('USR_NAME'), self.new_branch) pr_base = 'main' pr_body = "{} \n{} \n{}".format(self.issue_link, self.test_result, self.pipeline_link) + if not self.whether_single_path: + pr_body += f'\nBuildTargetingString: {self.package_name}\nSkip.CreateApiReview: true' res_create = api.pulls.create(pr_title, pr_head, pr_base, pr_body) # Add issue link on PR From 79a3594aad784b3da87b13f474d4679c18e53bda Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Wed, 16 Feb 2022 17:26:21 +0800 Subject: [PATCH 003/120] update comment --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index eb2f1a746e27..b8e4ef288b1f 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -412,7 +412,7 @@ def sdk_code_path(self) -> str: @property def whether_single_path(self) -> bool: - path = sum([os.path.isdir(listx) for listx in os.listdir(str(Path(f'sdk/{self.sdk_folder}'))]) + path = sum([os.path.isdir(listx) for listx in os.listdir(str(Path(f'sdk/{self.sdk_folder}')))]) return True if path==1 else False @return_origin_path From 7071377db0a8fcbdc037acc47beaed7ab13c0572 Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Thu, 17 Feb 2022 10:03:42 +0800 Subject: [PATCH 004/120] fix bug --- scripts/auto_release/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index b8e4ef288b1f..4e4aa35ca259 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -412,8 +412,9 @@ def sdk_code_path(self) -> str: @property def whether_single_path(self) -> bool: - path = sum([os.path.isdir(listx) for listx in os.listdir(str(Path(f'sdk/{self.sdk_folder}')))]) - return True if path==1 else False + path = str(Path(f'sdk/{self.sdk_folder}')) + num = sum([os.path.isdir(f'{path}/{listx}') for listx in os.listdir(path)]) + return True if num == 1 else False @return_origin_path def install_package_locally(self): @@ -460,7 +461,7 @@ def create_pr_proc(self): pr_base = 'main' pr_body = "{} \n{} \n{}".format(self.issue_link, self.test_result, self.pipeline_link) if not self.whether_single_path: - pr_body += f'\nBuildTargetingString: {self.package_name}\nSkip.CreateApiReview: true' + pr_body += f'\nBuildTargetingString\n azure-mgmt-{self.package_name}\nSkip.CreateApiReview\ntrue' res_create = api.pulls.create(pr_title, pr_head, pr_base, pr_body) # Add issue link on PR From 8f6fe79664db8c2957009fcc3058c85d68c3ccfd Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Thu, 17 Feb 2022 10:26:05 +0800 Subject: [PATCH 005/120] Update main.py --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 4e4aa35ca259..97620c4f9b64 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -414,7 +414,7 @@ def sdk_code_path(self) -> str: def whether_single_path(self) -> bool: path = str(Path(f'sdk/{self.sdk_folder}')) num = sum([os.path.isdir(f'{path}/{listx}') for listx in os.listdir(path)]) - return True if num == 1 else False + return num == 1 @return_origin_path def install_package_locally(self): From 8d23d87d7dea6d635316ac75fddc1f37aaeef644 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 17 Feb 2022 10:44:21 +0800 Subject: [PATCH 006/120] Update main.py --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 97620c4f9b64..ef846cc0534a 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -413,7 +413,7 @@ def sdk_code_path(self) -> str: @property def whether_single_path(self) -> bool: path = str(Path(f'sdk/{self.sdk_folder}')) - num = sum([os.path.isdir(f'{path}/{listx}') for listx in os.listdir(path)]) + num = sum([os.path.isdir(str(Path(f'{path}/{listx}'))) for listx in os.listdir(path)]) return num == 1 @return_origin_path From 44faa80efb2b2b16c65e93f55df1432feb4cb8f1 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 17 Feb 2022 10:46:51 +0800 Subject: [PATCH 007/120] Update main.py --- scripts/auto_release/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index ef846cc0534a..b6a791bda638 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -411,7 +411,7 @@ def sdk_code_path(self) -> str: return str(Path(f'sdk/{self.sdk_folder}/azure-mgmt-{self.package_name}')) @property - def whether_single_path(self) -> bool: + def is_single_path(self) -> bool: path = str(Path(f'sdk/{self.sdk_folder}')) num = sum([os.path.isdir(str(Path(f'{path}/{listx}'))) for listx in os.listdir(path)]) return num == 1 @@ -460,7 +460,7 @@ def create_pr_proc(self): pr_head = "{}:{}".format(os.getenv('USR_NAME'), self.new_branch) pr_base = 'main' pr_body = "{} \n{} \n{}".format(self.issue_link, self.test_result, self.pipeline_link) - if not self.whether_single_path: + if not self.is_single_path: pr_body += f'\nBuildTargetingString\n azure-mgmt-{self.package_name}\nSkip.CreateApiReview\ntrue' res_create = api.pulls.create(pr_title, pr_head, pr_base, pr_body) From 84768ebf73ec5cb679abd0a91a97651cb6ee2daf Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Fri, 18 Feb 2022 11:01:16 +0800 Subject: [PATCH 008/120] fix changelog format for new service --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index b6a791bda638..1b129be68cce 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -367,7 +367,7 @@ def edit_changelog_for_new_service(self): def edit_changelog_for_new_service_proc(content: List[str]): for i in range(0, len(content)): if '##' in content[i]: - content[i] = f'## {self.next_version}({current_time()})' + content[i] = f'## {self.next_version}({current_time()})\n\n' break modify_file(str(Path(self.sdk_code_path()) / 'CHANGELOG.md'), edit_changelog_for_new_service_proc) From 29f9a86b805ca6f507a0fde8fd6f70ea36d4c19d Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Fri, 18 Feb 2022 15:39:12 +0800 Subject: [PATCH 009/120] fix check_pprint_name bug --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 1b129be68cce..ea59218a1b1c 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -266,7 +266,7 @@ def edit_file_for_pprint_name(content: List[str]): for file in os.listdir(self.sdk_code_path()): if os.path.isfile(file): - modify_file(file, edit_file_for_pprint_name) + modify_file(str(Path(self.sdk_code_path()) / file), edit_file_for_pprint_name) log(f' replace \"MyService\" with \"{pprint_name}\" successfully ') def get_all_files_under_package_folder(self) -> List[str]: From 5b11c1267cd1e2f0480f0b8b67b3e021c9e4b042 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 18 Feb 2022 17:14:49 +0800 Subject: [PATCH 010/120] Update main.py --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index ea59218a1b1c..f3b229318e8c 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -367,7 +367,7 @@ def edit_changelog_for_new_service(self): def edit_changelog_for_new_service_proc(content: List[str]): for i in range(0, len(content)): if '##' in content[i]: - content[i] = f'## {self.next_version}({current_time()})\n\n' + content[i] = f'## {self.next_version}({current_time()})\n' break modify_file(str(Path(self.sdk_code_path()) / 'CHANGELOG.md'), edit_changelog_for_new_service_proc) From 6b13133eb4920748e9d34d317117b5bec2d4aad9 Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Fri, 18 Feb 2022 17:35:29 +0800 Subject: [PATCH 011/120] fix check_pprint_name bug --- scripts/auto_release/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index ea59218a1b1c..5f56b69f5677 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -265,8 +265,9 @@ def edit_file_for_pprint_name(content: List[str]): content[i] = content[i].replace('MyService', pprint_name) for file in os.listdir(self.sdk_code_path()): - if os.path.isfile(file): - modify_file(str(Path(self.sdk_code_path()) / file), edit_file_for_pprint_name) + file_path = str(Path(self.sdk_code_path()) / file) + if os.path.isfile(file_path): + modify_file(file_path, edit_file_for_pprint_name) log(f' replace \"MyService\" with \"{pprint_name}\" successfully ') def get_all_files_under_package_folder(self) -> List[str]: From c53585bd9a43bb28d99907fe0c9a52692c7d4a9e Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 21 Feb 2022 10:02:18 +0800 Subject: [PATCH 012/120] Update main.py --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 89f44635cbf6..be3277b51dba 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -368,7 +368,7 @@ def edit_changelog_for_new_service(self): def edit_changelog_for_new_service_proc(content: List[str]): for i in range(0, len(content)): if '##' in content[i]: - content[i] = f'## {self.next_version}({current_time()})\n' + content[i] = f'## {self.next_version} ({current_time()})\n' break modify_file(str(Path(self.sdk_code_path()) / 'CHANGELOG.md'), edit_changelog_for_new_service_proc) From 90ab52db6e0b0bb00dc8b5ee0b3ae5e62b8b496e Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 4 Mar 2022 10:34:22 +0800 Subject: [PATCH 013/120] Update main.py --- scripts/auto_release/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index be3277b51dba..adeaa514a600 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -549,7 +549,8 @@ def create_pr(self): # commit all code print_exec('git add sdk/') print_exec('git commit -m \"code and test\"') - print_check('git push origin HEAD -f') + print_exec('git remote add azure https://github.com/Azure/azure-sdk-for-python.git') + print_check('git push azure HEAD -f') # create PR self.create_pr_proc() From c12493377629bfa23b137ce47bc4cbd3929c0742 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 15:15:54 +0800 Subject: [PATCH 014/120] test --- scripts/auto_release/main.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index adeaa514a600..ae1ab4c4685a 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -44,6 +44,10 @@ def print_exec_output(cmd: str) -> List[str]: def print_check(cmd): log(cmd) subprocess.check_call(cmd, shell=True) + +def print_check_with_path(cmd, path): + log(cmd) + subprocess.check_call(cmd, shell=True, cwd=path) def preview_version_plus(preview_label: str, last_version: str) -> str: @@ -257,6 +261,11 @@ def edit_sdk_setup(content: List[str]): modify_file(str(Path(self.sdk_code_path()) / 'setup.py'), edit_sdk_setup) + # Use the template to update readme and setup by packaging_tools + def check_sdk_readme_and_setup(self): + path = self.sdk_folder + print_check_with_path(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}',path) + def check_pprint_name(self): pprint_name = self.package_name.capitalize() @@ -403,6 +412,8 @@ def check_ci_file(self): def check_file(self): self.check_pprint_name() + # self.check_sdk_readme_and_setup() + print(f'**{self.sdk_folder=}') self.check_sdk_setup() self.check_version() self.check_changelog_file() From e6f89a28194227fc445630a5219d68e92b6b865c Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 15:33:30 +0800 Subject: [PATCH 015/120] test --- scripts/auto_release/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index ae1ab4c4685a..f3bbf7354eba 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -263,8 +263,9 @@ def edit_sdk_setup(content: List[str]): # Use the template to update readme and setup by packaging_tools def check_sdk_readme_and_setup(self): - path = self.sdk_folder + path = str(Path(f'sdk/{self.sdk_folder}')) print_check_with_path(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}',path) + log(f' packaging_tools --build-conf successfully ') def check_pprint_name(self): pprint_name = self.package_name.capitalize() @@ -412,9 +413,8 @@ def check_ci_file(self): def check_file(self): self.check_pprint_name() - # self.check_sdk_readme_and_setup() - print(f'**{self.sdk_folder=}') - self.check_sdk_setup() + self.check_sdk_readme_and_setup() + # self.check_sdk_setup() self.check_version() self.check_changelog_file() self.check_ci_file() From 76faf366ed883ff42f43201e443e42d529b97329 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 15:38:38 +0800 Subject: [PATCH 016/120] change to azclibot --- scripts/auto_release/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index f3bbf7354eba..6c19b2ad524b 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -560,8 +560,8 @@ def create_pr(self): # commit all code print_exec('git add sdk/') print_exec('git commit -m \"code and test\"') - print_exec('git remote add azure https://github.com/Azure/azure-sdk-for-python.git') - print_check('git push azure HEAD -f') + print_exec('git remote add azclibot https://github.com/azclibot/azure-sdk-for-python.git') + print_check('git push azclibot HEAD -f') # create PR self.create_pr_proc() From ffb8635ac54beb77a939a8d6425d40449fa264a5 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 16:03:04 +0800 Subject: [PATCH 017/120] change to azure --- scripts/auto_release/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 6c19b2ad524b..e7b3c1f1fd25 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -560,8 +560,9 @@ def create_pr(self): # commit all code print_exec('git add sdk/') print_exec('git commit -m \"code and test\"') - print_exec('git remote add azclibot https://github.com/azclibot/azure-sdk-for-python.git') - print_check('git push azclibot HEAD -f') + print_exec('git remote add azure https://github.com/azure/azure-sdk-for-python.git') + res1 = print_exec_output('git push azure HEAD -f') + print(f'{res1=}') # create PR self.create_pr_proc() From 40a0f7469c08c85052d1b7970a8b776b5b39a20f Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 16:28:05 +0800 Subject: [PATCH 018/120] fix push --- scripts/auto_release/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index e7b3c1f1fd25..907319d6dd27 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -560,9 +560,7 @@ def create_pr(self): # commit all code print_exec('git add sdk/') print_exec('git commit -m \"code and test\"') - print_exec('git remote add azure https://github.com/azure/azure-sdk-for-python.git') - res1 = print_exec_output('git push azure HEAD -f') - print(f'{res1=}') + print_exec('git push origin HEAD -f') # create PR self.create_pr_proc() From 2f6b17443de2adc53d98e0663c596920e57c9cde Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 16:55:04 +0800 Subject: [PATCH 019/120] test --- scripts/auto_release/main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 907319d6dd27..f57c7fc22d08 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -85,8 +85,10 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): - usr = 'Azure' - branch = 'main' + # usr = 'Azure' + usr = 'azclibot' + # branch = 'main' + branch = 't2-advisor-2022-03-09-54263' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From d5ce82678a6873af4ecb7add4e8d357aa80cddf2 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 17:16:59 +0800 Subject: [PATCH 020/120] update --- scripts/auto_release/main.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index f57c7fc22d08..840d003da8f6 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -85,10 +85,8 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): - # usr = 'Azure' - usr = 'azclibot' - # branch = 'main' - branch = 't2-advisor-2022-03-09-54263' + usr = 'Azure' + branch = 'main' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') @@ -416,7 +414,6 @@ def check_ci_file(self): def check_file(self): self.check_pprint_name() self.check_sdk_readme_and_setup() - # self.check_sdk_setup() self.check_version() self.check_changelog_file() self.check_ci_file() @@ -562,7 +559,7 @@ def create_pr(self): # commit all code print_exec('git add sdk/') print_exec('git commit -m \"code and test\"') - print_exec('git push origin HEAD -f') + print_check('git push origin HEAD -f') # create PR self.create_pr_proc() From 79a553e181ba5df0ff1df165259a6804c5dbb7f3 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 17:35:48 +0800 Subject: [PATCH 021/120] update --- scripts/auto_release/main.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 840d003da8f6..3d56acdb7b67 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -44,10 +44,6 @@ def print_exec_output(cmd: str) -> List[str]: def print_check(cmd): log(cmd) subprocess.check_call(cmd, shell=True) - -def print_check_with_path(cmd, path): - log(cmd) - subprocess.check_call(cmd, shell=True, cwd=path) def preview_version_plus(preview_label: str, last_version: str) -> str: @@ -262,9 +258,10 @@ def edit_sdk_setup(content: List[str]): modify_file(str(Path(self.sdk_code_path()) / 'setup.py'), edit_sdk_setup) # Use the template to update readme and setup by packaging_tools + @return_origin_path def check_sdk_readme_and_setup(self): - path = str(Path(f'sdk/{self.sdk_folder}')) - print_check_with_path(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}',path) + os.chdir(Path(f'sdk/{self.sdk_folder}')) + print_check(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}') log(f' packaging_tools --build-conf successfully ') def check_pprint_name(self): @@ -414,6 +411,8 @@ def check_ci_file(self): def check_file(self): self.check_pprint_name() self.check_sdk_readme_and_setup() + self.check_sdk_readme() + self.check_sdk_setup() self.check_version() self.check_changelog_file() self.check_ci_file() From ec747f10069fc9a93a933dc1873cc3d21b626a3e Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 17:36:48 +0800 Subject: [PATCH 022/120] test --- scripts/auto_release/main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 3d56acdb7b67..301eb40de681 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -81,8 +81,10 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): - usr = 'Azure' - branch = 'main' + # usr = 'Azure' + # branch = 'main' + usr = 'azclibot' + branch = 't2-advisor-2022-03-09-54263' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From 6f51fc8d963e90275e951bbc9deac79d04aa25d7 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 17:48:34 +0800 Subject: [PATCH 023/120] test --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 301eb40de681..b58cec549b3e 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -241,7 +241,7 @@ def prepare_branch(self): # self.prepare_branch_with_base_branch() def check_sdk_readme(self): - sdk_readme = str(Path(f'sdk/{self.sdk_folder}/{self.package_name}/README.md')) + sdk_readme = str(Path(f'sdk/{self.sdk_folder}/azure-mgmt-{self.package_name}/README.md')) def edit_sdk_readme(content: List[str]): for i in range(0, len(content)): From 7107723a4d1ada144363da3cb3e0e03cecbb426a Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 17:57:28 +0800 Subject: [PATCH 024/120] reduction --- scripts/auto_release/main.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index b58cec549b3e..347e9c4bc671 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -81,10 +81,8 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): - # usr = 'Azure' - # branch = 'main' - usr = 'azclibot' - branch = 't2-advisor-2022-03-09-54263' + usr = 'Azure' + branch = 'main' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From 6885266bf40f30462904c8720dcdfb60e4dfb06d Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 11 Mar 2022 10:35:37 +0800 Subject: [PATCH 025/120] update main --- scripts/auto_release/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 347e9c4bc671..86667e0648aa 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -259,7 +259,7 @@ def edit_sdk_setup(content: List[str]): # Use the template to update readme and setup by packaging_tools @return_origin_path - def check_sdk_readme_and_setup(self): + def check_file_with_packaging_tool(self): os.chdir(Path(f'sdk/{self.sdk_folder}')) print_check(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}') log(f' packaging_tools --build-conf successfully ') @@ -409,8 +409,8 @@ def check_ci_file(self): self.check_ci_file_proc('azure-mgmt-core>=1.3.0,<2.0.0') def check_file(self): + self.check_file_with_packaging_tool() self.check_pprint_name() - self.check_sdk_readme_and_setup() self.check_sdk_readme() self.check_sdk_setup() self.check_version() From 6c9bbbd6d39b9511ff5cece8fc7b7a78eb09f6bf Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 11 Mar 2022 13:26:47 +0800 Subject: [PATCH 026/120] delete f --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 86667e0648aa..1eba85942dde 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -262,7 +262,7 @@ def edit_sdk_setup(content: List[str]): def check_file_with_packaging_tool(self): os.chdir(Path(f'sdk/{self.sdk_folder}')) print_check(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}') - log(f' packaging_tools --build-conf successfully ') + log(' packaging_tools --build-conf successfully ') def check_pprint_name(self): pprint_name = self.package_name.capitalize() From c2cb7e410d7bcda61ddc240e5a26a6ab0c0316c2 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Mon, 28 Mar 2022 10:13:52 +0800 Subject: [PATCH 027/120] Update main.py --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 1eba85942dde..ed36d5c9cd93 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -262,7 +262,7 @@ def edit_sdk_setup(content: List[str]): def check_file_with_packaging_tool(self): os.chdir(Path(f'sdk/{self.sdk_folder}')) print_check(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}') - log(' packaging_tools --build-conf successfully ') + log('packaging_tools --build-conf successfully ') def check_pprint_name(self): pprint_name = self.package_name.capitalize() From 5164abe59c5c24f5dc00370d24bc09827218fcaf Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Mon, 28 Mar 2022 10:40:56 +0800 Subject: [PATCH 028/120] test --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index ed36d5c9cd93..633f884f6e26 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -82,7 +82,7 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): usr = 'Azure' - branch = 'main' + branch = 'test_autorest_03_18' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From cdb30c42eb9c87048c722ef398bd27fd5550e654 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Mon, 28 Mar 2022 11:22:00 +0800 Subject: [PATCH 029/120] test --- scripts/auto_release/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 633f884f6e26..9655fd7e980d 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -173,10 +173,10 @@ def generate_code(self): # prepare input data input_data = { - 'headSha': self.get_latest_commit_in_swagger_repo(), + 'headSha': '90985ff9930614361e97034b3f149ea73efec6cb', 'repoHttpsUrl': "https://github.com/Azure/azure-rest-api-specs", - 'specFolder': self.spec_repo, - 'relatedReadmeMdFiles': [str(self.readme_local_folder())] + 'specFolder': 'https://github.com/Azure/azure-rest-api-specs/blob/main/specification/hdinsight/resource-manager/readme.md', + 'relatedReadmeMdFiles': ['specification/hdinsight/resource-manager/readme.md'] } self.autorest_result = str(Path(os.getenv('TEMP_FOLDER')) / 'temp.json') From e173adc9f1f872a55b9819e623b80c4d427883a4 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Mon, 28 Mar 2022 14:19:34 +0800 Subject: [PATCH 030/120] test --- scripts/auto_release/main.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 9655fd7e980d..40c98b38579b 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -173,10 +173,23 @@ def generate_code(self): # prepare input data input_data = { - 'headSha': '90985ff9930614361e97034b3f149ea73efec6cb', - 'repoHttpsUrl': "https://github.com/Azure/azure-rest-api-specs", - 'specFolder': 'https://github.com/Azure/azure-rest-api-specs/blob/main/specification/hdinsight/resource-manager/readme.md', - 'relatedReadmeMdFiles': ['specification/hdinsight/resource-manager/readme.md'] + "dryRun": False, + "specFolder": "../azure-rest-api-specs", + "headSha": "9ac082c33a7d0d8fa6768bd64e53394ada9baaf9", + "headRef": "master", + "repoHttpsUrl": "https://github.com/Azure/azure-rest-api-specs", + "trigger": "continuousIntegration", + "changedFiles": [ + "specification/hdinsight/resource-manager/readme.python.md" + ], "relatedReadmeMdFiles": [ + "specification/hdinsight/resource-manager/readme.md" + ], + "installInstructionInput": { + "isPublic": False, + "downloadUrlPrefix": "https://portal.azure-devex-tools.com/api/sdk-dl-pub?p=Azure/13991/azure-sdk-for-python-track2/", + "downloadCommandTemplate": "curl -L \"{URL}\" -o {FILENAME}", + "trigger": "continuousIntegration" + } } self.autorest_result = str(Path(os.getenv('TEMP_FOLDER')) / 'temp.json') From 04baa8aaa7a2e3d869e1d17fcff636f605ce0da8 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Mon, 28 Mar 2022 15:02:25 +0800 Subject: [PATCH 031/120] Update PythonSdkLiveTest.yml for Azure Pipelines --- scripts/auto_release/PythonSdkLiveTest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index c5b16b57b579..65e726c948c0 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -46,6 +46,8 @@ jobs: if [ "$(SPEC_README)" ]; then mkdir azure-rest-api-specs git clone https://github.com/Azure/azure-rest-api-specs.git $(pwd)/azure-rest-api-specs + git fetch origin test-hdinsight-03-17 + git checkout test-hdinsight-03-17 # install autorest sudo npm install -g n From 2a9f585748806276422a01c4ea29fbab21d706e6 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Mon, 28 Mar 2022 15:41:07 +0800 Subject: [PATCH 032/120] Update PythonSdkLiveTest.yml for Azure Pipelines --- scripts/auto_release/PythonSdkLiveTest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 65e726c948c0..a9b389822f09 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -46,8 +46,10 @@ jobs: if [ "$(SPEC_README)" ]; then mkdir azure-rest-api-specs git clone https://github.com/Azure/azure-rest-api-specs.git $(pwd)/azure-rest-api-specs + cd azure-rest-api-specs git fetch origin test-hdinsight-03-17 git checkout test-hdinsight-03-17 + cd .. # install autorest sudo npm install -g n From 8b71e0484c96d25d4fc0cf411e19f75f7ab4c23f Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Sat, 7 May 2022 17:44:23 +0800 Subject: [PATCH 033/120] remove surplus code --- scripts/auto_release/main.py | 34 +++++++--------------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index ed36d5c9cd93..52537517baaf 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -149,6 +149,7 @@ def __init__(self): self.pr_number = 0 self.container_name = '' self.private_package_link = [] # List[str] + self.tag_is_stable = False @return_origin_path def get_latest_commit_in_swagger_repo(self) -> str: @@ -183,9 +184,13 @@ def generate_code(self): with open(self.autorest_result, 'w') as file: json.dump(input_data, file) - # generate code + # generate code(be careful about the order) print_exec('python scripts/dev_setup.py -p azure-core') print_check(f'python -m packaging_tools.auto_codegen {self.autorest_result} {self.autorest_result}') + + self.tag_is_stable = self.get_autorest_result()["packages"][0]["tagIsStable"] + log(f"tag_is_stable is {self.tag_is_stable}") + print_check(f'python -m packaging_tools.auto_package {self.autorest_result} {self.autorest_result}') def get_package_name_with_autorest_result(self): @@ -282,33 +287,8 @@ def get_all_files_under_package_folder(self) -> List[str]: all_files(self.sdk_code_path(), files) return files - def judge_tag(self) -> bool: - files = self.get_all_files_under_package_folder() - default_api_version = '' # for multi-api - api_version = '' # for single-api - for file in files: - if '.py' not in file or '.pyc' in file: - continue - try: - with open(file, 'r') as file_in: - list_in = file_in.readlines() - except: - _LOG.info(f'can not open {file}') - continue - - for line in list_in: - if line.find('DEFAULT_API_VERSION = ') > -1: - default_api_version += line.split('=')[-1].strip('\n') # collect all default api version - if default_api_version == '' and line.find('api_version = ') > -1: - api_version += line.split('=')[-1].strip('\n') # collect all single api version - if default_api_version != '': - log(f'find default api version:{default_api_version}') - return 'preview' in default_api_version - log(f'find single api version:{api_version}') - return 'preview' in api_version - def calculate_next_version_proc(self, last_version: str): - preview_tag = self.judge_tag() + preview_tag = not self.tag_is_stable changelog = self.get_changelog() if changelog == '': return '0.0.0' From 68a527350a5e4f92994561cca78cc296d06de8e0 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Wed, 18 May 2022 11:00:33 +0800 Subject: [PATCH 034/120] Update PythonSdkLiveTest.yml for Azure Pipelines --- scripts/auto_release/PythonSdkLiveTest.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index a9b389822f09..522daee096af 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -47,8 +47,9 @@ jobs: mkdir azure-rest-api-specs git clone https://github.com/Azure/azure-rest-api-specs.git $(pwd)/azure-rest-api-specs cd azure-rest-api-specs - git fetch origin test-hdinsight-03-17 - git checkout test-hdinsight-03-17 + git remote add test https://github.com/openapi-env-test/azure-rest-api-specs.git + git fetch test bigcat/test-cosmos2 + git checkout bigcat/test-cosmos2 cd .. # install autorest From a517b293a484e81d3ee15b96648388d6b688beda Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Wed, 18 May 2022 11:02:33 +0800 Subject: [PATCH 035/120] Update main.py --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 40c98b38579b..223d25a22910 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -82,7 +82,7 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): usr = 'Azure' - branch = 'test_autorest_03_18' + branch = 'code-report-update-comparision-category' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From 3841e3265f22fb6ca882bdd31d59bc8c70a89a31 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Wed, 18 May 2022 11:06:33 +0800 Subject: [PATCH 036/120] Update main.py --- scripts/auto_release/main.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 223d25a22910..65741b5e814a 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -173,22 +173,10 @@ def generate_code(self): # prepare input data input_data = { - "dryRun": False, - "specFolder": "../azure-rest-api-specs", - "headSha": "9ac082c33a7d0d8fa6768bd64e53394ada9baaf9", - "headRef": "master", - "repoHttpsUrl": "https://github.com/Azure/azure-rest-api-specs", - "trigger": "continuousIntegration", - "changedFiles": [ - "specification/hdinsight/resource-manager/readme.python.md" - ], "relatedReadmeMdFiles": [ - "specification/hdinsight/resource-manager/readme.md" - ], - "installInstructionInput": { - "isPublic": False, - "downloadUrlPrefix": "https://portal.azure-devex-tools.com/api/sdk-dl-pub?p=Azure/13991/azure-sdk-for-python-track2/", - "downloadCommandTemplate": "curl -L \"{URL}\" -o {FILENAME}", - "trigger": "continuousIntegration" + 'headSha': self.get_latest_commit_in_swagger_repo(), + 'repoHttpsUrl': "https://github.com/Azure/azure-rest-api-specs", + 'specFolder': self.spec_repo, + 'relatedReadmeMdFiles': [str(self.readme_local_folder())] } } From 2fdba355b77cae817455ce4d1d0f85f21fca91b9 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Wed, 18 May 2022 11:12:14 +0800 Subject: [PATCH 037/120] Update main.py --- scripts/auto_release/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 65741b5e814a..9ffa8aa98ddc 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -177,7 +177,6 @@ def generate_code(self): 'repoHttpsUrl': "https://github.com/Azure/azure-rest-api-specs", 'specFolder': self.spec_repo, 'relatedReadmeMdFiles': [str(self.readme_local_folder())] - } } self.autorest_result = str(Path(os.getenv('TEMP_FOLDER')) / 'temp.json') From 3ad0f6437e8a02433827f70e030e3f9615ddad44 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Wed, 18 May 2022 14:41:46 +0800 Subject: [PATCH 038/120] Update main.py --- scripts/auto_release/main.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 9ffa8aa98ddc..127e6b9c7d10 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -173,10 +173,23 @@ def generate_code(self): # prepare input data input_data = { - 'headSha': self.get_latest_commit_in_swagger_repo(), - 'repoHttpsUrl': "https://github.com/Azure/azure-rest-api-specs", - 'specFolder': self.spec_repo, - 'relatedReadmeMdFiles': [str(self.readme_local_folder())] + "dryRun": False, + "specFolder": "../azure-rest-api-specs", + "headSha": "c6ed0986c1ee45df5c8f2bc56f2a9e802b01d2b0", + "headRef": "master", + "repoHttpsUrl": "https://github.com/Azure/azure-rest-api-specs", + "trigger": "continuousIntegration", + "changedFiles": [ + "specification/cosmos-db/resource-manager/readme.python.md" + ], "relatedReadmeMdFiles": [ + "specification/cosmos-db/resource-manager/readme.md" + ], + "installInstructionInput": { + "isPublic": False, + "downloadUrlPrefix": "https://portal.azure-devex-tools.com/api/sdk-dl-pub?p=Azure/13991/azure-sdk-for-python-track2/", + "downloadCommandTemplate": "curl -L \"{URL}\" -o {FILENAME}", + "trigger": "continuousIntegration" + } } self.autorest_result = str(Path(os.getenv('TEMP_FOLDER')) / 'temp.json') From a32eac743349b76c3ef1eb39b30abcd27c51c2c4 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Wed, 18 May 2022 17:33:05 +0800 Subject: [PATCH 039/120] Update main.py --- scripts/auto_release/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 127e6b9c7d10..88a159f2a23c 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -173,7 +173,7 @@ def generate_code(self): # prepare input data input_data = { - "dryRun": False, + "dryRun": false, "specFolder": "../azure-rest-api-specs", "headSha": "c6ed0986c1ee45df5c8f2bc56f2a9e802b01d2b0", "headRef": "master", @@ -185,7 +185,7 @@ def generate_code(self): "specification/cosmos-db/resource-manager/readme.md" ], "installInstructionInput": { - "isPublic": False, + "isPublic": false, "downloadUrlPrefix": "https://portal.azure-devex-tools.com/api/sdk-dl-pub?p=Azure/13991/azure-sdk-for-python-track2/", "downloadCommandTemplate": "curl -L \"{URL}\" -o {FILENAME}", "trigger": "continuousIntegration" From 7148c2a78a845fa4a15b0f3bf4b5ec76cc9556de Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Wed, 18 May 2022 17:41:44 +0800 Subject: [PATCH 040/120] Update main.py --- scripts/auto_release/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 88a159f2a23c..14a605b8b3fb 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -173,8 +173,9 @@ def generate_code(self): # prepare input data input_data = { - "dryRun": false, + "dryRun": False, "specFolder": "../azure-rest-api-specs", + "packageName": "azure-mgmt-cosmosdb", "headSha": "c6ed0986c1ee45df5c8f2bc56f2a9e802b01d2b0", "headRef": "master", "repoHttpsUrl": "https://github.com/Azure/azure-rest-api-specs", @@ -185,7 +186,7 @@ def generate_code(self): "specification/cosmos-db/resource-manager/readme.md" ], "installInstructionInput": { - "isPublic": false, + "isPublic": False, "downloadUrlPrefix": "https://portal.azure-devex-tools.com/api/sdk-dl-pub?p=Azure/13991/azure-sdk-for-python-track2/", "downloadCommandTemplate": "curl -L \"{URL}\" -o {FILENAME}", "trigger": "continuousIntegration" From 6d1fbbb363c0e441576ff5feeece4c1c1d470533 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Thu, 19 May 2022 13:36:28 +0800 Subject: [PATCH 041/120] Update main.py --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 14a605b8b3fb..353f976a4a4f 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -82,7 +82,7 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): usr = 'Azure' - branch = 'code-report-update-comparision-category' + branch = 'code-report-update-comparision-category-test1' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From e0083153f028dea4ad60c7940c6d3843b8fe8329 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Thu, 19 May 2022 14:53:35 +0800 Subject: [PATCH 042/120] Update main.py --- scripts/auto_release/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 353f976a4a4f..b4a6cbecc60a 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -175,7 +175,6 @@ def generate_code(self): input_data = { "dryRun": False, "specFolder": "../azure-rest-api-specs", - "packageName": "azure-mgmt-cosmosdb", "headSha": "c6ed0986c1ee45df5c8f2bc56f2a9e802b01d2b0", "headRef": "master", "repoHttpsUrl": "https://github.com/Azure/azure-rest-api-specs", @@ -200,6 +199,9 @@ def generate_code(self): # generate code print_exec('python scripts/dev_setup.py -p azure-core') print_check(f'python -m packaging_tools.auto_codegen {self.autorest_result} {self.autorest_result}') + with open(self.autorest_result, 'r') as fr: + print("#### self.autorest_result", fr.read()) + print_check(f'python -m packaging_tools.auto_package {self.autorest_result} {self.autorest_result}') def get_package_name_with_autorest_result(self): From 63bd619e9e0952e3261acfe354272219465ca091 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Thu, 19 May 2022 16:25:28 +0800 Subject: [PATCH 043/120] Update main.py --- scripts/auto_release/main.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index b4a6cbecc60a..79494788b970 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -173,23 +173,10 @@ def generate_code(self): # prepare input data input_data = { - "dryRun": False, - "specFolder": "../azure-rest-api-specs", - "headSha": "c6ed0986c1ee45df5c8f2bc56f2a9e802b01d2b0", - "headRef": "master", - "repoHttpsUrl": "https://github.com/Azure/azure-rest-api-specs", - "trigger": "continuousIntegration", - "changedFiles": [ - "specification/cosmos-db/resource-manager/readme.python.md" - ], "relatedReadmeMdFiles": [ - "specification/cosmos-db/resource-manager/readme.md" - ], - "installInstructionInput": { - "isPublic": False, - "downloadUrlPrefix": "https://portal.azure-devex-tools.com/api/sdk-dl-pub?p=Azure/13991/azure-sdk-for-python-track2/", - "downloadCommandTemplate": "curl -L \"{URL}\" -o {FILENAME}", - "trigger": "continuousIntegration" - } + 'headSha': self.get_latest_commit_in_swagger_repo(), + 'repoHttpsUrl': "https://github.com/Azure/azure-rest-api-specs", + 'specFolder': self.spec_repo, + 'relatedReadmeMdFiles': [str(self.readme_local_folder())] } self.autorest_result = str(Path(os.getenv('TEMP_FOLDER')) / 'temp.json') From 31a7e2df580060e07d06bc693ac738bded9c65de Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Thu, 19 May 2022 16:43:45 +0800 Subject: [PATCH 044/120] Update main.py --- scripts/auto_release/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 79494788b970..d61aa12328f3 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -193,6 +193,7 @@ def generate_code(self): def get_package_name_with_autorest_result(self): generate_result = self.get_autorest_result() + print(f"generate_result: {generate_result}") self.package_name = generate_result["packages"][0]["packageName"].split('-')[-1] def prepare_branch_with_readme(self): From dd0c4d8233c4be7b1108db19365f037c49353951 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Thu, 19 May 2022 16:52:16 +0800 Subject: [PATCH 045/120] Update PythonSdkLiveTest.yml for Azure Pipelines --- scripts/auto_release/PythonSdkLiveTest.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 522daee096af..c5b16b57b579 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -46,11 +46,6 @@ jobs: if [ "$(SPEC_README)" ]; then mkdir azure-rest-api-specs git clone https://github.com/Azure/azure-rest-api-specs.git $(pwd)/azure-rest-api-specs - cd azure-rest-api-specs - git remote add test https://github.com/openapi-env-test/azure-rest-api-specs.git - git fetch test bigcat/test-cosmos2 - git checkout bigcat/test-cosmos2 - cd .. # install autorest sudo npm install -g n From 225a28eac5f8a39b0a957142d8ee7c4af8cfec3a Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Fri, 20 May 2022 10:36:08 +0800 Subject: [PATCH 046/120] Update PythonSdkLiveTest.yml for Azure Pipelines --- scripts/auto_release/PythonSdkLiveTest.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index c5b16b57b579..516ffac1e2a7 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -46,6 +46,12 @@ jobs: if [ "$(SPEC_README)" ]; then mkdir azure-rest-api-specs git clone https://github.com/Azure/azure-rest-api-specs.git $(pwd)/azure-rest-api-specs + cd azure-rest-api-specs + git fetch origin bigcat/confluent-test-branch + git checkout bigcat/confluent-test-branch + cd .. + + # install autorest sudo npm install -g n From 79c8572dced30ee3abca73108dbf3791f96d7508 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Fri, 20 May 2022 10:56:09 +0800 Subject: [PATCH 047/120] Update PythonSdkLiveTest.yml for Azure Pipelines --- scripts/auto_release/PythonSdkLiveTest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 516ffac1e2a7..ee87e81bd321 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -47,8 +47,8 @@ jobs: mkdir azure-rest-api-specs git clone https://github.com/Azure/azure-rest-api-specs.git $(pwd)/azure-rest-api-specs cd azure-rest-api-specs - git fetch origin bigcat/confluent-test-branch - git checkout bigcat/confluent-test-branch + git fetch origin bigcat/confluent-test-branch-1 + git checkout bigcat/confluent-test-branch-1 cd .. From a41959d24189ab0cd718252a9e6974d62efd1448 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Mon, 23 May 2022 14:30:47 +0800 Subject: [PATCH 048/120] fix bug --- scripts/auto_release/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index f8a0cd4a2fb5..423780c44261 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -188,7 +188,8 @@ def generate_code(self): print_exec('python scripts/dev_setup.py -p azure-core') print_check(f'python -m packaging_tools.auto_codegen {self.autorest_result} {self.autorest_result}') - self.tag_is_stable = self.get_autorest_result()["packages"][0]["tagIsStable"] + generate_result = self.get_autorest_result() + self.tag_is_stable = generate_result["packages"][0]["tagIsStable"] log(f"tag_is_stable is {self.tag_is_stable}") print_check(f'python -m packaging_tools.auto_package {self.autorest_result} {self.autorest_result}') From aed57d80696e820afac0dd884f4a78692799863e Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Mon, 23 May 2022 14:45:37 +0800 Subject: [PATCH 049/120] update --- scripts/auto_release/PythonSdkLiveTest.yml | 5 ----- scripts/auto_release/main.py | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index ee87e81bd321..5942e0f969eb 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -46,11 +46,6 @@ jobs: if [ "$(SPEC_README)" ]; then mkdir azure-rest-api-specs git clone https://github.com/Azure/azure-rest-api-specs.git $(pwd)/azure-rest-api-specs - cd azure-rest-api-specs - git fetch origin bigcat/confluent-test-branch-1 - git checkout bigcat/confluent-test-branch-1 - cd .. - # install autorest diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 423780c44261..b59ac8d66052 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -82,7 +82,7 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): usr = 'Azure' - branch = 'code-report-update-comparision-category-test1' + branch = 'main' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') @@ -187,7 +187,7 @@ def generate_code(self): # generate code(be careful about the order) print_exec('python scripts/dev_setup.py -p azure-core') print_check(f'python -m packaging_tools.auto_codegen {self.autorest_result} {self.autorest_result}') - + generate_result = self.get_autorest_result() self.tag_is_stable = generate_result["packages"][0]["tagIsStable"] log(f"tag_is_stable is {self.tag_is_stable}") From f5bb4c5f507bf35438bff4554bf8ca54145f4e90 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Mon, 23 May 2022 14:54:02 +0800 Subject: [PATCH 050/120] test --- scripts/auto_release/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index b59ac8d66052..99e275273f0e 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -315,6 +315,7 @@ def calculate_next_version_proc(self, last_version: str): def get_autorest_result(self) -> Dict[Any, Any]: with open(self.autorest_result, 'r') as file_in: content = json.load(file_in) + print(f'**** content: {content}') return content def get_changelog(self) -> str: From 38d34ba46dc313927662bb107f12d3a7c1db6b47 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Mon, 23 May 2022 16:34:31 +0800 Subject: [PATCH 051/120] update code --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 99e275273f0e..522bb375f63f 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -189,7 +189,7 @@ def generate_code(self): print_check(f'python -m packaging_tools.auto_codegen {self.autorest_result} {self.autorest_result}') generate_result = self.get_autorest_result() - self.tag_is_stable = generate_result["packages"][0]["tagIsStable"] + self.tag_is_stable = list(generate_result.values())[0]['tagIsStable'] log(f"tag_is_stable is {self.tag_is_stable}") print_check(f'python -m packaging_tools.auto_package {self.autorest_result} {self.autorest_result}') From 047f03c0f33765b847643d4993b4e741fc2ea851 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Mon, 6 Jun 2022 10:09:58 +0800 Subject: [PATCH 052/120] update bot --- scripts/auto_release/PythonSdkLiveTest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 5942e0f969eb..b316809e93ce 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -40,7 +40,7 @@ jobs: # clone(REPO: https://github.com/Azure/azure-sdk-for-python.git, USR_NAME: Azure, USR_TOKEN: xxxxxxxxxxxxx) mkdir azure-sdk-for-python - git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} $(pwd)/azure-sdk-for-python + git clone ${REPO:0:8}$(USR_NAME):$(azuresdk-github-pat)@${REPO:8} $(pwd)/azure-sdk-for-python # prepare env if need to generate SDK code if [ "$(SPEC_README)" ]; then From 33da0454cccb313a3600f2d1e50a1af37e235c7c Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Mon, 6 Jun 2022 10:25:46 +0800 Subject: [PATCH 053/120] update main bot --- scripts/auto_release/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 522bb375f63f..2119b5be5abf 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -132,7 +132,6 @@ class CodegenTestPR: def __init__(self): self.issue_link = os.getenv('ISSUE_LINK') - self.usr_token = os.getenv('USR_TOKEN') self.pipeline_link = os.getenv('PIPELINE_LINK') self.bot_token = os.getenv('AZURESDK_BOT_TOKEN') self.spec_readme = os.getenv('SPEC_README', '') @@ -448,7 +447,7 @@ def run_test(self): self.run_test_proc() def create_pr_proc(self): - api = GhApi(owner='Azure', repo='azure-sdk-for-python', token=self.usr_token) + api = GhApi(owner='Azure', repo='azure-sdk-for-python', token=self.bot_token) pr_title = "[AutoRelease] {}(Do not merge)".format(self.new_branch) pr_head = "{}:{}".format(os.getenv('USR_NAME'), self.new_branch) pr_base = 'main' From a71853144d54fd34b446a362490a5fccb3c1b227 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Mon, 6 Jun 2022 10:38:59 +0800 Subject: [PATCH 054/120] Update PythonSdkLiveTest.yml for Azure Pipelines --- scripts/auto_release/PythonSdkLiveTest.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index b316809e93ce..7d9b2f3f9b78 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -78,7 +78,6 @@ jobs: export AZURE_CLIENT_SECRET=$(ENV_CLIENT_SECRET) export AZURE_SUBSCRIPTION_ID=$(ENV_SUBSCRIPTION_ID) export ISSUE_LINK=$(ISSUE_LINK) - export USR_TOKEN=$(USR_TOKEN) export BASE_BRANCH=$(BASE_BRANCH) export SCRIPT_PATH=$script_path export AZURESDK_BOT_TOKEN=$(azuresdk-github-pat) From 4bbcce9d5eff24353fc1c268c5df286245093add Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Mon, 6 Jun 2022 11:05:24 +0800 Subject: [PATCH 055/120] Update main.py --- scripts/auto_release/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 2119b5be5abf..f7c34442a009 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -195,7 +195,6 @@ def generate_code(self): def get_package_name_with_autorest_result(self): generate_result = self.get_autorest_result() - print(f"generate_result: {generate_result}") self.package_name = generate_result["packages"][0]["packageName"].split('-')[-1] def prepare_branch_with_readme(self): From bfd4d3e23f25473f74eb2f4df626bbf1bf444862 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Mon, 6 Jun 2022 11:06:25 +0800 Subject: [PATCH 056/120] Update main.py --- scripts/auto_release/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index f7c34442a009..b9c25fb6a903 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -313,7 +313,6 @@ def calculate_next_version_proc(self, last_version: str): def get_autorest_result(self) -> Dict[Any, Any]: with open(self.autorest_result, 'r') as file_in: content = json.load(file_in) - print(f'**** content: {content}') return content def get_changelog(self) -> str: From 82d4b0fd503f8a435b26fe7feebc13a88e9f46cb Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 7 Jun 2022 11:12:49 +0800 Subject: [PATCH 057/120] change branch for test --- scripts/auto_release/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index b9c25fb6a903..6ad8793dcbfe 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -81,8 +81,8 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): - usr = 'Azure' - branch = 'main' + usr = 'azure-sdk' + branch = 't2-communication-2022-06-06-53443' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From d4269a71f898259fd7bef033ef3c9cb5b38c544e Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 7 Jun 2022 11:29:01 +0800 Subject: [PATCH 058/120] for test --- scripts/auto_release/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 6ad8793dcbfe..e47e6c81d642 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -267,6 +267,8 @@ def edit_sdk_setup(content: List[str]): def check_file_with_packaging_tool(self): os.chdir(Path(f'sdk/{self.sdk_folder}')) print_check(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}') + with open('MANIFEST.in', 'r', encoding='utf-8') as f: + print(f'**** {f.read()}') log('packaging_tools --build-conf successfully ') def check_pprint_name(self): From 544efb839fe9574c64b2a13a065967a03fc09dfd Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 7 Jun 2022 11:40:20 +0800 Subject: [PATCH 059/120] for test --- scripts/auto_release/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index e47e6c81d642..5d85663df2b7 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -267,6 +267,8 @@ def edit_sdk_setup(content: List[str]): def check_file_with_packaging_tool(self): os.chdir(Path(f'sdk/{self.sdk_folder}')) print_check(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}') + print("**** path: ", Path(f'sdk/{self.sdk_folder}')) + print("**** path now:", os.getcwd()) with open('MANIFEST.in', 'r', encoding='utf-8') as f: print(f'**** {f.read()}') log('packaging_tools --build-conf successfully ') From 7d46cd824d40798d685dd9e102f423ea90391105 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 7 Jun 2022 11:48:40 +0800 Subject: [PATCH 060/120] for test --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 5d85663df2b7..c922054db71b 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -269,7 +269,7 @@ def check_file_with_packaging_tool(self): print_check(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}') print("**** path: ", Path(f'sdk/{self.sdk_folder}')) print("**** path now:", os.getcwd()) - with open('MANIFEST.in', 'r', encoding='utf-8') as f: + with open(Path(f'azure-mgmt-{self.package_name}/MANIFEST.in'), 'r', encoding='utf-8') as f: print(f'**** {f.read()}') log('packaging_tools --build-conf successfully ') From 62b39292c360ac7b6a2f4777f2aa2ac202fd2582 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Wed, 8 Jun 2022 15:08:52 +0800 Subject: [PATCH 061/120] add PyYAML --- tools/azure-sdk-tools/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/azure-sdk-tools/setup.py b/tools/azure-sdk-tools/setup.py index 53c644ce32ba..e22499c6995b 100644 --- a/tools/azure-sdk-tools/setup.py +++ b/tools/azure-sdk-tools/setup.py @@ -24,6 +24,7 @@ "azure-mgmt-storage", "azure-mgmt-keyvault", "python-dotenv", + "PyYAML==6.0" ] setup( From 146c007a7b335bf58f086d6a0c83413646512621 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 9 Jun 2022 17:54:37 +0800 Subject: [PATCH 062/120] code --- .../packaging_tools/auto_codegen.py | 7 +- .../packaging_tools/generate_utils.py | 75 ++++++++++++++++++- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index dc19dca8128b..7e693c277fe3 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -6,7 +6,7 @@ from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG from .generate_sdk import generate -from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview +from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, generate_dpg _LOGGER = logging.getLogger(__name__) @@ -23,7 +23,10 @@ def main(generate_input, generate_output): relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") config_file = CONFIG_FILE if 'resource-manager' in input_readme else CONFIG_FILE_DPG - config = generate(config_file, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) + if 'resource-manager' in input_readme: + config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) + else: + generate_dpg(input_readme, data.get('autorestConfig', '')) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 574d7b23eb6d..f414be22851f 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -6,7 +6,9 @@ from azure_devtools.ci_tools.git_tools import get_add_diff_file_list from pathlib import Path from subprocess import check_call -from typing import List +from typing import List, Union +from glob import glob +import yaml from .swaggertosdk.autorest_tools import build_autorest_options @@ -126,3 +128,74 @@ def judge_tag_preview(path: str) -> bool: _LOGGER.info(f'find single api version:{api_version}') return 'preview' in api_version + + +def extract_yaml_content(autorest_config: str) -> str: + num = [] + content = autorest_config.split('\r\n') + for i in range(len(content)): + if "```" in content[i]: + num.append(i) + if len(num) != 2: + raise Exception(f"autorestConfig content is not valid: {autorest_config}") + return '\n'.join(content[num[0] + 1: num[1]]) + + +def add_yaml_title(content: str, annotation: str = "", tag: str = "") -> str: + return f"{annotation}\n\n" + f"``` yaml {tag}\n" + content + "```\n" + + +def generate_dpg_config(autorest_config: str) -> str: + # remove useless lines + autorest_config = extract_yaml_content(autorest_config) + _LOGGER.info(f"autoresConfig after remove useles lines:\n{autorest_config}") + + # make dir if not exist + origin_config = yaml.safe_load(autorest_config) + _LOGGER.info(f"autoresConfig: {origin_config}") + swagger_folder = str(Path(origin_config["output-folder"], "swagger")) + if not os.path.exists(swagger_folder): + os.makedirs(swagger_folder) + + # generate autorest configuration + package_name = Path(origin_config["output-folder"]).parts[-1] + readme_content = { + "package-name": package_name, + "license-header": "MICROSOFT_MIT_NO_VERSION", + "clear-output-folder": True, + "no-namespace-folders": True, + "version-tolerant": True, + "package-version": "1.0.0b1", + "require": ["../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + "output-folder": "../" + package_name.replace('-', '/'), + "namespace": package_name.replace('-', '.') + } + + # output autorest configuration + swagger_readme = str(Path(swagger_folder, "README.md")) + with open(swagger_readme, "w") as file: + file.write(add_yaml_title(yaml.safe_dump(readme_content))) + return swagger_readme + + +def lookup_swagger_readme(rest_readme_path: str) -> str: + all_swagger_readme = glob(str(Path('sdk/*/*/swagger/README.md'))) + for readme in all_swagger_readme: + with open(readme, 'r') as file: + content = file.read() + if rest_readme_path in content: + return readme + return "" + + +def generate_dpg(rest_readme_path: str, autorest_config: str): + if autorest_config: + swagger_readme = generate_dpg_config(autorest_config) + else: + swagger_readme = lookup_swagger_readme(rest_readme_path) + if not swagger_readme: + return + + # extract global config + + # generate code From 8edbe377abe12d8ccdc3d1e8f851681e1e9e6b71 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 9 Jun 2022 18:06:28 +0800 Subject: [PATCH 063/120] README.md format --- tools/azure-sdk-tools/packaging_tools/generate_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index f414be22851f..9c5ed347d46b 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -142,7 +142,9 @@ def extract_yaml_content(autorest_config: str) -> str: def add_yaml_title(content: str, annotation: str = "", tag: str = "") -> str: - return f"{annotation}\n\n" + f"``` yaml {tag}\n" + content + "```\n" + if annotation: + annotation = f"{annotation}\n\n" + return f"# autorest configuration for Python\n\n{annotation}" + f"``` yaml {tag}\n" + content + "```\n" def generate_dpg_config(autorest_config: str) -> str: @@ -166,7 +168,7 @@ def generate_dpg_config(autorest_config: str) -> str: "no-namespace-folders": True, "version-tolerant": True, "package-version": "1.0.0b1", - "require": ["../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], "output-folder": "../" + package_name.replace('-', '/'), "namespace": package_name.replace('-', '.') } From ced64c782278a5154722fc8122fb9f801bf38a5c Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 10 Jun 2022 11:41:16 +0800 Subject: [PATCH 064/120] multi client --- .../packaging_tools/auto_codegen.py | 4 +- .../packaging_tools/generate_utils.py | 111 ++++++++++++++---- 2 files changed, 90 insertions(+), 25 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index 7e693c277fe3..9f2196cd6c58 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -6,7 +6,7 @@ from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG from .generate_sdk import generate -from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, generate_dpg +from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, gen_dpg _LOGGER = logging.getLogger(__name__) @@ -26,7 +26,7 @@ def main(generate_input, generate_output): if 'resource-manager' in input_readme: config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) else: - generate_dpg(input_readme, data.get('autorestConfig', '')) + gen_dpg(input_readme, data.get('autorestConfig', '')) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 9c5ed347d46b..2658920aba67 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -6,7 +6,7 @@ from azure_devtools.ci_tools.git_tools import get_add_diff_file_list from pathlib import Path from subprocess import check_call -from typing import List, Union +from typing import List, Dict, Any from glob import glob import yaml @@ -141,42 +141,107 @@ def extract_yaml_content(autorest_config: str) -> str: return '\n'.join(content[num[0] + 1: num[1]]) -def add_yaml_title(content: str, annotation: str = "", tag: str = "") -> str: - if annotation: - annotation = f"{annotation}\n\n" - return f"# autorest configuration for Python\n\n{annotation}" + f"``` yaml {tag}\n" + content + "```\n" +def add_config_title(content: str) -> str: + return f"# autorest configuration for Python\n\n{content}" -def generate_dpg_config(autorest_config: str) -> str: +def yaml_block(content: str, annotation: str = "", tag: str = "") -> str: + annotation = f"{annotation}\n\n" if annotation else annotation + return f"{annotation}" + f"``` yaml {tag}\n" + content + "```\n" + + + + + +def gen_package_name(origin_config: Dict[str, Any]) -> str: + return Path(origin_config["output-folder"]).parts[-1] + + +def gen_basic_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: + return { + "package-name": gen_package_name(origin_config), + "license-header": "MICROSOFT_MIT_NO_VERSION", + "clear-output-folder": True, + "no-namespace-folders": True, + "version-tolerant": True, + "package-version": origin_config.get("package-version", "1.0.0b1"), + "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + } + + +def gen_general_output_folder(package_name: str) -> str: + return "../" + package_name.replace('-', '/') + + +def gen_general_namespace(package_name: str) -> str: + return package_name.replace('-', '.') + + +def gen_dpg_config_single_client(origin_config: Dict[str, Any]) -> str: + package_name = Path(origin_config["output-folder"]).parts[-1] + readme_config = gen_basic_config(origin_config) + readme_config.update({ + "output-folder": gen_general_output_folder(package_name), + "namespace": gen_general_namespace(package_name), + }) + readme_content = yaml_block(yaml.safe_dump(readme_config), "### Settings") + return add_config_title(readme_content) + + +def gen_tag_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: + tag_config = {} + package_name = gen_package_name(origin_config) + for tag in origin_config["batch"]: + tag_name = tag["tag"] + extra_part = tag_name.split("-")[-1] + tag_config[tag_name] = { + "namespace": gen_general_namespace(package_name) + f".{extra_part}", + "output-folder": gen_general_output_folder(package_name) + f"/{extra_part}" + } + + return tag_config + + +def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: + # generate config + basic_config = gen_basic_config(origin_config) + batch_config = {"batch": origin_config["batch"]} + tag_config = gen_tag_config(origin_config) + + # convert to string + readme_content = yaml_block(yaml.dump(basic_config), "### Settings") + readme_content += yaml_block(yaml.dump(batch_config), "\n### Python multi-client") + for tag, value in tag_config.items(): + readme_content += yaml_block( + yaml.dump(value), + f"\n### Tag: {tag}", + f"$(tag) == '{tag}'", + ) + + return add_config_title(readme_content) + +def gen_dpg_config(autorest_config: str) -> str: # remove useless lines autorest_config = extract_yaml_content(autorest_config) - _LOGGER.info(f"autoresConfig after remove useles lines:\n{autorest_config}") + _LOGGER.info(f"autorestConfig after remove useless lines:\n{autorest_config}") # make dir if not exist origin_config = yaml.safe_load(autorest_config) - _LOGGER.info(f"autoresConfig: {origin_config}") + _LOGGER.info(f"autorestConfig: {origin_config}") swagger_folder = str(Path(origin_config["output-folder"], "swagger")) if not os.path.exists(swagger_folder): os.makedirs(swagger_folder) # generate autorest configuration - package_name = Path(origin_config["output-folder"]).parts[-1] - readme_content = { - "package-name": package_name, - "license-header": "MICROSOFT_MIT_NO_VERSION", - "clear-output-folder": True, - "no-namespace-folders": True, - "version-tolerant": True, - "package-version": "1.0.0b1", - "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], - "output-folder": "../" + package_name.replace('-', '/'), - "namespace": package_name.replace('-', '.') - } + if "batch:" in autorest_config: + readme_content = gen_dpg_config_multi_client(origin_config) + else: + readme_content = gen_dpg_config_single_client(origin_config) # output autorest configuration swagger_readme = str(Path(swagger_folder, "README.md")) with open(swagger_readme, "w") as file: - file.write(add_yaml_title(yaml.safe_dump(readme_content))) + file.write(readme_content) return swagger_readme @@ -190,9 +255,9 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: return "" -def generate_dpg(rest_readme_path: str, autorest_config: str): +def gen_dpg(rest_readme_path: str, autorest_config: str): if autorest_config: - swagger_readme = generate_dpg_config(autorest_config) + swagger_readme = gen_dpg_config(autorest_config) else: swagger_readme = lookup_swagger_readme(rest_readme_path) if not swagger_readme: From 0a29f58ac1515265150756cce3633600962ae85a Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 10 Jun 2022 13:42:49 +0800 Subject: [PATCH 065/120] code --- tools/azure-sdk-tools/packaging_tools/generate_utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 2658920aba67..1342d96f04e8 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -150,9 +150,6 @@ def yaml_block(content: str, annotation: str = "", tag: str = "") -> str: return f"{annotation}" + f"``` yaml {tag}\n" + content + "```\n" - - - def gen_package_name(origin_config: Dict[str, Any]) -> str: return Path(origin_config["output-folder"]).parts[-1] @@ -217,9 +214,10 @@ def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: f"\n### Tag: {tag}", f"$(tag) == '{tag}'", ) - + return add_config_title(readme_content) + def gen_dpg_config(autorest_config: str) -> str: # remove useless lines autorest_config = extract_yaml_content(autorest_config) From 378c1c032417edb29082c8c0bc8f2042f63a0bb1 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 10 Jun 2022 13:58:51 +0800 Subject: [PATCH 066/120] add python_tag --- scripts/auto_release/PythonSdkLiveTest.yml | 1 + scripts/auto_release/main.py | 3 +++ tools/azure-sdk-tools/packaging_tools/auto_codegen.py | 4 +++- tools/azure-sdk-tools/packaging_tools/generate_sdk.py | 3 ++- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 7d9b2f3f9b78..d3dd80fc8127 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -83,6 +83,7 @@ jobs: export AZURESDK_BOT_TOKEN=$(azuresdk-github-pat) export STORAGE_CONN_STR=$(storage-conn-str) export STORAGE_ENDPOINT=$(storage-endpoint) + export PYTHON_TAG=@(PYTHON_TAG) # run cd azure-sdk-for-python diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index c922054db71b..e6f55095dbe0 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -178,6 +178,9 @@ def generate_code(self): 'specFolder': self.spec_repo, 'relatedReadmeMdFiles': [str(self.readme_local_folder())] } + # if Python tag exists + if os.getenv('PYTHON_TAG'): + input_data['python_tag'] = os.getenv('PYTHON_TAG') self.autorest_result = str(Path(os.getenv('TEMP_FOLDER')) / 'temp.json') with open(self.autorest_result, 'w') as file: diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index dc19dca8128b..d8d616f8cae4 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -19,11 +19,13 @@ def main(generate_input, generate_output): sdk_folder = "." result = {} package_total = set() + python_tag = data.get('python_tag') if data.get('python_tag') else None for input_readme in data["relatedReadmeMdFiles"]: relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") config_file = CONFIG_FILE if 'resource-manager' in input_readme else CONFIG_FILE_DPG - config = generate(config_file, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) + config = generate(config_file, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, + python_tag=python_tag) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") diff --git a/tools/azure-sdk-tools/packaging_tools/generate_sdk.py b/tools/azure-sdk-tools/packaging_tools/generate_sdk.py index b403e8fa0ec4..73ba76c13850 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_sdk.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_sdk.py @@ -19,7 +19,8 @@ def generate( - config_path, sdk_folder, project_pattern, readme, restapi_git_folder, autorest_bin=None, force_generation=False + config_path, sdk_folder, project_pattern, readme, restapi_git_folder, autorest_bin=None, force_generation=False, + python_tag=None ): sdk_folder = Path(sdk_folder).expanduser() From 6eb0ed19f94b12d540ad23907074240f5ae3c8c6 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 10 Jun 2022 14:01:29 +0800 Subject: [PATCH 067/120] restore branch --- scripts/auto_release/main.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index e6f55095dbe0..af91183d99f0 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -81,8 +81,8 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): - usr = 'azure-sdk' - branch = 't2-communication-2022-06-06-53443' + usr = 'Azure' + branch = 'main' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') @@ -270,10 +270,6 @@ def edit_sdk_setup(content: List[str]): def check_file_with_packaging_tool(self): os.chdir(Path(f'sdk/{self.sdk_folder}')) print_check(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}') - print("**** path: ", Path(f'sdk/{self.sdk_folder}')) - print("**** path now:", os.getcwd()) - with open(Path(f'azure-mgmt-{self.package_name}/MANIFEST.in'), 'r', encoding='utf-8') as f: - print(f'**** {f.read()}') log('packaging_tools --build-conf successfully ') def check_pprint_name(self): From 1bd12ca40bf04d5897cec0b79ac8fb1a32bc23a2 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Fri, 10 Jun 2022 14:04:17 +0800 Subject: [PATCH 068/120] Update PythonSdkLiveTest.yml --- scripts/auto_release/PythonSdkLiveTest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index d3dd80fc8127..ac1e415c1faf 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -83,7 +83,7 @@ jobs: export AZURESDK_BOT_TOKEN=$(azuresdk-github-pat) export STORAGE_CONN_STR=$(storage-conn-str) export STORAGE_ENDPOINT=$(storage-endpoint) - export PYTHON_TAG=@(PYTHON_TAG) + export PYTHON_TAG=$(PYTHON_TAG) # run cd azure-sdk-for-python From 409ac725fabcf718d4782c2951c2746db0ca2607 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 10 Jun 2022 15:10:10 +0800 Subject: [PATCH 069/120] gen multi client --- .../packaging_tools/generate_utils.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 1342d96f04e8..7f0dfc2b130d 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -199,10 +199,18 @@ def gen_tag_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: return tag_config +def gen_batch_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: + batch_config = [] + for item in origin_config["batch"]: + for _, value in item.items(): + batch_config.append({value: True}) + return {"batch": batch_config} + + def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: # generate config basic_config = gen_basic_config(origin_config) - batch_config = {"batch": origin_config["batch"]} + batch_config = gen_batch_config(origin_config) tag_config = gen_tag_config(origin_config) # convert to string @@ -212,7 +220,7 @@ def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: readme_content += yaml_block( yaml.dump(value), f"\n### Tag: {tag}", - f"$(tag) == '{tag}'", + f"$({tag})", ) return add_config_title(readme_content) From 35e1a19f7712cbddad181e8952bec0d541ba9374 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 10 Jun 2022 15:18:00 +0800 Subject: [PATCH 070/120] update global_conf --- tools/azure-sdk-tools/packaging_tools/generate_sdk.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_sdk.py b/tools/azure-sdk-tools/packaging_tools/generate_sdk.py index 73ba76c13850..92ef2f5bbf28 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_sdk.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_sdk.py @@ -31,6 +31,8 @@ def generate( global_conf["autorest_options"] = solve_relative_path(global_conf.get("autorest_options", {}), sdk_folder) global_conf["envs"] = solve_relative_path(global_conf.get("envs", {}), sdk_folder) global_conf["advanced_options"] = solve_relative_path(global_conf.get("advanced_options", {}), sdk_folder) + if python_tag: + global_conf["autorest_options"]['tag'] = python_tag if restapi_git_folder: restapi_git_folder = Path(restapi_git_folder).expanduser() From 32bd55fc2fdde26858ec46815bb8d11d14ba5a9d Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Fri, 10 Jun 2022 15:34:38 +0800 Subject: [PATCH 071/120] Update auto_codegen.py --- tools/azure-sdk-tools/packaging_tools/auto_codegen.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index d8d616f8cae4..dc19dca8128b 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -19,13 +19,11 @@ def main(generate_input, generate_output): sdk_folder = "." result = {} package_total = set() - python_tag = data.get('python_tag') if data.get('python_tag') else None for input_readme in data["relatedReadmeMdFiles"]: relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") config_file = CONFIG_FILE if 'resource-manager' in input_readme else CONFIG_FILE_DPG - config = generate(config_file, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, - python_tag=python_tag) + config = generate(config_file, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") From 845a892f0bb19307886d268ec20eae1bd8036ae0 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Fri, 10 Jun 2022 15:35:12 +0800 Subject: [PATCH 072/120] Update generate_sdk.py --- tools/azure-sdk-tools/packaging_tools/generate_sdk.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_sdk.py b/tools/azure-sdk-tools/packaging_tools/generate_sdk.py index 92ef2f5bbf28..f526824c0f9d 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_sdk.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_sdk.py @@ -20,7 +20,6 @@ def generate( config_path, sdk_folder, project_pattern, readme, restapi_git_folder, autorest_bin=None, force_generation=False, - python_tag=None ): sdk_folder = Path(sdk_folder).expanduser() @@ -31,8 +30,6 @@ def generate( global_conf["autorest_options"] = solve_relative_path(global_conf.get("autorest_options", {}), sdk_folder) global_conf["envs"] = solve_relative_path(global_conf.get("envs", {}), sdk_folder) global_conf["advanced_options"] = solve_relative_path(global_conf.get("advanced_options", {}), sdk_folder) - if python_tag: - global_conf["autorest_options"]['tag'] = python_tag if restapi_git_folder: restapi_git_folder = Path(restapi_git_folder).expanduser() From 64de6b88e680adad2f12e4200616469fba75e42f Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Fri, 10 Jun 2022 15:35:39 +0800 Subject: [PATCH 073/120] Update generate_sdk.py --- tools/azure-sdk-tools/packaging_tools/generate_sdk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_sdk.py b/tools/azure-sdk-tools/packaging_tools/generate_sdk.py index f526824c0f9d..3f7aea0f08c1 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_sdk.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_sdk.py @@ -19,7 +19,7 @@ def generate( - config_path, sdk_folder, project_pattern, readme, restapi_git_folder, autorest_bin=None, force_generation=False, + config_path, sdk_folder, project_pattern, readme, restapi_git_folder, autorest_bin=None, force_generation=False, ): sdk_folder = Path(sdk_folder).expanduser() From 7e871b6629f11f0bc16e3d77576032f294995197 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Fri, 10 Jun 2022 15:36:01 +0800 Subject: [PATCH 074/120] Update generate_sdk.py --- tools/azure-sdk-tools/packaging_tools/generate_sdk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_sdk.py b/tools/azure-sdk-tools/packaging_tools/generate_sdk.py index 3f7aea0f08c1..b403e8fa0ec4 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_sdk.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_sdk.py @@ -19,7 +19,7 @@ def generate( - config_path, sdk_folder, project_pattern, readme, restapi_git_folder, autorest_bin=None, force_generation=False, + config_path, sdk_folder, project_pattern, readme, restapi_git_folder, autorest_bin=None, force_generation=False ): sdk_folder = Path(sdk_folder).expanduser() From f20bea0578718b1f1f64187381346b7b3d14761c Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 10 Jun 2022 15:36:12 +0800 Subject: [PATCH 075/120] single client gen --- swagger_to_sdk_config_dpg.json | 6 ++---- tools/azure-sdk-tools/packaging_tools/auto_codegen.py | 3 +-- .../azure-sdk-tools/packaging_tools/generate_utils.py | 11 ++++++++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/swagger_to_sdk_config_dpg.json b/swagger_to_sdk_config_dpg.json index 4900e744c9e8..1d011e81afdc 100644 --- a/swagger_to_sdk_config_dpg.json +++ b/swagger_to_sdk_config_dpg.json @@ -1,10 +1,8 @@ { "meta": { "autorest_options": { - "version": "3.7.2", - "use": ["@autorest/python@5.16.0", "@autorest/modelerfour@4.19.3"], - "python": "", - "sdkrel:python-sdks-folder": "./sdk/.", + "version": "3.8.1", + "use": ["@autorest/python@5.17.0", "@autorest/modelerfour@4.23.5"], "version-tolerant": "" }, "advanced_options": { diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index 9f2196cd6c58..d97a7891b518 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -4,7 +4,7 @@ from pathlib import Path from subprocess import check_call -from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG +from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, gen_dpg @@ -22,7 +22,6 @@ def main(generate_input, generate_output): for input_readme in data["relatedReadmeMdFiles"]: relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") - config_file = CONFIG_FILE if 'resource-manager' in input_readme else CONFIG_FILE_DPG if 'resource-manager' in input_readme: config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) else: diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 7f0dfc2b130d..8c05bc5ab505 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -10,7 +10,9 @@ from glob import glob import yaml -from .swaggertosdk.autorest_tools import build_autorest_options +from .swaggertosdk.autorest_tools import build_autorest_options, generate_code +from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE_DPG, read_config + _LOGGER = logging.getLogger(__name__) _SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII) @@ -226,6 +228,7 @@ def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: return add_config_title(readme_content) +# generate swagger/README.md and return relative path based on SDK repo root path def gen_dpg_config(autorest_config: str) -> str: # remove useless lines autorest_config = extract_yaml_content(autorest_config) @@ -262,6 +265,7 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: def gen_dpg(rest_readme_path: str, autorest_config: str): + # generate or find swagger/README.md if autorest_config: swagger_readme = gen_dpg_config(autorest_config) else: @@ -270,5 +274,10 @@ def gen_dpg(rest_readme_path: str, autorest_config: str): return # extract global config + global_config = read_config('.', CONFIG_FILE_DPG)["meta"] # generate code + current_path = os.getcwd() + os.chdir(Path(swagger_readme).parent) + generate_code(Path(swagger_readme).parts[-1], global_config, {}) + os.chdir(current_path) \ No newline at end of file From 091bf1f793947d72690edd4bf2d3a56e27e08d6e Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Fri, 10 Jun 2022 15:36:28 +0800 Subject: [PATCH 076/120] Update main.py --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index af91183d99f0..0029721727e3 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -82,7 +82,7 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): usr = 'Azure' - branch = 'main' + branch = 'test-condegen-update-0610' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From 95fe92e3ccfc462c5c4f6fce2d15776e0baf7eb8 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 10 Jun 2022 16:02:00 +0800 Subject: [PATCH 077/120] compatible with meta storage code --- .../packaging_tools/auto_codegen.py | 4 ++-- .../packaging_tools/generate_utils.py | 24 ++++++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index d97a7891b518..b84696648979 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -25,7 +25,7 @@ def main(generate_input, generate_output): if 'resource-manager' in input_readme: config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) else: - gen_dpg(input_readme, data.get('autorestConfig', '')) + config = gen_dpg(input_readme, data.get('autorestConfig', '')) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") @@ -53,7 +53,7 @@ def main(generate_input, generate_output): try: update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme) except Exception as e: - _LOGGER.info(str(e)) + _LOGGER.info(f"fail to update meta: {str(e)}") # Setup package locally check_call( diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 8c05bc5ab505..4d8798417834 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -18,7 +18,7 @@ _SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII) DEFAULT_DEST_FOLDER = "./dist" - +_DPG_README = "README.md" def get_package_names(sdk_folder): files = get_add_diff_file_list(sdk_folder) @@ -45,9 +45,13 @@ def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, readme_file = str(Path(spec_folder, input_readme)) global_conf = config["meta"] - local_conf = config["projects"][readme_file] + local_conf = config.get("projects", {}).get(readme_file, {}) - cmd = ["autorest", input_readme] + if "resource-manager" in input_readme: + cmd = ["autorest", input_readme] + else: + # autorest for DPG will be executed in package folder like: sdk/deviceupdate/azure-iot-deviceupdate/swagger + cmd = ["autorest", _DPG_README] cmd += build_autorest_options(global_conf, local_conf) # metadata @@ -248,14 +252,14 @@ def gen_dpg_config(autorest_config: str) -> str: readme_content = gen_dpg_config_single_client(origin_config) # output autorest configuration - swagger_readme = str(Path(swagger_folder, "README.md")) + swagger_readme = str(Path(swagger_folder, _DPG_README)) with open(swagger_readme, "w") as file: file.write(readme_content) return swagger_readme def lookup_swagger_readme(rest_readme_path: str) -> str: - all_swagger_readme = glob(str(Path('sdk/*/*/swagger/README.md'))) + all_swagger_readme = glob(str(Path(f'sdk/*/*/swagger/{_DPG_README}'))) for readme in all_swagger_readme: with open(readme, 'r') as file: content = file.read() @@ -264,7 +268,7 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: return "" -def gen_dpg(rest_readme_path: str, autorest_config: str): +def gen_dpg(rest_readme_path: str, autorest_config: str) -> Dict[str, Any]: # generate or find swagger/README.md if autorest_config: swagger_readme = gen_dpg_config(autorest_config) @@ -274,10 +278,12 @@ def gen_dpg(rest_readme_path: str, autorest_config: str): return # extract global config - global_config = read_config('.', CONFIG_FILE_DPG)["meta"] + global_config = read_config('.', CONFIG_FILE_DPG) # generate code current_path = os.getcwd() os.chdir(Path(swagger_readme).parent) - generate_code(Path(swagger_readme).parts[-1], global_config, {}) - os.chdir(current_path) \ No newline at end of file + generate_code(_DPG_README, global_config["meta"], {}) + os.chdir(current_path) + + return global_config From b99079e06da472c220c0dbfd3a7014e38ac9d892 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 10 Jun 2022 16:33:53 +0800 Subject: [PATCH 078/120] add log to lookup readme --- tools/azure-sdk-tools/packaging_tools/generate_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 4d8798417834..ced287a096ba 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -264,7 +264,9 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: with open(readme, 'r') as file: content = file.read() if rest_readme_path in content: + _LOGGER.info(f"find swagger readme: {readme}") return readme + _LOGGER.info(f"do not find swagger readme which contains {rest_readme_path}") return "" From 3f0e9ed0b35f543ea9b8b26bc60496bedb74f45b Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 10 Jun 2022 17:10:31 +0800 Subject: [PATCH 079/120] dpg package does not need _meta.json in MAINFEST.in --- .../packaging_tools/generate_utils.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index ced287a096ba..999aad2e7d87 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -78,23 +78,24 @@ def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, _LOGGER.info(f"Saved metadata to {metadata_file_path}") # Check whether MANIFEST.in includes _meta.json - require_meta = "include _meta.json\n" - manifest_file = os.path.join(package_folder, "MANIFEST.in") - if not os.path.exists(manifest_file): - _LOGGER.info(f"MANIFEST.in doesn't exist: {manifest_file}") - return - - includes = [] - write_flag = False - with open(manifest_file, "r") as f: - includes = f.readlines() - if require_meta not in includes: - includes = [require_meta] + includes - write_flag = True - - if write_flag: - with open(manifest_file, "w") as f: - f.write("".join(includes)) + if "resource-manager" in input_readme: + require_meta = "include _meta.json\n" + manifest_file = os.path.join(package_folder, "MANIFEST.in") + if not os.path.exists(manifest_file): + _LOGGER.info(f"MANIFEST.in doesn't exist: {manifest_file}") + return + + includes = [] + write_flag = False + with open(manifest_file, "r") as f: + includes = f.readlines() + if require_meta not in includes: + includes = [require_meta] + includes + write_flag = True + + if write_flag: + with open(manifest_file, "w") as f: + f.write("".join(includes)) # find all the files of one folder, including files in subdirectory From 8af7ccb2e6973082e4dd4bb49de95699ac1086c5 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Mon, 13 Jun 2022 09:19:14 +0800 Subject: [PATCH 080/120] Update main.py --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 0029721727e3..af91183d99f0 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -82,7 +82,7 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): usr = 'Azure' - branch = 'test-condegen-update-0610' + branch = 'main' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From 9dd3b1fbac17e3e3b42d50a95f3e21b2d4563a66 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Mon, 13 Jun 2022 11:38:34 +0800 Subject: [PATCH 081/120] Update PythonSdkLiveTest.yml for Azure Pipelines --- scripts/auto_release/PythonSdkLiveTest.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index ac1e415c1faf..f70e2c727afa 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -69,14 +69,14 @@ jobs: # import env variable export AZURE_TEST_RUN_LIVE=true - export TENANT_ID=$(ENV_TENANT_ID) - export CLIENT_ID=$(ENV_CLIENT_ID) - export CLIENT_SECRET=$(ENV_CLIENT_SECRET) - export SUBSCRIPTION_ID=$(ENV_SUBSCRIPTION_ID) - export AZURE_TENANT_ID=$(ENV_TENANT_ID) - export AZURE_CLIENT_ID=$(ENV_CLIENT_ID) - export AZURE_CLIENT_SECRET=$(ENV_CLIENT_SECRET) - export AZURE_SUBSCRIPTION_ID=$(ENV_SUBSCRIPTION_ID) + export TENANT_ID=$(ENV-TENANT-ID) + export CLIENT_ID=$(ENV-CLIENT-ID) + export CLIENT_SECRET=$(ENV-CLIENT-SECRET) + export SUBSCRIPTION_ID=$(ENV-SUBSCRIPTION-ID) + export AZURE_TENANT_ID=$(ENV-TENANT-ID) + export AZURE_CLIENT_ID=$(ENV-CLIENT-ID) + export AZURE_CLIENT_SECRET=$(ENV-CLIENT-SECRET) + export AZURE_SUBSCRIPTION_ID=$(ENV-SUBSCRIPTION-ID) export ISSUE_LINK=$(ISSUE_LINK) export BASE_BRANCH=$(BASE_BRANCH) export SCRIPT_PATH=$script_path From fb2a89dbd5c88580a8efad2eaccc6bda60447266 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 13 Jun 2022 15:22:36 +0800 Subject: [PATCH 082/120] add language, apiviewartifacts --- tools/azure-sdk-tools/packaging_tools/auto_package.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_package.py b/tools/azure-sdk-tools/packaging_tools/auto_package.py index 15e2c9090453..df1750003af3 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_package.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_package.py @@ -46,6 +46,11 @@ def main(generate_input, generate_output): # to distinguish with track1 if 'azure-mgmt-' in package_name: package["packageName"] = "track2_" + package["packageName"] + for artifact in package["artifacts"]: + if ".whl" in artifact: + package["apiViewArtifact"] = artifact + package["language"] = "Python" + break result["packages"].append(package) with open(generate_output, "w") as writer: From 1061cb35352c9794bfbf0d055ad1c6a8f0e3fc61 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 14 Jun 2022 11:27:02 +0800 Subject: [PATCH 083/120] uodate edit_sdk_setup --- scripts/auto_release/main.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index af91183d99f0..c7c03e84d767 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -9,8 +9,8 @@ from functools import wraps from typing import List, Any, Dict from packaging.version import Version -from ghapi.all import GhApi -from azure.storage.blob import BlobServiceClient, ContainerClient +# from ghapi.all import GhApi +# from azure.storage.blob import BlobServiceClient, ContainerClient from util import add_certificate _LOG = logging.getLogger() @@ -256,12 +256,28 @@ def edit_sdk_readme(content: List[str]): modify_file(sdk_readme, edit_sdk_readme) + @staticmethod + def get_need_mgmt_core(): + template_path = Path('../../tools/azure-sdk-tools/packaging_tools/templates/setup.py') + with open(template_path, 'r') as fr: + content = fr.readlines() + for line in content: + if 'msrest' in line: + target_msrest = line.strip().strip(',') + yield target_msrest + if 'azure-mgmt-core' in line: + target_mgmt_core = line.strip().strip(',') + yield target_mgmt_core + def check_sdk_setup(self): def edit_sdk_setup(content: List[str]): + target_msrest, target_mgmt_core = self.get_need_mgmt_core() + msrest_pattern = re.compile('msrest>=\d\.\d[1-2]\.\d[1-2]') + mgmt_core_pattern = re.compile('azure-mgmt-core>=\d\.\d\.\d,<\d\.\d\.\d') for i in range(0, len(content)): - content[i] = content[i].replace('msrestazure>=0.4.32,<2.0.0', 'azure-mgmt-core>=1.3.0,<2.0.0') - content[i] = content[i].replace('azure-mgmt-core>=1.2.0,<2.0.0', 'azure-mgmt-core>=1.3.0,<2.0.0') - content[i] = content[i].replace('msrest>=0.5.0', 'msrest>=0.6.21') + content[i] = content[i].replace('msrestazure>=0.4.32,<2.0.0', target_mgmt_core) + content[i] = re.sub(mgmt_core_pattern, target_mgmt_core, content[i]) + content[i] = re.sub(msrest_pattern, target_msrest, content[i]) modify_file(str(Path(self.sdk_code_path()) / 'setup.py'), edit_sdk_setup) @@ -561,5 +577,6 @@ def run(self): logging.basicConfig() main_logger.setLevel(logging.INFO) - instance = CodegenTestPR() - instance.run() + CodegenTestPR.get_need_mgmt_core() + # instance = CodegenTestPR() + # instance.run() From ce4bef811e1d98542a95ac279108a0a3fd59562f Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 14 Jun 2022 11:30:57 +0800 Subject: [PATCH 084/120] fix --- scripts/auto_release/main.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index c7c03e84d767..10f560c695a2 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -9,8 +9,8 @@ from functools import wraps from typing import List, Any, Dict from packaging.version import Version -# from ghapi.all import GhApi -# from azure.storage.blob import BlobServiceClient, ContainerClient +from ghapi.all import GhApi +from azure.storage.blob import BlobServiceClient, ContainerClient from util import add_certificate _LOG = logging.getLogger() @@ -577,6 +577,5 @@ def run(self): logging.basicConfig() main_logger.setLevel(logging.INFO) - CodegenTestPR.get_need_mgmt_core() - # instance = CodegenTestPR() - # instance.run() + instance = CodegenTestPR() + instance.run() From 916bd3a2fc7ae661b5ff5db45acf5c615564e120 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 14 Jun 2022 11:40:01 +0800 Subject: [PATCH 085/120] test --- scripts/auto_release/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 10f560c695a2..c2f572b8c361 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -330,6 +330,7 @@ def calculate_next_version_proc(self, last_version: str): return next_version def get_autorest_result(self) -> Dict[Any, Any]: + print("***", os.getcwd()) with open(self.autorest_result, 'r') as file_in: content = json.load(file_in) return content From 4278ee3c7e734b776f3ae6bcb4dec6272feaf750 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 14 Jun 2022 11:54:25 +0800 Subject: [PATCH 086/120] test --- scripts/auto_release/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index c2f572b8c361..4a63cf074828 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -258,7 +258,8 @@ def edit_sdk_readme(content: List[str]): @staticmethod def get_need_mgmt_core(): - template_path = Path('../../tools/azure-sdk-tools/packaging_tools/templates/setup.py') + print("***", os.getcwd()) + template_path = Path('tools/azure-sdk-tools/packaging_tools/templates/setup.py') with open(template_path, 'r') as fr: content = fr.readlines() for line in content: @@ -330,7 +331,6 @@ def calculate_next_version_proc(self, last_version: str): return next_version def get_autorest_result(self) -> Dict[Any, Any]: - print("***", os.getcwd()) with open(self.autorest_result, 'r') as file_in: content = json.load(file_in) return content From 5fc5ccf21beee02a4a1578d64a468c7932ed11bb Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 14 Jun 2022 13:47:52 +0800 Subject: [PATCH 087/120] fix bug --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 4a63cf074828..af9e6ce13316 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -272,7 +272,7 @@ def get_need_mgmt_core(): def check_sdk_setup(self): def edit_sdk_setup(content: List[str]): - target_msrest, target_mgmt_core = self.get_need_mgmt_core() + target_msrest, target_mgmt_core = list(self.get_need_mgmt_core()) msrest_pattern = re.compile('msrest>=\d\.\d[1-2]\.\d[1-2]') mgmt_core_pattern = re.compile('azure-mgmt-core>=\d\.\d\.\d,<\d\.\d\.\d') for i in range(0, len(content)): From c895704b08d6936c32a22b2e16b0697b3b461ce6 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 14 Jun 2022 13:58:42 +0800 Subject: [PATCH 088/120] debug --- scripts/auto_release/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index af9e6ce13316..29ddac669e06 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -272,6 +272,7 @@ def get_need_mgmt_core(): def check_sdk_setup(self): def edit_sdk_setup(content: List[str]): + print("**** get_need_mgmt_core: ", list(self.get_need_mgmt_core())) target_msrest, target_mgmt_core = list(self.get_need_mgmt_core()) msrest_pattern = re.compile('msrest>=\d\.\d[1-2]\.\d[1-2]') mgmt_core_pattern = re.compile('azure-mgmt-core>=\d\.\d\.\d,<\d\.\d\.\d') From 4c22e5cd09e746d7a912d54533370e8dee2cce3a Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 14 Jun 2022 14:08:51 +0800 Subject: [PATCH 089/120] fix bug --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 29ddac669e06..189f72fb9f5c 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -263,7 +263,7 @@ def get_need_mgmt_core(): with open(template_path, 'r') as fr: content = fr.readlines() for line in content: - if 'msrest' in line: + if 'msrest>' in line: target_msrest = line.strip().strip(',') yield target_msrest if 'azure-mgmt-core' in line: From cded964566bd762bc91a8f46ee3407780430d6c6 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 14 Jun 2022 14:18:21 +0800 Subject: [PATCH 090/120] fix bug --- scripts/auto_release/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 189f72fb9f5c..a9ddfc2ce19f 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -264,10 +264,10 @@ def get_need_mgmt_core(): content = fr.readlines() for line in content: if 'msrest>' in line: - target_msrest = line.strip().strip(',') + target_msrest = line.strip().strip(',').strip('\'') yield target_msrest if 'azure-mgmt-core' in line: - target_mgmt_core = line.strip().strip(',') + target_mgmt_core = line.strip().strip(',').strip('\'') yield target_mgmt_core def check_sdk_setup(self): From cc60f92450b8c1d509f0b3d03f9148aaf65b10b7 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Tue, 14 Jun 2022 14:38:12 +0800 Subject: [PATCH 091/120] Update main.py --- scripts/auto_release/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index a9ddfc2ce19f..2cfd11d8d97f 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -258,7 +258,6 @@ def edit_sdk_readme(content: List[str]): @staticmethod def get_need_mgmt_core(): - print("***", os.getcwd()) template_path = Path('tools/azure-sdk-tools/packaging_tools/templates/setup.py') with open(template_path, 'r') as fr: content = fr.readlines() @@ -272,7 +271,6 @@ def get_need_mgmt_core(): def check_sdk_setup(self): def edit_sdk_setup(content: List[str]): - print("**** get_need_mgmt_core: ", list(self.get_need_mgmt_core())) target_msrest, target_mgmt_core = list(self.get_need_mgmt_core()) msrest_pattern = re.compile('msrest>=\d\.\d[1-2]\.\d[1-2]') mgmt_core_pattern = re.compile('azure-mgmt-core>=\d\.\d\.\d,<\d\.\d\.\d') From 1f67dfea37a512c48d0faba36ae03d91fea6fe2c Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 14 Jun 2022 15:35:29 +0800 Subject: [PATCH 092/120] move to ci check --- scripts/auto_release/main.py | 45 +++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 2cfd11d8d97f..89f87322b8fb 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -256,28 +256,12 @@ def edit_sdk_readme(content: List[str]): modify_file(sdk_readme, edit_sdk_readme) - @staticmethod - def get_need_mgmt_core(): - template_path = Path('tools/azure-sdk-tools/packaging_tools/templates/setup.py') - with open(template_path, 'r') as fr: - content = fr.readlines() - for line in content: - if 'msrest>' in line: - target_msrest = line.strip().strip(',').strip('\'') - yield target_msrest - if 'azure-mgmt-core' in line: - target_mgmt_core = line.strip().strip(',').strip('\'') - yield target_mgmt_core - def check_sdk_setup(self): def edit_sdk_setup(content: List[str]): - target_msrest, target_mgmt_core = list(self.get_need_mgmt_core()) - msrest_pattern = re.compile('msrest>=\d\.\d[1-2]\.\d[1-2]') - mgmt_core_pattern = re.compile('azure-mgmt-core>=\d\.\d\.\d,<\d\.\d\.\d') for i in range(0, len(content)): - content[i] = content[i].replace('msrestazure>=0.4.32,<2.0.0', target_mgmt_core) - content[i] = re.sub(mgmt_core_pattern, target_mgmt_core, content[i]) - content[i] = re.sub(msrest_pattern, target_msrest, content[i]) + content[i] = content[i].replace('msrestazure>=0.4.32,<2.0.0', 'azure-mgmt-core>=1.3.0,<2.0.0') + content[i] = content[i].replace('azure-mgmt-core>=1.2.0,<2.0.0', 'azure-mgmt-core>=1.3.0,<2.0.0') + content[i] = content[i].replace('msrest>=0.5.0', 'msrest>=0.6.21') modify_file(str(Path(self.sdk_code_path()) / 'setup.py'), edit_sdk_setup) @@ -391,12 +375,29 @@ def check_changelog_file(self): else: self.edit_changelog() + @staticmethod + def get_need_dependency(): + template_path = Path('tools/azure-sdk-tools/packaging_tools/templates/setup.py') + with open(template_path, 'r') as fr: + content = fr.readlines() + for line in content: + if 'msrest>' in line: + target_msrest = line.strip().strip(',').strip('\'') + yield target_msrest + if 'azure-mgmt-core' in line: + target_mgmt_core = line.strip().strip(',').strip('\'') + yield target_mgmt_core + def check_ci_file_proc(self, dependency: str): def edit_ci_file(content: List[str]): new_line = f'#override azure-mgmt-{self.package_name} {dependency}' + dependency_name = dependency.split('>')[0] for i in range(len(content)): if new_line in content[i]: return + if f'azure-mgmt-{self.package_name} {dependency_name}' in content[i]: + content[i] = new_line + return prefix = '' if '\n' in content[-1] else '\n' content.append(prefix + new_line + '\n') @@ -404,8 +405,10 @@ def edit_ci_file(content: List[str]): print_exec('git add shared_requirements.txt') def check_ci_file(self): - self.check_ci_file_proc('msrest>=0.6.21') - self.check_ci_file_proc('azure-mgmt-core>=1.3.0,<2.0.0') + # eg: target_msrest = 'msrest>=0.6.21', target_mgmt_core = 'azure-mgmt-core>=1.3.0,<2.0.0' + target_msrest, target_mgmt_core = list(self.get_need_dependency()) + self.check_ci_file_proc(target_msrest) + self.check_ci_file_proc(target_mgmt_core) def check_file(self): self.check_file_with_packaging_tool() From 88323f7c6d16504f169ce65ceee1755961aaae70 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Wed, 15 Jun 2022 10:36:50 +0800 Subject: [PATCH 093/120] Update main.py --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 89f87322b8fb..0dbdb7d66451 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -396,7 +396,7 @@ def edit_ci_file(content: List[str]): if new_line in content[i]: return if f'azure-mgmt-{self.package_name} {dependency_name}' in content[i]: - content[i] = new_line + content[i] = new_line + '\n' return prefix = '' if '\n' in content[-1] else '\n' content.append(prefix + new_line + '\n') From 4d8045e0bcb88f4a316a9f43e0d193faba9fcb8c Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Wed, 22 Jun 2022 14:14:16 +0800 Subject: [PATCH 094/120] for test --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 7c4aff504b5a..7878beb77b91 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -82,7 +82,7 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): usr = 'Azure' - branch = 'main' + branch = 'jftest-generate-servicebus-ci' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From b11ff6f34a898fb55c1c84cf456109665eee93f9 Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Wed, 22 Jun 2022 14:54:01 +0800 Subject: [PATCH 095/120] restore --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 7878beb77b91..7c4aff504b5a 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -82,7 +82,7 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): usr = 'Azure' - branch = 'jftest-generate-servicebus-ci' + branch = 'main' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From 9449e44f8549ef3178e7e2568306757a5c9a14c2 Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Wed, 6 Jul 2022 14:04:22 +0800 Subject: [PATCH 096/120] update msg for failed version --- scripts/auto_release/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 7c4aff504b5a..b25f05c468ca 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -285,7 +285,8 @@ def calculate_next_version_proc(self, last_version: str): preview_tag = not self.tag_is_stable changelog = self.get_changelog() if changelog == '': - return '0.0.0' + msg = 'it should be stable' if self.tag_is_stable else 'it should be perview' + return f'0.0.0 ({msg})' preview_version = 'rc' in last_version or 'b' in last_version # | preview tag | stable tag # preview version(1.0.0rc1/1.0.0b1) | 1.0.0rc2(track1)/1.0.0b2(track2) | 1.0.0 From aff78b2d9443cb01c1610edd3b76ad28b79e095c Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Thu, 7 Jul 2022 13:43:33 +0800 Subject: [PATCH 097/120] for testing --- scripts/auto_release/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index b25f05c468ca..d1a7066b0a24 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -81,8 +81,8 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): - usr = 'Azure' - branch = 'main' + usr = 'openapi-env-test' + branch = 'python-track2-test' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From 2fbda64399d616b2f555d24634fe8c4eef4dc0eb Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Thu, 7 Jul 2022 16:57:39 +0800 Subject: [PATCH 098/120] for testing --- scripts/auto_release/main.py | 2 +- swagger_to_sdk_config_autorest.json | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index d1a7066b0a24..ec3d4b80ffa5 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -562,7 +562,7 @@ def create_pr(self): def run(self): self.prepare_branch() self.check_file() - self.run_test() + # self.run_test() self.create_pr() diff --git a/swagger_to_sdk_config_autorest.json b/swagger_to_sdk_config_autorest.json index 90bbc26bdf05..1fd31529a5cf 100644 --- a/swagger_to_sdk_config_autorest.json +++ b/swagger_to_sdk_config_autorest.json @@ -1,11 +1,12 @@ { "meta": { "autorest_options": { - "version": "3.7.2", - "use": ["@autorest/python@5.16.0", "@autorest/modelerfour@4.19.3"], + "version": "3.8.1", + "use": ["@autorest/python@6.0.1", "@autorest/modelerfour@4.23.5"], "python": "", "sdkrel:python-sdks-folder": "./sdk/.", - "python3-only": "" + "version-tolerant": false, + "models-mode": "msrest" }, "advanced_options": { "create_sdk_pull_requests": true, From 74c9e0caa345d1d357716d2845374274fc8d9652 Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Fri, 8 Jul 2022 09:50:05 +0800 Subject: [PATCH 099/120] restore --- swagger_to_sdk_config_autorest.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/swagger_to_sdk_config_autorest.json b/swagger_to_sdk_config_autorest.json index 1fd31529a5cf..90bbc26bdf05 100644 --- a/swagger_to_sdk_config_autorest.json +++ b/swagger_to_sdk_config_autorest.json @@ -1,12 +1,11 @@ { "meta": { "autorest_options": { - "version": "3.8.1", - "use": ["@autorest/python@6.0.1", "@autorest/modelerfour@4.23.5"], + "version": "3.7.2", + "use": ["@autorest/python@5.16.0", "@autorest/modelerfour@4.19.3"], "python": "", "sdkrel:python-sdks-folder": "./sdk/.", - "version-tolerant": false, - "models-mode": "msrest" + "python3-only": "" }, "advanced_options": { "create_sdk_pull_requests": true, From 3ec222f92424dcea14f8871e00bf3eecc59d40df Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Fri, 8 Jul 2022 09:52:09 +0800 Subject: [PATCH 100/120] restore --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index ec3d4b80ffa5..d1a7066b0a24 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -562,7 +562,7 @@ def create_pr(self): def run(self): self.prepare_branch() self.check_file() - # self.run_test() + self.run_test() self.create_pr() From c11dac07e376e91dc1556d65619b1b9653a757b5 Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Fri, 8 Jul 2022 14:04:28 +0800 Subject: [PATCH 101/120] pin fastcore version --- scripts/auto_release/requirement.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/requirement.txt b/scripts/auto_release/requirement.txt index d8a62d6a6a1f..698652f950c2 100644 --- a/scripts/auto_release/requirement.txt +++ b/scripts/auto_release/requirement.txt @@ -3,4 +3,5 @@ python-dotenv==0.15.0 ghapi==0.1.19 packaging==21.3 pytest==6.2.5 -azure-storage-blob==12.9.0 \ No newline at end of file +azure-storage-blob==12.9.0 +fastcore==1.3.25 \ No newline at end of file From f889bead6dec6ea8f566937615ab1c96bc0d3890 Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Fri, 15 Jul 2022 13:55:08 +0800 Subject: [PATCH 102/120] test --- scripts/auto_release/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index d1a7066b0a24..92356ea358a2 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -81,8 +81,8 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): - usr = 'openapi-env-test' - branch = 'python-track2-test' + usr = 'BigCat20196' + branch = 'bigcat/update_auto_codegen' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From db2f4a3f183a5f4b46dc35a3f931a8298b8fd054 Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Fri, 22 Jul 2022 11:06:24 +0800 Subject: [PATCH 103/120] debug --- scripts/auto_release/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 92356ea358a2..66bddb5c7e32 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -81,8 +81,8 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): - usr = 'BigCat20196' - branch = 'bigcat/update_auto_codegen' + usr = 'Azure' + branch = 'main' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') @@ -191,6 +191,7 @@ def generate_code(self): print_check(f'python -m packaging_tools.auto_codegen {self.autorest_result} {self.autorest_result}') generate_result = self.get_autorest_result() + print(f"*** generate_result: {generate_result}") self.tag_is_stable = list(generate_result.values())[0]['tagIsStable'] log(f"tag_is_stable is {self.tag_is_stable}") From 5741f7886cec18e9a026afde92c865d52b3ab147 Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Fri, 22 Jul 2022 11:26:26 +0800 Subject: [PATCH 104/120] debug --- scripts/auto_release/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 66bddb5c7e32..b25f05c468ca 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -191,7 +191,6 @@ def generate_code(self): print_check(f'python -m packaging_tools.auto_codegen {self.autorest_result} {self.autorest_result}') generate_result = self.get_autorest_result() - print(f"*** generate_result: {generate_result}") self.tag_is_stable = list(generate_result.values())[0]['tagIsStable'] log(f"tag_is_stable is {self.tag_is_stable}") From c8877f643bbf55f499369edbd46237fe4af9cbd1 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Tue, 26 Jul 2022 11:06:44 +0800 Subject: [PATCH 105/120] optimize --- .../azure-sdk-tools/packaging_tools/generate_utils.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 065e569e4b69..c565d7f0520e 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -166,18 +166,13 @@ def gen_basic_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: return { "package-name": gen_package_name(origin_config), "license-header": "MICROSOFT_MIT_NO_VERSION", - "clear-output-folder": True, - "no-namespace-folders": True, - "version-tolerant": True, "package-version": origin_config.get("package-version", "1.0.0b1"), "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + "package-mode": "dataplane", + "output-folder": "../", } -def gen_general_output_folder(package_name: str) -> str: - return "../" + package_name.replace('-', '/') - - def gen_general_namespace(package_name: str) -> str: return package_name.replace('-', '.') @@ -186,7 +181,6 @@ def gen_dpg_config_single_client(origin_config: Dict[str, Any]) -> str: package_name = Path(origin_config["output-folder"]).parts[-1] readme_config = gen_basic_config(origin_config) readme_config.update({ - "output-folder": gen_general_output_folder(package_name), "namespace": gen_general_namespace(package_name), }) readme_content = yaml_block(yaml.safe_dump(readme_config), "### Settings") @@ -201,7 +195,6 @@ def gen_tag_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: extra_part = tag_name.split("-")[-1] tag_config[tag_name] = { "namespace": gen_general_namespace(package_name) + f".{extra_part}", - "output-folder": gen_general_output_folder(package_name) + f"/{extra_part}" } return tag_config From 15f033c0acf2854633d3affae2fd9ad90b4f54ff Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 26 Jul 2022 11:38:23 +0800 Subject: [PATCH 106/120] Update setup.py --- tools/azure-sdk-tools/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/setup.py b/tools/azure-sdk-tools/setup.py index a568c5f5cac5..db116f4616f0 100644 --- a/tools/azure-sdk-tools/setup.py +++ b/tools/azure-sdk-tools/setup.py @@ -25,7 +25,7 @@ "azure-mgmt-storage", "azure-mgmt-keyvault", "python-dotenv", - "PyYAML==6.0" + "PyYAML" ] setup( From 374e1f38cee8d7c5b3e0f5c0d59186f72bdfb8e6 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Tue, 26 Jul 2022 18:32:19 +0800 Subject: [PATCH 107/120] sdk generation pipeline support dpg Signed-off-by: Zhou Zheng --- scripts/sdk_generate.sh | 4 ++-- scripts/sdk_init.sh | 4 ++-- .../packaging_tools/auto_codegen.py | 2 +- .../packaging_tools/generate_utils.py | 17 +++++++++-------- .../packaging_tools/sdk_generator.py | 15 ++++++++++----- .../packaging_tools/sdk_package.py | 11 +++++++++++ 6 files changed, 35 insertions(+), 18 deletions(-) diff --git a/scripts/sdk_generate.sh b/scripts/sdk_generate.sh index 69201e03613b..591889aa16f5 100755 --- a/scripts/sdk_generate.sh +++ b/scripts/sdk_generate.sh @@ -9,8 +9,8 @@ PATH="$VIRTUAL_ENV/bin:$PATH" export PATH # node version degrade -sudo npm install -g n -sudo n 14.15.0 +npm install -g n +n 14.15.0 echo "$PATH" export PATH="/usr/local/n/versions/node/14.15.0/bin:$PATH" diff --git a/scripts/sdk_init.sh b/scripts/sdk_init.sh index 916a51068cff..890373cea858 100755 --- a/scripts/sdk_init.sh +++ b/scripts/sdk_init.sh @@ -1,8 +1,8 @@ #!/bin/bash # install python3.8 -sudo apt-get install python3.8 -sudo apt-get install python3.8-venv +apt-get install python3.8 +apt-get install python3.8-venv # init env TMPDIR="$(dirname $(dirname $(readlink -f "$0")))/venv" diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index 14f87f40a813..2ce77831c70b 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -28,7 +28,7 @@ def main(generate_input, generate_output): config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, python_tag=python_tag) else: - config = gen_dpg(input_readme, data.get('autorestConfig', '')) + config = gen_dpg(input_readme, data.get('autorestConfig', ''), spec_folder) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index c565d7f0520e..a5e740907c8a 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -162,12 +162,13 @@ def gen_package_name(origin_config: Dict[str, Any]) -> str: return Path(origin_config["output-folder"]).parts[-1] -def gen_basic_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: +def gen_basic_config(origin_config: Dict[str, Any], spec_folder: str = "../../../../../azure-rest-api-specs/") -> Dict[str, Any]: + spec_root = re.sub('specification', '', spec_folder) return { "package-name": gen_package_name(origin_config), "license-header": "MICROSOFT_MIT_NO_VERSION", "package-version": origin_config.get("package-version", "1.0.0b1"), - "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + "require": [spec_root + line for line in origin_config["require"]], "package-mode": "dataplane", "output-folder": "../", } @@ -177,9 +178,9 @@ def gen_general_namespace(package_name: str) -> str: return package_name.replace('-', '.') -def gen_dpg_config_single_client(origin_config: Dict[str, Any]) -> str: +def gen_dpg_config_single_client(origin_config: Dict[str, Any], spec_folder: str) -> str: package_name = Path(origin_config["output-folder"]).parts[-1] - readme_config = gen_basic_config(origin_config) + readme_config = gen_basic_config(origin_config, spec_folder) readme_config.update({ "namespace": gen_general_namespace(package_name), }) @@ -228,7 +229,7 @@ def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: # generate swagger/README.md and return relative path based on SDK repo root path -def gen_dpg_config(autorest_config: str) -> str: +def gen_dpg_config(autorest_config: str, spec_folder: str) -> str: # remove useless lines autorest_config = extract_yaml_content(autorest_config) _LOGGER.info(f"autorestConfig after remove useless lines:\n{autorest_config}") @@ -244,7 +245,7 @@ def gen_dpg_config(autorest_config: str) -> str: if "batch:" in autorest_config: readme_content = gen_dpg_config_multi_client(origin_config) else: - readme_content = gen_dpg_config_single_client(origin_config) + readme_content = gen_dpg_config_single_client(origin_config, spec_folder) # output autorest configuration swagger_readme = str(Path(swagger_folder, _DPG_README)) @@ -265,10 +266,10 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: return "" -def gen_dpg(rest_readme_path: str, autorest_config: str) -> Dict[str, Any]: +def gen_dpg(rest_readme_path: str, autorest_config: str, spec_folder: str) -> Dict[str, Any]: # generate or find swagger/README.md if autorest_config: - swagger_readme = gen_dpg_config(autorest_config) + swagger_readme = gen_dpg_config(autorest_config, spec_folder) else: swagger_readme = lookup_swagger_readme(rest_readme_path) if not swagger_readme: diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py index c2b06e51322e..a730bdaebe8d 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py @@ -4,9 +4,9 @@ from pathlib import Path from subprocess import check_call -from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG +from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate -from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview +from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, gen_dpg _LOGGER = logging.getLogger(__name__) @@ -24,8 +24,10 @@ def main(generate_input, generate_output): input_readme = data["relatedReadmeMdFile"] relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") - config_file = CONFIG_FILE if 'resource-manager' in input_readme else CONFIG_FILE_DPG - config = generate(config_file, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) + if 'resource-manager' in input_readme: + config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) + else: + config = gen_dpg(input_readme, data.get('autorestConfig', ''), spec_folder) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") @@ -39,10 +41,12 @@ def main(generate_input, generate_output): package_entry = {} package_entry["packageName"] = package_name package_entry["path"] = [folder_name] + package_entry["readmeMd"] = [input_readme] package_entry["tagIsStable"] = not judge_tag_preview(sdk_code_path) result[package_name] = package_entry else: result[package_name]["path"].append(folder_name) + result[package_name]["readmeMd"].append(input_readme) # Generate some necessary file for new service init_new_service(package_name, folder_name) @@ -51,7 +55,7 @@ def main(generate_input, generate_output): try: update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme) except Exception as e: - _LOGGER.info(str(e)) + _LOGGER.info(f"fail to update meta: {str(e)}") # Setup package locally check_call( @@ -62,6 +66,7 @@ def main(generate_input, generate_output): # remove duplicates for value in result.values(): value["path"] = list(set(value["path"])) + value["readmeMd"] = list(set(value["readmeMd"])) with open(generate_output, "w") as writer: json.dump(result, writer) diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_package.py b/tools/azure-sdk-tools/packaging_tools/sdk_package.py index cbf5c5d799bf..fdb0d2e9a70d 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_package.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_package.py @@ -13,6 +13,7 @@ def main(generate_input, generate_output): with open(generate_input, "r") as reader: data = json.load(reader) + _LOGGER.info(f"auto_package input: {data}") sdk_folder = "." result = {"packages": []} @@ -38,9 +39,19 @@ def main(generate_input, generate_output): dist_path = Path(sdk_folder, folder_name, package_name, "dist") package["artifacts"] = [str(dist_path / package_file) for package_file in os.listdir(dist_path)] package["result"] = "succeeded" + # Installation package + package["installInstructions"] = { + "full": "You can install the use using pip install of the artificats.", + "lite": f"pip install {package_name}", + } # to distinguish with track1 if 'azure-mgmt-' in package_name: package["packageName"] = "track2_" + package["packageName"] + for artifact in package["artifacts"]: + if ".whl" in artifact: + package["apiViewArtifact"] = artifact + package["language"] = "Python" + break package["packageFolder"] = package["path"][0] result["packages"].append(package) From 937bd2102fa94744d2e88bc1470bfa710a0e7d64 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Tue, 26 Jul 2022 18:53:38 +0800 Subject: [PATCH 108/120] auto codegen don't use specFolder to define swagger repo's location Signed-off-by: Zhou Zheng --- tools/azure-sdk-tools/packaging_tools/auto_codegen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index 2ce77831c70b..b35318a340d2 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -28,7 +28,7 @@ def main(generate_input, generate_output): config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, python_tag=python_tag) else: - config = gen_dpg(input_readme, data.get('autorestConfig', ''), spec_folder) + config = gen_dpg(input_readme, data.get('autorestConfig', ''), "../../../../../azure-rest-api-specs/") package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") From dab3a92f62aff6c4250c2dcd9514d6acb0975ab0 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Tue, 26 Jul 2022 19:08:45 +0800 Subject: [PATCH 109/120] fetch automation_generate.sh recent change to sdk_generate.sh Signed-off-by: Zhou Zheng --- tools/azure-sdk-tools/packaging_tools/sdk_generator.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py index a730bdaebe8d..0ef787271d34 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py @@ -6,7 +6,8 @@ from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate -from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, gen_dpg +from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, + format_samples, gen_dpg) _LOGGER = logging.getLogger(__name__) @@ -19,13 +20,15 @@ def main(generate_input, generate_output): spec_folder = data["specFolder"] sdk_folder = "." result = {} + python_tag = data.get('python_tag') package_total = set() input_readme = data["relatedReadmeMdFile"] relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") if 'resource-manager' in input_readme: - config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) + config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, + python_tag=python_tag) else: config = gen_dpg(input_readme, data.get('autorestConfig', ''), spec_folder) package_names = get_package_names(sdk_folder) @@ -50,6 +53,7 @@ def main(generate_input, generate_output): # Generate some necessary file for new service init_new_service(package_name, folder_name) + format_samples(sdk_code_path) # Update metadata try: From 7607ffa2f1d2764213ca9be6e0823a9ee3011613 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Wed, 27 Jul 2022 11:04:56 +0800 Subject: [PATCH 110/120] set default spec flord as global constant Signed-off-by: Zhou Zheng --- tools/azure-sdk-tools/packaging_tools/auto_codegen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index b35318a340d2..620f39a21a5e 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -11,6 +11,7 @@ _LOGGER = logging.getLogger(__name__) +DEFAULT_SPEC_FOLDER = "../../../../../azure-rest-api-specs/" def main(generate_input, generate_output): with open(generate_input, "r") as reader: @@ -28,7 +29,7 @@ def main(generate_input, generate_output): config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, python_tag=python_tag) else: - config = gen_dpg(input_readme, data.get('autorestConfig', ''), "../../../../../azure-rest-api-specs/") + config = gen_dpg(input_readme, data.get('autorestConfig', ''), DEFAULT_SPEC_FOLDER) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") From 0bc45a0dd0dc520eae4d7532523aabf1d9ba03c2 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Wed, 27 Jul 2022 11:37:21 +0800 Subject: [PATCH 111/120] set DEFAULT_SPEC_FOLDER in generate_utils Signed-off-by: Zhou Zheng --- tools/azure-sdk-tools/packaging_tools/auto_codegen.py | 4 +--- tools/azure-sdk-tools/packaging_tools/generate_utils.py | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index 620f39a21a5e..a55ffe9e658a 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -7,12 +7,10 @@ from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, - format_samples, gen_dpg) + format_samples, gen_dpg, DEFAULT_SPEC_FOLDER) _LOGGER = logging.getLogger(__name__) -DEFAULT_SPEC_FOLDER = "../../../../../azure-rest-api-specs/" - def main(generate_input, generate_output): with open(generate_input, "r") as reader: data = json.load(reader) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index a5e740907c8a..452cc7c03106 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -19,6 +19,7 @@ _SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII) DEFAULT_DEST_FOLDER = "./dist" +DEFAULT_SPEC_FOLDER = "../../../../../azure-rest-api-specs/" _DPG_README = "README.md" def get_package_names(sdk_folder): @@ -162,7 +163,7 @@ def gen_package_name(origin_config: Dict[str, Any]) -> str: return Path(origin_config["output-folder"]).parts[-1] -def gen_basic_config(origin_config: Dict[str, Any], spec_folder: str = "../../../../../azure-rest-api-specs/") -> Dict[str, Any]: +def gen_basic_config(origin_config: Dict[str, Any], spec_folder: str = DEFAULT_SPEC_FOLDER) -> Dict[str, Any]: spec_root = re.sub('specification', '', spec_folder) return { "package-name": gen_package_name(origin_config), From ec10281371b5dbc7978b52706cc2164ad9e7d01d Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 28 Jul 2022 16:25:19 +0800 Subject: [PATCH 112/120] code --- .../packaging_tools/auto_codegen.py | 4 +-- .../packaging_tools/generate_utils.py | 29 +++++-------------- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index a55ffe9e658a..de2d39ace085 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -7,7 +7,7 @@ from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, - format_samples, gen_dpg, DEFAULT_SPEC_FOLDER) + format_samples, gen_dpg, dpg_relative_folder) _LOGGER = logging.getLogger(__name__) @@ -27,7 +27,7 @@ def main(generate_input, generate_output): config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, python_tag=python_tag) else: - config = gen_dpg(input_readme, data.get('autorestConfig', ''), DEFAULT_SPEC_FOLDER) + config = gen_dpg(input_readme, data.get('autorestConfig', ''), dpg_relative_folder(spec_folder)) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 02d88a980e94..4037f1357a6d 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -19,12 +19,11 @@ _SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII) DEFAULT_DEST_FOLDER = "./dist" -<<<<<<< HEAD -DEFAULT_SPEC_FOLDER = "../../../../../azure-rest-api-specs/" -======= ->>>>>>> f5eded26f09270ae60c840dab2ce275d523eb8f6 _DPG_README = "README.md" +def dpg_relative_folder(spec_folder: str) -> str: + return ("../" * 4) + spec_folder + "/" + def get_package_names(sdk_folder): files = get_add_diff_file_list(sdk_folder) matches = {_SDK_FOLDER_RE.search(f) for f in files} @@ -166,21 +165,13 @@ def gen_package_name(origin_config: Dict[str, Any]) -> str: return Path(origin_config["output-folder"]).parts[-1] -<<<<<<< HEAD -def gen_basic_config(origin_config: Dict[str, Any], spec_folder: str = DEFAULT_SPEC_FOLDER) -> Dict[str, Any]: +def gen_basic_config(origin_config: Dict[str, Any], spec_folder: str) -> Dict[str, Any]: spec_root = re.sub('specification', '', spec_folder) -======= -def gen_basic_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: ->>>>>>> f5eded26f09270ae60c840dab2ce275d523eb8f6 return { "package-name": gen_package_name(origin_config), "license-header": "MICROSOFT_MIT_NO_VERSION", "package-version": origin_config.get("package-version", "1.0.0b1"), -<<<<<<< HEAD "require": [spec_root + line for line in origin_config["require"]], -======= - "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], ->>>>>>> f5eded26f09270ae60c840dab2ce275d523eb8f6 "package-mode": "dataplane", "output-folder": "../", } @@ -190,15 +181,9 @@ def gen_general_namespace(package_name: str) -> str: return package_name.replace('-', '.') -<<<<<<< HEAD def gen_dpg_config_single_client(origin_config: Dict[str, Any], spec_folder: str) -> str: package_name = Path(origin_config["output-folder"]).parts[-1] readme_config = gen_basic_config(origin_config, spec_folder) -======= -def gen_dpg_config_single_client(origin_config: Dict[str, Any]) -> str: - package_name = Path(origin_config["output-folder"]).parts[-1] - readme_config = gen_basic_config(origin_config) ->>>>>>> f5eded26f09270ae60c840dab2ce275d523eb8f6 readme_config.update({ "namespace": gen_general_namespace(package_name), }) @@ -227,9 +212,9 @@ def gen_batch_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: return {"batch": batch_config} -def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: +def gen_dpg_config_multi_client(origin_config: Dict[str, Any], spec_folder: str) -> str: # generate config - basic_config = gen_basic_config(origin_config) + basic_config = gen_basic_config(origin_config, spec_folder) batch_config = gen_batch_config(origin_config) tag_config = gen_tag_config(origin_config) @@ -261,7 +246,7 @@ def gen_dpg_config(autorest_config: str, spec_folder: str) -> str: # generate autorest configuration if "batch:" in autorest_config: - readme_content = gen_dpg_config_multi_client(origin_config) + readme_content = gen_dpg_config_multi_client(origin_config, spec_folder) else: readme_content = gen_dpg_config_single_client(origin_config, spec_folder) From b41f34ac23a3a4c981fe37776da54124ff7a58fe Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Mon, 8 Aug 2022 13:49:02 +0800 Subject: [PATCH 113/120] fix auto-release utils --- scripts/auto_release/util.py | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/scripts/auto_release/util.py b/scripts/auto_release/util.py index c9e9305dce9d..a2fe9a858d1a 100644 --- a/scripts/auto_release/util.py +++ b/scripts/auto_release/util.py @@ -1,31 +1,15 @@ from pathlib import Path -CERTIFICATION = ''' ------BEGIN CERTIFICATE----- -MIIDSDCCAjCgAwIBAgIUPMKpJ/j10eQrcQBNnkImIaOYHakwDQYJKoZIhvcNAQEL -BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIxMDgwNTAwMzU1NloXDTIyMDgw -NTAwMzU1NlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF -AAOCAQ8AMIIBCgKCAQEAxe/ZseXgOTVoF7uTjX5Leknk95jIoyGc+VlxA8BhzGOr -r4u6VNQZRCMq+svHY36tW4+u/xHNe2kvbwy2mnS8cFFLfst+94qBZVJDBxSGZ9I/ -wekErNsjFsik4UrMvcC+ZlGPh7hb3f7tSx29tn1DIkAUXVnbZ6TT5s+mYRQpZ6fW -6kR3RNfc0A1IUM7Zs9yfNEr0O2H41P2HcLKoOPtvd7GvTQm9Ofh3srKvII+sZn/J -WH7r76oRQMX904mOMdryQwZLObsqX4dXIEbafKVSecB3PBVIhv8gVtJhcZbQP1pI -mMiWd6PHv46ZhGf7+cKnYUSa8Ia2t/wetK1wd00dFwIDAQABo4GRMIGOMA8GA1Ud -EwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGmMBYGA1UdJQEB/wQMMAoGCCsGAQUF -BwMBMBcGA1UdEQEB/wQNMAuCCWxvY2FsaG9zdDA6BgorBgEEAYI3VAEBBCwMKkFT -UC5ORVQgQ29yZSBIVFRQUyBkZXZlbG9wbWVudCBjZXJ0aWZpY2F0ZTANBgkqhkiG -9w0BAQsFAAOCAQEAIj2VlBVcXGSly6KCBg6lgwFi+henWfSox77iuGAaAxDjN3jd -9lZahW4MPNLHKSrPRb4YNSLZ2jh7zdcttQrqd4qH65o1q56q5JrCmli99iIzY9Y8 -RdYyxK4Zzr31wjpsyFiWQfqJTuSFUUg9uDDj0negwEZLIGlt7nr12wflt2+QOJtD -byMeSZLbB5dPzn341DK0qfJEJMMgL0XsPEVZ3TQ6Alc9zq5wI608C/mXnz3xJE05 -UTYD8pRJJ/DyG0empvOVE8Sg93msHPquAbgqO9aqCpykgg/a8CFvI4wRdfvGEFlv -8XJKL8Y/PFsmFeO3axq3zUYKFVdc9Un4dFIaag== ------END CERTIFICATE----- -''' + +def _find_certificate(): + devcert_path = Path('eng/common/testproxy/dotnet-devcert.crt') + with open(devcert_path, 'r') as fr: + return fr.read() def add_certificate(): + certification = _find_certificate() cacert_path = Path('../venv-sdk/lib/python3.8/site-packages/certifi/cacert.pem') with open(cacert_path, 'a+') as f: f.seek(0, 0) - f.write(CERTIFICATION) + f.write(certification) From 1787c9ac8b4497f9985aee28ee12b60909a95629 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Wed, 10 Aug 2022 11:12:15 +0800 Subject: [PATCH 114/120] code --- scripts/auto_release/main.py | 18 ++++++++++++++---- scripts/release_helper/python.py | 8 +++++++- .../packaging_tools/change_log.py | 6 +++--- .../packaging_tools/code_report.py | 1 + 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index a423955e8c96..7de4c0cbb640 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -54,9 +54,9 @@ def preview_version_plus(preview_label: str, last_version: str) -> str: def stable_version_plus(changelog: str, last_version: str): flag = [False, False, False] # breaking, feature, bugfix - flag[0] = '**Breaking changes**' in changelog - flag[1] = '**Features**' in changelog - flag[2] = '**Bugfixes**' in changelog + flag[0] = '### Breaking changes' in changelog + flag[1] = '### Features Added' in changelog + flag[2] = '### Bugs Fixed' in changelog num = last_version.split('.') if flag[0]: @@ -380,6 +380,16 @@ def get_need_dependency(): target_mgmt_core = line.strip().strip(',').strip('\'') yield target_mgmt_core + @staticmethod + def insert_line_num(content: List[str]) -> int: + start_num = 0 + end_num = len(content) + for i in range(end_num): + if content[i].find("#override azure-mgmt-") > -1: + start_num = i + break + return (int(str(time.time()).split('.')[-1]) % max(end_num - start_num, 1)) + start_num + def check_ci_file_proc(self, dependency: str): def edit_ci_file(content: List[str]): new_line = f'#override azure-mgmt-{self.package_name} {dependency}' @@ -391,7 +401,7 @@ def edit_ci_file(content: List[str]): content[i] = new_line + '\n' return prefix = '' if '\n' in content[-1] else '\n' - content.append(prefix + new_line + '\n') + content.insert(self.insert_line_num(content), prefix + new_line + '\n') modify_file(str(Path('shared_requirements.txt')), edit_ci_file) print_exec('git add shared_requirements.txt') diff --git a/scripts/release_helper/python.py b/scripts/release_helper/python.py index 3f020b14c09d..9beed7ae9f39 100644 --- a/scripts/release_helper/python.py +++ b/scripts/release_helper/python.py @@ -19,6 +19,7 @@ _7_DAY_ATTENTION = '7days attention' _MultiAPI = 'MultiAPI' _ON_TIME = 'on time' +_HOLD_ON = 'HoldOn' # record published issues _FILE_OUT = 'published_issues_python.csv' @@ -114,7 +115,11 @@ def attention_policy(self): def on_time_policy(self): if _ON_TIME in self.issue_package.labels_name: - self.bot_advice.append('on time') + self.bot_advice.append('On time') + + def hold_on_policy(self): + if _HOLD_ON in self.issue_package.labels_name: + self.bot_advice.append('Hold on') def remind_policy(self): if self.delay_time >= 15 and _7_DAY_ATTENTION in self.issue_package.labels_name and self.date_from_target < 0: @@ -133,6 +138,7 @@ def auto_bot_advice(self): self.multi_api_policy() self.attention_policy() self.on_time_policy() + self.hold_on_policy() self.remind_policy() diff --git a/tools/azure-sdk-tools/packaging_tools/change_log.py b/tools/azure-sdk-tools/packaging_tools/change_log.py index 07edeadaf9b1..118aa15b1669 100644 --- a/tools/azure-sdk-tools/packaging_tools/change_log.py +++ b/tools/azure-sdk-tools/packaging_tools/change_log.py @@ -31,11 +31,11 @@ def build_md(self): self.sort() buffer = [] if self.features: - _build_md(sorted(set(self.features), key=self.features.index), "**Features**", buffer) + _build_md(sorted(set(self.features), key=self.features.index), "### Features Added", buffer) if self.breaking_changes: - _build_md(sorted(set(self.breaking_changes), key=self.breaking_changes.index), "**Breaking changes**", buffer) + _build_md(sorted(set(self.breaking_changes), key=self.breaking_changes.index), "### Breaking changes", buffer) if not (self.features or self.breaking_changes) and self.optional_features: - _build_md(sorted(set(self.optional_features), key=self.optional_features.index), "**Features**", buffer) + _build_md(sorted(set(self.optional_features), key=self.optional_features.index), "### Features Added", buffer) return "\n".join(buffer).strip() diff --git a/tools/azure-sdk-tools/packaging_tools/code_report.py b/tools/azure-sdk-tools/packaging_tools/code_report.py index bb5799209e8c..40a15848b065 100644 --- a/tools/azure-sdk-tools/packaging_tools/code_report.py +++ b/tools/azure-sdk-tools/packaging_tools/code_report.py @@ -165,6 +165,7 @@ def merge_report(report_paths): merged_report["models"]["exceptions"].update(report_json["models"]["exceptions"]) merged_report["models"]["models"].update(report_json["models"]["models"]) merged_report["operations"].update(report_json["operations"]) + merged_report["client"] = report_json["client"] return merged_report From 7e05eca978879b64f8adce41067d08d9e7cacca0 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 10 Aug 2022 11:35:01 +0800 Subject: [PATCH 115/120] Update main.py --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 7de4c0cbb640..660ee7744c1b 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -82,7 +82,7 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): usr = 'Azure' - branch = 'main' + branch = 'changelog-sharerequirements' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From e7c45b1ceac0af02fca2d8b98025e1c78432f4d8 Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Wed, 17 Aug 2022 14:54:50 +0800 Subject: [PATCH 116/120] add python commit --- scripts/auto_release/PythonSdkLiveTest.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index f70e2c727afa..2e51899b4a22 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -47,6 +47,9 @@ jobs: mkdir azure-rest-api-specs git clone https://github.com/Azure/azure-rest-api-specs.git $(pwd)/azure-rest-api-specs + if [ "$(PYTHON_COMMIT)" ]; then + git reset --hard $(PYTHON_COMMIT) + fi # install autorest sudo npm install -g n From f2aa1d7f2ab06c824704ab54b25be0fe2c554918 Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Wed, 17 Aug 2022 15:17:39 +0800 Subject: [PATCH 117/120] update branch --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 27423d0c765e..86766e1c2b78 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -82,7 +82,7 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): usr = 'Azure' - branch = 'changelog-sharerequirements' + branch = 'main' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From 6cfbbf0a1dc7b8b27890fa99940c2f660775b595 Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Wed, 17 Aug 2022 15:34:37 +0800 Subject: [PATCH 118/120] debug --- scripts/auto_release/PythonSdkLiveTest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 2e51899b4a22..9764fa47fc69 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -49,6 +49,7 @@ jobs: if [ "$(PYTHON_COMMIT)" ]; then git reset --hard $(PYTHON_COMMIT) + echo $(PYTHON_COMMIT) fi # install autorest From aadf9895286a9d347892d451a2f45e1c0ceac3d9 Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Wed, 17 Aug 2022 15:53:14 +0800 Subject: [PATCH 119/120] update --- scripts/auto_release/PythonSdkLiveTest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 9764fa47fc69..89b755cf7a82 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -48,8 +48,9 @@ jobs: git clone https://github.com/Azure/azure-rest-api-specs.git $(pwd)/azure-rest-api-specs if [ "$(PYTHON_COMMIT)" ]; then + cd azure-rest-api-specs git reset --hard $(PYTHON_COMMIT) - echo $(PYTHON_COMMIT) + cd ../ fi # install autorest From 921f5dcea5795776f63af8c22a8761b1c076dcfa Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Wed, 17 Aug 2022 16:13:02 +0800 Subject: [PATCH 120/120] Update PythonSdkLiveTest.yml --- scripts/auto_release/PythonSdkLiveTest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 89b755cf7a82..c70f5cea1416 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -47,9 +47,9 @@ jobs: mkdir azure-rest-api-specs git clone https://github.com/Azure/azure-rest-api-specs.git $(pwd)/azure-rest-api-specs - if [ "$(PYTHON_COMMIT)" ]; then + if [ "$(REST_REPO_HASH)" ]; then cd azure-rest-api-specs - git reset --hard $(PYTHON_COMMIT) + git reset --hard $(REST_REPO_HASH) cd ../ fi