From 62b39292c360ac7b6a2f4777f2aa2ac202fd2582 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Wed, 8 Jun 2022 15:08:52 +0800 Subject: [PATCH 01/22] add PyYAML --- tools/azure-sdk-tools/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/azure-sdk-tools/setup.py b/tools/azure-sdk-tools/setup.py index 53c644ce32ba..e22499c6995b 100644 --- a/tools/azure-sdk-tools/setup.py +++ b/tools/azure-sdk-tools/setup.py @@ -24,6 +24,7 @@ "azure-mgmt-storage", "azure-mgmt-keyvault", "python-dotenv", + "PyYAML==6.0" ] setup( From 146c007a7b335bf58f086d6a0c83413646512621 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 9 Jun 2022 17:54:37 +0800 Subject: [PATCH 02/22] code --- .../packaging_tools/auto_codegen.py | 7 +- .../packaging_tools/generate_utils.py | 75 ++++++++++++++++++- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index dc19dca8128b..7e693c277fe3 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -6,7 +6,7 @@ from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG from .generate_sdk import generate -from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview +from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, generate_dpg _LOGGER = logging.getLogger(__name__) @@ -23,7 +23,10 @@ def main(generate_input, generate_output): relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") config_file = CONFIG_FILE if 'resource-manager' in input_readme else CONFIG_FILE_DPG - config = generate(config_file, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) + if 'resource-manager' in input_readme: + config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) + else: + generate_dpg(input_readme, data.get('autorestConfig', '')) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 574d7b23eb6d..f414be22851f 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -6,7 +6,9 @@ from azure_devtools.ci_tools.git_tools import get_add_diff_file_list from pathlib import Path from subprocess import check_call -from typing import List +from typing import List, Union +from glob import glob +import yaml from .swaggertosdk.autorest_tools import build_autorest_options @@ -126,3 +128,74 @@ def judge_tag_preview(path: str) -> bool: _LOGGER.info(f'find single api version:{api_version}') return 'preview' in api_version + + +def extract_yaml_content(autorest_config: str) -> str: + num = [] + content = autorest_config.split('\r\n') + for i in range(len(content)): + if "```" in content[i]: + num.append(i) + if len(num) != 2: + raise Exception(f"autorestConfig content is not valid: {autorest_config}") + return '\n'.join(content[num[0] + 1: num[1]]) + + +def add_yaml_title(content: str, annotation: str = "", tag: str = "") -> str: + return f"{annotation}\n\n" + f"``` yaml {tag}\n" + content + "```\n" + + +def generate_dpg_config(autorest_config: str) -> str: + # remove useless lines + autorest_config = extract_yaml_content(autorest_config) + _LOGGER.info(f"autoresConfig after remove useles lines:\n{autorest_config}") + + # make dir if not exist + origin_config = yaml.safe_load(autorest_config) + _LOGGER.info(f"autoresConfig: {origin_config}") + swagger_folder = str(Path(origin_config["output-folder"], "swagger")) + if not os.path.exists(swagger_folder): + os.makedirs(swagger_folder) + + # generate autorest configuration + package_name = Path(origin_config["output-folder"]).parts[-1] + readme_content = { + "package-name": package_name, + "license-header": "MICROSOFT_MIT_NO_VERSION", + "clear-output-folder": True, + "no-namespace-folders": True, + "version-tolerant": True, + "package-version": "1.0.0b1", + "require": ["../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + "output-folder": "../" + package_name.replace('-', '/'), + "namespace": package_name.replace('-', '.') + } + + # output autorest configuration + swagger_readme = str(Path(swagger_folder, "README.md")) + with open(swagger_readme, "w") as file: + file.write(add_yaml_title(yaml.safe_dump(readme_content))) + return swagger_readme + + +def lookup_swagger_readme(rest_readme_path: str) -> str: + all_swagger_readme = glob(str(Path('sdk/*/*/swagger/README.md'))) + for readme in all_swagger_readme: + with open(readme, 'r') as file: + content = file.read() + if rest_readme_path in content: + return readme + return "" + + +def generate_dpg(rest_readme_path: str, autorest_config: str): + if autorest_config: + swagger_readme = generate_dpg_config(autorest_config) + else: + swagger_readme = lookup_swagger_readme(rest_readme_path) + if not swagger_readme: + return + + # extract global config + + # generate code From 8edbe377abe12d8ccdc3d1e8f851681e1e9e6b71 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 9 Jun 2022 18:06:28 +0800 Subject: [PATCH 03/22] README.md format --- tools/azure-sdk-tools/packaging_tools/generate_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index f414be22851f..9c5ed347d46b 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -142,7 +142,9 @@ def extract_yaml_content(autorest_config: str) -> str: def add_yaml_title(content: str, annotation: str = "", tag: str = "") -> str: - return f"{annotation}\n\n" + f"``` yaml {tag}\n" + content + "```\n" + if annotation: + annotation = f"{annotation}\n\n" + return f"# autorest configuration for Python\n\n{annotation}" + f"``` yaml {tag}\n" + content + "```\n" def generate_dpg_config(autorest_config: str) -> str: @@ -166,7 +168,7 @@ def generate_dpg_config(autorest_config: str) -> str: "no-namespace-folders": True, "version-tolerant": True, "package-version": "1.0.0b1", - "require": ["../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], "output-folder": "../" + package_name.replace('-', '/'), "namespace": package_name.replace('-', '.') } From ced64c782278a5154722fc8122fb9f801bf38a5c Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 10 Jun 2022 11:41:16 +0800 Subject: [PATCH 04/22] multi client --- .../packaging_tools/auto_codegen.py | 4 +- .../packaging_tools/generate_utils.py | 111 ++++++++++++++---- 2 files changed, 90 insertions(+), 25 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index 7e693c277fe3..9f2196cd6c58 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -6,7 +6,7 @@ from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG from .generate_sdk import generate -from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, generate_dpg +from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, gen_dpg _LOGGER = logging.getLogger(__name__) @@ -26,7 +26,7 @@ def main(generate_input, generate_output): if 'resource-manager' in input_readme: config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) else: - generate_dpg(input_readme, data.get('autorestConfig', '')) + gen_dpg(input_readme, data.get('autorestConfig', '')) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 9c5ed347d46b..2658920aba67 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -6,7 +6,7 @@ from azure_devtools.ci_tools.git_tools import get_add_diff_file_list from pathlib import Path from subprocess import check_call -from typing import List, Union +from typing import List, Dict, Any from glob import glob import yaml @@ -141,42 +141,107 @@ def extract_yaml_content(autorest_config: str) -> str: return '\n'.join(content[num[0] + 1: num[1]]) -def add_yaml_title(content: str, annotation: str = "", tag: str = "") -> str: - if annotation: - annotation = f"{annotation}\n\n" - return f"# autorest configuration for Python\n\n{annotation}" + f"``` yaml {tag}\n" + content + "```\n" +def add_config_title(content: str) -> str: + return f"# autorest configuration for Python\n\n{content}" -def generate_dpg_config(autorest_config: str) -> str: +def yaml_block(content: str, annotation: str = "", tag: str = "") -> str: + annotation = f"{annotation}\n\n" if annotation else annotation + return f"{annotation}" + f"``` yaml {tag}\n" + content + "```\n" + + + + + +def gen_package_name(origin_config: Dict[str, Any]) -> str: + return Path(origin_config["output-folder"]).parts[-1] + + +def gen_basic_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: + return { + "package-name": gen_package_name(origin_config), + "license-header": "MICROSOFT_MIT_NO_VERSION", + "clear-output-folder": True, + "no-namespace-folders": True, + "version-tolerant": True, + "package-version": origin_config.get("package-version", "1.0.0b1"), + "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + } + + +def gen_general_output_folder(package_name: str) -> str: + return "../" + package_name.replace('-', '/') + + +def gen_general_namespace(package_name: str) -> str: + return package_name.replace('-', '.') + + +def gen_dpg_config_single_client(origin_config: Dict[str, Any]) -> str: + package_name = Path(origin_config["output-folder"]).parts[-1] + readme_config = gen_basic_config(origin_config) + readme_config.update({ + "output-folder": gen_general_output_folder(package_name), + "namespace": gen_general_namespace(package_name), + }) + readme_content = yaml_block(yaml.safe_dump(readme_config), "### Settings") + return add_config_title(readme_content) + + +def gen_tag_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: + tag_config = {} + package_name = gen_package_name(origin_config) + for tag in origin_config["batch"]: + tag_name = tag["tag"] + extra_part = tag_name.split("-")[-1] + tag_config[tag_name] = { + "namespace": gen_general_namespace(package_name) + f".{extra_part}", + "output-folder": gen_general_output_folder(package_name) + f"/{extra_part}" + } + + return tag_config + + +def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: + # generate config + basic_config = gen_basic_config(origin_config) + batch_config = {"batch": origin_config["batch"]} + tag_config = gen_tag_config(origin_config) + + # convert to string + readme_content = yaml_block(yaml.dump(basic_config), "### Settings") + readme_content += yaml_block(yaml.dump(batch_config), "\n### Python multi-client") + for tag, value in tag_config.items(): + readme_content += yaml_block( + yaml.dump(value), + f"\n### Tag: {tag}", + f"$(tag) == '{tag}'", + ) + + return add_config_title(readme_content) + +def gen_dpg_config(autorest_config: str) -> str: # remove useless lines autorest_config = extract_yaml_content(autorest_config) - _LOGGER.info(f"autoresConfig after remove useles lines:\n{autorest_config}") + _LOGGER.info(f"autorestConfig after remove useless lines:\n{autorest_config}") # make dir if not exist origin_config = yaml.safe_load(autorest_config) - _LOGGER.info(f"autoresConfig: {origin_config}") + _LOGGER.info(f"autorestConfig: {origin_config}") swagger_folder = str(Path(origin_config["output-folder"], "swagger")) if not os.path.exists(swagger_folder): os.makedirs(swagger_folder) # generate autorest configuration - package_name = Path(origin_config["output-folder"]).parts[-1] - readme_content = { - "package-name": package_name, - "license-header": "MICROSOFT_MIT_NO_VERSION", - "clear-output-folder": True, - "no-namespace-folders": True, - "version-tolerant": True, - "package-version": "1.0.0b1", - "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], - "output-folder": "../" + package_name.replace('-', '/'), - "namespace": package_name.replace('-', '.') - } + if "batch:" in autorest_config: + readme_content = gen_dpg_config_multi_client(origin_config) + else: + readme_content = gen_dpg_config_single_client(origin_config) # output autorest configuration swagger_readme = str(Path(swagger_folder, "README.md")) with open(swagger_readme, "w") as file: - file.write(add_yaml_title(yaml.safe_dump(readme_content))) + file.write(readme_content) return swagger_readme @@ -190,9 +255,9 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: return "" -def generate_dpg(rest_readme_path: str, autorest_config: str): +def gen_dpg(rest_readme_path: str, autorest_config: str): if autorest_config: - swagger_readme = generate_dpg_config(autorest_config) + swagger_readme = gen_dpg_config(autorest_config) else: swagger_readme = lookup_swagger_readme(rest_readme_path) if not swagger_readme: From 0a29f58ac1515265150756cce3633600962ae85a Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Fri, 10 Jun 2022 13:42:49 +0800 Subject: [PATCH 05/22] code --- tools/azure-sdk-tools/packaging_tools/generate_utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 2658920aba67..1342d96f04e8 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -150,9 +150,6 @@ def yaml_block(content: str, annotation: str = "", tag: str = "") -> str: return f"{annotation}" + f"``` yaml {tag}\n" + content + "```\n" - - - def gen_package_name(origin_config: Dict[str, Any]) -> str: return Path(origin_config["output-folder"]).parts[-1] @@ -217,9 +214,10 @@ def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: f"\n### Tag: {tag}", f"$(tag) == '{tag}'", ) - + return add_config_title(readme_content) + def gen_dpg_config(autorest_config: str) -> str: # remove useless lines autorest_config = extract_yaml_content(autorest_config) From 409ac725fabcf718d4782c2951c2746db0ca2607 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 10 Jun 2022 15:10:10 +0800 Subject: [PATCH 06/22] gen multi client --- .../packaging_tools/generate_utils.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 1342d96f04e8..7f0dfc2b130d 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -199,10 +199,18 @@ def gen_tag_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: return tag_config +def gen_batch_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: + batch_config = [] + for item in origin_config["batch"]: + for _, value in item.items(): + batch_config.append({value: True}) + return {"batch": batch_config} + + def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: # generate config basic_config = gen_basic_config(origin_config) - batch_config = {"batch": origin_config["batch"]} + batch_config = gen_batch_config(origin_config) tag_config = gen_tag_config(origin_config) # convert to string @@ -212,7 +220,7 @@ def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: readme_content += yaml_block( yaml.dump(value), f"\n### Tag: {tag}", - f"$(tag) == '{tag}'", + f"$({tag})", ) return add_config_title(readme_content) From f20bea0578718b1f1f64187381346b7b3d14761c Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 10 Jun 2022 15:36:12 +0800 Subject: [PATCH 07/22] single client gen --- swagger_to_sdk_config_dpg.json | 6 ++---- tools/azure-sdk-tools/packaging_tools/auto_codegen.py | 3 +-- .../azure-sdk-tools/packaging_tools/generate_utils.py | 11 ++++++++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/swagger_to_sdk_config_dpg.json b/swagger_to_sdk_config_dpg.json index 4900e744c9e8..1d011e81afdc 100644 --- a/swagger_to_sdk_config_dpg.json +++ b/swagger_to_sdk_config_dpg.json @@ -1,10 +1,8 @@ { "meta": { "autorest_options": { - "version": "3.7.2", - "use": ["@autorest/python@5.16.0", "@autorest/modelerfour@4.19.3"], - "python": "", - "sdkrel:python-sdks-folder": "./sdk/.", + "version": "3.8.1", + "use": ["@autorest/python@5.17.0", "@autorest/modelerfour@4.23.5"], "version-tolerant": "" }, "advanced_options": { diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index 9f2196cd6c58..d97a7891b518 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -4,7 +4,7 @@ from pathlib import Path from subprocess import check_call -from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG +from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, gen_dpg @@ -22,7 +22,6 @@ def main(generate_input, generate_output): for input_readme in data["relatedReadmeMdFiles"]: relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") - config_file = CONFIG_FILE if 'resource-manager' in input_readme else CONFIG_FILE_DPG if 'resource-manager' in input_readme: config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) else: diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 7f0dfc2b130d..8c05bc5ab505 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -10,7 +10,9 @@ from glob import glob import yaml -from .swaggertosdk.autorest_tools import build_autorest_options +from .swaggertosdk.autorest_tools import build_autorest_options, generate_code +from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE_DPG, read_config + _LOGGER = logging.getLogger(__name__) _SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII) @@ -226,6 +228,7 @@ def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: return add_config_title(readme_content) +# generate swagger/README.md and return relative path based on SDK repo root path def gen_dpg_config(autorest_config: str) -> str: # remove useless lines autorest_config = extract_yaml_content(autorest_config) @@ -262,6 +265,7 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: def gen_dpg(rest_readme_path: str, autorest_config: str): + # generate or find swagger/README.md if autorest_config: swagger_readme = gen_dpg_config(autorest_config) else: @@ -270,5 +274,10 @@ def gen_dpg(rest_readme_path: str, autorest_config: str): return # extract global config + global_config = read_config('.', CONFIG_FILE_DPG)["meta"] # generate code + current_path = os.getcwd() + os.chdir(Path(swagger_readme).parent) + generate_code(Path(swagger_readme).parts[-1], global_config, {}) + os.chdir(current_path) \ No newline at end of file From 95fe92e3ccfc462c5c4f6fce2d15776e0baf7eb8 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 10 Jun 2022 16:02:00 +0800 Subject: [PATCH 08/22] compatible with meta storage code --- .../packaging_tools/auto_codegen.py | 4 ++-- .../packaging_tools/generate_utils.py | 24 ++++++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index d97a7891b518..b84696648979 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -25,7 +25,7 @@ def main(generate_input, generate_output): if 'resource-manager' in input_readme: config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) else: - gen_dpg(input_readme, data.get('autorestConfig', '')) + config = gen_dpg(input_readme, data.get('autorestConfig', '')) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") @@ -53,7 +53,7 @@ def main(generate_input, generate_output): try: update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme) except Exception as e: - _LOGGER.info(str(e)) + _LOGGER.info(f"fail to update meta: {str(e)}") # Setup package locally check_call( diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 8c05bc5ab505..4d8798417834 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -18,7 +18,7 @@ _SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII) DEFAULT_DEST_FOLDER = "./dist" - +_DPG_README = "README.md" def get_package_names(sdk_folder): files = get_add_diff_file_list(sdk_folder) @@ -45,9 +45,13 @@ def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, readme_file = str(Path(spec_folder, input_readme)) global_conf = config["meta"] - local_conf = config["projects"][readme_file] + local_conf = config.get("projects", {}).get(readme_file, {}) - cmd = ["autorest", input_readme] + if "resource-manager" in input_readme: + cmd = ["autorest", input_readme] + else: + # autorest for DPG will be executed in package folder like: sdk/deviceupdate/azure-iot-deviceupdate/swagger + cmd = ["autorest", _DPG_README] cmd += build_autorest_options(global_conf, local_conf) # metadata @@ -248,14 +252,14 @@ def gen_dpg_config(autorest_config: str) -> str: readme_content = gen_dpg_config_single_client(origin_config) # output autorest configuration - swagger_readme = str(Path(swagger_folder, "README.md")) + swagger_readme = str(Path(swagger_folder, _DPG_README)) with open(swagger_readme, "w") as file: file.write(readme_content) return swagger_readme def lookup_swagger_readme(rest_readme_path: str) -> str: - all_swagger_readme = glob(str(Path('sdk/*/*/swagger/README.md'))) + all_swagger_readme = glob(str(Path(f'sdk/*/*/swagger/{_DPG_README}'))) for readme in all_swagger_readme: with open(readme, 'r') as file: content = file.read() @@ -264,7 +268,7 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: return "" -def gen_dpg(rest_readme_path: str, autorest_config: str): +def gen_dpg(rest_readme_path: str, autorest_config: str) -> Dict[str, Any]: # generate or find swagger/README.md if autorest_config: swagger_readme = gen_dpg_config(autorest_config) @@ -274,10 +278,12 @@ def gen_dpg(rest_readme_path: str, autorest_config: str): return # extract global config - global_config = read_config('.', CONFIG_FILE_DPG)["meta"] + global_config = read_config('.', CONFIG_FILE_DPG) # generate code current_path = os.getcwd() os.chdir(Path(swagger_readme).parent) - generate_code(Path(swagger_readme).parts[-1], global_config, {}) - os.chdir(current_path) \ No newline at end of file + generate_code(_DPG_README, global_config["meta"], {}) + os.chdir(current_path) + + return global_config From b99079e06da472c220c0dbfd3a7014e38ac9d892 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 10 Jun 2022 16:33:53 +0800 Subject: [PATCH 09/22] add log to lookup readme --- tools/azure-sdk-tools/packaging_tools/generate_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 4d8798417834..ced287a096ba 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -264,7 +264,9 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: with open(readme, 'r') as file: content = file.read() if rest_readme_path in content: + _LOGGER.info(f"find swagger readme: {readme}") return readme + _LOGGER.info(f"do not find swagger readme which contains {rest_readme_path}") return "" From 3f0e9ed0b35f543ea9b8b26bc60496bedb74f45b Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 10 Jun 2022 17:10:31 +0800 Subject: [PATCH 10/22] dpg package does not need _meta.json in MAINFEST.in --- .../packaging_tools/generate_utils.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index ced287a096ba..999aad2e7d87 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -78,23 +78,24 @@ def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, _LOGGER.info(f"Saved metadata to {metadata_file_path}") # Check whether MANIFEST.in includes _meta.json - require_meta = "include _meta.json\n" - manifest_file = os.path.join(package_folder, "MANIFEST.in") - if not os.path.exists(manifest_file): - _LOGGER.info(f"MANIFEST.in doesn't exist: {manifest_file}") - return - - includes = [] - write_flag = False - with open(manifest_file, "r") as f: - includes = f.readlines() - if require_meta not in includes: - includes = [require_meta] + includes - write_flag = True - - if write_flag: - with open(manifest_file, "w") as f: - f.write("".join(includes)) + if "resource-manager" in input_readme: + require_meta = "include _meta.json\n" + manifest_file = os.path.join(package_folder, "MANIFEST.in") + if not os.path.exists(manifest_file): + _LOGGER.info(f"MANIFEST.in doesn't exist: {manifest_file}") + return + + includes = [] + write_flag = False + with open(manifest_file, "r") as f: + includes = f.readlines() + if require_meta not in includes: + includes = [require_meta] + includes + write_flag = True + + if write_flag: + with open(manifest_file, "w") as f: + f.write("".join(includes)) # find all the files of one folder, including files in subdirectory From fb2a89dbd5c88580a8efad2eaccc6bda60447266 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 13 Jun 2022 15:22:36 +0800 Subject: [PATCH 11/22] add language, apiviewartifacts --- tools/azure-sdk-tools/packaging_tools/auto_package.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_package.py b/tools/azure-sdk-tools/packaging_tools/auto_package.py index 15e2c9090453..df1750003af3 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_package.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_package.py @@ -46,6 +46,11 @@ def main(generate_input, generate_output): # to distinguish with track1 if 'azure-mgmt-' in package_name: package["packageName"] = "track2_" + package["packageName"] + for artifact in package["artifacts"]: + if ".whl" in artifact: + package["apiViewArtifact"] = artifact + package["language"] = "Python" + break result["packages"].append(package) with open(generate_output, "w") as writer: From c8877f643bbf55f499369edbd46237fe4af9cbd1 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Tue, 26 Jul 2022 11:06:44 +0800 Subject: [PATCH 12/22] optimize --- .../azure-sdk-tools/packaging_tools/generate_utils.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 065e569e4b69..c565d7f0520e 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -166,18 +166,13 @@ def gen_basic_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: return { "package-name": gen_package_name(origin_config), "license-header": "MICROSOFT_MIT_NO_VERSION", - "clear-output-folder": True, - "no-namespace-folders": True, - "version-tolerant": True, "package-version": origin_config.get("package-version", "1.0.0b1"), "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + "package-mode": "dataplane", + "output-folder": "../", } -def gen_general_output_folder(package_name: str) -> str: - return "../" + package_name.replace('-', '/') - - def gen_general_namespace(package_name: str) -> str: return package_name.replace('-', '.') @@ -186,7 +181,6 @@ def gen_dpg_config_single_client(origin_config: Dict[str, Any]) -> str: package_name = Path(origin_config["output-folder"]).parts[-1] readme_config = gen_basic_config(origin_config) readme_config.update({ - "output-folder": gen_general_output_folder(package_name), "namespace": gen_general_namespace(package_name), }) readme_content = yaml_block(yaml.safe_dump(readme_config), "### Settings") @@ -201,7 +195,6 @@ def gen_tag_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: extra_part = tag_name.split("-")[-1] tag_config[tag_name] = { "namespace": gen_general_namespace(package_name) + f".{extra_part}", - "output-folder": gen_general_output_folder(package_name) + f"/{extra_part}" } return tag_config From 15f033c0acf2854633d3affae2fd9ad90b4f54ff Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 26 Jul 2022 11:38:23 +0800 Subject: [PATCH 13/22] Update setup.py --- tools/azure-sdk-tools/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/setup.py b/tools/azure-sdk-tools/setup.py index a568c5f5cac5..db116f4616f0 100644 --- a/tools/azure-sdk-tools/setup.py +++ b/tools/azure-sdk-tools/setup.py @@ -25,7 +25,7 @@ "azure-mgmt-storage", "azure-mgmt-keyvault", "python-dotenv", - "PyYAML==6.0" + "PyYAML" ] setup( From 374e1f38cee8d7c5b3e0f5c0d59186f72bdfb8e6 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Tue, 26 Jul 2022 18:32:19 +0800 Subject: [PATCH 14/22] sdk generation pipeline support dpg Signed-off-by: Zhou Zheng --- scripts/sdk_generate.sh | 4 ++-- scripts/sdk_init.sh | 4 ++-- .../packaging_tools/auto_codegen.py | 2 +- .../packaging_tools/generate_utils.py | 17 +++++++++-------- .../packaging_tools/sdk_generator.py | 15 ++++++++++----- .../packaging_tools/sdk_package.py | 11 +++++++++++ 6 files changed, 35 insertions(+), 18 deletions(-) diff --git a/scripts/sdk_generate.sh b/scripts/sdk_generate.sh index 69201e03613b..591889aa16f5 100755 --- a/scripts/sdk_generate.sh +++ b/scripts/sdk_generate.sh @@ -9,8 +9,8 @@ PATH="$VIRTUAL_ENV/bin:$PATH" export PATH # node version degrade -sudo npm install -g n -sudo n 14.15.0 +npm install -g n +n 14.15.0 echo "$PATH" export PATH="/usr/local/n/versions/node/14.15.0/bin:$PATH" diff --git a/scripts/sdk_init.sh b/scripts/sdk_init.sh index 916a51068cff..890373cea858 100755 --- a/scripts/sdk_init.sh +++ b/scripts/sdk_init.sh @@ -1,8 +1,8 @@ #!/bin/bash # install python3.8 -sudo apt-get install python3.8 -sudo apt-get install python3.8-venv +apt-get install python3.8 +apt-get install python3.8-venv # init env TMPDIR="$(dirname $(dirname $(readlink -f "$0")))/venv" diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index 14f87f40a813..2ce77831c70b 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -28,7 +28,7 @@ def main(generate_input, generate_output): config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, python_tag=python_tag) else: - config = gen_dpg(input_readme, data.get('autorestConfig', '')) + config = gen_dpg(input_readme, data.get('autorestConfig', ''), spec_folder) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index c565d7f0520e..a5e740907c8a 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -162,12 +162,13 @@ def gen_package_name(origin_config: Dict[str, Any]) -> str: return Path(origin_config["output-folder"]).parts[-1] -def gen_basic_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: +def gen_basic_config(origin_config: Dict[str, Any], spec_folder: str = "../../../../../azure-rest-api-specs/") -> Dict[str, Any]: + spec_root = re.sub('specification', '', spec_folder) return { "package-name": gen_package_name(origin_config), "license-header": "MICROSOFT_MIT_NO_VERSION", "package-version": origin_config.get("package-version", "1.0.0b1"), - "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + "require": [spec_root + line for line in origin_config["require"]], "package-mode": "dataplane", "output-folder": "../", } @@ -177,9 +178,9 @@ def gen_general_namespace(package_name: str) -> str: return package_name.replace('-', '.') -def gen_dpg_config_single_client(origin_config: Dict[str, Any]) -> str: +def gen_dpg_config_single_client(origin_config: Dict[str, Any], spec_folder: str) -> str: package_name = Path(origin_config["output-folder"]).parts[-1] - readme_config = gen_basic_config(origin_config) + readme_config = gen_basic_config(origin_config, spec_folder) readme_config.update({ "namespace": gen_general_namespace(package_name), }) @@ -228,7 +229,7 @@ def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: # generate swagger/README.md and return relative path based on SDK repo root path -def gen_dpg_config(autorest_config: str) -> str: +def gen_dpg_config(autorest_config: str, spec_folder: str) -> str: # remove useless lines autorest_config = extract_yaml_content(autorest_config) _LOGGER.info(f"autorestConfig after remove useless lines:\n{autorest_config}") @@ -244,7 +245,7 @@ def gen_dpg_config(autorest_config: str) -> str: if "batch:" in autorest_config: readme_content = gen_dpg_config_multi_client(origin_config) else: - readme_content = gen_dpg_config_single_client(origin_config) + readme_content = gen_dpg_config_single_client(origin_config, spec_folder) # output autorest configuration swagger_readme = str(Path(swagger_folder, _DPG_README)) @@ -265,10 +266,10 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: return "" -def gen_dpg(rest_readme_path: str, autorest_config: str) -> Dict[str, Any]: +def gen_dpg(rest_readme_path: str, autorest_config: str, spec_folder: str) -> Dict[str, Any]: # generate or find swagger/README.md if autorest_config: - swagger_readme = gen_dpg_config(autorest_config) + swagger_readme = gen_dpg_config(autorest_config, spec_folder) else: swagger_readme = lookup_swagger_readme(rest_readme_path) if not swagger_readme: diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py index c2b06e51322e..a730bdaebe8d 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py @@ -4,9 +4,9 @@ from pathlib import Path from subprocess import check_call -from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG +from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate -from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview +from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, gen_dpg _LOGGER = logging.getLogger(__name__) @@ -24,8 +24,10 @@ def main(generate_input, generate_output): input_readme = data["relatedReadmeMdFile"] relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") - config_file = CONFIG_FILE if 'resource-manager' in input_readme else CONFIG_FILE_DPG - config = generate(config_file, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) + if 'resource-manager' in input_readme: + config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) + else: + config = gen_dpg(input_readme, data.get('autorestConfig', ''), spec_folder) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") @@ -39,10 +41,12 @@ def main(generate_input, generate_output): package_entry = {} package_entry["packageName"] = package_name package_entry["path"] = [folder_name] + package_entry["readmeMd"] = [input_readme] package_entry["tagIsStable"] = not judge_tag_preview(sdk_code_path) result[package_name] = package_entry else: result[package_name]["path"].append(folder_name) + result[package_name]["readmeMd"].append(input_readme) # Generate some necessary file for new service init_new_service(package_name, folder_name) @@ -51,7 +55,7 @@ def main(generate_input, generate_output): try: update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme) except Exception as e: - _LOGGER.info(str(e)) + _LOGGER.info(f"fail to update meta: {str(e)}") # Setup package locally check_call( @@ -62,6 +66,7 @@ def main(generate_input, generate_output): # remove duplicates for value in result.values(): value["path"] = list(set(value["path"])) + value["readmeMd"] = list(set(value["readmeMd"])) with open(generate_output, "w") as writer: json.dump(result, writer) diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_package.py b/tools/azure-sdk-tools/packaging_tools/sdk_package.py index cbf5c5d799bf..fdb0d2e9a70d 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_package.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_package.py @@ -13,6 +13,7 @@ def main(generate_input, generate_output): with open(generate_input, "r") as reader: data = json.load(reader) + _LOGGER.info(f"auto_package input: {data}") sdk_folder = "." result = {"packages": []} @@ -38,9 +39,19 @@ def main(generate_input, generate_output): dist_path = Path(sdk_folder, folder_name, package_name, "dist") package["artifacts"] = [str(dist_path / package_file) for package_file in os.listdir(dist_path)] package["result"] = "succeeded" + # Installation package + package["installInstructions"] = { + "full": "You can install the use using pip install of the artificats.", + "lite": f"pip install {package_name}", + } # to distinguish with track1 if 'azure-mgmt-' in package_name: package["packageName"] = "track2_" + package["packageName"] + for artifact in package["artifacts"]: + if ".whl" in artifact: + package["apiViewArtifact"] = artifact + package["language"] = "Python" + break package["packageFolder"] = package["path"][0] result["packages"].append(package) From 937bd2102fa94744d2e88bc1470bfa710a0e7d64 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Tue, 26 Jul 2022 18:53:38 +0800 Subject: [PATCH 15/22] auto codegen don't use specFolder to define swagger repo's location Signed-off-by: Zhou Zheng --- tools/azure-sdk-tools/packaging_tools/auto_codegen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index 2ce77831c70b..b35318a340d2 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -28,7 +28,7 @@ def main(generate_input, generate_output): config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, python_tag=python_tag) else: - config = gen_dpg(input_readme, data.get('autorestConfig', ''), spec_folder) + config = gen_dpg(input_readme, data.get('autorestConfig', ''), "../../../../../azure-rest-api-specs/") package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") From dab3a92f62aff6c4250c2dcd9514d6acb0975ab0 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Tue, 26 Jul 2022 19:08:45 +0800 Subject: [PATCH 16/22] fetch automation_generate.sh recent change to sdk_generate.sh Signed-off-by: Zhou Zheng --- tools/azure-sdk-tools/packaging_tools/sdk_generator.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py index a730bdaebe8d..0ef787271d34 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py @@ -6,7 +6,8 @@ from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate -from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, gen_dpg +from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, + format_samples, gen_dpg) _LOGGER = logging.getLogger(__name__) @@ -19,13 +20,15 @@ def main(generate_input, generate_output): spec_folder = data["specFolder"] sdk_folder = "." result = {} + python_tag = data.get('python_tag') package_total = set() input_readme = data["relatedReadmeMdFile"] relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") if 'resource-manager' in input_readme: - config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) + config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, + python_tag=python_tag) else: config = gen_dpg(input_readme, data.get('autorestConfig', ''), spec_folder) package_names = get_package_names(sdk_folder) @@ -50,6 +53,7 @@ def main(generate_input, generate_output): # Generate some necessary file for new service init_new_service(package_name, folder_name) + format_samples(sdk_code_path) # Update metadata try: From 7607ffa2f1d2764213ca9be6e0823a9ee3011613 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Wed, 27 Jul 2022 11:04:56 +0800 Subject: [PATCH 17/22] set default spec flord as global constant Signed-off-by: Zhou Zheng --- tools/azure-sdk-tools/packaging_tools/auto_codegen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index b35318a340d2..620f39a21a5e 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -11,6 +11,7 @@ _LOGGER = logging.getLogger(__name__) +DEFAULT_SPEC_FOLDER = "../../../../../azure-rest-api-specs/" def main(generate_input, generate_output): with open(generate_input, "r") as reader: @@ -28,7 +29,7 @@ def main(generate_input, generate_output): config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, python_tag=python_tag) else: - config = gen_dpg(input_readme, data.get('autorestConfig', ''), "../../../../../azure-rest-api-specs/") + config = gen_dpg(input_readme, data.get('autorestConfig', ''), DEFAULT_SPEC_FOLDER) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") From 0bc45a0dd0dc520eae4d7532523aabf1d9ba03c2 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Wed, 27 Jul 2022 11:37:21 +0800 Subject: [PATCH 18/22] set DEFAULT_SPEC_FOLDER in generate_utils Signed-off-by: Zhou Zheng --- tools/azure-sdk-tools/packaging_tools/auto_codegen.py | 4 +--- tools/azure-sdk-tools/packaging_tools/generate_utils.py | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index 620f39a21a5e..a55ffe9e658a 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -7,12 +7,10 @@ from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, - format_samples, gen_dpg) + format_samples, gen_dpg, DEFAULT_SPEC_FOLDER) _LOGGER = logging.getLogger(__name__) -DEFAULT_SPEC_FOLDER = "../../../../../azure-rest-api-specs/" - def main(generate_input, generate_output): with open(generate_input, "r") as reader: data = json.load(reader) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index a5e740907c8a..452cc7c03106 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -19,6 +19,7 @@ _SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII) DEFAULT_DEST_FOLDER = "./dist" +DEFAULT_SPEC_FOLDER = "../../../../../azure-rest-api-specs/" _DPG_README = "README.md" def get_package_names(sdk_folder): @@ -162,7 +163,7 @@ def gen_package_name(origin_config: Dict[str, Any]) -> str: return Path(origin_config["output-folder"]).parts[-1] -def gen_basic_config(origin_config: Dict[str, Any], spec_folder: str = "../../../../../azure-rest-api-specs/") -> Dict[str, Any]: +def gen_basic_config(origin_config: Dict[str, Any], spec_folder: str = DEFAULT_SPEC_FOLDER) -> Dict[str, Any]: spec_root = re.sub('specification', '', spec_folder) return { "package-name": gen_package_name(origin_config), From ec10281371b5dbc7978b52706cc2164ad9e7d01d Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Thu, 28 Jul 2022 16:25:19 +0800 Subject: [PATCH 19/22] code --- .../packaging_tools/auto_codegen.py | 4 +-- .../packaging_tools/generate_utils.py | 29 +++++-------------- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index a55ffe9e658a..de2d39ace085 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -7,7 +7,7 @@ from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, - format_samples, gen_dpg, DEFAULT_SPEC_FOLDER) + format_samples, gen_dpg, dpg_relative_folder) _LOGGER = logging.getLogger(__name__) @@ -27,7 +27,7 @@ def main(generate_input, generate_output): config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, python_tag=python_tag) else: - config = gen_dpg(input_readme, data.get('autorestConfig', ''), DEFAULT_SPEC_FOLDER) + config = gen_dpg(input_readme, data.get('autorestConfig', ''), dpg_relative_folder(spec_folder)) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 02d88a980e94..4037f1357a6d 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -19,12 +19,11 @@ _SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII) DEFAULT_DEST_FOLDER = "./dist" -<<<<<<< HEAD -DEFAULT_SPEC_FOLDER = "../../../../../azure-rest-api-specs/" -======= ->>>>>>> f5eded26f09270ae60c840dab2ce275d523eb8f6 _DPG_README = "README.md" +def dpg_relative_folder(spec_folder: str) -> str: + return ("../" * 4) + spec_folder + "/" + def get_package_names(sdk_folder): files = get_add_diff_file_list(sdk_folder) matches = {_SDK_FOLDER_RE.search(f) for f in files} @@ -166,21 +165,13 @@ def gen_package_name(origin_config: Dict[str, Any]) -> str: return Path(origin_config["output-folder"]).parts[-1] -<<<<<<< HEAD -def gen_basic_config(origin_config: Dict[str, Any], spec_folder: str = DEFAULT_SPEC_FOLDER) -> Dict[str, Any]: +def gen_basic_config(origin_config: Dict[str, Any], spec_folder: str) -> Dict[str, Any]: spec_root = re.sub('specification', '', spec_folder) -======= -def gen_basic_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: ->>>>>>> f5eded26f09270ae60c840dab2ce275d523eb8f6 return { "package-name": gen_package_name(origin_config), "license-header": "MICROSOFT_MIT_NO_VERSION", "package-version": origin_config.get("package-version", "1.0.0b1"), -<<<<<<< HEAD "require": [spec_root + line for line in origin_config["require"]], -======= - "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], ->>>>>>> f5eded26f09270ae60c840dab2ce275d523eb8f6 "package-mode": "dataplane", "output-folder": "../", } @@ -190,15 +181,9 @@ def gen_general_namespace(package_name: str) -> str: return package_name.replace('-', '.') -<<<<<<< HEAD def gen_dpg_config_single_client(origin_config: Dict[str, Any], spec_folder: str) -> str: package_name = Path(origin_config["output-folder"]).parts[-1] readme_config = gen_basic_config(origin_config, spec_folder) -======= -def gen_dpg_config_single_client(origin_config: Dict[str, Any]) -> str: - package_name = Path(origin_config["output-folder"]).parts[-1] - readme_config = gen_basic_config(origin_config) ->>>>>>> f5eded26f09270ae60c840dab2ce275d523eb8f6 readme_config.update({ "namespace": gen_general_namespace(package_name), }) @@ -227,9 +212,9 @@ def gen_batch_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: return {"batch": batch_config} -def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: +def gen_dpg_config_multi_client(origin_config: Dict[str, Any], spec_folder: str) -> str: # generate config - basic_config = gen_basic_config(origin_config) + basic_config = gen_basic_config(origin_config, spec_folder) batch_config = gen_batch_config(origin_config) tag_config = gen_tag_config(origin_config) @@ -261,7 +246,7 @@ def gen_dpg_config(autorest_config: str, spec_folder: str) -> str: # generate autorest configuration if "batch:" in autorest_config: - readme_content = gen_dpg_config_multi_client(origin_config) + readme_content = gen_dpg_config_multi_client(origin_config, spec_folder) else: readme_content = gen_dpg_config_single_client(origin_config, spec_folder) From 43a77998982af37c8ee916f715aff78c7343d63e Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 28 Jul 2022 17:16:08 +0800 Subject: [PATCH 20/22] Update sdk_generate.sh --- scripts/sdk_generate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/sdk_generate.sh b/scripts/sdk_generate.sh index 591889aa16f5..69201e03613b 100755 --- a/scripts/sdk_generate.sh +++ b/scripts/sdk_generate.sh @@ -9,8 +9,8 @@ PATH="$VIRTUAL_ENV/bin:$PATH" export PATH # node version degrade -npm install -g n -n 14.15.0 +sudo npm install -g n +sudo n 14.15.0 echo "$PATH" export PATH="/usr/local/n/versions/node/14.15.0/bin:$PATH" From 098dd1449aa883e0557b1dc7d0006fcb9a598e67 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 28 Jul 2022 17:16:25 +0800 Subject: [PATCH 21/22] Update sdk_init.sh --- scripts/sdk_init.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/sdk_init.sh b/scripts/sdk_init.sh index 890373cea858..916a51068cff 100755 --- a/scripts/sdk_init.sh +++ b/scripts/sdk_init.sh @@ -1,8 +1,8 @@ #!/bin/bash # install python3.8 -apt-get install python3.8 -apt-get install python3.8-venv +sudo apt-get install python3.8 +sudo apt-get install python3.8-venv # init env TMPDIR="$(dirname $(dirname $(readlink -f "$0")))/venv" From afc279ab21ce7d144c1eadfd062eacad35b2996f Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 28 Jul 2022 17:17:34 +0800 Subject: [PATCH 22/22] Update generate_utils.py --- tools/azure-sdk-tools/packaging_tools/generate_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 4037f1357a6d..6112d827ecf6 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -166,7 +166,6 @@ def gen_package_name(origin_config: Dict[str, Any]) -> str: def gen_basic_config(origin_config: Dict[str, Any], spec_folder: str) -> Dict[str, Any]: - spec_root = re.sub('specification', '', spec_folder) return { "package-name": gen_package_name(origin_config), "license-header": "MICROSOFT_MIT_NO_VERSION",