diff --git a/scripts/sdk_generate.sh b/scripts/sdk_generate.sh index 69201e03613b..591889aa16f5 100755 --- a/scripts/sdk_generate.sh +++ b/scripts/sdk_generate.sh @@ -9,8 +9,8 @@ PATH="$VIRTUAL_ENV/bin:$PATH" export PATH # node version degrade -sudo npm install -g n -sudo n 14.15.0 +npm install -g n +n 14.15.0 echo "$PATH" export PATH="/usr/local/n/versions/node/14.15.0/bin:$PATH" diff --git a/scripts/sdk_init.sh b/scripts/sdk_init.sh index 916a51068cff..890373cea858 100755 --- a/scripts/sdk_init.sh +++ b/scripts/sdk_init.sh @@ -1,8 +1,8 @@ #!/bin/bash # install python3.8 -sudo apt-get install python3.8 -sudo apt-get install python3.8-venv +apt-get install python3.8 +apt-get install python3.8-venv # init env TMPDIR="$(dirname $(dirname $(readlink -f "$0")))/venv" diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index 14f87f40a813..a55ffe9e658a 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -7,11 +7,10 @@ from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, - format_samples, gen_dpg) + format_samples, gen_dpg, DEFAULT_SPEC_FOLDER) _LOGGER = logging.getLogger(__name__) - def main(generate_input, generate_output): with open(generate_input, "r") as reader: data = json.load(reader) @@ -28,7 +27,7 @@ def main(generate_input, generate_output): config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, python_tag=python_tag) else: - config = gen_dpg(input_readme, data.get('autorestConfig', '')) + config = gen_dpg(input_readme, data.get('autorestConfig', ''), DEFAULT_SPEC_FOLDER) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index c565d7f0520e..452cc7c03106 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -19,6 +19,7 @@ _SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII) DEFAULT_DEST_FOLDER = "./dist" +DEFAULT_SPEC_FOLDER = "../../../../../azure-rest-api-specs/" _DPG_README = "README.md" def get_package_names(sdk_folder): @@ -162,12 +163,13 @@ def gen_package_name(origin_config: Dict[str, Any]) -> str: return Path(origin_config["output-folder"]).parts[-1] -def gen_basic_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: +def gen_basic_config(origin_config: Dict[str, Any], spec_folder: str = DEFAULT_SPEC_FOLDER) -> Dict[str, Any]: + spec_root = re.sub('specification', '', spec_folder) return { "package-name": gen_package_name(origin_config), "license-header": "MICROSOFT_MIT_NO_VERSION", "package-version": origin_config.get("package-version", "1.0.0b1"), - "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + "require": [spec_root + line for line in origin_config["require"]], "package-mode": "dataplane", "output-folder": "../", } @@ -177,9 +179,9 @@ def gen_general_namespace(package_name: str) -> str: return package_name.replace('-', '.') -def gen_dpg_config_single_client(origin_config: Dict[str, Any]) -> str: +def gen_dpg_config_single_client(origin_config: Dict[str, Any], spec_folder: str) -> str: package_name = Path(origin_config["output-folder"]).parts[-1] - readme_config = gen_basic_config(origin_config) + readme_config = gen_basic_config(origin_config, spec_folder) readme_config.update({ "namespace": gen_general_namespace(package_name), }) @@ -228,7 +230,7 @@ def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: # generate swagger/README.md and return relative path based on SDK repo root path -def gen_dpg_config(autorest_config: str) -> str: +def gen_dpg_config(autorest_config: str, spec_folder: str) -> str: # remove useless lines autorest_config = extract_yaml_content(autorest_config) _LOGGER.info(f"autorestConfig after remove useless lines:\n{autorest_config}") @@ -244,7 +246,7 @@ def gen_dpg_config(autorest_config: str) -> str: if "batch:" in autorest_config: readme_content = gen_dpg_config_multi_client(origin_config) else: - readme_content = gen_dpg_config_single_client(origin_config) + readme_content = gen_dpg_config_single_client(origin_config, spec_folder) # output autorest configuration swagger_readme = str(Path(swagger_folder, _DPG_README)) @@ -265,10 +267,10 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: return "" -def gen_dpg(rest_readme_path: str, autorest_config: str) -> Dict[str, Any]: +def gen_dpg(rest_readme_path: str, autorest_config: str, spec_folder: str) -> Dict[str, Any]: # generate or find swagger/README.md if autorest_config: - swagger_readme = gen_dpg_config(autorest_config) + swagger_readme = gen_dpg_config(autorest_config, spec_folder) else: swagger_readme = lookup_swagger_readme(rest_readme_path) if not swagger_readme: diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py index c2b06e51322e..0ef787271d34 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py @@ -4,9 +4,10 @@ from pathlib import Path from subprocess import check_call -from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG +from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate -from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview +from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, + format_samples, gen_dpg) _LOGGER = logging.getLogger(__name__) @@ -19,13 +20,17 @@ def main(generate_input, generate_output): spec_folder = data["specFolder"] sdk_folder = "." result = {} + python_tag = data.get('python_tag') package_total = set() input_readme = data["relatedReadmeMdFile"] relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") - config_file = CONFIG_FILE if 'resource-manager' in input_readme else CONFIG_FILE_DPG - config = generate(config_file, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) + if 'resource-manager' in input_readme: + config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, + python_tag=python_tag) + else: + config = gen_dpg(input_readme, data.get('autorestConfig', ''), spec_folder) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") @@ -39,19 +44,22 @@ def main(generate_input, generate_output): package_entry = {} package_entry["packageName"] = package_name package_entry["path"] = [folder_name] + package_entry["readmeMd"] = [input_readme] package_entry["tagIsStable"] = not judge_tag_preview(sdk_code_path) result[package_name] = package_entry else: result[package_name]["path"].append(folder_name) + result[package_name]["readmeMd"].append(input_readme) # Generate some necessary file for new service init_new_service(package_name, folder_name) + format_samples(sdk_code_path) # Update metadata try: update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme) except Exception as e: - _LOGGER.info(str(e)) + _LOGGER.info(f"fail to update meta: {str(e)}") # Setup package locally check_call( @@ -62,6 +70,7 @@ def main(generate_input, generate_output): # remove duplicates for value in result.values(): value["path"] = list(set(value["path"])) + value["readmeMd"] = list(set(value["readmeMd"])) with open(generate_output, "w") as writer: json.dump(result, writer) diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_package.py b/tools/azure-sdk-tools/packaging_tools/sdk_package.py index cbf5c5d799bf..fdb0d2e9a70d 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_package.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_package.py @@ -13,6 +13,7 @@ def main(generate_input, generate_output): with open(generate_input, "r") as reader: data = json.load(reader) + _LOGGER.info(f"auto_package input: {data}") sdk_folder = "." result = {"packages": []} @@ -38,9 +39,19 @@ def main(generate_input, generate_output): dist_path = Path(sdk_folder, folder_name, package_name, "dist") package["artifacts"] = [str(dist_path / package_file) for package_file in os.listdir(dist_path)] package["result"] = "succeeded" + # Installation package + package["installInstructions"] = { + "full": "You can install the use using pip install of the artificats.", + "lite": f"pip install {package_name}", + } # to distinguish with track1 if 'azure-mgmt-' in package_name: package["packageName"] = "track2_" + package["packageName"] + for artifact in package["artifacts"]: + if ".whl" in artifact: + package["apiViewArtifact"] = artifact + package["language"] = "Python" + break package["packageFolder"] = package["path"][0] result["packages"].append(package) diff --git a/tools/azure-sdk-tools/setup.py b/tools/azure-sdk-tools/setup.py index a568c5f5cac5..db116f4616f0 100644 --- a/tools/azure-sdk-tools/setup.py +++ b/tools/azure-sdk-tools/setup.py @@ -25,7 +25,7 @@ "azure-mgmt-storage", "azure-mgmt-keyvault", "python-dotenv", - "PyYAML==6.0" + "PyYAML" ] setup(