From 83be3af8b3145b51ce5485f093a3aa3e59609067 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Sun, 12 Jul 2020 00:03:49 +0800 Subject: [PATCH 01/20] dynamic extension install poc --- src/azure-cli-core/azure/cli/core/__init__.py | 15 ++------ src/azure-cli-core/azure/cli/core/_session.py | 3 ++ .../azure/cli/core/extension/operations.py | 20 +++++------ src/azure-cli-core/azure/cli/core/parser.py | 35 +++++++++++++++---- src/azure-cli-core/azure/cli/core/util.py | 13 +++++++ .../cli/command_modules/extension/custom.py | 4 +-- 6 files changed, 59 insertions(+), 31 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/__init__.py b/src/azure-cli-core/azure/cli/core/__init__.py index c28f6727ce0..15f1b5b3960 100644 --- a/src/azure-cli-core/azure/cli/core/__init__.py +++ b/src/azure-cli-core/azure/cli/core/__init__.py @@ -373,18 +373,6 @@ def _get_extension_suppressions(mod_loaders): res.append(sup) return res - def _roughly_parse_command(args): - # Roughly parse the command part: --name vm1 - # Similar to knack.invocation.CommandInvoker._rudimentary_get_command, but we don't need to bother with - # positional args - nouns = [] - for arg in args: - if arg and arg[0] != '-': - nouns.append(arg) - else: - break - return ' '.join(nouns).lower() - # Clear the tables to make this method idempotent self.command_group_table.clear() self.command_table.clear() @@ -404,8 +392,9 @@ def _roughly_parse_command(args): _update_command_table_from_extensions([], index_extensions) logger.debug("Loaded %d groups, %d commands.", len(self.command_group_table), len(self.command_table)) + from azure.cli.core.util import roughly_parse_command # The index may be outdated. Make sure the command appears in the loaded command table - command_str = _roughly_parse_command(args) + command_str = roughly_parse_command(args) if command_str in self.command_table: logger.debug("Found a match in the command table for '%s'", command_str) return self.command_table diff --git a/src/azure-cli-core/azure/cli/core/_session.py b/src/azure-cli-core/azure/cli/core/_session.py index 0df29ceb768..6c283d55cd2 100644 --- a/src/azure-cli-core/azure/cli/core/_session.py +++ b/src/azure-cli-core/azure/cli/core/_session.py @@ -113,3 +113,6 @@ def __len__(self): # it could be lagged behind and can be used to check whether # an upgrade of azure-cli happens VERSIONS = Session() + +# EXT_CMD_INDEX provides command to extension name mapping +EXT_CMD_INDEX = Session() diff --git a/src/azure-cli-core/azure/cli/core/extension/operations.py b/src/azure-cli-core/azure/cli/core/extension/operations.py index 8a929e24d63..65626552124 100644 --- a/src/azure-cli-core/azure/cli/core/extension/operations.py +++ b/src/azure-cli-core/azure/cli/core/extension/operations.py @@ -85,8 +85,8 @@ def _validate_whl_extension(ext_file): check_version_compatibility(azext_metadata) -def _add_whl_ext(cmd, source, ext_sha256=None, pip_extra_index_urls=None, pip_proxy=None, system=None): # pylint: disable=too-many-statements - cmd.cli_ctx.get_progress_controller().add(message='Analyzing') +def _add_whl_ext(cli_ctx, source, ext_sha256=None, pip_extra_index_urls=None, pip_proxy=None, system=None): # pylint: disable=too-many-statements + cli_ctx.get_progress_controller().add(message='Analyzing') if not source.endswith('.whl'): raise ValueError('Unknown extension type. Only Python wheels are supported.') url_parse_result = urlparse(source) @@ -107,7 +107,7 @@ def _add_whl_ext(cmd, source, ext_sha256=None, pip_extra_index_urls=None, pip_pr ext_file = os.path.join(tmp_dir, whl_filename) logger.debug('Downloading %s to %s', source, ext_file) try: - cmd.cli_ctx.get_progress_controller().add(message='Downloading') + cli_ctx.get_progress_controller().add(message='Downloading') _whl_download_from_url(url_parse_result, ext_file) except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err: raise CLIError('Please ensure you have network connection. Error detail: {}'.format(str(err))) @@ -129,7 +129,7 @@ def _add_whl_ext(cmd, source, ext_sha256=None, pip_extra_index_urls=None, pip_pr raise CLIError("The checksum of the extension does not match the expected value. " "Use --debug for more information.") try: - cmd.cli_ctx.get_progress_controller().add(message='Validating') + cli_ctx.get_progress_controller().add(message='Validating') _validate_whl_extension(ext_file) except AssertionError: logger.debug(traceback.format_exc()) @@ -139,7 +139,7 @@ def _add_whl_ext(cmd, source, ext_sha256=None, pip_extra_index_urls=None, pip_pr logger.debug('Validation successful on %s', ext_file) # Check for distro consistency check_distro_consistency() - cmd.cli_ctx.get_progress_controller().add(message='Installing') + cli_ctx.get_progress_controller().add(message='Installing') # Install with pip extension_path = build_extension_path(extension_name, system) pip_args = ['install', '--target', extension_path, ext_file] @@ -205,7 +205,7 @@ def check_version_compatibility(azext_metadata): raise CLIError(min_max_msg_fmt) -def add_extension(cmd, source=None, extension_name=None, index_url=None, yes=None, # pylint: disable=unused-argument +def add_extension(cli_ctx, source=None, extension_name=None, index_url=None, yes=None, # pylint: disable=unused-argument pip_extra_index_urls=None, pip_proxy=None, system=None, version=None): ext_sha256 = None @@ -213,7 +213,7 @@ def add_extension(cmd, source=None, extension_name=None, index_url=None, yes=Non version = None if version == 'latest' else version if extension_name: - cmd.cli_ctx.get_progress_controller().add(message='Searching') + cli_ctx.get_progress_controller().add(message='Searching') ext = None try: ext = get_extension(extension_name) @@ -235,7 +235,7 @@ def add_extension(cmd, source=None, extension_name=None, index_url=None, yes=Non err = "No matching extensions for '{}'. Use --debug for more information.".format(extension_name) raise CLIError(err) - extension_name = _add_whl_ext(cmd=cmd, source=source, ext_sha256=ext_sha256, + extension_name = _add_whl_ext(cli_ctx=cli_ctx, source=source, ext_sha256=ext_sha256, pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy, system=system) try: ext = get_extension(extension_name) @@ -288,7 +288,7 @@ def show_extension(extension_name): raise CLIError(e) -def update_extension(cmd, extension_name, index_url=None, pip_extra_index_urls=None, pip_proxy=None): +def update_extension(cli_ctx, extension_name, index_url=None, pip_extra_index_urls=None, pip_proxy=None): try: ext = get_extension(extension_name, ext_type=WheelExtension) cur_version = ext.get_version() @@ -306,7 +306,7 @@ def update_extension(cmd, extension_name, index_url=None, pip_extra_index_urls=N shutil.rmtree(extension_path) # Install newer version try: - _add_whl_ext(cmd=cmd, source=download_url, ext_sha256=ext_sha256, + _add_whl_ext(cli_ctx=cli_ctx, source=download_url, ext_sha256=ext_sha256, pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy) logger.debug('Deleting backup of old extension at %s', backup_dir) shutil.rmtree(backup_dir) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index b933554739d..213b80ac44a 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -280,18 +280,41 @@ def parse_known_args(self, args=None, namespace=None): self._namespace, self._raw_arguments = super().parse_known_args(args=args, namespace=namespace) return self._namespace, self._raw_arguments + def _search_in_extension_commands(self, cli_ctx, command_str): + from azure.cli.core._session import EXT_CMD_INDEX + import os + EXT_CMD_INDEX.load(os.path.join(cli_ctx.config.config_dir, 'extCmdIndex.json')) + cmd_chain = EXT_CMD_INDEX + for part in command_str.split('.'): + try: + if isinstance(cmd_chain[part], str): + return cmd_chain[part] + cmd_chain = cmd_chain[part] + except KeyError: + return None + def _check_value(self, action, value): # Override to customize the error message when a argument is not among the available choices # converted value must be one of the choices (if specified) + # cli_ctx = self.cli_ctx if action.choices is not None and value not in action.choices: if not self.command_source: + from azure.cli.core.util import roughly_parse_command + command_str = roughly_parse_command(self.prog.split()[1:] + self._raw_arguments, delimiter='.') + ext_name = self._search_in_extension_commands(cli_ctx, command_str) + if ext_name: + from azure.cli.core.extension.operations import add_extension + from knack.prompting import prompt_y_n + go_on = prompt_y_n('You are running commands from the extension {}. Would you like to install it first?'.format(ext_name), default='y') + if go_on: + add_extension(self.cli_ctx, extension_name=ext_name) + logger.warning('Please rerun your command.') + self.exit(6) + else: + logger.error("Command failed due to corresponding extension not installed. Please run 'az extension add -n {}' first.".format(ext_name)) + self.exit(2) # parser has no `command_source`, value is part of command itself - extensions_link = 'https://docs.microsoft.com/en-us/cli/azure/azure-cli-extensions-overview' - error_msg = ("{prog}: '{value}' is not in the '{prog}' command group. See '{prog} --help'. " - "If the command is from an extension, " - "please make sure the corresponding extension is installed. " - "To learn more about extensions, please visit " - "{extensions_link}").format(prog=self.prog, value=value, extensions_link=extensions_link) + error_msg = ("{prog}: '{value}' is not in the '{prog}' command group. See '{prog} --help'. ") else: # `command_source` indicates command values have been parsed, value is an argument parameter = action.option_strings[0] if action.option_strings else action.dest diff --git a/src/azure-cli-core/azure/cli/core/util.py b/src/azure-cli-core/azure/cli/core/util.py index bbe092092cf..b4e4aeb0870 100644 --- a/src/azure-cli-core/azure/cli/core/util.py +++ b/src/azure-cli-core/azure/cli/core/util.py @@ -990,3 +990,16 @@ def get_linux_distro(): release_info[k.lower()] = v.strip('"') return release_info.get('name', None), release_info.get('version_id', None) + + +def roughly_parse_command(args, delimiter=' '): + # Roughly parse the command part: --name vm1 + # Similar to knack.invocation.CommandInvoker._rudimentary_get_command, but we don't need to bother with + # positional args + nouns = [] + for arg in args: + if arg and arg[0] != '-': + nouns.append(arg) + else: + break + return delimiter.join(nouns).lower() diff --git a/src/azure-cli/azure/cli/command_modules/extension/custom.py b/src/azure-cli/azure/cli/command_modules/extension/custom.py index 51f9e24a095..bd85a88add6 100644 --- a/src/azure-cli/azure/cli/command_modules/extension/custom.py +++ b/src/azure-cli/azure/cli/command_modules/extension/custom.py @@ -13,7 +13,7 @@ def add_extension_cmd(cmd, source=None, extension_name=None, index_url=None, yes=None, pip_extra_index_urls=None, pip_proxy=None, system=None): - return add_extension(cmd=cmd, source=source, extension_name=extension_name, index_url=index_url, yes=yes, + return add_extension(cmd.cli_ctx, source=source, extension_name=extension_name, index_url=index_url, yes=yes, pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy, system=system) @@ -30,7 +30,7 @@ def show_extension_cmd(extension_name): def update_extension_cmd(cmd, extension_name, index_url=None, pip_extra_index_urls=None, pip_proxy=None): - return update_extension(cmd, extension_name, index_url=index_url, pip_extra_index_urls=pip_extra_index_urls, + return update_extension(cmd.cli_ctx, extension_name, index_url=index_url, pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy) From 9b3fa62bf761faf149d93f8f6e0b1a0819804a28 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Sun, 12 Jul 2020 18:58:02 +0800 Subject: [PATCH 02/20] fix cli_ctx --- src/azure-cli-core/azure/cli/core/parser.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index 213b80ac44a..a03a43deee1 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -280,10 +280,11 @@ def parse_known_args(self, args=None, namespace=None): self._namespace, self._raw_arguments = super().parse_known_args(args=args, namespace=namespace) return self._namespace, self._raw_arguments - def _search_in_extension_commands(self, cli_ctx, command_str): + def _search_in_extension_commands(self, command_str): from azure.cli.core._session import EXT_CMD_INDEX import os - EXT_CMD_INDEX.load(os.path.join(cli_ctx.config.config_dir, 'extCmdIndex.json')) + # self.cli_ctx is None when self.prog is not 'az', such as 'az iot', use cli_ctx from cli_help which is not lost. + EXT_CMD_INDEX.load(os.path.join((self.cli_ctx or self.cli_help.cli_ctx).config.config_dir, 'extCmdIndex.json')) cmd_chain = EXT_CMD_INDEX for part in command_str.split('.'): try: @@ -292,29 +293,30 @@ def _search_in_extension_commands(self, cli_ctx, command_str): cmd_chain = cmd_chain[part] except KeyError: return None + return None def _check_value(self, action, value): # Override to customize the error message when a argument is not among the available choices # converted value must be one of the choices (if specified) - # cli_ctx = self.cli_ctx if action.choices is not None and value not in action.choices: if not self.command_source: from azure.cli.core.util import roughly_parse_command command_str = roughly_parse_command(self.prog.split()[1:] + self._raw_arguments, delimiter='.') - ext_name = self._search_in_extension_commands(cli_ctx, command_str) + ext_name = self._search_in_extension_commands(command_str) if ext_name: from azure.cli.core.extension.operations import add_extension from knack.prompting import prompt_y_n go_on = prompt_y_n('You are running commands from the extension {}. Would you like to install it first?'.format(ext_name), default='y') if go_on: - add_extension(self.cli_ctx, extension_name=ext_name) + add_extension(self.cli_ctx or self.cli_help.cli_ctx, extension_name=ext_name) logger.warning('Please rerun your command.') - self.exit(6) + self.exit(6) # TODO define the right exit code else: - logger.error("Command failed due to corresponding extension not installed. Please run 'az extension add -n {}' first.".format(ext_name)) + logger.error("Command failed due to corresponding extension not installed. Please run 'az extension add -n %s' first.", ext_name) self.exit(2) # parser has no `command_source`, value is part of command itself - error_msg = ("{prog}: '{value}' is not in the '{prog}' command group. See '{prog} --help'. ") + error_msg = ("{prog}: '{value}' is not in the '{prog}' command group. See '{prog} --help'. ").format( + prog=self.prog, value=value) else: # `command_source` indicates command values have been parsed, value is an argument parameter = action.option_strings[0] if action.option_strings else action.dest From ef4f251c34fd5f7b25d6d5c606fe84e9d5172fe3 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Tue, 14 Jul 2020 14:05:15 +0800 Subject: [PATCH 03/20] continue run with subprocess --- src/azure-cli-core/azure/cli/core/parser.py | 26 +++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index a03a43deee1..e40635ede41 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -301,16 +301,28 @@ def _check_value(self, action, value): if action.choices is not None and value not in action.choices: if not self.command_source: from azure.cli.core.util import roughly_parse_command - command_str = roughly_parse_command(self.prog.split()[1:] + self._raw_arguments, delimiter='.') + cmd_list = self.prog.split() + self._raw_arguments + command_str = roughly_parse_command(cmd_list[1:], delimiter='.') ext_name = self._search_in_extension_commands(command_str) if ext_name: - from azure.cli.core.extension.operations import add_extension - from knack.prompting import prompt_y_n - go_on = prompt_y_n('You are running commands from the extension {}. Would you like to install it first?'.format(ext_name), default='y') + cli_ctx = self.cli_ctx or self.cli_help.cli_ctx + ask_before_dynamic_extension_install = cli_ctx.config.getboolean('extension', 'ask_before_dynamic_extension_install', True) + if ask_before_dynamic_extension_install: + from knack.prompting import prompt_y_n + go_on = prompt_y_n('You are running a command from the extension {}. Would you like to install it first?'.format(ext_name), default='y') + else: + go_on = True if go_on: - add_extension(self.cli_ctx or self.cli_help.cli_ctx, extension_name=ext_name) - logger.warning('Please rerun your command.') - self.exit(6) # TODO define the right exit code + from azure.cli.core.extension.operations import add_extension + add_extension(cli_ctx, extension_name=ext_name) + run_after_extension_installed = cli_ctx.config.getboolean('extension', 'run_after_extension_installed', True) # TODO default ot False + if run_after_extension_installed: + import subprocess + exit_code = subprocess.call(cmd_list) + self.exit(exit_code) + else: + logger.warning('Please rerun your command.') + self.exit(2) # TODO define the right exit code else: logger.error("Command failed due to corresponding extension not installed. Please run 'az extension add -n %s' first.", ext_name) self.exit(2) From 3c552a7ceeb3d925437fe4c8f0c884f795dd7795 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Wed, 15 Jul 2020 10:02:55 +0800 Subject: [PATCH 04/20] add config --- src/azure-cli-core/azure/cli/core/parser.py | 53 +++++++++++---------- src/azure-cli-core/azure/cli/core/util.py | 4 +- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index e40635ede41..ea7ef026cdb 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -286,7 +286,7 @@ def _search_in_extension_commands(self, command_str): # self.cli_ctx is None when self.prog is not 'az', such as 'az iot', use cli_ctx from cli_help which is not lost. EXT_CMD_INDEX.load(os.path.join((self.cli_ctx or self.cli_help.cli_ctx).config.config_dir, 'extCmdIndex.json')) cmd_chain = EXT_CMD_INDEX - for part in command_str.split('.'): + for part in command_str.split(): try: if isinstance(cmd_chain[part], str): return cmd_chain[part] @@ -302,15 +302,19 @@ def _check_value(self, action, value): if not self.command_source: from azure.cli.core.util import roughly_parse_command cmd_list = self.prog.split() + self._raw_arguments - command_str = roughly_parse_command(cmd_list[1:], delimiter='.') + command_str = roughly_parse_command(cmd_list[1:]) ext_name = self._search_in_extension_commands(command_str) if ext_name: + telemetry.set_command_details(command_str, + parameters=AzCliCommandInvoker._extract_parameter_names(cmd_list), # pylint: disable=protected-access + extension_name=ext_name) # TODO add extension_version cli_ctx = self.cli_ctx or self.cli_help.cli_ctx - ask_before_dynamic_extension_install = cli_ctx.config.getboolean('extension', 'ask_before_dynamic_extension_install', True) + ask_before_dynamic_extension_install = cli_ctx.config.getboolean('extension', 'ask_before_dynamic_extension_install', False) if ask_before_dynamic_extension_install: from knack.prompting import prompt_y_n go_on = prompt_y_n('You are running a command from the extension {}. Would you like to install it first?'.format(ext_name), default='y') else: + logger.warning('You are running a command from the extension %s. It will be installed first.', ext_name) go_on = True if go_on: from azure.cli.core.extension.operations import add_extension @@ -318,17 +322,17 @@ def _check_value(self, action, value): run_after_extension_installed = cli_ctx.config.getboolean('extension', 'run_after_extension_installed', True) # TODO default ot False if run_after_extension_installed: import subprocess - exit_code = subprocess.call(cmd_list) + exit_code = subprocess.call(cmd_list, shell=True) + telemetry.set_user_fault("Extension {} dynamically installed and commands will be rerun automatically.".format(ext_name)) self.exit(exit_code) else: - logger.warning('Please rerun your command.') - self.exit(2) # TODO define the right exit code + error_msg = 'Extension installed. Please rerun your command.' else: - logger.error("Command failed due to corresponding extension not installed. Please run 'az extension add -n %s' first.", ext_name) - self.exit(2) - # parser has no `command_source`, value is part of command itself - error_msg = ("{prog}: '{value}' is not in the '{prog}' command group. See '{prog} --help'. ").format( - prog=self.prog, value=value) + error_msg = "Command failed due to corresponding extension not installed. Please run 'az extension add -n {}' first.".format(ext_name) + else: + # parser has no `command_source`, value is part of command itself + error_msg = ("{prog}: '{value}' is not in the '{prog}' command group. See '{prog} --help'. ").format( + prog=self.prog, value=value) else: # `command_source` indicates command values have been parsed, value is an argument parameter = action.option_strings[0] if action.option_strings else action.dest @@ -337,17 +341,18 @@ def _check_value(self, action, value): telemetry.set_user_fault(error_msg) with CommandLoggerContext(logger): logger.error(error_msg) - candidates = difflib.get_close_matches(value, action.choices, cutoff=0.7) - if candidates: - print_args = { - 's': 's' if len(candidates) > 1 else '', - 'verb': 'are' if len(candidates) > 1 else 'is', - 'value': value - } - self._suggestion_msg.append("\nThe most similar choice{s} to '{value}' {verb}:".format(**print_args)) - self._suggestion_msg.append('\n'.join(['\t' + candidate for candidate in candidates])) - - failure_recovery_recommendations = self._get_failure_recovery_recommendations(action) - self._suggestion_msg.extend(failure_recovery_recommendations) - self._print_suggestion_msg(sys.stderr) + if not ext_name: + candidates = difflib.get_close_matches(value, action.choices, cutoff=0.7) + if candidates: + print_args = { + 's': 's' if len(candidates) > 1 else '', + 'verb': 'are' if len(candidates) > 1 else 'is', + 'value': value + } + self._suggestion_msg.append("\nThe most similar choice{s} to '{value}' {verb}:".format(**print_args)) + self._suggestion_msg.append('\n'.join(['\t' + candidate for candidate in candidates])) + + failure_recovery_recommendations = self._get_failure_recovery_recommendations(action) + self._suggestion_msg.extend(failure_recovery_recommendations) + self._print_suggestion_msg(sys.stderr) self.exit(2) diff --git a/src/azure-cli-core/azure/cli/core/util.py b/src/azure-cli-core/azure/cli/core/util.py index b4e4aeb0870..88961e48604 100644 --- a/src/azure-cli-core/azure/cli/core/util.py +++ b/src/azure-cli-core/azure/cli/core/util.py @@ -992,7 +992,7 @@ def get_linux_distro(): return release_info.get('name', None), release_info.get('version_id', None) -def roughly_parse_command(args, delimiter=' '): +def roughly_parse_command(args): # Roughly parse the command part: --name vm1 # Similar to knack.invocation.CommandInvoker._rudimentary_get_command, but we don't need to bother with # positional args @@ -1002,4 +1002,4 @@ def roughly_parse_command(args, delimiter=' '): nouns.append(arg) else: break - return delimiter.join(nouns).lower() + return ' '.join(nouns).lower() From 3f800bd5548299734a5356b38d0f647a8a89f6dd Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Thu, 16 Jul 2020 18:12:31 +0800 Subject: [PATCH 05/20] donwload remote index --- .../azure/cli/core/extension/operations.py | 8 ++++---- src/azure-cli-core/azure/cli/core/parser.py | 20 +++++++++++++++---- .../cli/command_modules/extension/__init__.py | 2 +- .../cli/command_modules/extension/custom.py | 4 ++-- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/extension/operations.py b/src/azure-cli-core/azure/cli/core/extension/operations.py index 65626552124..d8dcfbced0d 100644 --- a/src/azure-cli-core/azure/cli/core/extension/operations.py +++ b/src/azure-cli-core/azure/cli/core/extension/operations.py @@ -205,15 +205,15 @@ def check_version_compatibility(azext_metadata): raise CLIError(min_max_msg_fmt) -def add_extension(cli_ctx, source=None, extension_name=None, index_url=None, yes=None, # pylint: disable=unused-argument +def add_extension(cmd=None, cli_ctx=None, source=None, extension_name=None, index_url=None, yes=None, # pylint: disable=unused-argument pip_extra_index_urls=None, pip_proxy=None, system=None, version=None): ext_sha256 = None version = None if version == 'latest' else version - + cmd_cli_ctx = cli_ctx or cmd.cli_ctx if extension_name: - cli_ctx.get_progress_controller().add(message='Searching') + cmd_cli_ctx.get_progress_controller().add(message='Searching') ext = None try: ext = get_extension(extension_name) @@ -235,7 +235,7 @@ def add_extension(cli_ctx, source=None, extension_name=None, index_url=None, yes err = "No matching extensions for '{}'. Use --debug for more information.".format(extension_name) raise CLIError(err) - extension_name = _add_whl_ext(cli_ctx=cli_ctx, source=source, ext_sha256=ext_sha256, + extension_name = _add_whl_ext(cli_ctx=cmd_cli_ctx, source=source, ext_sha256=ext_sha256, pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy, system=system) try: ext = get_extension(extension_name) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index ea7ef026cdb..17ceb796e33 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -284,7 +284,18 @@ def _search_in_extension_commands(self, command_str): from azure.cli.core._session import EXT_CMD_INDEX import os # self.cli_ctx is None when self.prog is not 'az', such as 'az iot', use cli_ctx from cli_help which is not lost. - EXT_CMD_INDEX.load(os.path.join((self.cli_ctx or self.cli_help.cli_ctx).config.config_dir, 'extCmdIndex.json')) + VALID_SECOND = 3600 * 24 * 10 + EXT_CMD_INDEX.load(os.path.join((self.cli_ctx or self.cli_help.cli_ctx).config.config_dir, 'extCmdIndex.json'), VALID_SECOND) + if not EXT_CMD_INDEX.data: + import requests + from azure.cli.core.util import should_disable_connection_verify + response = requests.get('https://fengsa.blob.core.windows.net/index/extCmdIndex.json', verify=(not should_disable_connection_verify())) + if response.status_code == 200: + EXT_CMD_INDEX.data = response.json() + EXT_CMD_INDEX.save_with_retry() + else: + logger.info("Error when retrieveing extension command index. Response code:%s", response.status_code) + return None cmd_chain = EXT_CMD_INDEX for part in command_str.split(): try: @@ -300,6 +311,7 @@ def _check_value(self, action, value): # converted value must be one of the choices (if specified) if action.choices is not None and value not in action.choices: if not self.command_source: + # Check if the command is from an extension from azure.cli.core.util import roughly_parse_command cmd_list = self.prog.split() + self._raw_arguments command_str = roughly_parse_command(cmd_list[1:]) @@ -309,7 +321,7 @@ def _check_value(self, action, value): parameters=AzCliCommandInvoker._extract_parameter_names(cmd_list), # pylint: disable=protected-access extension_name=ext_name) # TODO add extension_version cli_ctx = self.cli_ctx or self.cli_help.cli_ctx - ask_before_dynamic_extension_install = cli_ctx.config.getboolean('extension', 'ask_before_dynamic_extension_install', False) + ask_before_dynamic_extension_install = cli_ctx.config.getboolean('extension', 'ask_before_dynamic_extension_install', True) if ask_before_dynamic_extension_install: from knack.prompting import prompt_y_n go_on = prompt_y_n('You are running a command from the extension {}. Would you like to install it first?'.format(ext_name), default='y') @@ -318,8 +330,8 @@ def _check_value(self, action, value): go_on = True if go_on: from azure.cli.core.extension.operations import add_extension - add_extension(cli_ctx, extension_name=ext_name) - run_after_extension_installed = cli_ctx.config.getboolean('extension', 'run_after_extension_installed', True) # TODO default ot False + add_extension(cli_ctx=cli_ctx, extension_name=ext_name) + run_after_extension_installed = cli_ctx.config.getboolean('extension', 'run_after_extension_installed', True) if run_after_extension_installed: import subprocess exit_code = subprocess.call(cmd_list, shell=True) diff --git a/src/azure-cli/azure/cli/command_modules/extension/__init__.py b/src/azure-cli/azure/cli/command_modules/extension/__init__.py index 815bd347a39..704fee1e2b0 100644 --- a/src/azure-cli/azure/cli/command_modules/extension/__init__.py +++ b/src/azure-cli/azure/cli/command_modules/extension/__init__.py @@ -39,7 +39,7 @@ def validate_extension_add(namespace): extension_custom = CliCommandType(operations_tmpl='azure.cli.command_modules.extension.custom#{}') with self.command_group('extension', extension_custom) as g: - g.command('add', 'add_extension', confirmation=ext_add_has_confirmed, validator=validate_extension_add) + g.command('add', 'add_extension_cmd', confirmation=ext_add_has_confirmed, validator=validate_extension_add) g.command('remove', 'remove_extension') g.command('list', 'list_extensions') g.show_command('show', 'show_extension') diff --git a/src/azure-cli/azure/cli/command_modules/extension/custom.py b/src/azure-cli/azure/cli/command_modules/extension/custom.py index bd85a88add6..5a0881a40b5 100644 --- a/src/azure-cli/azure/cli/command_modules/extension/custom.py +++ b/src/azure-cli/azure/cli/command_modules/extension/custom.py @@ -13,8 +13,8 @@ def add_extension_cmd(cmd, source=None, extension_name=None, index_url=None, yes=None, pip_extra_index_urls=None, pip_proxy=None, system=None): - return add_extension(cmd.cli_ctx, source=source, extension_name=extension_name, index_url=index_url, yes=yes, - pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy, system=system) + return add_extension(cli_ctx=cmd.cli_ctx, source=source, extension_name=extension_name, index_url=index_url, + yes=yes, pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy, system=system) def remove_extension_cmd(extension_name): From e2b667baad49dac65eafc20c626562bbf52235a0 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Fri, 17 Jul 2020 15:17:14 +0800 Subject: [PATCH 06/20] check close matches first --- src/azure-cli-core/azure/cli/core/parser.py | 98 +++++++++++-------- .../cli/command_modules/extension/__init__.py | 2 +- 2 files changed, 59 insertions(+), 41 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index 17ceb796e33..adac0c8b395 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -283,13 +283,16 @@ def parse_known_args(self, args=None, namespace=None): def _search_in_extension_commands(self, command_str): from azure.cli.core._session import EXT_CMD_INDEX import os - # self.cli_ctx is None when self.prog is not 'az', such as 'az iot', use cli_ctx from cli_help which is not lost. + # self.cli_ctx is None when self.prog is not 'az', such as 'az iot'. + # use cli_ctx from cli_help which is not lost. VALID_SECOND = 3600 * 24 * 10 - EXT_CMD_INDEX.load(os.path.join((self.cli_ctx or self.cli_help.cli_ctx).config.config_dir, 'extCmdIndex.json'), VALID_SECOND) + EXT_CMD_INDEX.load(os.path.join((self.cli_ctx or self.cli_help.cli_ctx).config.config_dir, + 'extCmdIndex.json'), VALID_SECOND) if not EXT_CMD_INDEX.data: import requests from azure.cli.core.util import should_disable_connection_verify - response = requests.get('https://fengsa.blob.core.windows.net/index/extCmdIndex.json', verify=(not should_disable_connection_verify())) + response = requests.get('https://fengsa.blob.core.windows.net/index/extCmdIndex.json', + verify=(not should_disable_connection_verify())) if response.status_code == 200: EXT_CMD_INDEX.data = response.json() EXT_CMD_INDEX.save_with_retry() @@ -306,62 +309,77 @@ def _search_in_extension_commands(self, command_str): return None return None - def _check_value(self, action, value): + def _check_value(self, action, value): # pylint: disable=too-many-statements # Override to customize the error message when a argument is not among the available choices # converted value must be one of the choices (if specified) - if action.choices is not None and value not in action.choices: + if action.choices is not None and value not in action.choices: # pylint: disable=too-many-nested-blocks + caused_by_extension_not_installed = False if not self.command_source: - # Check if the command is from an extension - from azure.cli.core.util import roughly_parse_command - cmd_list = self.prog.split() + self._raw_arguments - command_str = roughly_parse_command(cmd_list[1:]) - ext_name = self._search_in_extension_commands(command_str) - if ext_name: - telemetry.set_command_details(command_str, - parameters=AzCliCommandInvoker._extract_parameter_names(cmd_list), # pylint: disable=protected-access - extension_name=ext_name) # TODO add extension_version - cli_ctx = self.cli_ctx or self.cli_help.cli_ctx - ask_before_dynamic_extension_install = cli_ctx.config.getboolean('extension', 'ask_before_dynamic_extension_install', True) - if ask_before_dynamic_extension_install: - from knack.prompting import prompt_y_n - go_on = prompt_y_n('You are running a command from the extension {}. Would you like to install it first?'.format(ext_name), default='y') - else: - logger.warning('You are running a command from the extension %s. It will be installed first.', ext_name) - go_on = True - if go_on: - from azure.cli.core.extension.operations import add_extension - add_extension(cli_ctx=cli_ctx, extension_name=ext_name) - run_after_extension_installed = cli_ctx.config.getboolean('extension', 'run_after_extension_installed', True) - if run_after_extension_installed: - import subprocess - exit_code = subprocess.call(cmd_list, shell=True) - telemetry.set_user_fault("Extension {} dynamically installed and commands will be rerun automatically.".format(ext_name)) - self.exit(exit_code) + candidates = difflib.get_close_matches(value, action.choices, cutoff=0.7) + error_msg = None + if not candidates: + # Check if the command is from an extension + from azure.cli.core.util import roughly_parse_command + cmd_list = self.prog.split() + self._raw_arguments + command_str = roughly_parse_command(cmd_list[1:]) + ext_name = self._search_in_extension_commands(command_str) + if ext_name: + caused_by_extension_not_installed = True + telemetry.set_command_details(command_str, + parameters=AzCliCommandInvoker._extract_parameter_names(cmd_list), # pylint: disable=protected-access + extension_name=ext_name) # TODO add extension_version + cli_ctx = self.cli_ctx or self.cli_help.cli_ctx + ask_before_dynamic_extension_install = cli_ctx.config.getboolean( + 'extension', 'ask_before_dynamic_extension_install', True) + if ask_before_dynamic_extension_install: + from knack.prompting import prompt_y_n + go_on = prompt_y_n( + 'You are running a command from the extension {}. Would you like to install it first?' + .format(ext_name), default='y') else: - error_msg = 'Extension installed. Please rerun your command.' - else: - error_msg = "Command failed due to corresponding extension not installed. Please run 'az extension add -n {}' first.".format(ext_name) - else: + logger.warning('You are running a command from the extension %s. ' + 'It will be installed first.', ext_name) + go_on = True + if go_on: + from azure.cli.core.extension.operations import add_extension + add_extension(cli_ctx=cli_ctx, extension_name=ext_name) + run_after_extension_installed = cli_ctx.config.getboolean('extension', + 'run_after_extension_installed', + True) + if run_after_extension_installed: + import subprocess + exit_code = subprocess.call(cmd_list, shell=True) + telemetry.set_user_fault("Extension {} dynamically installed and commands will be " + "rerun automatically.".format(ext_name)) + self.exit(exit_code) + else: + error_msg = 'Extension installed. Please rerun your command.' + else: + error_msg = "Command failed due to corresponding extension not installed. " \ + "Please run 'az extension add -n {}' first.".format(ext_name) + if not error_msg: # parser has no `command_source`, value is part of command itself - error_msg = ("{prog}: '{value}' is not in the '{prog}' command group. See '{prog} --help'. ").format( - prog=self.prog, value=value) + error_msg = "{prog}: '{value}' is not in the '{prog}' command group. See '{prog} --help'. " \ + .format(prog=self.prog, value=value) else: # `command_source` indicates command values have been parsed, value is an argument parameter = action.option_strings[0] if action.option_strings else action.dest error_msg = "{prog}: '{value}' is not a valid value for '{param}'. See '{prog} --help'.".format( prog=self.prog, value=value, param=parameter) + candidates = difflib.get_close_matches(value, action.choices, cutoff=0.7) + telemetry.set_user_fault(error_msg) with CommandLoggerContext(logger): logger.error(error_msg) - if not ext_name: - candidates = difflib.get_close_matches(value, action.choices, cutoff=0.7) + if not caused_by_extension_not_installed: if candidates: print_args = { 's': 's' if len(candidates) > 1 else '', 'verb': 'are' if len(candidates) > 1 else 'is', 'value': value } - self._suggestion_msg.append("\nThe most similar choice{s} to '{value}' {verb}:".format(**print_args)) + self._suggestion_msg.append("\nThe most similar choice{s} to '{value}' {verb}:" + .format(**print_args)) self._suggestion_msg.append('\n'.join(['\t' + candidate for candidate in candidates])) failure_recovery_recommendations = self._get_failure_recovery_recommendations(action) diff --git a/src/azure-cli/azure/cli/command_modules/extension/__init__.py b/src/azure-cli/azure/cli/command_modules/extension/__init__.py index 704fee1e2b0..63d1bc1f624 100644 --- a/src/azure-cli/azure/cli/command_modules/extension/__init__.py +++ b/src/azure-cli/azure/cli/command_modules/extension/__init__.py @@ -44,7 +44,7 @@ def validate_extension_add(namespace): g.command('list', 'list_extensions') g.show_command('show', 'show_extension') g.command('list-available', 'list_available_extensions', table_transformer=transform_extension_list_available) - g.command('update', 'update_extension') + g.command('update', 'update_extension_cmd') return self.command_table From 1b6857f850170d29f430ed2e6fa0560ce9429c20 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Fri, 17 Jul 2020 15:56:31 +0800 Subject: [PATCH 07/20] change config name --- src/azure-cli-core/azure/cli/core/parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index adac0c8b395..3150ad53d66 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -330,7 +330,7 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements extension_name=ext_name) # TODO add extension_version cli_ctx = self.cli_ctx or self.cli_help.cli_ctx ask_before_dynamic_extension_install = cli_ctx.config.getboolean( - 'extension', 'ask_before_dynamic_extension_install', True) + 'extension', 'ask_before_dynamic_install', True) if ask_before_dynamic_extension_install: from knack.prompting import prompt_y_n go_on = prompt_y_n( @@ -344,7 +344,7 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements from azure.cli.core.extension.operations import add_extension add_extension(cli_ctx=cli_ctx, extension_name=ext_name) run_after_extension_installed = cli_ctx.config.getboolean('extension', - 'run_after_extension_installed', + 'run_after_dynamic_install', True) if run_after_extension_installed: import subprocess From 3ee541541ba2b7c6709bf3112864009d773ee60b Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Fri, 17 Jul 2020 16:48:07 +0800 Subject: [PATCH 08/20] add option to turn off dynamic install --- src/azure-cli-core/azure/cli/core/parser.py | 33 +++++++++++++-------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index 3150ad53d66..cfacbfb9c1a 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -283,9 +283,9 @@ def parse_known_args(self, args=None, namespace=None): def _search_in_extension_commands(self, command_str): from azure.cli.core._session import EXT_CMD_INDEX import os - # self.cli_ctx is None when self.prog is not 'az', such as 'az iot'. - # use cli_ctx from cli_help which is not lost. VALID_SECOND = 3600 * 24 * 10 + # self.cli_ctx is None when self.prog is beyond 'az', such as 'az iot'. + # use cli_ctx from cli_help which is not lost. EXT_CMD_INDEX.load(os.path.join((self.cli_ctx or self.cli_help.cli_ctx).config.config_dir, 'extCmdIndex.json'), VALID_SECOND) if not EXT_CMD_INDEX.data: @@ -317,7 +317,12 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements if not self.command_source: candidates = difflib.get_close_matches(value, action.choices, cutoff=0.7) error_msg = None - if not candidates: + # self.cli_ctx is None when self.prog is beyond 'az', such as 'az iot'. + # use cli_ctx from cli_help which is not lost. + cli_ctx = self.cli_ctx or self.cli_help.cli_ctx + use_dynamic_install = cli_ctx.config.get( + 'extension', 'use_dynamic_install', 'yes_prompt') + if use_dynamic_install.lower() != 'no' and not candidates: # Check if the command is from an extension from azure.cli.core.util import roughly_parse_command cmd_list = self.prog.split() + self._raw_arguments @@ -328,18 +333,15 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements telemetry.set_command_details(command_str, parameters=AzCliCommandInvoker._extract_parameter_names(cmd_list), # pylint: disable=protected-access extension_name=ext_name) # TODO add extension_version - cli_ctx = self.cli_ctx or self.cli_help.cli_ctx - ask_before_dynamic_extension_install = cli_ctx.config.getboolean( - 'extension', 'ask_before_dynamic_install', True) - if ask_before_dynamic_extension_install: + if use_dynamic_install.lower() == 'yes_without_prompt': + logger.warning('You are running a command from the extension %s. ' + 'It will be installed first.', ext_name) + go_on = True + else: from knack.prompting import prompt_y_n go_on = prompt_y_n( 'You are running a command from the extension {}. Would you like to install it first?' .format(ext_name), default='y') - else: - logger.warning('You are running a command from the extension %s. ' - 'It will be installed first.', ext_name) - go_on = True if go_on: from azure.cli.core.extension.operations import add_extension add_extension(cli_ctx=cli_ctx, extension_name=ext_name) @@ -359,8 +361,15 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements "Please run 'az extension add -n {}' first.".format(ext_name) if not error_msg: # parser has no `command_source`, value is part of command itself - error_msg = "{prog}: '{value}' is not in the '{prog}' command group. See '{prog} --help'. " \ + error_msg = "{prog}: '{value}' is not in the '{prog}' command group. See '{prog} --help'." \ .format(prog=self.prog, value=value) + if use_dynamic_install.lower() == 'no': + extensions_link = 'https://docs.microsoft.com/en-us/cli/azure/azure-cli-extensions-overview' + error_msg = ("{msg} " + "If the command is from an extension, " + "please make sure the corresponding extension is installed. " + "To learn more about extensions, please visit " + "{extensions_link}").format(msg=error_msg, extensions_link=extensions_link) else: # `command_source` indicates command values have been parsed, value is an argument parameter = action.option_strings[0] if action.option_strings else action.dest From 72b1506801756cca0243f12c614a71ce68c6d158 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Fri, 17 Jul 2020 16:48:07 +0800 Subject: [PATCH 09/20] add option to turn off dynamic install --- src/azure-cli-core/azure/cli/core/parser.py | 33 +++++++++++++-------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index 3150ad53d66..cfacbfb9c1a 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -283,9 +283,9 @@ def parse_known_args(self, args=None, namespace=None): def _search_in_extension_commands(self, command_str): from azure.cli.core._session import EXT_CMD_INDEX import os - # self.cli_ctx is None when self.prog is not 'az', such as 'az iot'. - # use cli_ctx from cli_help which is not lost. VALID_SECOND = 3600 * 24 * 10 + # self.cli_ctx is None when self.prog is beyond 'az', such as 'az iot'. + # use cli_ctx from cli_help which is not lost. EXT_CMD_INDEX.load(os.path.join((self.cli_ctx or self.cli_help.cli_ctx).config.config_dir, 'extCmdIndex.json'), VALID_SECOND) if not EXT_CMD_INDEX.data: @@ -317,7 +317,12 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements if not self.command_source: candidates = difflib.get_close_matches(value, action.choices, cutoff=0.7) error_msg = None - if not candidates: + # self.cli_ctx is None when self.prog is beyond 'az', such as 'az iot'. + # use cli_ctx from cli_help which is not lost. + cli_ctx = self.cli_ctx or self.cli_help.cli_ctx + use_dynamic_install = cli_ctx.config.get( + 'extension', 'use_dynamic_install', 'yes_prompt') + if use_dynamic_install.lower() != 'no' and not candidates: # Check if the command is from an extension from azure.cli.core.util import roughly_parse_command cmd_list = self.prog.split() + self._raw_arguments @@ -328,18 +333,15 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements telemetry.set_command_details(command_str, parameters=AzCliCommandInvoker._extract_parameter_names(cmd_list), # pylint: disable=protected-access extension_name=ext_name) # TODO add extension_version - cli_ctx = self.cli_ctx or self.cli_help.cli_ctx - ask_before_dynamic_extension_install = cli_ctx.config.getboolean( - 'extension', 'ask_before_dynamic_install', True) - if ask_before_dynamic_extension_install: + if use_dynamic_install.lower() == 'yes_without_prompt': + logger.warning('You are running a command from the extension %s. ' + 'It will be installed first.', ext_name) + go_on = True + else: from knack.prompting import prompt_y_n go_on = prompt_y_n( 'You are running a command from the extension {}. Would you like to install it first?' .format(ext_name), default='y') - else: - logger.warning('You are running a command from the extension %s. ' - 'It will be installed first.', ext_name) - go_on = True if go_on: from azure.cli.core.extension.operations import add_extension add_extension(cli_ctx=cli_ctx, extension_name=ext_name) @@ -359,8 +361,15 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements "Please run 'az extension add -n {}' first.".format(ext_name) if not error_msg: # parser has no `command_source`, value is part of command itself - error_msg = "{prog}: '{value}' is not in the '{prog}' command group. See '{prog} --help'. " \ + error_msg = "{prog}: '{value}' is not in the '{prog}' command group. See '{prog} --help'." \ .format(prog=self.prog, value=value) + if use_dynamic_install.lower() == 'no': + extensions_link = 'https://docs.microsoft.com/en-us/cli/azure/azure-cli-extensions-overview' + error_msg = ("{msg} " + "If the command is from an extension, " + "please make sure the corresponding extension is installed. " + "To learn more about extensions, please visit " + "{extensions_link}").format(msg=error_msg, extensions_link=extensions_link) else: # `command_source` indicates command values have been parsed, value is an argument parameter = action.option_strings[0] if action.option_strings else action.dest From bf8f7ee9f0d3871669eefc913584f1932ac83f6e Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Sat, 18 Jul 2020 00:10:24 +0800 Subject: [PATCH 10/20] fix when cli_ctx is None --- .../azure/cli/core/extension/operations.py | 5 +++-- src/azure-cli-core/azure/cli/core/parser.py | 13 ++++++++----- .../azure/cli/command_modules/extension/custom.py | 4 ++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/extension/operations.py b/src/azure-cli-core/azure/cli/core/extension/operations.py index 64dbed11256..f3bdc518298 100644 --- a/src/azure-cli-core/azure/cli/core/extension/operations.py +++ b/src/azure-cli-core/azure/cli/core/extension/operations.py @@ -289,8 +289,9 @@ def show_extension(extension_name): raise CLIError(e) -def update_extension(cli_ctx, extension_name, index_url=None, pip_extra_index_urls=None, pip_proxy=None): +def update_extension(cmd=None, extension_name=None, cli_ctx=None, index_url=None, pip_extra_index_urls=None, pip_proxy=None): try: + cmd_cli_ctx = cli_ctx or cmd.cli_ctx ext = get_extension(extension_name, ext_type=WheelExtension) cur_version = ext.get_version() try: @@ -307,7 +308,7 @@ def update_extension(cli_ctx, extension_name, index_url=None, pip_extra_index_ur shutil.rmtree(extension_path) # Install newer version try: - _add_whl_ext(cli_ctx=cli_ctx, source=download_url, ext_sha256=ext_sha256, + _add_whl_ext(cli_ctx=cmd_cli_ctx, source=download_url, ext_sha256=ext_sha256, pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy) logger.debug('Deleting backup of old extension at %s', backup_dir) shutil.rmtree(backup_dir) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index cfacbfb9c1a..a418ef1d6cf 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -286,18 +286,21 @@ def _search_in_extension_commands(self, command_str): VALID_SECOND = 3600 * 24 * 10 # self.cli_ctx is None when self.prog is beyond 'az', such as 'az iot'. # use cli_ctx from cli_help which is not lost. - EXT_CMD_INDEX.load(os.path.join((self.cli_ctx or self.cli_help.cli_ctx).config.config_dir, + cli_ctx = self.cli_ctx or (self.cli_help.cli_ctx if self.cli_help else None) + if not cli_ctx: + return None + EXT_CMD_INDEX.load(os.path.join(cli_ctx.config.config_dir, 'extCmdIndex.json'), VALID_SECOND) if not EXT_CMD_INDEX.data: import requests from azure.cli.core.util import should_disable_connection_verify - response = requests.get('https://fengsa.blob.core.windows.net/index/extCmdIndex.json', + response = requests.get('https://fengsa.blob.core.windows.net/index/extCmdIndex.json', # TODO use prod url verify=(not should_disable_connection_verify())) if response.status_code == 200: EXT_CMD_INDEX.data = response.json() EXT_CMD_INDEX.save_with_retry() else: - logger.info("Error when retrieveing extension command index. Response code:%s", response.status_code) + logger.info("Error when retrieveing extension command index. Response code: %s", response.status_code) return None cmd_chain = EXT_CMD_INDEX for part in command_str.split(): @@ -319,9 +322,9 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements error_msg = None # self.cli_ctx is None when self.prog is beyond 'az', such as 'az iot'. # use cli_ctx from cli_help which is not lost. - cli_ctx = self.cli_ctx or self.cli_help.cli_ctx + cli_ctx = self.cli_ctx or (self.cli_help.cli_ctx if self.cli_help else None) use_dynamic_install = cli_ctx.config.get( - 'extension', 'use_dynamic_install', 'yes_prompt') + 'extension', 'use_dynamic_install', 'yes_prompt') if cli_ctx else 'no' if use_dynamic_install.lower() != 'no' and not candidates: # Check if the command is from an extension from azure.cli.core.util import roughly_parse_command diff --git a/src/azure-cli/azure/cli/command_modules/extension/custom.py b/src/azure-cli/azure/cli/command_modules/extension/custom.py index 5a0881a40b5..96e708e0f84 100644 --- a/src/azure-cli/azure/cli/command_modules/extension/custom.py +++ b/src/azure-cli/azure/cli/command_modules/extension/custom.py @@ -30,8 +30,8 @@ def show_extension_cmd(extension_name): def update_extension_cmd(cmd, extension_name, index_url=None, pip_extra_index_urls=None, pip_proxy=None): - return update_extension(cmd.cli_ctx, extension_name, index_url=index_url, pip_extra_index_urls=pip_extra_index_urls, - pip_proxy=pip_proxy) + return update_extension(cli_ctx=cmd.cli_ctx, extension_name=extension_name, index_url=index_url, + pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy) def list_available_extensions_cmd(index_url=None, show_details=False): From 4bd9965dd9341a6da4bcefc95be62f4e2492166f Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Tue, 21 Jul 2020 18:22:58 +0800 Subject: [PATCH 11/20] add no prompt msg --- src/azure-cli-core/azure/cli/core/parser.py | 31 ++++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index 7bab4bf2210..d8ebff11afe 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -309,13 +309,19 @@ def _search_in_extension_commands(self, command_str): if not EXT_CMD_INDEX.data: import requests from azure.cli.core.util import should_disable_connection_verify - response = requests.get('https://fengsa.blob.core.windows.net/index/extCmdIndex.json', # TODO use prod url - verify=(not should_disable_connection_verify())) + try: + response = requests.get( + 'https://azurecliextensionsync.blob.core.windows.net/cmd-index/extCmdIndex.json', + verify=(not should_disable_connection_verify()), + timeout=300) + except Exception as ex: # pylint: disable=broad-except + logger.info("Request failed for extension command index: %s", str(ex)) + return None if response.status_code == 200: EXT_CMD_INDEX.data = response.json() EXT_CMD_INDEX.save_with_retry() else: - logger.info("Error when retrieveing extension command index. Response code: %s", response.status_code) + logger.info("Error when retrieving extension command index. Response code: %s", response.status_code) return None cmd_chain = EXT_CMD_INDEX for part in command_str.split(): @@ -356,10 +362,21 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements 'It will be installed first.', ext_name) go_on = True else: - from knack.prompting import prompt_y_n - go_on = prompt_y_n( - 'You are running a command from the extension {}. Would you like to install it first?' - .format(ext_name), default='y') + from knack.prompting import prompt_y_n, NoTTYException + NO_PROMPT_CONFIG_MSG = "Run 'az config set extension.use_dynamic_install=" \ + "yes_without_prompt' to allow installing extensions with no prompt." + try: + go_on = prompt_y_n( + 'You are running a command from the extension {}. ' + 'Would you like to install it first?' + .format(ext_name), default='y') + if go_on: + logger.warning(NO_PROMPT_CONFIG_MSG) + except NoTTYException: + logger.warning("You are running a command from the extension %s.\n " + "Unable to prompt for extension install confirmation as no tty " + "available. %s", ext_name, NO_PROMPT_CONFIG_MSG) + go_on = False if go_on: from azure.cli.core.extension.operations import add_extension add_extension(cli_ctx=cli_ctx, extension_name=ext_name) From 28360a0ea8005b015422f3c68dacf176530638a6 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Tue, 21 Jul 2020 22:12:03 +0800 Subject: [PATCH 12/20] modify message --- src/azure-cli-core/azure/cli/core/parser.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index d8ebff11afe..a6f010b4395 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -357,6 +357,15 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements telemetry.set_command_details(command_str, parameters=AzCliCommandInvoker._extract_parameter_names(cmd_list), # pylint: disable=protected-access extension_name=ext_name) + run_after_extension_installed = cli_ctx.config.getboolean('extension', + 'run_after_dynamic_install', + True) + prompt_msg = 'You are running a command from the extension {}. ' \ + 'Would you like to install it first?'.format(ext_name) + if run_after_extension_installed: + prompt_msg = '{} The command will continue to run after the extension is installed.' \ + .format(prompt_msg) + if use_dynamic_install.lower() == 'yes_without_prompt': logger.warning('You are running a command from the extension %s. ' 'It will be installed first.', ext_name) @@ -364,12 +373,9 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements else: from knack.prompting import prompt_y_n, NoTTYException NO_PROMPT_CONFIG_MSG = "Run 'az config set extension.use_dynamic_install=" \ - "yes_without_prompt' to allow installing extensions with no prompt." + "yes_without_prompt' to allow installing extensions without prompt." try: - go_on = prompt_y_n( - 'You are running a command from the extension {}. ' - 'Would you like to install it first?' - .format(ext_name), default='y') + go_on = prompt_y_n(prompt_msg, default='y') if go_on: logger.warning(NO_PROMPT_CONFIG_MSG) except NoTTYException: @@ -380,9 +386,6 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements if go_on: from azure.cli.core.extension.operations import add_extension add_extension(cli_ctx=cli_ctx, extension_name=ext_name) - run_after_extension_installed = cli_ctx.config.getboolean('extension', - 'run_after_dynamic_install', - True) if run_after_extension_installed: import subprocess exit_code = subprocess.call(cmd_list, shell=True) From 18efb124894a2741f33444e735640b258134ef8e Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Thu, 23 Jul 2020 10:46:25 +0800 Subject: [PATCH 13/20] default to no --- src/azure-cli-core/azure/cli/core/parser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index a6f010b4395..15d8d5135cb 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -345,7 +345,9 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements # use cli_ctx from cli_help which is not lost. cli_ctx = self.cli_ctx or (self.cli_help.cli_ctx if self.cli_help else None) use_dynamic_install = cli_ctx.config.get( - 'extension', 'use_dynamic_install', 'yes_prompt') if cli_ctx else 'no' + 'extension', 'use_dynamic_install', 'no') if cli_ctx else 'no' + if use_dynamic_install.lower() not in ['no', 'yes_prompt', 'yes_without_prompt']: + use_dynamic_install = 'no' if use_dynamic_install.lower() != 'no' and not candidates: # Check if the command is from an extension from azure.cli.core.util import roughly_parse_command From de3fda35209bd28ddd34302f72f74a3b819a18e6 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Thu, 23 Jul 2020 13:35:35 +0800 Subject: [PATCH 14/20] refactor error msg --- src/azure-cli-core/azure/cli/core/parser.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index 15d8d5135cb..76f30c9489b 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -362,18 +362,17 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements run_after_extension_installed = cli_ctx.config.getboolean('extension', 'run_after_dynamic_install', True) - prompt_msg = 'You are running a command from the extension {}. ' \ - 'Would you like to install it first?'.format(ext_name) - if run_after_extension_installed: - prompt_msg = '{} The command will continue to run after the extension is installed.' \ - .format(prompt_msg) - if use_dynamic_install.lower() == 'yes_without_prompt': logger.warning('You are running a command from the extension %s. ' 'It will be installed first.', ext_name) go_on = True else: from knack.prompting import prompt_y_n, NoTTYException + prompt_msg = 'You are running a command from the extension {}. ' \ + 'Would you like to install it first?'.format(ext_name) + if run_after_extension_installed: + prompt_msg = '{} The command will continue to run after the extension is installed.' \ + .format(prompt_msg) NO_PROMPT_CONFIG_MSG = "Run 'az config set extension.use_dynamic_install=" \ "yes_without_prompt' to allow installing extensions without prompt." try: @@ -395,7 +394,7 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements "rerun automatically.".format(ext_name)) self.exit(exit_code) else: - error_msg = 'Extension installed. Please rerun your command.' + error_msg = 'Extension {} installed. Please rerun your command.'.format(ext_name) else: error_msg = "Command failed due to corresponding extension not installed. " \ "Please run 'az extension add -n {}' first.".format(ext_name) From 00cc9cb2bf35410c6b69e7858a798f32c6c02e27 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Thu, 23 Jul 2020 16:01:43 +0800 Subject: [PATCH 15/20] fix style --- src/azure-cli-core/azure/cli/core/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index 76f30c9489b..dbe715e5841 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -369,7 +369,7 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements else: from knack.prompting import prompt_y_n, NoTTYException prompt_msg = 'You are running a command from the extension {}. ' \ - 'Would you like to install it first?'.format(ext_name) + 'Would you like to install it first?'.format(ext_name) if run_after_extension_installed: prompt_msg = '{} The command will continue to run after the extension is installed.' \ .format(prompt_msg) From 1abde7cb4b607a08b2674ec012bf9fcd24ea8b0c Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Mon, 27 Jul 2020 19:20:06 +0800 Subject: [PATCH 16/20] resolve UX comments --- src/azure-cli-core/azure/cli/core/parser.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index dbe715e5841..c2a7cdab695 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -361,15 +361,15 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements extension_name=ext_name) run_after_extension_installed = cli_ctx.config.getboolean('extension', 'run_after_dynamic_install', - True) + False) if use_dynamic_install.lower() == 'yes_without_prompt': - logger.warning('You are running a command from the extension %s. ' + logger.warning('The command requires the extension %s. ' 'It will be installed first.', ext_name) go_on = True else: from knack.prompting import prompt_y_n, NoTTYException - prompt_msg = 'You are running a command from the extension {}. ' \ - 'Would you like to install it first?'.format(ext_name) + prompt_msg = 'The command requires the extension {}. ' \ + 'Do you want to install it now?'.format(ext_name) if run_after_extension_installed: prompt_msg = '{} The command will continue to run after the extension is installed.' \ .format(prompt_msg) @@ -380,7 +380,7 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements if go_on: logger.warning(NO_PROMPT_CONFIG_MSG) except NoTTYException: - logger.warning("You are running a command from the extension %s.\n " + logger.warning("The command requires the extension %s.\n " "Unable to prompt for extension install confirmation as no tty " "available. %s", ext_name, NO_PROMPT_CONFIG_MSG) go_on = False @@ -396,8 +396,8 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements else: error_msg = 'Extension {} installed. Please rerun your command.'.format(ext_name) else: - error_msg = "Command failed due to corresponding extension not installed. " \ - "Please run 'az extension add -n {}' first.".format(ext_name) + error_msg = "The command requires the extension {ext_name}. " \ + "To install, run 'az extension add -n {ext_name}'.".format(ext_name=ext_name) if not error_msg: # parser has no `command_source`, value is part of command itself error_msg = "{prog}: '{value}' is not in the '{prog}' command group. See '{prog} --help'." \ From e9530f5b32ee56e478835c6e64a2d76e1e8ec9d7 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Thu, 30 Jul 2020 14:45:16 +0800 Subject: [PATCH 17/20] make changes backward compatible --- src/azure-cli-core/azure/cli/core/extension/operations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/extension/operations.py b/src/azure-cli-core/azure/cli/core/extension/operations.py index f3bdc518298..126d3aad9ad 100644 --- a/src/azure-cli-core/azure/cli/core/extension/operations.py +++ b/src/azure-cli-core/azure/cli/core/extension/operations.py @@ -206,9 +206,9 @@ def check_version_compatibility(azext_metadata): raise CLIError(min_max_msg_fmt) -def add_extension(cmd=None, cli_ctx=None, source=None, extension_name=None, index_url=None, yes=None, # pylint: disable=unused-argument +def add_extension(cmd=None, source=None, extension_name=None, index_url=None, yes=None, # pylint: disable=unused-argument pip_extra_index_urls=None, pip_proxy=None, system=None, - version=None): + version=None, cli_ctx=None): ext_sha256 = None version = None if version == 'latest' else version @@ -289,7 +289,7 @@ def show_extension(extension_name): raise CLIError(e) -def update_extension(cmd=None, extension_name=None, cli_ctx=None, index_url=None, pip_extra_index_urls=None, pip_proxy=None): +def update_extension(cmd=None, extension_name=None, index_url=None, pip_extra_index_urls=None, pip_proxy=None, cli_ctx=None): try: cmd_cli_ctx = cli_ctx or cmd.cli_ctx ext = get_extension(extension_name, ext_type=WheelExtension) From 181c3ec200d42086bdd018688e712b33da176f96 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Thu, 30 Jul 2020 20:04:16 +0800 Subject: [PATCH 18/20] add test --- src/azure-cli-core/azure/cli/core/_session.py | 4 +- src/azure-cli-core/azure/cli/core/parser.py | 75 +++++++++++-------- .../azure/cli/core/tests/test_parser.py | 18 +++++ 3 files changed, 62 insertions(+), 35 deletions(-) diff --git a/src/azure-cli-core/azure/cli/core/_session.py b/src/azure-cli-core/azure/cli/core/_session.py index 6c283d55cd2..712be84fa8b 100644 --- a/src/azure-cli-core/azure/cli/core/_session.py +++ b/src/azure-cli-core/azure/cli/core/_session.py @@ -114,5 +114,5 @@ def __len__(self): # an upgrade of azure-cli happens VERSIONS = Session() -# EXT_CMD_INDEX provides command to extension name mapping -EXT_CMD_INDEX = Session() +# EXT_CMD_TREE provides command to extension name mapping +EXT_CMD_TREE = Session() diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index c2a7cdab695..158c5986eb4 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -280,23 +280,8 @@ def parse_known_args(self, args=None, namespace=None): self._namespace, self._raw_arguments = super().parse_known_args(args=args, namespace=namespace) return self._namespace, self._raw_arguments - def _search_in_extension_commands(self, command_str): - """Search the command in an extension commands dict which mimics a trie. - If the value of the dict item is a string, then the key represents the end of a complete command - and the value is the name of the extension that the command belongs to. - An example of the dict read from extCmdIndex.json: - { - "aks": { - "create": "aks-preview", - "update": "aks-preview", - "app": { - "up": "deploy-to-azure" - }, - "use-dev-spaces": "dev-spaces" - } - } - """ - from azure.cli.core._session import EXT_CMD_INDEX + def _get_extension_command_tree(self): + from azure.cli.core._session import EXT_CMD_TREE import os VALID_SECOND = 3600 * 24 * 10 # self.cli_ctx is None when self.prog is beyond 'az', such as 'az iot'. @@ -304,26 +289,45 @@ def _search_in_extension_commands(self, command_str): cli_ctx = self.cli_ctx or (self.cli_help.cli_ctx if self.cli_help else None) if not cli_ctx: return None - EXT_CMD_INDEX.load(os.path.join(cli_ctx.config.config_dir, - 'extCmdIndex.json'), VALID_SECOND) - if not EXT_CMD_INDEX.data: + EXT_CMD_TREE.load(os.path.join(cli_ctx.config.config_dir, 'extensionCommandTree.json'), VALID_SECOND) + if not EXT_CMD_TREE.data: import requests from azure.cli.core.util import should_disable_connection_verify try: response = requests.get( - 'https://azurecliextensionsync.blob.core.windows.net/cmd-index/extCmdIndex.json', + 'https://azurecliextensionsync.blob.core.windows.net/cmd-index/extensionCommandTree.json', verify=(not should_disable_connection_verify()), timeout=300) except Exception as ex: # pylint: disable=broad-except - logger.info("Request failed for extension command index: %s", str(ex)) + logger.info("Request failed for extension command tree: %s", str(ex)) return None if response.status_code == 200: - EXT_CMD_INDEX.data = response.json() - EXT_CMD_INDEX.save_with_retry() + EXT_CMD_TREE.data = response.json() + EXT_CMD_TREE.save_with_retry() else: - logger.info("Error when retrieving extension command index. Response code: %s", response.status_code) + logger.info("Error when retrieving extension command tree. Response code: %s", response.status_code) return None - cmd_chain = EXT_CMD_INDEX + return EXT_CMD_TREE + + def _search_in_extension_commands(self, command_str): + """Search the command in an extension commands dict which mimics a prefix tree. + If the value of the dict item is a string, then the key represents the end of a complete command + and the value is the name of the extension that the command belongs to. + An example of the dict read from extensionCommandTree.json: + { + "aks": { + "create": "aks-preview", + "update": "aks-preview", + "app": { + "up": "deploy-to-azure" + }, + "use-dev-spaces": "dev-spaces" + }, + ... + } + """ + + cmd_chain = self._get_extension_command_tree() for part in command_str.split(): try: if isinstance(cmd_chain[part], str): @@ -333,6 +337,14 @@ def _search_in_extension_commands(self, command_str): return None return None + def _get_extension_use_dynamic_install_config(self): + cli_ctx = self.cli_ctx or (self.cli_help.cli_ctx if self.cli_help else None) + use_dynamic_install = cli_ctx.config.get( + 'extension', 'use_dynamic_install', 'no').lower() if cli_ctx else 'no' + if use_dynamic_install not in ['no', 'yes_prompt', 'yes_without_prompt']: + use_dynamic_install = 'no' + return use_dynamic_install + def _check_value(self, action, value): # pylint: disable=too-many-statements # Override to customize the error message when a argument is not among the available choices # converted value must be one of the choices (if specified) @@ -344,11 +356,8 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements # self.cli_ctx is None when self.prog is beyond 'az', such as 'az iot'. # use cli_ctx from cli_help which is not lost. cli_ctx = self.cli_ctx or (self.cli_help.cli_ctx if self.cli_help else None) - use_dynamic_install = cli_ctx.config.get( - 'extension', 'use_dynamic_install', 'no') if cli_ctx else 'no' - if use_dynamic_install.lower() not in ['no', 'yes_prompt', 'yes_without_prompt']: - use_dynamic_install = 'no' - if use_dynamic_install.lower() != 'no' and not candidates: + use_dynamic_install = self._get_extension_use_dynamic_install_config() + if use_dynamic_install != 'no' and not candidates: # Check if the command is from an extension from azure.cli.core.util import roughly_parse_command cmd_list = self.prog.split() + self._raw_arguments @@ -361,8 +370,8 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements extension_name=ext_name) run_after_extension_installed = cli_ctx.config.getboolean('extension', 'run_after_dynamic_install', - False) - if use_dynamic_install.lower() == 'yes_without_prompt': + False) if cli_ctx else False + if use_dynamic_install == 'yes_without_prompt': logger.warning('The command requires the extension %s. ' 'It will be installed first.', ext_name) go_on = True diff --git a/src/azure-cli-core/azure/cli/core/tests/test_parser.py b/src/azure-cli-core/azure/cli/core/tests/test_parser.py index 7d9eae89dad..47708a6d397 100644 --- a/src/azure-cli-core/azure/cli/core/tests/test_parser.py +++ b/src/azure-cli-core/azure/cli/core/tests/test_parser.py @@ -215,6 +215,12 @@ def mock_log_error(_, msg): def mock_get_close_matches(*args, **kwargs): choice_lists.append(original_get_close_matches(*args, **kwargs)) + def mock_ext_cmd_tree_load(*args, **kwargs): + return {"test": {"new-ext": {"create": "new-ext-name", "reset": "another-ext-name"}}} + + def mock_add_extension(*args, **kwargs): + pass + # run multiple faulty commands and save error logs, as well as close matches with mock.patch('logging.Logger.error', mock_log_error), \ mock.patch('difflib.get_close_matches', mock_get_close_matches): @@ -248,6 +254,18 @@ def mock_get_close_matches(*args, **kwargs): for choice in ['enum_1', 'enum_2']: self.assertIn(choice, choices) + # test dynamic extension install + with mock.patch('logging.Logger.error', mock_log_error), \ + mock.patch('azure.cli.core.extension.operations.add_extension', mock_add_extension), \ + mock.patch('azure.cli.core.parser.AzCliCommandParser._get_extension_command_tree', mock_ext_cmd_tree_load), \ + mock.patch('azure.cli.core.parser.AzCliCommandParser._get_extension_use_dynamic_install_config', return_value='yes_without_prompt'): + with self.assertRaises(SystemExit): + parser.parse_args('test new-ext create --opt enum_2'.split()) + self.assertIn("Extension new-ext-name installed. Please rerun your command.", logger_msgs[5]) + with self.assertRaises(SystemExit): + parser.parse_args('test new-ext reset pos1 pos2'.split()) # test positional args + self.assertIn("Extension another-ext-name installed. Please rerun your command.", logger_msgs[6]) + @mock.patch('importlib.import_module', _mock_import_lib) @mock.patch('pkgutil.iter_modules', _mock_iter_modules) @mock.patch('azure.cli.core.commands._load_command_loader', _mock_load_command_loader) From 2e68097498be46df35e9465339575600ccecdb9d Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Thu, 30 Jul 2020 21:30:50 +0800 Subject: [PATCH 19/20] fix shell --- src/azure-cli-core/azure/cli/core/parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index 158c5986eb4..6a35573297b 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -398,7 +398,8 @@ def _check_value(self, action, value): # pylint: disable=too-many-statements add_extension(cli_ctx=cli_ctx, extension_name=ext_name) if run_after_extension_installed: import subprocess - exit_code = subprocess.call(cmd_list, shell=True) + import platform + exit_code = subprocess.call(cmd_list, shell=platform.system() == 'Windows') telemetry.set_user_fault("Extension {} dynamically installed and commands will be " "rerun automatically.".format(ext_name)) self.exit(exit_code) From 4b16148162894198808b5260f801e1ba3cfb6eeb Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Thu, 30 Jul 2020 21:51:31 +0800 Subject: [PATCH 20/20] fix style --- src/azure-cli-core/azure/cli/core/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/azure-cli-core/azure/cli/core/parser.py b/src/azure-cli-core/azure/cli/core/parser.py index 6a35573297b..6e56daa5015 100644 --- a/src/azure-cli-core/azure/cli/core/parser.py +++ b/src/azure-cli-core/azure/cli/core/parser.py @@ -345,7 +345,7 @@ def _get_extension_use_dynamic_install_config(self): use_dynamic_install = 'no' return use_dynamic_install - def _check_value(self, action, value): # pylint: disable=too-many-statements + def _check_value(self, action, value): # pylint: disable=too-many-statements, too-many-locals # Override to customize the error message when a argument is not among the available choices # converted value must be one of the choices (if specified) if action.choices is not None and value not in action.choices: # pylint: disable=too-many-nested-blocks