[core] Optimize help loader v1.

src/azure-cli-core/azure/cli/core/_help.py

@@ -138,6 +138,12 @@ def new_normalize_text(s):
         HelpObject._normalize_text = new_normalize_text  # pylint: disable=protected-access
 
         self._register_help_loaders()
+        self._name_to_content = {}
+
+    # override
+    def show_help(self, cli_name, nouns, parser, is_group):
+        self.update_loaders_with_help_file_contents(nouns)
+        super(AzCliHelp, self).show_help(cli_name, nouns, parser, is_group)
 
     def _register_help_loaders(self):
         import azure.cli.core._help_loaders as help_loaders
@@ -157,6 +163,25 @@ def is_loader_cls(cls):
 
         self.versioned_loaders = versioned_loaders
 
+    def update_loaders_with_help_file_contents(self, nouns):
+        loader_file_names_dict = {}
+        file_name_set = set()
+        for ldr_cls_name, loader in self.versioned_loaders.items():
+            new_file_names = loader.get_noun_help_file_names(nouns) or []
+            loader_file_names_dict[ldr_cls_name] = new_file_names
+            file_name_set.update(new_file_names)
+
+        for file_name in file_name_set:
+            if file_name not in self._name_to_content:
+                with open(file_name, 'r') as f:
+                    self._name_to_content[file_name] = f.read()
+
+        for ldr_cls_name, file_names in loader_file_names_dict.items():
+            file_contents = {}
+            for name in file_names:
+                file_contents[name] = self._name_to_content[name]
+            self.versioned_loaders[ldr_cls_name].update_file_contents(file_contents)
+
 
 class CliHelpFile(KnackHelpFile):
 

src/azure-cli-core/azure/cli/core/_help_loaders.py

@@ -4,10 +4,13 @@
 # --------------------------------------------------------------------------------------------
 
 import abc
+import os
+import yaml
 from knack.util import CLIError
 from knack.log import get_logger
 from azure.cli.core._help import (HelpExample, CliHelpFile)
 
+
 logger = get_logger(__name__)
 
 try:
@@ -20,25 +23,38 @@
 class BaseHelpLoader(ABC):
     def __init__(self, help_ctx=None):
         self.help_ctx = help_ctx
-        self._data = None
+        self._entry_data = None
+        self._file_content_dict = {}
 
     def versioned_load(self, help_obj, parser):
-        self.load_raw_data(help_obj, parser)
+        if not self._file_content_dict:
+            return
+        self._entry_data = None
+        # Cycle through versioned_load helpers
+        self.load_entry_data(help_obj, parser)
         if self._data_is_applicable():
             self.load_help_body(help_obj)
             self.load_help_parameters(help_obj)
             self.load_help_examples(help_obj)
+        self._entry_data = None
+
+    def update_file_contents(self, file_contents):
+        self._file_content_dict.update(file_contents)
+
+    @abc.abstractmethod
+    def get_noun_help_file_names(self, nouns):
+        pass
 
     @property
     @abc.abstractmethod
     def version(self):
         pass
 
     def _data_is_applicable(self):
-        return self._data and self.version == self._data.get('version')
+        return self._entry_data and self.version == self._entry_data.get('version')
 
     @abc.abstractmethod
-    def load_raw_data(self, help_obj, parser):
+    def load_entry_data(self, help_obj, parser):
         pass
 
     @abc.abstractmethod
@@ -75,26 +91,14 @@ def _update_help_obj_params(help_obj, data_params, params_equal, attr_key_tups):
             loaded_params.append(param_obj)
         help_obj.parameters = loaded_params
 
-    # get the yaml help
-    @staticmethod
-    def _get_yaml_help_for_nouns(nouns, cmd_loader_map_ref):
-        import inspect
-        import os
-
-        def _parse_yaml_from_string(text, help_file_path):
-            import yaml
-
-            dir_name, base_name = os.path.split(help_file_path)
 
-            pretty_file_path = os.path.join(os.path.basename(dir_name), base_name)
+class YamlLoaderMixin(object):  # pylint:disable=too-few-public-methods
+    """A class containing helper methods for Yaml Loaders."""
 
-            try:
-                data = yaml.load(text)
-                if not data:
-                    raise CLIError("Error: Help file {} is empty".format(pretty_file_path))
-                return data
-            except yaml.YAMLError as e:
-                raise CLIError("Error parsing {}:\n\n{}".format(pretty_file_path, e))
+    # get the list of yaml help file names for the command or group
+    @staticmethod
+    def _get_yaml_help_files_list(nouns, cmd_loader_map_ref):
+        import inspect
 
         command_nouns = " ".join(nouns)
         # if command in map, get the loader. Path of loader is path of helpfile.
@@ -121,11 +125,22 @@ def _parse_yaml_from_string(text, help_file_path):
                 for file in files:
                     if file.endswith("help.yaml") or file.endswith("help.yml"):
                         help_file_path = os.path.join(dir_name, file)
-                        with open(help_file_path, "r") as f:
-                            text = f.read()
-                            results.append(_parse_yaml_from_string(text, help_file_path))
+                        results.append(help_file_path)
         return results
 
+    @staticmethod
+    def _parse_yaml_from_string(text, help_file_path):
+        dir_name, base_name = os.path.split(help_file_path)
+        pretty_file_path = os.path.join(os.path.basename(dir_name), base_name)
+
+        if not text:
+            raise CLIError("No content passed for {}.".format(pretty_file_path))
+
+        try:
+            return yaml.load(text)
+        except yaml.YAMLError as e:
+            raise CLIError("Error parsing {}:\n\n{}".format(pretty_file_path, e))
+
 
 class HelpLoaderV0(BaseHelpLoader):
 
@@ -136,7 +151,10 @@ def version(self):
     def versioned_load(self, help_obj, parser):
         super(CliHelpFile, help_obj).load(parser)  # pylint:disable=bad-super-call
 
-    def load_raw_data(self, help_obj, parser):
+    def get_noun_help_file_names(self, nouns):
+        pass
+
+    def load_entry_data(self, help_obj, parser):
         pass
 
     def load_help_body(self, help_obj):
@@ -149,7 +167,7 @@ def load_help_examples(self, help_obj):
         pass
 
 
-class HelpLoaderV1(BaseHelpLoader):
+class HelpLoaderV1(BaseHelpLoader, YamlLoaderMixin):
     core_attrs_to_keys = [("short_summary", "summary"), ("long_summary", "description")]
     body_attrs_to_keys = core_attrs_to_keys + [("links", "links")]
     param_attrs_to_keys = core_attrs_to_keys + [("value_sources", "value-sources")]
@@ -158,16 +176,29 @@ class HelpLoaderV1(BaseHelpLoader):
     def version(self):
         return 1
 
-    def load_raw_data(self, help_obj, parser):
+    def get_noun_help_file_names(self, nouns):
+        cmd_loader_map_ref = self.help_ctx.cli_ctx.invocation.commands_loader.cmd_to_loader_map
+        return self._get_yaml_help_files_list(nouns, cmd_loader_map_ref)
+
+    def update_file_contents(self, file_contents):
+        for file_name in file_contents:
+            if file_name not in self._file_content_dict:
+                data_dict = {file_name: self._parse_yaml_from_string(file_contents[file_name], file_name)}
+                self._file_content_dict.update(data_dict)
+
+    def load_entry_data(self, help_obj, parser):
         prog = parser.prog if hasattr(parser, "prog") else parser._prog_prefix  # pylint: disable=protected-access
         command_nouns = prog.split()[1:]
         cmd_loader_map_ref = self.help_ctx.cli_ctx.invocation.commands_loader.cmd_to_loader_map
-        data_list = self._get_yaml_help_for_nouns(command_nouns, cmd_loader_map_ref)
-        self._data = self._get_entry_data(help_obj.command, data_list)
+
+        files_list = self._get_yaml_help_files_list(command_nouns, cmd_loader_map_ref)
+        data_list = [self._file_content_dict[name] for name in files_list]
+
+        self._entry_data = self._get_entry_data(help_obj.command, data_list)
 
     def load_help_body(self, help_obj):
         help_obj.long_summary = ""  # similar to knack...
-        self._update_obj_from_data_dict(help_obj, self._data, self.body_attrs_to_keys)
+        self._update_obj_from_data_dict(help_obj, self._entry_data, self.body_attrs_to_keys)
 
     def load_help_parameters(self, help_obj):
         def params_equal(param, param_dict):
@@ -176,18 +207,18 @@ def params_equal(param, param_dict):
             # for positionals, help file must name must match param name shown when -h is run
             return param_dict['name'] == param.name
 
-        if help_obj.type == "command" and hasattr(help_obj, "parameters") and self._data.get("arguments"):
+        if help_obj.type == "command" and hasattr(help_obj, "parameters") and self._entry_data.get("arguments"):
             loaded_params = []
             for param_obj in help_obj.parameters:
-                loaded_param = next((n for n in self._data["arguments"] if params_equal(param_obj, n)), None)
+                loaded_param = next((n for n in self._entry_data["arguments"] if params_equal(param_obj, n)), None)
                 if loaded_param:
                     self._update_obj_from_data_dict(param_obj, loaded_param, self.param_attrs_to_keys)
                 loaded_params.append(param_obj)
             help_obj.parameters = loaded_params
 
     def load_help_examples(self, help_obj):
-        if help_obj.type == "command" and self._data.get("examples"):
-            help_obj.examples = [HelpExample(**ex) for ex in self._data["examples"] if help_obj._should_include_example(ex)]  # pylint: disable=line-too-long, protected-access
+        if help_obj.type == "command" and self._entry_data.get("examples"):
+            help_obj.examples = [HelpExample(**ex) for ex in self._entry_data["examples"] if help_obj._should_include_example(ex)]  # pylint: disable=line-too-long, protected-access
 
     @staticmethod
     def _get_entry_data(cmd_name, data_list):

src/azure-cli-core/azure/cli/core/file_util.py

@@ -27,6 +27,7 @@ def get_all_help(cli_ctx):
     help_files = []
     for cmd, parser in zip(sub_parser_keys, sub_parser_values):
         try:
+            help_ctx.update_loaders_with_help_file_contents(cmd.split())
             help_file = CliGroupHelpFile(help_ctx, cmd, parser) if _is_group(parser) \
                 else CliCommandHelpFile(help_ctx, cmd, parser)
             help_file.load(parser)

src/azure-cli-core/azure/cli/core/tests/test_help_loaders.py

@@ -52,8 +52,62 @@ def dummy_handler(arg1, arg2=None, arg3=None, arg4=None):
 # region Test Help Loader
 
 
+class JsonLoaderMixin(object):
+    """A class containing helper methods for Json Loaders."""
+
+    # get the list of json help file names for the command or group
+    @staticmethod
+    def _get_json_help_files_list(nouns, cmd_loader_map_ref):
+        import inspect
+        import os
+
+        command_nouns = " ".join(nouns)
+        # if command in map, get the loader. Path of loader is path of helpfile.
+        ldr_or_none = cmd_loader_map_ref.get(command_nouns, [None])[0]
+        if ldr_or_none:
+            loaders = {ldr_or_none}
+        else:
+            loaders = set()
+
+        # otherwise likely a group, try to find all command loaders under group as the group help could be defined
+        # in either.
+        if not loaders:
+            for cmd_name, cmd_ldr in cmd_loader_map_ref.items():
+                # if first word in loader name is the group, this is a command in the command group
+                if cmd_name.startswith(command_nouns + " "):
+                    loaders.add(cmd_ldr[0])
+
+        results = []
+        if loaders:
+            for loader in loaders:
+                loader_file_path = inspect.getfile(loader.__class__)
+                dir_name = os.path.dirname(loader_file_path)
+                files = os.listdir(dir_name)
+                for file in files:
+                    if file.endswith("help.json"):
+                        help_file_path = os.path.join(dir_name, file)
+                        results.append(help_file_path)
+        return results
+
+    @staticmethod
+    def _parse_json_from_string(text, help_file_path):
+        import os
+        import json
+
+        dir_name, base_name = os.path.split(help_file_path)
+        pretty_file_path = os.path.join(os.path.basename(dir_name), base_name)
+
+        if not text:
+            raise CLIError("No content passed for {}.".format(pretty_file_path))
+
+        try:
+            return json.loads(text)
+        except ValueError as e:
+            raise CLIError("Error parsing {}:\n\n{}".format(pretty_file_path, e))
+
+
 # test Help Loader, loads from help.json
-class DummyHelpLoader(HelpLoaderV1):
+class DummyHelpLoader(HelpLoaderV1, JsonLoaderMixin):
     # This loader has different keys in the data object. Except for "arguments" and "examples".
     core_attrs_to_keys = [("short_summary", "short"), ("long_summary", "long")]
     body_attrs_to_keys = core_attrs_to_keys + [("links", "hyper-links")]
@@ -63,57 +117,25 @@ class DummyHelpLoader(HelpLoaderV1):
     def version(self):
         return 2
 
-    def load_raw_data(self, help_obj, parser):
-        prog = parser.prog if hasattr(parser, "prog") else parser._prog_prefix
-        command_nouns = prog.split()[1:]
+    def get_noun_help_file_names(self, nouns):
         cmd_loader_map_ref = self.help_ctx.cli_ctx.invocation.commands_loader.cmd_to_loader_map
-        all_data = self.get_json_help_for_nouns(command_nouns, cmd_loader_map_ref)
-        self._data = self._get_entry_data(help_obj.command, all_data)
-
-    def load_help_body(self, help_obj):
-        self._update_obj_from_data_dict(help_obj, self._data, self.body_attrs_to_keys)
+        return self._get_json_help_files_list(nouns, cmd_loader_map_ref)
 
-    # get the json help
-    @staticmethod
-    def get_json_help_for_nouns(nouns, cmd_loader_map_ref):
-        import inspect
-        import os
+    def update_file_contents(self, file_contents):
+        for file_name in file_contents:
+            if file_name not in self._file_content_dict:
+                data_dict = {file_name: self._parse_json_from_string(file_contents[file_name], file_name)}
+                self._file_content_dict.update(data_dict)
 
-        def _parse_json_from_string(text, help_file_path):
-            import json
-
-            dir_name, base_name = os.path.split(help_file_path)
+    def load_entry_data(self, help_obj, parser):
+        prog = parser.prog if hasattr(parser, "prog") else parser._prog_prefix  # pylint: disable=protected-access
+        command_nouns = prog.split()[1:]
+        cmd_loader_map_ref = self.help_ctx.cli_ctx.invocation.commands_loader.cmd_to_loader_map
 
-            pretty_file_path = os.path.join(os.path.basename(dir_name), base_name)
+        files_list = self._get_json_help_files_list(command_nouns, cmd_loader_map_ref)
+        data_list = [self._file_content_dict[name] for name in files_list]
 
-            try:
-                data = json.loads(text)
-                if not data:
-                    raise CLIError("Error: Help file {} is empty".format(pretty_file_path))
-                return data
-            except ValueError as e:
-                raise CLIError("Error parsing {}:\n\n{}".format(pretty_file_path, e))
+        self._entry_data = self._get_entry_data(help_obj.command, data_list)
 
-        command_nouns = " ".join(nouns)
-        # if command in map, get the loader. Path of loader is path of helpfile.
-        loader = cmd_loader_map_ref.get(command_nouns, [None])[0]
-
-        # otherwise likely a group, get the loader
-        if not loader:
-            for k, v in cmd_loader_map_ref.items():
-                # if loader name starts with noun / group, this is a command in the command group
-                if k.startswith(command_nouns):
-                    loader = v[0]
-                    break
-
-        if loader:
-            loader_file_path = inspect.getfile(loader.__class__)
-            dir_name = os.path.dirname(loader_file_path)
-            files = os.listdir(dir_name)
-            for file in files:
-                if file.endswith("help.json"):
-                    help_file_path = os.path.join(dir_name, file)
-                    with open(help_file_path, "r") as f:
-                        text = f.read()
-                        return [_parse_json_from_string(text, help_file_path)]
-        return None
+    def load_help_body(self, help_obj):
+        self._update_obj_from_data_dict(help_obj, self._entry_data, self.body_attrs_to_keys)