Expanded Cmd2Lexer to highlight argparse flags in red and values in yellow

We can roll this back if people don't like it.

Author: tleonhardt

cmd2/pt_utils.py

@@ -197,7 +197,27 @@ def get_line(lineno: int) -> list[tuple[str, str]]:
 
                 # Add the rest of the line
                 if cmd_end < len(line):
-                    tokens.append(('', line[cmd_end:]))
+                    rest = line[cmd_end:]
+                    # Regex to match whitespace, flags, quoted strings, or other words
+                    arg_pattern = re.compile(r'(\s+)|(--?[^\s\'"]+)|("[^"]*"|\'[^\']*\')|([^\s\'"]+)')
+
+                    # Get redirection tokens and terminators to avoid highlighting them as values
+                    exclude_tokens = set(constants.REDIRECTION_TOKENS)
+                    if hasattr(self.cmd_app, 'statement_parser'):
+                        exclude_tokens.update(self.cmd_app.statement_parser.terminators)
+
+                    for m in arg_pattern.finditer(rest):
+                        space, flag, quoted, word = m.groups()
+                        text = m.group(0)
+
+                        if space:
+                            tokens.append(('', text))
+                        elif flag:
+                            tokens.append(('ansired', text))
+                        elif (quoted or word) and text not in exclude_tokens:
+                            tokens.append(('ansiyellow', text))
+                        else:
+                            tokens.append(('', text))
             elif line:
                 # No command match found, add the entire line unstyled
                 tokens.append(('', line))

tests/test_pt_utils.py

@@ -50,7 +50,7 @@ def test_lex_document_command(self, mock_cmd_app):
         get_line = lexer.lex_document(document)
         tokens = get_line(0)
 
-        assert tokens == [('ansigreen', 'help'), ('', ' something')]
+        assert tokens == [('ansigreen', 'help'), ('', ' '), ('ansiyellow', 'something')]
 
     def test_lex_document_alias(self, mock_cmd_app):
         """Test lexing an alias."""
@@ -62,7 +62,7 @@ def test_lex_document_alias(self, mock_cmd_app):
         get_line = lexer.lex_document(document)
         tokens = get_line(0)
 
-        assert tokens == [('ansicyan', 'ls'), ('', ' -l')]
+        assert tokens == [('ansicyan', 'ls'), ('', ' '), ('ansired', '-l')]
 
     def test_lex_document_macro(self, mock_cmd_app):
         """Test lexing a macro."""
@@ -74,7 +74,7 @@ def test_lex_document_macro(self, mock_cmd_app):
         get_line = lexer.lex_document(document)
         tokens = get_line(0)
 
-        assert tokens == [('ansimagenta', 'my_macro'), ('', ' arg1')]
+        assert tokens == [('ansimagenta', 'my_macro'), ('', ' '), ('ansiyellow', 'arg1')]
 
     def test_lex_document_leading_whitespace(self, mock_cmd_app):
         """Test lexing with leading whitespace."""
@@ -86,7 +86,7 @@ def test_lex_document_leading_whitespace(self, mock_cmd_app):
         get_line = lexer.lex_document(document)
         tokens = get_line(0)
 
-        assert tokens == [('', '   '), ('ansigreen', 'help'), ('', ' something')]
+        assert tokens == [('', '   '), ('ansigreen', 'help'), ('', ' '), ('ansiyellow', 'something')]
 
     def test_lex_document_unknown_command(self, mock_cmd_app):
         """Test lexing an unknown command."""
@@ -97,7 +97,7 @@ def test_lex_document_unknown_command(self, mock_cmd_app):
         get_line = lexer.lex_document(document)
         tokens = get_line(0)
 
-        assert tokens == [('', 'unknown'), ('', ' command')]
+        assert tokens == [('', 'unknown'), ('', ' '), ('ansiyellow', 'command')]
 
     def test_lex_document_no_command(self, mock_cmd_app):
         """Test lexing an empty line or line with only whitespace."""
@@ -110,6 +110,30 @@ def test_lex_document_no_command(self, mock_cmd_app):
 
         assert tokens == [('', '   ')]
 
+    def test_lex_document_arguments(self, mock_cmd_app):
+        """Test lexing a command with flags and values."""
+        mock_cmd_app.all_commands = ["help"]
+        lexer = pt_utils.Cmd2Lexer(cast(Any, mock_cmd_app))
+
+        line = "help -v --name \"John Doe\" > out.txt"
+        document = Document(line)
+        get_line = lexer.lex_document(document)
+        tokens = get_line(0)
+
+        assert tokens == [
+            ('ansigreen', 'help'),
+            ('', ' '),
+            ('ansired', '-v'),
+            ('', ' '),
+            ('ansired', '--name'),
+            ('', ' '),
+            ('ansiyellow', '"John Doe"'),
+            ('', ' '),
+            ('', '>'),
+            ('', ' '),
+            ('ansiyellow', 'out.txt'),
+        ]
+
 
 class TestCmd2Completer:
     def test_get_completions_basic(self, mock_cmd_app):