QMK CLI and JSON keymap support (qmk#6176)

* Script to generate keymap.c from JSON file.

* Support for keymap.json

* Add a warning about the keymap.c getting overwritten.

* Fix keymap generating

* Install the python deps

* Flesh out more of the python environment

* Remove defunct json2keymap

* Style everything with yapf

* Polish up python support

* Hide json keymap.c into the .build dir

* Polish up qmk-compile-json

* Make milc work with positional arguments

* Fix a couple small things

* Fix some errors and make the CLI more understandable

* Make the qmk wrapper more robust

* Add basic QMK Doctor

* Clean up docstrings and flesh them out as needed

* remove unused compile_firmware() function

Author: skullydazed

.editorconfig

@@ -16,6 +16,10 @@ insert_final_newline = true
 trim_trailing_whitespace = false
 indent_size = 4
 
+[{qmk,*.py}]
+charset = utf-8
+max_line_length = 200
+
 # Make these match what we have in .gitattributes
 [*.mk]
 end_of_line = lf

.gitignore

@@ -70,3 +70,6 @@ util/Win_Check_Output.txt
 secrets.tar
 id_rsa_*
 /.vs
+
+# python things
+__pycache__

bin/qmk

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""CLI wrapper for running QMK commands.
+"""
+import os
+import subprocess
+import sys
+from glob import glob
+from time import strftime
+from importlib import import_module
+from importlib.util import find_spec
+
+# Add the QMK python libs to our path
+script_dir = os.path.dirname(os.path.realpath(__file__))
+qmk_dir = os.path.abspath(os.path.join(script_dir, '..'))
+python_lib_dir = os.path.abspath(os.path.join(qmk_dir, 'lib', 'python'))
+sys.path.append(python_lib_dir)
+
+# Change to the root of our checkout
+os.environ['ORIG_CWD'] = os.getcwd()
+os.chdir(qmk_dir)
+
+# Make sure our modules have been setup
+with open('requirements.txt', 'r') as fd:
+    for line in fd.readlines():
+        line = line.strip().replace('<', '=').replace('>', '=')
+
+        if line[0] == '#':
+            continue
+
+        if '#' in line:
+            line = line.split('#')[0]
+
+        module = line.split('=')[0] if '=' in line else line
+        if not find_spec(module):
+            print('Your QMK build environment is not fully setup!\n')
+            print('Please run `./util/qmk_install.sh` to setup QMK.')
+            exit(255)
+
+# Figure out our version
+command = ['git', 'describe', '--abbrev=6', '--dirty', '--always', '--tags']
+result = subprocess.run(command, text=True, capture_output=True)
+
+if result.returncode == 0:
+    os.environ['QMK_VERSION'] = 'QMK ' + result.stdout.strip()
+else:
+    os.environ['QMK_VERSION'] = 'QMK ' + strftime('%Y-%m-%d-%H:%M:%S')
+
+# Setup the CLI
+import milc
+milc.EMOJI_LOGLEVELS['INFO'] = '{fg_blue}ψ{style_reset_all}'
+
+# If we were invoked as `qmk <cmd>` massage sys.argv into `qmk-<cmd>`.
+# This means we can't accept arguments to the qmk script itself.
+script_name = os.path.basename(sys.argv[0])
+if script_name == 'qmk':
+    if len(sys.argv) == 1:
+        milc.cli.log.error('No subcommand specified!\n')
+
+    if len(sys.argv) == 1 or sys.argv[1] in ['-h', '--help']:
+        milc.cli.echo('usage: qmk <subcommand> [...]')
+        milc.cli.echo('\nsubcommands:')
+        subcommands = glob(os.path.join(qmk_dir, 'bin', 'qmk-*'))
+        for subcommand in sorted(subcommands):
+            subcommand = os.path.basename(subcommand).split('-', 1)[1]
+            milc.cli.echo('\t%s', subcommand)
+        milc.cli.echo('\nqmk <subcommand> --help for more information')
+        exit(1)
+
+    if sys.argv[1] in ['-V', '--version']:
+        milc.cli.echo(os.environ['QMK_VERSION'])
+        exit(0)
+
+    sys.argv[0] = script_name = '-'.join((script_name, sys.argv[1]))
+    del sys.argv[1]
+
+# Look for which module to import
+if script_name == 'qmk':
+    milc.cli.print_help()
+    exit(0)
+elif not script_name.startswith('qmk-'):
+    milc.cli.log.error('Invalid symlink, must start with "qmk-": %s', script_name)
+else:
+    subcommand = script_name.replace('-', '.').replace('_', '.').split('.')
+    subcommand.insert(1, 'cli')
+    subcommand = '.'.join(subcommand)
+
+    try:
+        import_module(subcommand)
+    except ModuleNotFoundError as e:
+        if e.__class__.__name__ != subcommand:
+            raise
+
+        milc.cli.log.error('Invalid subcommand! Could not import %s.', subcommand)
+        exit(1)
+
+if __name__ == '__main__':
+    milc.cli()

bin/qmk-compile-json

@@ -0,0 +1 @@
+qmk

bin/qmk-doctor

@@ -0,0 +1 @@
+qmk

bin/qmk-hello

@@ -0,0 +1 @@
+qmk

bin/qmk-json-keymap

@@ -0,0 +1 @@
+qmk

build_json.mk

@@ -0,0 +1,27 @@
+# Look for a json keymap file
+ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_5)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_4)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_3)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_2)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_1)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
+endif
+
+# Generate the keymap.c
+ifneq ("$(KEYMAP_JSON)","")
+    _ = $(shell bin/qmk-json-keymap -f $(KEYMAP_JSON) -o $(KEYMAP_C))
+endif

build_keyboard.mk

@@ -98,31 +98,38 @@ MAIN_KEYMAP_PATH_3 := $(KEYBOARD_PATH_3)/keymaps/$(KEYMAP)
 MAIN_KEYMAP_PATH_4 := $(KEYBOARD_PATH_4)/keymaps/$(KEYMAP)
 MAIN_KEYMAP_PATH_5 := $(KEYBOARD_PATH_5)/keymaps/$(KEYMAP)
 
-ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_5)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_5)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_4)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_4)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_3)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_3)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_2)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_2)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_1)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_1)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
-else ifneq ($(LAYOUTS),)
-    include build_layout.mk
-else
-    $(error Could not find keymap)
-    # this state should never be reached
+# Check for keymap.json first, so we can regenerate keymap.c
+include build_json.mk
+
+ifeq ("$(wildcard $(KEYMAP_PATH))", "")
+    # Look through the possible keymap folders until we find a matching keymap.c
+    ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
+        -include $(MAIN_KEYMAP_PATH_5)/rules.mk
+        KEYMAP_C := $(MAIN_KEYMAP_PATH_5)/keymap.c
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.c)","")
+        -include $(MAIN_KEYMAP_PATH_4)/rules.mk
+        KEYMAP_C := $(MAIN_KEYMAP_PATH_4)/keymap.c
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.c)","")
+        -include $(MAIN_KEYMAP_PATH_3)/rules.mk
+        KEYMAP_C := $(MAIN_KEYMAP_PATH_3)/keymap.c
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.c)","")
+        -include $(MAIN_KEYMAP_PATH_2)/rules.mk
+        KEYMAP_C := $(MAIN_KEYMAP_PATH_2)/keymap.c
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
+    else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.c)","")
+        -include $(MAIN_KEYMAP_PATH_1)/rules.mk
+        KEYMAP_C := $(MAIN_KEYMAP_PATH_1)/keymap.c
+        KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
+    else ifneq ($(LAYOUTS),)
+        # If we haven't found a keymap yet fall back to community layouts
+        include build_layout.mk
+    else
+        $(error Could not find keymap)
+        # this state should never be reached
+    endif
 endif
 
 ifeq ($(strip $(CTPC)), yes)
@@ -313,7 +320,6 @@ ifneq ("$(wildcard $(USER_PATH)/config.h)","")
     CONFIG_H += $(USER_PATH)/config.h
 endif
 
-
 # Object files directory
 #     To put object files in current directory, use a dot (.), do NOT make
 #     this an empty or blank macro!
@@ -323,7 +329,7 @@ ifneq ("$(wildcard $(KEYMAP_PATH)/config.h)","")
     CONFIG_H += $(KEYMAP_PATH)/config.h
 endif
 
-# # project specific files
+# project specific files
 SRC += $(KEYBOARD_SRC) \
     $(KEYMAP_C) \
     $(QUANTUM_SRC)

docs/_summary.md

@@ -8,6 +8,7 @@
 
 * [QMK Basics](README.md)
   * [QMK Introduction](getting_started_introduction.md)
+  * [QMK CLI](cli.md)
   * [Contributing to QMK](contributing.md)
   * [How to Use Github](getting_started_github.md)
   * [Getting Help](getting_started_getting_help.md)
@@ -34,13 +35,16 @@
   * [Keyboard Guidelines](hardware_keyboard_guidelines.md)
   * [Config Options](config_options.md)
   * [Keycodes](keycodes.md)
+  * [Coding Conventions - C](coding_conventions_c.md)
+  * [Coding Conventions - Python](coding_conventions_python.md)
   * [Documentation Best Practices](documentation_best_practices.md)
   * [Documentation Templates](documentation_templates.md)
   * [Glossary](reference_glossary.md)
   * [Unit Testing](unit_testing.md)
   * [Useful Functions](ref_functions.md)
   * [Configurator Support](reference_configurator_support.md)
   * [info.json Format](reference_info_json.md)
+  * [Python Development](python_development.md)
 
 * [Features](features.md)
   * [Basic Keycodes](keycodes_basic.md)