Major rework, no regex/globbing, more walking

Instead of using regexes and globbing to find the rules.mk and keymap.c
files, walk the directory tree to find them.
Also, do away with the concept of revision.

lib/python/qmk/keymap.py

@@ -2,8 +2,6 @@
 """
 import os
 from traceback import format_exc
-import re
-import glob
 
 import qmk.path
 import qmk.makefile
@@ -100,27 +98,6 @@ def write(keyboard, keymap, layout, layers):
 
     return keymap_file
 
-def find_keymaps(base_path, revision = "", community = False):
-    """ Find the available keymaps for a keyboard and revision pair.
-
-    Args:
-        base_path: The base path of the keyboard.
-
-        revision: The keyboard's revision.
-
-        community: Set to True for the layouts under layouts/community.
-
-    Returns:
-        a set with the keymaps's names
-    """
-    path_wildcard = os.path.join(base_path, "**", "keymap.c")
-    if community:
-        path_regex = re.compile(r"^" + re.escape(base_path) + "(\S+)" + os.path.sep + "keymap\.c")
-    else:
-        path_regex = re.compile(r"^" + re.escape(base_path) + "(?:" + re.escape(revision) + os.path.sep + ")?keymaps" + os.path.sep + "(\S+)" + os.path.sep + "keymap\.c")
-    names = [path_regex.sub(lambda name: name.group(1), path) for path in glob.iglob(path_wildcard, recursive = True) if path_regex.search(path)]
-    return set(names)
-
 def list_keymaps(keyboard_name):
     """ List the available keymaps for a keyboard.
 
@@ -130,25 +107,28 @@ def list_keymaps(keyboard_name):
     Returns:
         a set with the names of the available keymaps
     """
-    if os.path.sep in keyboard_name:
-        keyboard, revision = os.path.split(os.path.normpath(keyboard_name))
-    else:
-        keyboard = keyboard_name
-        revision = ""
-
     # parse all the rules.mk files for the keyboard
-    rules_mk = qmk.makefile.get_rules_mk(keyboard, revision)
+    rules_mk = qmk.makefile.get_rules_mk(keyboard_name)
     names = set()
 
     if rules_mk:
-        # get the keymaps from the keyboard's directory
-        kb_base_path = os.path.join(os.getcwd(), "keyboards", keyboard) + os.path.sep
-        names = find_keymaps(kb_base_path, revision)
+        # qmk_firmware/keyboards
+        keyboards_dir = os.path.join(os.getcwd(), "keyboards")
+        # path to the keyboard's directory
+        kb_path = os.path.join(keyboards_dir, keyboard_name)
+        # walk up the directory tree until keyboards_dir
+        # and collect all directories' name with keymap.c file in it
+        while kb_path != keyboards_dir:
+            keymaps_dir = os.path.join(kb_path, "keymaps")
+            if os.path.exists(keymaps_dir):
+                names = names.union([keymap for keymap in os.listdir(keymaps_dir) if os.path.isfile(os.path.join(keymaps_dir, keymap, "keymap.c"))])
+            kb_path = os.path.dirname(kb_path)
 
         # if community layouts are supported, get them
         if "LAYOUTS" in rules_mk:
-            for layout in rules_mk["LAYOUTS"]["value"].split():
-                cl_base_path = os.path.join(os.getcwd(), "layouts", "community", layout) + os.path.sep
-                names = names.union(find_keymaps(cl_base_path, revision, community = True))
+            for layout in rules_mk["LAYOUTS"].split():
+                cl_path = os.path.join(os.getcwd(), "layouts", "community", layout)
+                if os.path.exists(cl_path):
+                    names = names.union([keymap for keymap in os.listdir(cl_path) if os.path.isfile(os.path.join(cl_path, keymap, "keymap.c"))])
 
     return sorted(names)

lib/python/qmk/makefile.py

@@ -1,72 +1,80 @@
 """ Functions for working with Makefiles
 """
 import os
-import glob
-import re
 
 import qmk.path
 from qmk.errors import NoSuchKeyboardError
 
-def parse_rules_mk(file_path):
-    """ Parse a rules.mk file
+def parse_rules_mk_file(file, rules_mk=None):
+    """Turn a rules.mk file into a dictionary.
 
     Args:
-        file_path: path to the rules.mk file
+        file: path to the rules.mk file
+        rules_mk: already parsed rules.mk the new file should be merged with
 
     Returns:
         a dictionary with the file's content
     """
-    # regex to match lines uncommented lines and get the data
-    # group(1) = option's name
-    # group(2) = operator (eg.: '=', '+=')
-    # group(3) = value(s)
-    rules_mk_regex = re.compile(r"^\s*(\w+)\s*([\?\:\+\-]?=)\s*(\S.*?)(?=\s*(\#|$))")
-    parsed_file = dict()
-    mk_content = qmk.path.file_lines(file_path)
-    for line in mk_content:
-        found = rules_mk_regex.search(line)
-        if found:
-            parsed_file[found.group(1)] = dict(operator = found.group(2), value = found.group(3))
-    return parsed_file
+    if not rules_mk:
+        rules_mk = {}
 
-def merge_rules_mk_files(base, revision):
-    """ Merge a keyboard revision's rules.mk file with
-    the 'base' rules.mk file
+    if os.path.exists(file):
+        rules_mk_lines = qmk.path.file_lines(file)
 
-    Args:
-        base: the base rules.mk file's content as dictionary
-        revision: the revision's rules.mk file's content as dictionary
+        for line in rules_mk_lines:
+            # Filter out comments
+            if line.strip().startswith("#"):
+                continue
 
-    Returns:
-        a dictionary with the merged content
-    """
-    return {**base, **revision}
+            # Strip in-line comments
+            if '#' in line:
+                line = line[:line.index('#')].strip()
+
+            if '=' in line:
+                # Append
+                if '+=' in line:
+                    key, value = line.split('+=', 1)
+                    if key.strip() not in rules_mk:
+                        rules_mk[key.strip()] = value.strip()
+                    else:
+                        rules_mk[key.strip()] += ' ' + value.strip()
+                # Set if absent
+                elif "?=" in line:
+                    key, value = line.split('?=', 1)
+                    if key.strip() not in rules_mk:
+                        rules_mk[key.strip()] = value.strip()
+                else:
+                    if ":=" in line:
+                        line.replace(":","")
+                    key, value = line.split('=', 1)
+                    rules_mk[key.strip()] = value.strip()
 
-def get_rules_mk(keyboard, revision = ""):
+    return rules_mk
+
+def get_rules_mk(keyboard):
     """ Get a rules.mk for a keyboard
 
     Args:
         keyboard: name of the keyboard
-        revision: revision of the keyboard
+
+    Raises:
+        NoSuchKeyboardError: when the keyboard does not exists
 
     Returns:
         a dictionary with the content of the rules.mk file
     """
-    base_path = os.path.join(os.getcwd(), "keyboards", keyboard) + os.path.sep
-    rules_mk = dict()
-    if os.path.exists(base_path + os.path.sep + revision):
-        rules_mk_path_wildcard = os.path.join(base_path, "**", "rules.mk")
-        rules_mk_regex = re.compile(r"^" + base_path + "(?:" + revision + os.path.sep + ")?rules.mk")
-        paths = [path for path in glob.iglob(rules_mk_path_wildcard, recursive = True) if rules_mk_regex.search(path)]
-        for file_path in paths:
-            rules_mk[revision if revision in file_path else "base"] = parse_rules_mk(file_path)
+    # Start with qmk_firmware/keyboards
+    kb_path = os.path.join(os.getcwd(), "keyboards")
+    # walk down the directory tree
+    # and collect all rules.mk files
+    if os.path.exists(os.path.join(kb_path, keyboard)):
+        rules_mk = dict()
+        for directory in keyboard.split(os.path.sep):
+            kb_path = os.path.join(kb_path, directory)
+            rules_mk_path = os.path.join(kb_path, "rules.mk")
+            if os.path.exists(rules_mk_path):
+                rules_mk = parse_rules_mk_file(rules_mk_path, rules_mk)
     else:
         raise NoSuchKeyboardError("The requested keyboard and/or revision does not exist.")
 
-    # if the base or the revision directory does not contain a rules.mk
-    if len(rules_mk) == 1:
-        rules_mk = rules_mk[revision]
-    # if both directories contain rules.mk files
-    elif len(rules_mk) == 2:
-        rules_mk = merge_rules_mk_files(rules_mk["base"], rules_mk[revision])
     return rules_mk