Creates a layouts/ folder for keymaps shared between keyboards (#1609)

* include variables and .h files as pp directives

* start layout compilation

* split ergodoxes up

* don't compile all layouts for everything

* might seg fault

* reset layouts variable

* actually reset layouts

* include rules.mk instead

* remove includes from rules.mk

* update variable setting

* load visualizer from path

* adds some more examples

* adds more layouts

* more boards added

* more boards added

* adds documentation for layouts

* use lowercase names for LAYOUT_

* add layout.json files for each layout

* add community folder, default keymaps for layouts

* touch-up default layouts

* touch-up layouts, some keyboard rules.mk

* update documentation for layouts

* fix up serial/i2c switches

2 files changed, 713 insertions, 0 deletions

diff --git a/keyboards/ergodox_ez/util/compile_keymap.py b/keyboards/ergodox_ez/util/compile_keymap.py
new file mode 100644
index 000000000..7076a6ecb
--- /dev/null
+++ b/keyboards/ergodox_ez/util/compile_keymap.py
@@ -0,0 +1,710 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Compiler for keymap.c files
+
+This scrip will generate a keymap.c file from a simple
+markdown file with a specific layout.
+
+Usage:
+    python compile_keymap.py INPUT_PATH [OUTPUT_PATH]
+"""
+from __future__ import division
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import os
+import io
+import re
+import sys
+import json
+import unicodedata
+import collections
+import itertools as it
+
+PY2 = sys.version_info.major == 2
+
+if PY2:
+    chr = unichr
+
+
+KEYBOARD_LAYOUTS = {
+    # These map positions in the parsed layout to
+    # positions in the KEYMAP MATRIX
+    'ergodox_ez': [
+        [ 0,  1,  2,  3,  4,  5,  6],  [38, 39, 40, 41, 42, 43, 44],
+        [ 7,  8,  9, 10, 11, 12, 13],  [45, 46, 47, 48, 49, 50, 51],
+        [14, 15, 16, 17, 18, 19    ],  [    52, 53, 54, 55, 56, 57],
+        [20, 21, 22, 23, 24, 25, 26],  [58, 59, 60, 61, 62, 63, 64],
+        [27, 28, 29, 30, 31        ],  [        65, 66, 67, 68, 69],
+        [                    32, 33],  [70, 71                    ],
+        [                        34],  [72                        ],
+        [                35, 36, 37],  [73, 74, 75                ],
+    ]
+}
+
+ROW_INDENTS = {
+    'ergodox_ez': [0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 5, 0, 6, 0, 4, 0]
+}
+
+BLANK_LAYOUTS = [
+# Compact Layout
+"""
+.------------------------------------.------------------------------------.
+|     |    |    |    |    |    |     |     |    |    |    |    |    |     |
+!-----+----+----+----+----+----------!-----+----+----+----+----+----+-----!
+|     |    |    |    |    |    |     |     |    |    |    |    |    |     |
+!-----+----+----+----x----x----!     !     !----x----x----+----+----+-----!
+|     |    |    |    |    |    |-----!-----!    |    |    |    |    |     |
+!-----+----+----+----x----x----!     !     !----x----x----+----+----+-----!
+|     |    |    |    |    |    |     |     |    |    |    |    |    |     |
+'-----+----+----+----+----+----------'----------+----+----+----+----+-----'
+ |    |    |    |    |    |                     !    |    |    |    |    |
+ '------------------------'                     '------------------------'
+                        .-----------. .-----------.
+                        |     |     | !     |     |
+                  .-----+-----+-----! !-----+-----+-----.
+                  !     !     |     | !     |     !     !
+                  !     !     !-----! !-----!     !     !
+                  |     |     |     | !     |     |     |
+                  '-----------------' '-----------------'
+""",
+
+# Wide Layout
+"""
+.---------------------------------------------. .---------------------------------------------.
+|       |     |     |     |     |     |       | !       |     |     |     |     |     |       |
+!-------+-----+-----+-----+-----+-------------! !-------+-----+-----+-----+-----+-----+-------!
+|       |     |     |     |     |     |       | !       |     |     |     |     |     |       |
+!-------+-----+-----+-----x-----x-----!       ! !       !-----x-----x-----+-----+-----+-------!
+|       |     |     |     |     |     |-------! !-------!     |     |     |     |     |       |
+!-------+-----+-----+-----x-----x-----!       ! !       !-----x-----x-----+-----+-----+-------!
+|       |     |     |     |     |     |       | !       |     |     |     |     |     |       |
+'-------+-----+-----+-----+-----+-------------' '-------------+-----+-----+-----+-----+-------'
+ |      |     |     |     |     |                             !     |     |     |     |      |
+ '------------------------------'                             '------------------------------'
+                              .---------------. .---------------.
+                              |       |       | !       |       |
+                      .-------+-------+-------! !-------+-------+-------.
+                      !       !       |       | !       |       !       !
+                      !       !       !-------! !-------!       !       !
+                      |       |       |       | !       |       |       |
+                      '-----------------------' '-----------------------'
+""",
+]
+
+
+DEFAULT_CONFIG = {
+    "keymaps_includes": [
+        "keymap_common.h",
+    ],
+    'filler': "-+.'!:x",
+    'separator': "|",
+    'default_key_prefix': ["KC_"],
+}
+
+
+SECTIONS = [
+    'layout_config',
+    'layers',
+]
+
+
+#       Markdown Parsing
+
+ONELINE_COMMENT_RE = re.compile(r"""
+    ^                       # comment must be at the start of the line
+    \s*                     # arbitrary whitespace
+    //                      # start of the comment
+    (.*)                    # the comment
+    $                       # until the end of line
+""", re.MULTILINE | re.VERBOSE
+)
+
+INLINE_COMMENT_RE = re.compile(r"""
+    ([\,\"\[\]\{\}\d])      # anythig that might end a expression
+    \s+                     # comment must be preceded by whitespace
+    //                      # start of the comment
+    \s                      # and succeded by whitespace
+    (?:[^\"\]\}\{\[]*)      # the comment (except things which might be json)
+    $                       # until the end of line
+""", re.MULTILINE | re.VERBOSE)
+
+TRAILING_COMMA_RE = re.compile(r"""
+    ,                       # the comma
+    (?:\s*)                 # arbitrary whitespace
+    $                       # only works if the trailing comma is followed by newline
+    (\s*)                   # arbitrary whitespace
+    ([\]\}])                # end of an array or object
+""", re.MULTILINE | re.VERBOSE)
+
+
+def loads(raw_data):
+    if isinstance(raw_data, bytes):
+        raw_data = raw_data.decode('utf-8')
+
+    raw_data = ONELINE_COMMENT_RE.sub(r"", raw_data)
+    raw_data = INLINE_COMMENT_RE.sub(r"\1", raw_data)
+    raw_data = TRAILING_COMMA_RE.sub(r"\1\2", raw_data)
+    return json.loads(raw_data)
+
+
+def parse_config(path):
+    def reset_section():
+        section.update({
+            'name': section.get('name', ""),
+            'sub_name': "",
+            'start_line': -1,
+            'end_line': -1,
+            'code_lines': [],
+        })
+
+    def start_section(line_index, line):
+        end_section()
+        if line.startswith("# "):
+            name = line[2:]
+        elif line.startswith("## "):
+            name = line[3:]
+        else:
+            name = ""
+
+        name = name.strip().replace(" ", "_").lower()
+        if name in SECTIONS:
+            section['name'] = name
+        else:
+            section['sub_name'] = name
+        section['start_line'] = line_index
+
+    def end_section():
+        if section['start_line'] >= 0:
+            if section['name'] == 'layout_config':
+                config.update(loads("\n".join(
+                    section['code_lines']
+                )))
+            elif section['sub_name'].startswith('layer'):
+                layer_name = section['sub_name']
+                config['layer_lines'][layer_name] = section['code_lines']
+
+        reset_section()
+
+    def amend_section(line_index, line):
+        section['end_line'] = line_index
+        section['code_lines'].append(line)
+
+    config = DEFAULT_CONFIG.copy()
+    config.update({
+        'layer_lines': collections.OrderedDict(),
+        'macro_ids': {'UM'},
+        'unicode_macros': {},
+    })
+
+    section = {}
+    reset_section()
+
+    with io.open(path, encoding="utf-8") as fh:
+        for i, line in enumerate(fh):
+            if line.startswith("#"):
+                start_section(i, line)
+            elif line.startswith("    "):
+                amend_section(i, line[4:])
+            else:
+                # TODO: maybe parse description
+                pass
+
+    end_section()
+    assert 'layout' in config
+    return config
+
+#       header file parsing
+
+IF0_RE = re.compile(r"""
+    ^
+    #if 0
+    $.*?
+    #endif
+""", re.MULTILINE | re.DOTALL | re.VERBOSE)
+
+
+COMMENT_RE = re.compile(r"""
+    /\*
+    .*?
+    \*/"
+""", re.MULTILINE | re.DOTALL | re.VERBOSE)
+
+
+def read_header_file(path):
+    with io.open(path, encoding="utf-8") as fh:
+        data = fh.read()
+    data, _ = COMMENT_RE.subn("", data)
+    data, _ = IF0_RE.subn("", data)
+    return data
+
+
+def regex_partial(re_str_fmt, flags):
+    def partial(*args, **kwargs):
+        re_str = re_str_fmt.format(*args, **kwargs)
+        return re.compile(re_str, flags)
+    return partial
+
+
+KEYDEF_REP = regex_partial(r"""
+    #define
+    \s
+    (
+        (?:{})          # the prefixes
+        (?:\w+)         # the key name
+    )                   # capture group end
+""", re.MULTILINE | re.DOTALL | re.VERBOSE)
+
+
+ENUM_RE = re.compile(r"""
+    (
+        enum
+        \s\w+\s
+        \{
+        .*?             # the enum content
+        \}
+        ;
+    )                   # capture group end
+""", re.MULTILINE | re.DOTALL | re.VERBOSE)
+
+
+ENUM_KEY_REP = regex_partial(r"""
+    (
+        {}              # the prefixes
+        \w+             # the key name
+    )                   # capture group end
+""", re.MULTILINE | re.DOTALL | re.VERBOSE)
+
+
+def parse_keydefs(config, data):
+    prefix_options = "|".join(config['key_prefixes'])
+    keydef_re = KEYDEF_REP(prefix_options)
+    enum_key_re = ENUM_KEY_REP(prefix_options)
+    for match in keydef_re.finditer(data):
+        yield match.groups()[0]
+
+    for enum_match in ENUM_RE.finditer(data):
+        enum = enum_match.groups()[0]
+        for key_match in enum_key_re.finditer(enum):
+            yield key_match.groups()[0]
+
+
+def parse_valid_keys(config, out_path):
+    basepath = os.path.abspath(os.path.join(os.path.dirname(out_path)))
+    dirpaths = []
+    subpaths = []
+    while len(subpaths) < 6:
+        path = os.path.join(basepath, *subpaths)
+        dirpaths.append(path)
+        dirpaths.append(os.path.join(path, "tmk_core", "common"))
+        dirpaths.append(os.path.join(path, "quantum"))
+        subpaths.append('..')
+
+    includes = set(config['keymaps_includes'])
+    includes.add("keycode.h")
+
+    valid_keycodes = set()
+    for dirpath, include in it.product(dirpaths, includes):
+        include_path = os.path.join(dirpath, include)
+        if os.path.exists(include_path):
+            header_data = read_header_file(include_path)
+            valid_keycodes.update(
+                parse_keydefs(config, header_data)
+            )
+    return valid_keycodes
+
+
+#       Keymap Parsing
+
+def iter_raw_codes(layer_lines, filler, separator):
+    filler_re = re.compile("[" + filler + " ]")
+    for line in layer_lines:
+        line, _ = filler_re.subn("", line.strip())
+        if not line:
+            continue
+        codes = line.split(separator)
+        for code in codes[1:-1]:
+            yield code
+
+
+def iter_indexed_codes(raw_codes, key_indexes):
+    key_rows = {}
+    key_indexes_flat = []
+
+    for row_index, key_indexes in enumerate(key_indexes):
+        for key_index in key_indexes:
+            key_rows[key_index] = row_index
+        key_indexes_flat.extend(key_indexes)
+    assert len(raw_codes) == len(key_indexes_flat)
+    for raw_code, key_index in zip(raw_codes, key_indexes_flat):
+        # we keep track of the row mostly for layout purposes
+        yield raw_code, key_index, key_rows[key_index]
+
+
+LAYER_CHANGE_RE = re.compile(r"""
+    (DF|TG|MO)\(\d+\)
+""", re.VERBOSE)
+
+
+MACRO_RE = re.compile(r"""
+    M\(\w+\)
+""", re.VERBOSE)
+
+
+UNICODE_RE = re.compile(r"""
+    U[0-9A-F]{4}
+""", re.VERBOSE)
+
+
+NON_CODE = re.compile(r"""
+    ^[^A-Z0-9_]$
+""", re.VERBOSE)
+
+
+def parse_uni_code(raw_code):
+    macro_id = "UC_" + (
+        unicodedata.name(raw_code)
+        .replace(" ", "_")
+        .replace("-", "_")
+    )
+    code = "M({})".format(macro_id)
+    uc_hex = "{:04X}".format(ord(raw_code))
+    return code, macro_id, uc_hex
+
+
+def parse_key_code(raw_code, key_prefixes, valid_keycodes):
+    if raw_code in valid_keycodes:
+        return raw_code
+
+    for prefix in key_prefixes:
+        code = prefix + raw_code
+        if code in valid_keycodes:
+            return code
+
+
+def parse_code(raw_code, key_prefixes, valid_keycodes):
+    if not raw_code:
+        return 'KC_TRNS', None, None
+
+    if LAYER_CHANGE_RE.match(raw_code):
+        return raw_code, None, None
+
+    if MACRO_RE.match(raw_code):
+        macro_id = raw_code[2:-1]
+        return raw_code, macro_id, None
+
+    if UNICODE_RE.match(raw_code):
+        hex_code = raw_code[1:]
+        return parse_uni_code(chr(int(hex_code, 16)))
+
+    if NON_CODE.match(raw_code):
+        return parse_uni_code(raw_code)
+
+    code = parse_key_code(raw_code, key_prefixes, valid_keycodes)
+    return code, None, None
+
+
+def parse_keymap(config, key_indexes, layer_lines, valid_keycodes):
+    keymap = {}
+    raw_codes = list(iter_raw_codes(
+        layer_lines, config['filler'], config['separator']
+    ))
+    indexed_codes = iter_indexed_codes(raw_codes, key_indexes)
+    key_prefixes = config['key_prefixes']
+    for raw_code, key_index, row_index in indexed_codes:
+        code, macro_id, uc_hex = parse_code(
+            raw_code, key_prefixes, valid_keycodes
+        )
+        # TODO: line numbers for invalid codes
+        err_msg = "Could not parse key '{}' on row {}".format(
+            raw_code, row_index
+        )
+        assert code is not None, err_msg
+        # print(repr(raw_code), repr(code), macro_id, uc_hex)
+        if macro_id:
+            config['macro_ids'].add(macro_id)
+        if uc_hex:
+            config['unicode_macros'][macro_id] = uc_hex
+        keymap[key_index] = (code, row_index)
+    return keymap
+
+
+def parse_keymaps(config, valid_keycodes):
+    keymaps = collections.OrderedDict()
+    key_indexes = config.get(
+        'key_indexes', KEYBOARD_LAYOUTS[config['layout']]
+    )
+    # TODO: maybe validate key_indexes
+
+    for layer_name, layer_lines, in config['layer_lines'].items():
+        keymaps[layer_name] = parse_keymap(
+            config, key_indexes, layer_lines, valid_keycodes
+        )
+    return keymaps
+
+#       keymap.c output
+
+USERCODE = """
+// Runs just one time when the keyboard initializes.
+void matrix_init_user(void) {
+
+};
+
+// Runs constantly in the background, in a loop.
+void matrix_scan_user(void) {
+    uint8_t layer = biton32(layer_state);
+
+    ergodox_board_led_off();
+    ergodox_right_led_1_off();
+    ergodox_right_led_2_off();
+    ergodox_right_led_3_off();
+    switch (layer) {
+        case L1:
+            ergodox_right_led_1_on();
+            break;
+        case L2:
+            ergodox_right_led_2_on();
+            break;
+        case L3:
+            ergodox_right_led_3_on();
+            break;
+        case L4:
+            ergodox_right_led_1_on();
+            ergodox_right_led_2_on();
+            break;
+        case L5:
+            ergodox_right_led_1_on();
+            ergodox_right_led_3_on();
+            break;
+        // case L6:
+        //     ergodox_right_led_2_on();
+        //     ergodox_right_led_3_on();
+        //     break;
+        // case L7:
+        //     ergodox_right_led_1_on();
+        //     ergodox_right_led_2_on();
+        //     ergodox_right_led_3_on();
+        //     break;
+        default:
+            ergodox_board_led_off();
+            break;
+    }
+};
+"""
+
+MACROCODE = """
+#define UC_MODE_WIN 0
+#define UC_MODE_LINUX 1
+#define UC_MODE_OSX 2
+
+// TODO: allow default mode to be configured
+static uint16_t unicode_mode = UC_MODE_WIN;
+
+uint16_t hextokeycode(uint8_t hex) {{
+    if (hex == 0x0) {{
+        return KC_P0;
+    }}
+    if (hex < 0xA) {{
+        return KC_P1 + (hex - 0x1);
+    }}
+    return KC_A + (hex - 0xA);
+}}
+
+void unicode_action_function(uint16_t hi, uint16_t lo) {{
+    switch (unicode_mode) {{
+    case UC_MODE_WIN:
+        register_code(KC_LALT);
+
+        register_code(KC_PPLS);
+        unregister_code(KC_PPLS);
+
+        register_code(hextokeycode((hi & 0xF0) >> 4));
+        unregister_code(hextokeycode((hi & 0xF0) >> 4));
+        register_code(hextokeycode((hi & 0x0F)));
+        unregister_code(hextokeycode((hi & 0x0F)));
+        register_code(hextokeycode((lo & 0xF0) >> 4));
+        unregister_code(hextokeycode((lo & 0xF0) >> 4));
+        register_code(hextokeycode((lo & 0x0F)));
+        unregister_code(hextokeycode((lo & 0x0F)));
+
+        unregister_code(KC_LALT);
+        break;
+    case UC_MODE_LINUX:
+        register_code(KC_LCTL);
+        register_code(KC_LSFT);
+
+        register_code(KC_U);
+        unregister_code(KC_U);
+
+        register_code(hextokeycode((hi & 0xF0) >> 4));
+        unregister_code(hextokeycode((hi & 0xF0) >> 4));
+        register_code(hextokeycode((hi & 0x0F)));
+        unregister_code(hextokeycode((hi & 0x0F)));
+        register_code(hextokeycode((lo & 0xF0) >> 4));
+        unregister_code(hextokeycode((lo & 0xF0) >> 4));
+        register_code(hextokeycode((lo & 0x0F)));
+        unregister_code(hextokeycode((lo & 0x0F)));
+
+        unregister_code(KC_LCTL);
+        unregister_code(KC_LSFT);
+        break;
+    case UC_MODE_OSX:
+        break;
+    }}
+}}
+
+const macro_t *action_get_macro(keyrecord_t *record, uint8_t id, uint8_t opt) {{
+    if (!record->event.pressed) {{
+        return MACRO_NONE;
+    }}
+    // MACRODOWN only works in this function
+    switch(id) {{
+        case UM:
+            unicode_mode = (unicode_mode + 1) % 2;
+            break;
+{macro_cases}
+{unicode_macro_cases}
+        default:
+            break;
+    }}
+    return MACRO_NONE;
+}};
+"""
+
+
+UNICODE_MACRO_TEMPLATE = """
+case {macro_id}:
+    unicode_action_function(0x{hi:02x}, 0x{lo:02x});
+    break;
+""".strip()
+
+
+def unicode_macro_cases(config):
+    for macro_id, uc_hex in config['unicode_macros'].items():
+        hi = int(uc_hex, 16) >> 8
+        lo = int(uc_hex, 16) & 0xFF
+        unimacro_keys = ", ".join(
+            "T({})".format(
+                "KP_" + digit if digit.isdigit() else digit
+            ) for digit in uc_hex
+        )
+        yield UNICODE_MACRO_TEMPLATE.format(
+            macro_id=macro_id, hi=hi, lo=lo
+        )
+
+
+def iter_keymap_lines(keymap, row_indents=None):
+    col_widths = {}
+    col = 0
+    # first pass, figure out the column widths
+    prev_row_index = None
+    for code, row_index in keymap.values():
+        if row_index != prev_row_index:
+            col = 0
+            if row_indents:
+                col = row_indents[row_index]
+        col_widths[col] = max(len(code), col_widths.get(col, 0))
+        prev_row_index = row_index
+        col += 1
+
+    # second pass, yield the cell values
+    col = 0
+    prev_row_index = None
+    for key_index in sorted(keymap):
+        code, row_index = keymap[key_index]
+        if row_index != prev_row_index:
+            col = 0
+            yield "\n"
+            if row_indents:
+                for indent_col in range(row_indents[row_index]):
+                    pad = " " * (col_widths[indent_col] - 4)
+                    yield (" /*-*/" + pad)
+                col = row_indents[row_index]
+        else:
+            yield pad
+        yield " {}".format(code)
+        if key_index < len(keymap) - 1:
+            yield ","
+            # This will be yielded on the next iteration when
+            # we know that we're not at the end of a line.
+            pad = " " * (col_widths[col] - len(code))
+        prev_row_index = row_index
+        col += 1
+
+
+def iter_keymap_parts(config, keymaps):
+    # includes
+    for include_path in config['keymaps_includes']:
+        yield '#include "{}"\n'.format(include_path)
+
+    yield "\n"
+
+    # definitions
+    for i, macro_id in enumerate(sorted(config['macro_ids'])):
+        yield "#define {} {}\n".format(macro_id, i)
+
+    yield "\n"
+
+    for i, layer_name in enumerate(config['layer_lines']):
+        yield '#define L{0:<3} {0:<5}  // {1}\n'.format(i, layer_name)
+
+    yield "\n"
+
+    # keymaps
+    yield "const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\n"
+
+    for i, layer_name in enumerate(config['layer_lines']):
+        # comment
+        layer_lines = config['layer_lines'][layer_name]
+        prefixed_lines = " * " + " * ".join(layer_lines)
+        yield "/*\n{} */\n".format(prefixed_lines)
+
+        # keymap codes
+        keymap = keymaps[layer_name]
+        row_indents = ROW_INDENTS.get(config['layout'])
+        keymap_lines = "".join(iter_keymap_lines(keymap, row_indents))
+        yield "[L{0}] = KEYMAP({1}\n),\n".format(i, keymap_lines)
+
+    yield "};\n\n"
+
+    # no idea what this is for
+    yield "const uint16_t PROGMEM fn_actions[] = {};\n"
+
+    # macros
+    yield MACROCODE.format(
+        macro_cases="",
+        unicode_macro_cases="\n".join(unicode_macro_cases(config)),
+    )
+
+    # TODO: dynamically create blinking lights
+    yield USERCODE
+
+
+def main(argv=sys.argv[1:]):
+    if not argv or '-h' in argv or '--help' in argv:
+        print(__doc__)
+        return 0
+
+    in_path = os.path.abspath(argv[0])
+    if not os.path.exists(in_path):
+        print("No such file '{}'".format(in_path))
+        return 1
+
+    if len(argv) > 1:
+        out_path = os.path.abspath(argv[1])
+    else:
+        dirname = os.path.dirname(in_path)
+        out_path = os.path.join(dirname, "keymap.c")
+
+    config = parse_config(in_path)
+    valid_keys = parse_valid_keys(config, out_path)
+    keymaps = parse_keymaps(config, valid_keys)
+
+    with io.open(out_path, mode="w", encoding="utf-8") as fh:
+        for part in iter_keymap_parts(config, keymaps):
+            fh.write(part)
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/keyboards/ergodox_ez/util/readme.md b/keyboards/ergodox_ez/util/readme.md
new file mode 100644
index 000000000..26c5e5d99
--- /dev/null
+++ b/keyboards/ergodox_ez/util/readme.md
@@ -0,0 +1,3 @@
+# ErgoDox EZ Utilities
+
+The Python script in this directory, by [mbarkhau](https://github.com/mbarkhau) allows you to write out a basic ErgoDox EZ keymap using Markdown notation, and then transpile it to C, which you can then compile. It's experimental, but if you're not comfortable using C, it's a nice option.