New keycode macro (XP) for shifted character pairs using UNICODEMAP + bug fixes and improvements (#4803)

* Expose unicode_saved_mods

* Add UNICODEMAP shift pair functionality and XS keycode

* Add XS to keycode reference documentation

* Pick pair index based on both Shift and Caps Lock state

* Add XS to Unicode feature docs

* Clean up process_unicode* headers

* Extract unicode_map index calculation into function

* Pick pair index as XOR rather than OR of Shift and Caps states

* unicode_input_start() has to be called before the unicode_map index is calculated

* Replace unicodemap_input_error() with more generic unicode_input_cancel()

* Replace register+tap+unregister with tap_code16(LCTL(LSFT(KC_U)))

* UNICODE_OSX_KEY → UNICODE_KEY_OSX, UNICODE_WINC_KEY → UNICODE_KEY_WINC

* Make keycode range checks more robust

* Fix keycode range checks for different input modes

* Add UNICODE_KEY_LNX, update docs

* QK_UNICODEMAP_SHIFT → QK_UNICODEMAP_PAIR

* XS → XP, update docs

* Tweak Unicode docs

* Use recently added MOD_MASK_SHIFT and IS_HOST_LED_ON helpers

* Update Unicode table in docs/keycodes.md

* Update Unicode docs per review comments

* Replace references to Mac OS X with macOS in Unicode docs

* As of v0.9.0, WinCompose supports all possible code points

* Expand descriptions in XP docs

* Update keycode table and cycling docs

* Further expand cycling docs

6 files changed, 64 insertions, 35 deletions

diff --git a/quantum/process_keycode/process_unicode.c b/quantum/process_keycode/process_unicode.c
index 19beb8452..2c914013a 100644
--- a/quantum/process_keycode/process_unicode.c
+++ b/quantum/process_keycode/process_unicode.c
@@ -13,15 +13,15 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
+
 #include "process_unicode.h"
 #include "action_util.h"
 #include "eeprom.h"
 
 bool process_unicode(uint16_t keycode, keyrecord_t *record) {
-  if (keycode > QK_UNICODE && record->event.pressed) {
-    uint16_t unicode = keycode & 0x7FFF;
+  if (keycode >= QK_UNICODE && keycode <= QK_UNICODE_MAX && record->event.pressed) {
     unicode_input_start();
-    register_hex(unicode);
+    register_hex(keycode & 0x7FFF);
     unicode_input_finish();
   }
   return true;
diff --git a/quantum/process_keycode/process_unicode.h b/quantum/process_keycode/process_unicode.h
index 0913e9910..22765ad56 100644
--- a/quantum/process_keycode/process_unicode.h
+++ b/quantum/process_keycode/process_unicode.h
@@ -13,9 +13,9 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
+
 #pragma once
 
-#include "quantum.h"
 #include "process_unicode_common.h"
 
 bool process_unicode(uint16_t keycode, keyrecord_t *record);
diff --git a/quantum/process_keycode/process_unicode_common.c b/quantum/process_keycode/process_unicode_common.c
index d0a9cf232..21ac2291d 100644
--- a/quantum/process_keycode/process_unicode_common.c
+++ b/quantum/process_keycode/process_unicode_common.c
@@ -20,6 +20,8 @@
 #include <string.h>
 
 unicode_config_t unicode_config;
+uint8_t          unicode_saved_mods;
+
 #if UNICODE_SELECTED_MODES != -1
 static uint8_t selected[] = { UNICODE_SELECTED_MODES };
 static uint8_t selected_count = sizeof selected / sizeof *selected;
@@ -75,30 +77,24 @@ void persist_unicode_input_mode(void) {
   eeprom_update_byte(EECONFIG_UNICODEMODE, unicode_config.input_mode);
 }
 
-static uint8_t saved_mods;
-
 __attribute__((weak))
 void unicode_input_start(void) {
-  saved_mods = get_mods(); // Save current mods
+  unicode_saved_mods = get_mods(); // Save current mods
   clear_mods(); // Unregister mods to start from a clean state
 
   switch (unicode_config.input_mode) {
   case UC_OSX:
-    register_code(UNICODE_OSX_KEY);
+    register_code(UNICODE_KEY_OSX);
     break;
   case UC_LNX:
-    register_code(KC_LCTL);
-    register_code(KC_LSFT);
-    tap_code(KC_U); // TODO: Replace with tap_code16(LCTL(LSFT(KC_U))); and test
-    unregister_code(KC_LSFT);
-    unregister_code(KC_LCTL);
+    tap_code16(UNICODE_KEY_LNX);
     break;
   case UC_WIN:
     register_code(KC_LALT);
     tap_code(KC_PPLS);
     break;
   case UC_WINC:
-    tap_code(UNICODE_WINC_KEY);
+    tap_code(UNICODE_KEY_WINC);
     tap_code(KC_U);
     break;
   }
@@ -110,7 +106,7 @@ __attribute__((weak))
 void unicode_input_finish(void) {
   switch (unicode_config.input_mode) {
   case UC_OSX:
-    unregister_code(UNICODE_OSX_KEY);
+    unregister_code(UNICODE_KEY_OSX);
     break;
   case UC_LNX:
     tap_code(KC_SPC);
@@ -123,7 +119,25 @@ void unicode_input_finish(void) {
     break;
   }
 
-  set_mods(saved_mods); // Reregister previously set mods
+  set_mods(unicode_saved_mods); // Reregister previously set mods
+}
+
+__attribute__((weak))
+void unicode_input_cancel(void) {
+  switch (unicode_config.input_mode) {
+  case UC_OSX:
+    unregister_code(UNICODE_KEY_OSX);
+    break;
+  case UC_LNX:
+  case UC_WINC:
+    tap_code(KC_ESC);
+    break;
+  case UC_WIN:
+    unregister_code(KC_LALT);
+    break;
+  }
+
+  set_mods(unicode_saved_mods); // Reregister previously set mods
 }
 
 __attribute__((weak))
diff --git a/quantum/process_keycode/process_unicode_common.h b/quantum/process_keycode/process_unicode_common.h
index e608ab76b..7340800e5 100644
--- a/quantum/process_keycode/process_unicode_common.h
+++ b/quantum/process_keycode/process_unicode_common.h
@@ -23,11 +23,14 @@
 #endif
 
 // Keycodes used for starting Unicode input on different platforms
-#ifndef UNICODE_OSX_KEY
-  #define UNICODE_OSX_KEY  KC_LALT
+#ifndef UNICODE_KEY_OSX
+  #define UNICODE_KEY_OSX  KC_LALT
 #endif
-#ifndef UNICODE_WINC_KEY
-  #define UNICODE_WINC_KEY KC_RALT
+#ifndef UNICODE_KEY_LNX
+  #define UNICODE_KEY_LNX  LCTL(LSFT(KC_U))
+#endif
+#ifndef UNICODE_KEY_WINC
+  #define UNICODE_KEY_WINC KC_RALT
 #endif
 
 // Comma-delimited, ordered list of input modes selected for use (e.g. in cycle)
@@ -63,6 +66,7 @@ typedef union {
 } unicode_config_t;
 
 extern unicode_config_t unicode_config;
+extern uint8_t          unicode_saved_mods;
 
 void unicode_input_mode_init(void);
 uint8_t get_unicode_input_mode(void);
@@ -72,6 +76,7 @@ void persist_unicode_input_mode(void);
 
 void unicode_input_start(void);
 void unicode_input_finish(void);
+void unicode_input_cancel(void);
 
 void register_hex(uint16_t hex);
 void send_unicode_hex_string(const char *str);
diff --git a/quantum/process_keycode/process_unicodemap.c b/quantum/process_keycode/process_unicodemap.c
index 327402761..b88787986 100644
--- a/quantum/process_keycode/process_unicodemap.c
+++ b/quantum/process_keycode/process_unicodemap.c
@@ -15,7 +15,6 @@
  */
 
 #include "process_unicodemap.h"
-#include "process_unicode_common.h"
 
 void register_hex32(uint32_t hex) {
   bool onzerostart = true;
@@ -38,28 +37,39 @@ void register_hex32(uint32_t hex) {
 }
 
 __attribute__((weak))
-void unicodemap_input_error() {}
+uint16_t unicodemap_index(uint16_t keycode) {
+  if (keycode >= QK_UNICODEMAP_PAIR) {
+    // Keycode is a pair: extract index based on Shift / Caps Lock state
+    uint16_t index = keycode - QK_UNICODEMAP_PAIR;
+
+    bool shift = unicode_saved_mods & MOD_MASK_SHIFT, caps = IS_HOST_LED_ON(USB_LED_CAPS_LOCK);
+    if (shift ^ caps) { index >>= 7; }
+
+    return index & 0x7F;
+  } else {
+    // Keycode is a regular index
+    return keycode - QK_UNICODEMAP;
+  }
+}
 
 bool process_unicodemap(uint16_t keycode, keyrecord_t *record) {
-  if ((keycode & QK_UNICODEMAP) == QK_UNICODEMAP && record->event.pressed) {
-    uint16_t index = keycode - QK_UNICODEMAP;
-    uint32_t code = pgm_read_dword(unicode_map + index);
+  if (keycode >= QK_UNICODEMAP && keycode <= QK_UNICODEMAP_PAIR_MAX && record->event.pressed) {
+    unicode_input_start();
+
+    uint32_t code = pgm_read_dword(unicode_map + unicodemap_index(keycode));
     uint8_t input_mode = get_unicode_input_mode();
 
-    if (code > 0xFFFF && code <= 0x10FFFF && input_mode == UC_OSX) {
-      // Convert to UTF-16 surrogate pair
+    if (code > 0x10FFFF || (code > 0xFFFF && input_mode == UC_WIN)) {
+      // Character is out of range supported by the platform
+      unicode_input_cancel();
+    } else if (code > 0xFFFF && input_mode == UC_OSX) {
+      // Convert to UTF-16 surrogate pair on Mac
       code -= 0x10000;
       uint32_t lo = code & 0x3FF, hi = (code & 0xFFC00) >> 10;
-
-      unicode_input_start();
       register_hex32(hi + 0xD800);
       register_hex32(lo + 0xDC00);
       unicode_input_finish();
-    } else if ((code > 0x10FFFF && input_mode == UC_OSX) || (code > 0xFFFFF && input_mode == UC_LNX)) {
-      // Character is out of range supported by the OS
-      unicodemap_input_error();
     } else {
-      unicode_input_start();
       register_hex32(code);
       unicode_input_finish();
     }
diff --git a/quantum/process_keycode/process_unicodemap.h b/quantum/process_keycode/process_unicodemap.h
index fe4f97915..51709c5dc 100644
--- a/quantum/process_keycode/process_unicodemap.h
+++ b/quantum/process_keycode/process_unicodemap.h
@@ -16,10 +16,10 @@
 
 #pragma once
 
-#include "quantum.h"
 #include "process_unicode_common.h"
 
 extern const uint32_t PROGMEM unicode_map[];
 
-void unicodemap_input_error(void);
+void register_hex32(uint32_t hex);
+uint16_t unicodemap_index(uint16_t keycode);
 bool process_unicodemap(uint16_t keycode, keyrecord_t *record);