Extend allowed range of tappable keycodes to include modifiers (#5809)

* Extend allowed range of tappable keycodes to include modifiers

* Get rid of the magic numbers altogether

* Remove some more magic numbers

* Extract LM() functionality from ACT_LAYER_TAP

* Use ACTION() macro everywhere

2 files changed, 25 insertions, 16 deletions

diff --git a/tmk_core/common/action.c b/tmk_core/common/action.c
index 5172e8650..f47fd20fc 100644
--- a/tmk_core/common/action.c
+++ b/tmk_core/common/action.c
@@ -437,20 +437,19 @@ void process_action(keyrecord_t *record, action_t action)
                 }
             }
             break;
+        case ACT_LAYER_MODS:
+            if (event.pressed) {
+                layer_on(action.layer_mods.layer);
+                register_mods(action.layer_mods.mods);
+            } else {
+                unregister_mods(action.layer_mods.mods);
+                layer_off(action.layer_mods.layer);
+            }
+            break;
     #ifndef NO_ACTION_TAPPING
         case ACT_LAYER_TAP:
         case ACT_LAYER_TAP_EXT:
             switch (action.layer_tap.code) {
-                case 0xe0 ... 0xef:
-                    /* layer On/Off with modifiers(left only) */
-                    if (event.pressed) {
-                        layer_on(action.layer_tap.val);
-                        register_mods(action.layer_tap.code & 0x0f);
-                    } else {
-                        layer_off(action.layer_tap.val);
-                        unregister_mods(action.layer_tap.code & 0x0f);
-                    }
-                    break;
                 case OP_TAP_TOGGLE:
                     /* tap toggle */
                     if (event.pressed) {
@@ -652,6 +651,7 @@ void process_action(keyrecord_t *record, action_t action)
     // if this event is a layer action, update the leds
     switch (action.kind.id) {
         case ACT_LAYER:
+        case ACT_LAYER_MODS:
         #ifndef NO_ACTION_TAPPING
         case ACT_LAYER_TAP:
         case ACT_LAYER_TAP_EXT:
@@ -957,7 +957,7 @@ bool is_tap_action(action_t action)
         case ACT_LAYER_TAP:
         case ACT_LAYER_TAP_EXT:
             switch (action.layer_tap.code) {
-                case 0x00 ... 0xdf:
+                case KC_NO ... KC_RGUI:
                 case OP_TAP_TOGGLE:
                 case OP_ONESHOT:
                     return true;
@@ -965,7 +965,7 @@ bool is_tap_action(action_t action)
             return false;
         case ACT_SWAP_HANDS:
             switch (action.swap.code) {
-                case 0x00 ... 0xdf:
+                case KC_NO ... KC_RGUI:
                 case OP_SH_TAP_TOGGLE:
                     return true;
             }
@@ -1014,6 +1014,7 @@ void debug_action(action_t action)
         case ACT_USAGE:             dprint("ACT_USAGE");             break;
         case ACT_MOUSEKEY:          dprint("ACT_MOUSEKEY");          break;
         case ACT_LAYER:             dprint("ACT_LAYER");             break;
+        case ACT_LAYER_MODS:        dprint("ACT_LAYER_MODS");        break;
         case ACT_LAYER_TAP:         dprint("ACT_LAYER_TAP");         break;
         case ACT_LAYER_TAP_EXT:     dprint("ACT_LAYER_TAP_EXT");     break;
         case ACT_MACRO:             dprint("ACT_MACRO");             break;
diff --git a/tmk_core/common/action_code.h b/tmk_core/common/action_code.h
index d836b7a8a..5b5d0e566 100644
--- a/tmk_core/common/action_code.h
+++ b/tmk_core/common/action_code.h
@@ -66,7 +66,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *   EBBBB: bits and extra bit
  *   ee:    on event(01:press, 10:release, 11:both)
  *
- * 1001|xxxx|xxxx xxxx   (reserved)
+ * ACT_LAYER_MODS(1001):
+ * 1001|LLLL| mods       Layer with modifiers held
  *
  * ACT_LAYER_TAP(101x):
  * 101E|LLLL| keycode    On/Off with tap key    (0x00-DF)[TAP]
@@ -110,6 +111,7 @@ enum action_kind_id {
     ACT_SWAP_HANDS      = 0b0110,
     /* Layer Actions */
     ACT_LAYER           = 0b1000,
+    ACT_LAYER_MODS      = 0b1001,
     ACT_LAYER_TAP       = 0b1010, /* Layer  0-15 */
     ACT_LAYER_TAP_EXT   = 0b1011, /* Layer 16-31 */
     /* Extensions */
@@ -153,6 +155,12 @@ typedef union {
         uint8_t  op     :2;
         uint8_t  kind   :4;
     } layer_bitop;
+    struct action_layer_mods
+    {
+        uint8_t  mods   :8;
+        uint8_t  layer  :4;
+        uint8_t  kind   :4;
+    } layer_mods;
     struct action_layer_tap {
         uint8_t  code   :8;
         uint8_t  val    :5;
@@ -261,8 +269,8 @@ enum layer_param_tap_op {
     OP_SET_CLEAR,
     OP_ONESHOT,
 };
-#define ACTION_LAYER_BITOP(op, part, bits, on)      (ACT_LAYER<<12 | (op)<<10 | (on)<<8 | (part)<<5 | ((bits)&0x1f))
-#define ACTION_LAYER_TAP(layer, key)                (ACT_LAYER_TAP<<12 | (layer)<<8 | (key))
+#define ACTION_LAYER_BITOP(op, part, bits, on)      ACTION(ACT_LAYER, (op)<<10 | (on)<<8 | (part)<<5 | ((bits)&0x1f))
+#define ACTION_LAYER_TAP(layer, key)                ACTION(ACT_LAYER_TAP, (layer)<<8 | (key))
 /* Default Layer */
 #define ACTION_DEFAULT_LAYER_SET(layer)             ACTION_DEFAULT_LAYER_BIT_SET((layer)/4, 1<<((layer)%4))
 /* Layer Operation */
@@ -277,7 +285,7 @@ enum layer_param_tap_op {
 #define ACTION_LAYER_OFF_ON(layer)                  ACTION_LAYER_TAP((layer), OP_OFF_ON)
 #define ACTION_LAYER_SET_CLEAR(layer)               ACTION_LAYER_TAP((layer), OP_SET_CLEAR)
 #define ACTION_LAYER_ONESHOT(layer)                 ACTION_LAYER_TAP((layer), OP_ONESHOT)
-#define ACTION_LAYER_MODS(layer, mods)              ACTION_LAYER_TAP((layer), 0xe0 | ((mods)&0x0f))
+#define ACTION_LAYER_MODS(layer, mods)              ACTION(ACT_LAYER_MODS, (layer) << 8 | (mods))
 /* With Tapping */
 #define ACTION_LAYER_TAP_KEY(layer, key)            ACTION_LAYER_TAP((layer), (key))
 #define ACTION_LAYER_TAP_TOGGLE(layer)              ACTION_LAYER_TAP((layer), OP_TAP_TOGGLE)