fix incorrect register selection for `vpbroadcastm{b2q,w2d}` with `rex.b` set

5 files changed, 13 insertions, 2 deletions

diff --git a/CHANGELOG b/CHANGELOG
index 5d8ae65..f471751 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -13,6 +13,9 @@
   instructions are otherwise decoded correctly.)
 * fix incorrect register selection for `vpmovm2*` with `rex.b` set (would select
   registers "k8" through "k15", but should be masked down to "k0".."k7".)
+* fix incorrect register selection for `vpbroadcastm{b2q,w2d}` with `rex.b`
+  set. basically the same bug as `vpmovm2*`; would select registers "k8".."k15",
+  but should have been masked to "k0".."k7".
 * fix incorrect register selection for `vpmov*2m` with `rex.r` set. similar to
   above, except 64-bit only: in 32- and 16-bit modes, this case was and continues
   to be a `bound` instruction. in 64-bit mode, this would disassemble as a
diff --git a/src/shared/evex.in b/src/shared/evex.in
index 58ec14a..a43a97e 100644
--- a/src/shared/evex.in
+++ b/src/shared/evex.in
@@ -2872,6 +2872,8 @@ pub(crate) fn read_evex_operands<
       if mem_oper == OperandSpec::RegMMM {
         instruction.mem_size = 0;
         instruction.regs[1].bank = RegisterBank::K;
+        // only 8 mask registers, `xed` suggests that any invalid bits are masked out???
+        instruction.regs[1].num &= 0b111;
       } else {
         return Err(DecodeError::InvalidOperand);
       }
@@ -2897,6 +2899,8 @@ pub(crate) fn read_evex_operands<
       if mem_oper == OperandSpec::RegMMM {
         instruction.mem_size = 0;
         instruction.regs[1].bank = RegisterBank::K;
+        // only 8 mask registers, `xed` suggests that any invalid bits are masked out???
+        instruction.regs[1].num &= 0b111;
       } else {
         return Err(DecodeError::InvalidOperand);
       }
diff --git a/test/long_mode/mod.rs b/test/long_mode/mod.rs
index b087d74..c0ec1a5 100644
--- a/test/long_mode/mod.rs
+++ b/test/long_mode/mod.rs
@@ -1466,6 +1466,8 @@ fn evex() {
     test_display(&[0x62, 0xf2, 0x7d, 0x08, 0x2a, 0x44, 0x40, 0x01], "vmovntdqa xmm0, xmmword [rax + rax * 2 + 0x10]");
     test_invalid(&[0x62, 0xf2, 0x7d, 0x1d, 0x66, 0x50, 0x01, 0x11]);
 
+    // vpbroadcastmw2d. similar to `vpmovm2*`, out-of-range `k` are just masked down.
+    test_display(&[0x62, 0xd2, 0x7e, 0x28, 0x3a, 0xca], "vpbroadcastmw2d ymm1, k2");
     // vpmovm2b (and larger forms). for some reason the source operand is a mask register but uses
     // modrm bits as a register selector. out-of-range `k` seem to just get masked down..
     test_display(&[0x62, 0xd2, 0x7e, 0x08, 0x28, 0xc2], "vpmovm2b xmm0, k2");
diff --git a/test/protected_mode/mod.rs b/test/protected_mode/mod.rs
index 88d6c49..fd77b5e 100644
--- a/test/protected_mode/mod.rs
+++ b/test/protected_mode/mod.rs
@@ -1332,11 +1332,12 @@ fn test_misc() {
 
 #[test]
 fn evex() {
+    // vpbroadcastmw2d. similar to `vpmovm2*`, out-of-range `k` are just masked down.
+    test_display(&[0x62, 0xd2, 0x7e, 0x28, 0x3a, 0xca], "vpbroadcastmw2d ymm1, k2");
     // vpmovm2b (and larger forms). for some reason the source operand is a mask register but uses
     // modrm bits as a register selector. out-of-range `k` seem to just get masked down..
     test_display(&[0x62, 0xd2, 0x7e, 0x08, 0x28, 0xc2], "vpmovm2b xmm0, k2");
     test_display(&[0x62, 0xf2, 0x7e, 0x08, 0x28, 0xc1], "vpmovm2b xmm0, k1");
-
     // vpmovb2m (and larger forms). out-of-range `k` are invalid in 64-bit mode, are part of the
     // `bound` instruction for 32- and 16-bit modes.
     test_display(&[0x62, 0x72, 0x7e /* , 0x28, 0x29, 0xfd */], "bound esi, qword [edx + 0x7e]");
diff --git a/test/real_mode/mod.rs b/test/real_mode/mod.rs
index b422887..bc045d8 100644
--- a/test/real_mode/mod.rs
+++ b/test/real_mode/mod.rs
@@ -18401,11 +18401,12 @@ fn test_invalid_sequences() {
     test_invalid(&[0xf3, 0xf2, 0x0f, 0xae, 0x8f, 0x54, 0x3c, 0x58, 0xb7]);
     test_invalid(&[0xff, 0xd8]);
 
+    // vpbroadcastmw2d. similar to `vpmovm2*`, out-of-range `k` are just masked down.
+    test_display(&[0x62, 0xd2, 0x7e, 0x28, 0x3a, 0xca], "vpbroadcastmw2d ymm1, k2");
     // vpmovm2b (and larger forms). for some reason the source operand is a mask register but uses
     // modrm bits as a register selector. out-of-range `k` seem to just get masked down..
     test_display(&[0x62, 0xd2, 0x7e, 0x08, 0x28, 0xc2], "vpmovm2b xmm0, k2");
     test_display(&[0x62, 0xf2, 0x7e, 0x08, 0x28, 0xc1], "vpmovm2b xmm0, k1");
-
     // vpmovb2m (and larger forms). out-of-range `k` are invalid in 64-bit mode, are part of the
     // `bound` instruction for 32- and 16-bit modes.
     test_display(&[0x62, 0x72, 0x7e /* , 0x28, 0x29, 0xfd */], "bound si, dword [bp + si * 1 + 0x7e]");