fix operand handling for the psl/psr family of xmm shifts/rotates

these instructions ignored rex bits even for xmm reigsters, which is
incorrect (so says xed)

2 files changed, 30 insertions, 31 deletions

diff --git a/src/long_mode/mod.rs b/src/long_mode/mod.rs
index 27c3ca9..34db72a 100644
--- a/src/long_mode/mod.rs
+++ b/src/long_mode/mod.rs
@@ -5259,8 +5259,8 @@ enum OperandCode {
 //    ModRM_0x0f38 = OperandCodeBuilder::new().read_modrm().operand_case(53).bits(),
 //    ModRM_0x0f3a = OperandCodeBuilder::new().read_modrm().operand_case(54).bits(),
     ModRM_0x0f71 = OperandCodeBuilder::new().read_E().operand_case(OperandCase::ModRM_0x0f71).bits(),
-    ModRM_0x0f72 = OperandCodeBuilder::new().read_modrm().operand_case(OperandCase::ModRM_0x0f72).bits(),
-    ModRM_0x0f73 = OperandCodeBuilder::new().read_modrm().operand_case(OperandCase::ModRM_0x0f73).bits(),
+    ModRM_0x0f72 = OperandCodeBuilder::new().read_E().operand_case(OperandCase::ModRM_0x0f72).bits(),
+    ModRM_0x0f73 = OperandCodeBuilder::new().read_E().operand_case(OperandCase::ModRM_0x0f73).bits(),
     ModRM_0xf20f78 = OperandCodeBuilder::new().read_modrm().operand_case(OperandCase::ModRM_0xf20f78).bits(),
     ModRM_0x660f78 = OperandCodeBuilder::new().read_modrm().operand_case(OperandCase::ModRM_0x660f78).bits(),
     ModRM_0xf30f1e = OperandCodeBuilder::new().operand_case(OperandCase::ModRM_0xf30f1e).bits(),
@@ -5425,7 +5425,7 @@ enum OperandCode {
     Mdq_Gdq = OperandCodeBuilder::new().read_E().mem_reg().operand_case(OperandCase::Mdq_Gdq).bits(),
     G_E_xmm_Ib = OperandCodeBuilder::new().read_E().operand_case(OperandCase::G_E_xmm_Ib).bits(),
     G_E_xmm_Ub = OperandCodeBuilder::new().read_E().operand_case(OperandCase::G_E_xmm_Ub).bits(),
-    G_U_xmm_Ub = OperandCodeBuilder::new().read_E().deny_regmmm().operand_case(OperandCase::G_U_xmm_Ub).bits(),
+    G_U_xmm_Ub = OperandCodeBuilder::new().read_E().operand_case(OperandCase::G_U_xmm_Ub).bits(),
     AL_Ob = OperandCodeBuilder::new().operand_case(OperandCase::AL_Ob).bits(),
     AL_Xb = OperandCodeBuilder::new().operand_case(OperandCase::AL_Xb).bits(),
     AX_Ov = OperandCodeBuilder::new().operand_case(OperandCase::AX_Ov).bits(),
@@ -7833,6 +7833,9 @@ fn read_operands<
             }
         }
         OperandCase::G_U_xmm_Ub => {
+            if mem_oper != OperandSpec::RegMMM {
+                return Err(DecodeError::InvalidOperand);
+            }
             instruction.operands[1] = mem_oper;
             instruction.regs[0].bank = RegisterBank::D;
             instruction.regs[1].bank = RegisterBank::X;
@@ -8602,7 +8605,6 @@ fn read_operands<
             }
 
             if instruction.prefixes.operand_size() {
-                // TODO: should this respect rex.w? it did not before, does now.
                 instruction.regs[1].bank = RegisterBank::X;
             } else {
                 instruction.regs[1].bank = RegisterBank::MM;
@@ -8619,14 +8621,11 @@ fn read_operands<
                 return Err(DecodeError::InvalidOperand);
             }
 
-            instruction.operand_count = 2;
-
-            let modrm = read_modrm(words)?;
-            if modrm & 0xc0 != 0xc0 {
+            if mem_oper != OperandSpec::RegMMM {
                 return Err(DecodeError::InvalidOperand);
             }
 
-            let r = (modrm >> 3) & 7;
+            let r = instruction.regs[0].num & 0b0111;
             match r {
                 2 => {
                     instruction.opcode = Opcode::PSRLD;
@@ -8643,11 +8642,12 @@ fn read_operands<
             }
 
             if instruction.prefixes.operand_size() {
-                instruction.regs[1] = RegSpec { bank: RegisterBank::X, num: modrm & 7 };
+                instruction.regs[1].bank = RegisterBank::X;
             } else {
-                instruction.regs[1] = RegSpec { bank: RegisterBank::MM, num: modrm & 7 };
+                instruction.regs[1].bank = RegisterBank::MM;
+                instruction.regs[1].num &= 0b0111;
             }
-            instruction.operands[0] = OperandSpec::RegMMM;
+            instruction.operands[0] = mem_oper;
             instruction.imm = read_imm_signed(words, 1)? as u64;
             instruction.operands[1] = OperandSpec::ImmU8;
         },
@@ -8656,14 +8656,11 @@ fn read_operands<
                 return Err(DecodeError::InvalidOperand);
             }
 
-            instruction.operand_count = 2;
-
-            let modrm = read_modrm(words)?;
-            if modrm & 0xc0 != 0xc0 {
+            if mem_oper != OperandSpec::RegMMM {
                 return Err(DecodeError::InvalidOperand);
             }
 
-            let r = (modrm >> 3) & 7;
+            let r = instruction.regs[0].num & 0b0111;
             match r {
                 2 => {
                     instruction.opcode = Opcode::PSRLQ;
@@ -8689,11 +8686,12 @@ fn read_operands<
             }
 
             if instruction.prefixes.operand_size() {
-                instruction.regs[1] = RegSpec { bank: RegisterBank::X, num: modrm & 7 };
+                instruction.regs[1].bank = RegisterBank::X;
             } else {
-                instruction.regs[1] = RegSpec { bank: RegisterBank::MM, num: modrm & 7 };
+                instruction.regs[1].bank = RegisterBank::MM;
+                instruction.regs[1].num &= 0b0111;
             }
-            instruction.operands[0] = OperandSpec::RegMMM;
+            instruction.operands[0] = mem_oper;
             instruction.imm = read_imm_signed(words, 1)? as u64;
             instruction.operands[1] = OperandSpec::ImmU8;
         },
@@ -8817,6 +8815,7 @@ fn read_operands<
         OperandCase::G_mm_E => {
             instruction.regs[0].bank = RegisterBank::MM;
             instruction.regs[0].num &= 0b111;
+            instruction.operands[1] = mem_oper;
             if mem_oper == OperandSpec::RegMMM {
                 instruction.regs[1].bank = RegisterBank::MM;
                 instruction.regs[1].num &= 0b111;
@@ -8865,7 +8864,7 @@ fn read_operands<
             }
         }
         OperandCase::E_G_mm => {
-            instruction.operands[1] = instruction.operands[0];
+            instruction.operands[1] = OperandSpec::RegRRR;
             instruction.operands[0] = mem_oper;
             instruction.regs[0].bank = RegisterBank::MM;
             instruction.regs[0].num &= 0b111;
diff --git a/test/long_mode/mod.rs b/test/long_mode/mod.rs
index ab1cc23..dd3d3d3 100644
--- a/test/long_mode/mod.rs
+++ b/test/long_mode/mod.rs
@@ -440,25 +440,25 @@ fn test_sse2() {
     test_instr(&[0xf2, 0x0f, 0x70, 0xc0, 0x4e], "pshuflw xmm0, xmm0, 0x4e");
     test_instr(&[0xf3, 0x0f, 0x70, 0xc0, 0x4e], "pshufhw xmm0, xmm0, 0x4e");
     test_invalid(&[0x66, 0x4f, 0x0f, 0x71, 0x10, 0x8f]);
-    test_instr(&[0x66, 0x4f, 0x0f, 0x71, 0xd0, 0x8f], "psrlw xmm0, 0x8f");
+    test_instr(&[0x66, 0x4f, 0x0f, 0x71, 0xd0, 0x8f], "psrlw xmm8, 0x8f");
     test_invalid(&[0x66, 0x4f, 0x0f, 0x71, 0x20, 0x8f]);
-    test_instr(&[0x66, 0x4f, 0x0f, 0x71, 0xe0, 0x8f], "psraw xmm0, 0x8f");
+    test_instr(&[0x66, 0x4f, 0x0f, 0x71, 0xe0, 0x8f], "psraw xmm8, 0x8f");
     test_invalid(&[0x66, 0x4f, 0x0f, 0x71, 0x30, 0x8f]);
-    test_instr(&[0x66, 0x4f, 0x0f, 0x71, 0xf0, 0x8f], "psllw xmm0, 0x8f");
+    test_instr(&[0x66, 0x4f, 0x0f, 0x71, 0xf0, 0x8f], "psllw xmm8, 0x8f");
     test_invalid(&[0x66, 0x4f, 0x0f, 0x72, 0x10, 0x8f]);
-    test_instr(&[0x66, 0x4f, 0x0f, 0x72, 0xd0, 0x8f], "psrld xmm0, 0x8f");
+    test_instr(&[0x66, 0x4f, 0x0f, 0x72, 0xd0, 0x8f], "psrld xmm8, 0x8f");
     test_invalid(&[0x66, 0x4f, 0x0f, 0x72, 0x20, 0x8f]);
-    test_instr(&[0x66, 0x4f, 0x0f, 0x72, 0xe0, 0x8f], "psrad xmm0, 0x8f");
+    test_instr(&[0x66, 0x4f, 0x0f, 0x72, 0xe0, 0x8f], "psrad xmm8, 0x8f");
     test_invalid(&[0x66, 0x4f, 0x0f, 0x72, 0x30, 0x8f]);
-    test_instr(&[0x66, 0x4f, 0x0f, 0x72, 0xf0, 0x8f], "pslld xmm0, 0x8f");
+    test_instr(&[0x66, 0x4f, 0x0f, 0x72, 0xf0, 0x8f], "pslld xmm8, 0x8f");
     test_invalid(&[0x66, 0x4f, 0x0f, 0x73, 0x10, 0x8f]);
     test_invalid(&[0x66, 0x4f, 0x0f, 0x73, 0x18, 0x8f]);
-    test_instr(&[0x66, 0x4f, 0x0f, 0x73, 0xd0, 0x8f], "psrlq xmm0, 0x8f");
-    test_instr(&[0x66, 0x4f, 0x0f, 0x73, 0xd8, 0x8f], "psrldq xmm0, 0x8f");
+    test_instr(&[0x66, 0x4f, 0x0f, 0x73, 0xd0, 0x8f], "psrlq xmm8, 0x8f");
+    test_instr(&[0x66, 0x4f, 0x0f, 0x73, 0xd8, 0x8f], "psrldq xmm8, 0x8f");
     test_invalid(&[0x66, 0x4f, 0x0f, 0x73, 0x30, 0x8f]);
     test_invalid(&[0x66, 0x4f, 0x0f, 0x73, 0x38, 0x8f]);
-    test_instr(&[0x66, 0x4f, 0x0f, 0x73, 0xf0, 0x8f], "psllq xmm0, 0x8f");
-    test_instr(&[0x66, 0x4f, 0x0f, 0x73, 0xf8, 0x8f], "pslldq xmm0, 0x8f");
+    test_instr(&[0x66, 0x4f, 0x0f, 0x73, 0xf0, 0x8f], "psllq xmm8, 0x8f");
+    test_instr(&[0x66, 0x4f, 0x0f, 0x73, 0xf8, 0x8f], "pslldq xmm8, 0x8f");
     test_instr(&[0x66, 0x0f, 0x7e, 0xc1], "movd ecx, xmm0");
     test_instr(&[0x66, 0x48, 0x0f, 0x7e, 0xc1], "movq rcx, xmm0");
     test_instr(&[0x66, 0x48, 0x0f, 0x7e, 0x01], "movq qword [rcx], xmm0");