cleanup pass on vex-encoded instructions is going to be exciting

6 files changed, 116 insertions, 21 deletions

diff --git a/CHANGELOG b/CHANGELOG
index b516dc0..8e115a2 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -21,6 +21,8 @@
 * reject 0f0d prefetch/nop with a register operand, which was incorrectly decoded before.
 * maskmovq and maskmovdqu now have their implicit memory access size reported correctly.
 * monitor now reports a 1-byte memory access size. the monitored range in practice depends on CPUID.
+* fix incorrect operand order for VEX-encoded vmovupd opcode 0x11.
+* reject a few VEX-encoded instructions that are specific about allowed W-bits.
 
 ## 2.0.0
 
diff --git a/src/long_mode/vex.rs b/src/long_mode/vex.rs
index 2ee9749..b0b900a 100644
--- a/src/long_mode/vex.rs
+++ b/src/long_mode/vex.rs
@@ -1755,7 +1755,7 @@ fn read_vex_instruction<
 //                        0x0a => (Opcode::VROUNDSS, VEXOperandCode::G_V_E_xmm_imm8),
 //                        0x0b => (Opcode::VROUNDSD, VEXOperandCode::G_V_E_xmm_imm8),
                         0x10 => (Opcode::VMOVUPD, VEXOperandCode::G_E_xyLmm),
-                        0x11 => (Opcode::VMOVUPD, VEXOperandCode::G_E_xyLmm),
+                        0x11 => (Opcode::VMOVUPD, VEXOperandCode::E_G_xyLmm),
                         0x12 => (Opcode::VMOVLPD, if L {
                             return Err(DecodeError::InvalidOpcode);
                         } else {
@@ -2073,10 +2073,26 @@ fn read_vex_instruction<
                     0x09 => (Opcode::VPSIGNW, VEXOperandCode::G_V_E_xyLmm),
                     0x0A => (Opcode::VPSIGND, VEXOperandCode::G_V_E_xyLmm),
                     0x0B => (Opcode::VPMULHRSW, VEXOperandCode::G_V_E_xyLmm),
-                    0x0C => (Opcode::VPERMILPS, VEXOperandCode::G_V_E_xyLmm),
-                    0x0D => (Opcode::VPERMILPD, VEXOperandCode::G_V_E_xyLmm),
-                    0x0E => (Opcode::VTESTPS, VEXOperandCode::G_E_xyLmm),
-                    0x0F => (Opcode::VTESTPD, VEXOperandCode::G_E_xyLmm),
+                    0x0C => (Opcode::VPERMILPS, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_V_E_xyLmm
+                    }),
+                    0x0D => (Opcode::VPERMILPD, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_V_E_xyLmm
+                    }),
+                    0x0E => (Opcode::VTESTPS, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_E_xyLmm
+                    }),
+                    0x0F => (Opcode::VTESTPD, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_E_xyLmm
+                    }),
                     0x13 => (Opcode::VCVTPH2PS, VEXOperandCode::G_E_xyLmm),
                     0x16 => (Opcode::VPERMPS, if L {
                         if instruction.prefixes.vex_unchecked().w() {
@@ -2614,8 +2630,16 @@ fn read_vex_instruction<
                     } else {
                         VEXOperandCode::G_V_E_xyLmm_imm8
                     }),
-                    0x04 => (Opcode::VPERMILPS, VEXOperandCode::G_E_xyLmm_imm8),
-                    0x05 => (Opcode::VPERMILPD, VEXOperandCode::G_E_xyLmm_imm8),
+                    0x04 => (Opcode::VPERMILPS, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_E_xyLmm_imm8
+                    }),
+                    0x05 => (Opcode::VPERMILPD, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_E_xyLmm_imm8
+                    }),
                     0x06 => (Opcode::VPERM2F128, if L {
                         if instruction.prefixes.vex_unchecked().w() {
                             return Err(DecodeError::InvalidOpcode);
diff --git a/src/protected_mode/vex.rs b/src/protected_mode/vex.rs
index 337e9bc..9144fef 100644
--- a/src/protected_mode/vex.rs
+++ b/src/protected_mode/vex.rs
@@ -1650,7 +1650,7 @@ fn read_vex_instruction<
 //                        0x0a => (Opcode::VROUNDSS, VEXOperandCode::G_V_E_xmm_imm8),
 //                        0x0b => (Opcode::VROUNDSD, VEXOperandCode::G_V_E_xmm_imm8),
                         0x10 => (Opcode::VMOVUPD, VEXOperandCode::G_E_xyLmm),
-                        0x11 => (Opcode::VMOVUPD, VEXOperandCode::G_E_xyLmm),
+                        0x11 => (Opcode::VMOVUPD, VEXOperandCode::E_G_xyLmm),
                         0x12 => (Opcode::VMOVLPD, if L {
                             return Err(DecodeError::InvalidOpcode);
                         } else {
@@ -1944,10 +1944,26 @@ fn read_vex_instruction<
                     0x09 => (Opcode::VPSIGNW, VEXOperandCode::G_V_E_xyLmm),
                     0x0A => (Opcode::VPSIGND, VEXOperandCode::G_V_E_xyLmm),
                     0x0B => (Opcode::VPMULHRSW, VEXOperandCode::G_V_E_xyLmm),
-                    0x0C => (Opcode::VPERMILPS, VEXOperandCode::G_V_E_xyLmm),
-                    0x0D => (Opcode::VPERMILPD, VEXOperandCode::G_V_E_xyLmm),
-                    0x0E => (Opcode::VTESTPS, VEXOperandCode::G_E_xyLmm),
-                    0x0F => (Opcode::VTESTPD, VEXOperandCode::G_E_xyLmm),
+                    0x0C => (Opcode::VPERMILPS, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_V_E_xyLmm
+                    }),
+                    0x0D => (Opcode::VPERMILPD, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_V_E_xyLmm
+                    }),
+                    0x0E => (Opcode::VTESTPS, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_E_xyLmm
+                    }),
+                    0x0F => (Opcode::VTESTPD, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_E_xyLmm
+                    }),
                     0x13 => (Opcode::VCVTPH2PS, VEXOperandCode::G_E_xyLmm),
                     0x16 => (Opcode::VPERMPS, if L {
                         if instruction.prefixes.vex_unchecked().w() {
@@ -2485,8 +2501,16 @@ fn read_vex_instruction<
                     } else {
                         VEXOperandCode::G_V_E_xyLmm_imm8
                     }),
-                    0x04 => (Opcode::VPERMILPS, VEXOperandCode::G_E_xyLmm_imm8),
-                    0x05 => (Opcode::VPERMILPD, VEXOperandCode::G_E_xyLmm_imm8),
+                    0x04 => (Opcode::VPERMILPS, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_E_xyLmm_imm8
+                    }),
+                    0x05 => (Opcode::VPERMILPD, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_E_xyLmm_imm8
+                    }),
                     0x06 => (Opcode::VPERM2F128, if L {
                         if instruction.prefixes.vex_unchecked().w() {
                             return Err(DecodeError::InvalidOpcode);
diff --git a/src/real_mode/vex.rs b/src/real_mode/vex.rs
index afa9565..cf85db9 100644
--- a/src/real_mode/vex.rs
+++ b/src/real_mode/vex.rs
@@ -1650,7 +1650,7 @@ fn read_vex_instruction<
 //                        0x0a => (Opcode::VROUNDSS, VEXOperandCode::G_V_E_xmm_imm8),
 //                        0x0b => (Opcode::VROUNDSD, VEXOperandCode::G_V_E_xmm_imm8),
                         0x10 => (Opcode::VMOVUPD, VEXOperandCode::G_E_xyLmm),
-                        0x11 => (Opcode::VMOVUPD, VEXOperandCode::G_E_xyLmm),
+                        0x11 => (Opcode::VMOVUPD, VEXOperandCode::E_G_xyLmm),
                         0x12 => (Opcode::VMOVLPD, if L {
                             return Err(DecodeError::InvalidOpcode);
                         } else {
@@ -1944,10 +1944,26 @@ fn read_vex_instruction<
                     0x09 => (Opcode::VPSIGNW, VEXOperandCode::G_V_E_xyLmm),
                     0x0A => (Opcode::VPSIGND, VEXOperandCode::G_V_E_xyLmm),
                     0x0B => (Opcode::VPMULHRSW, VEXOperandCode::G_V_E_xyLmm),
-                    0x0C => (Opcode::VPERMILPS, VEXOperandCode::G_V_E_xyLmm),
-                    0x0D => (Opcode::VPERMILPD, VEXOperandCode::G_V_E_xyLmm),
-                    0x0E => (Opcode::VTESTPS, VEXOperandCode::G_E_xyLmm),
-                    0x0F => (Opcode::VTESTPD, VEXOperandCode::G_E_xyLmm),
+                    0x0C => (Opcode::VPERMILPS, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_V_E_xyLmm
+                    }),
+                    0x0D => (Opcode::VPERMILPD, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_V_E_xyLmm
+                    }),
+                    0x0E => (Opcode::VTESTPS, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_E_xyLmm
+                    }),
+                    0x0F => (Opcode::VTESTPD, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_E_xyLmm
+                    }),
                     0x13 => (Opcode::VCVTPH2PS, VEXOperandCode::G_E_xyLmm),
                     0x16 => (Opcode::VPERMPS, if L {
                         if instruction.prefixes.vex_unchecked().w() {
@@ -2485,8 +2501,16 @@ fn read_vex_instruction<
                     } else {
                         VEXOperandCode::G_V_E_xyLmm_imm8
                     }),
-                    0x04 => (Opcode::VPERMILPS, VEXOperandCode::G_E_xyLmm_imm8),
-                    0x05 => (Opcode::VPERMILPD, VEXOperandCode::G_E_xyLmm_imm8),
+                    0x04 => (Opcode::VPERMILPS, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_E_xyLmm_imm8
+                    }),
+                    0x05 => (Opcode::VPERMILPD, if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
+                    } else {
+                        VEXOperandCode::G_E_xyLmm_imm8
+                    }),
                     0x06 => (Opcode::VPERM2F128, if L {
                         if instruction.prefixes.vex_unchecked().w() {
                             return Err(DecodeError::InvalidOpcode);
diff --git a/test/long_mode/mod.rs b/test/long_mode/mod.rs
index 6d12c8d..bf49443 100644
--- a/test/long_mode/mod.rs
+++ b/test/long_mode/mod.rs
@@ -1751,10 +1751,14 @@ fn test_vex() {
     test_instr(&[0xc4, 0b000_00010, 0b0_0111_101, 0x0c, 0b11_001_010], "vpermilps ymm9, ymm8, ymm10");
     test_instr(&[0xc4, 0b000_00010, 0b0_0111_001, 0x0d, 0b11_001_010], "vpermilpd xmm9, xmm8, xmm10");
     test_instr(&[0xc4, 0b000_00010, 0b0_0111_101, 0x0d, 0b11_001_010], "vpermilpd ymm9, ymm8, ymm10");
+    test_invalid(&[0xc4, 0b000_00010, 0b1_0111_001, 0x0d, 0b11_001_010]);
+    test_invalid(&[0xc4, 0b000_00010, 0b1_0111_101, 0x0d, 0b11_001_010]);
     test_instr(&[0xc4, 0b000_00010, 0b0_1111_001, 0x0e, 0b11_001_010], "vtestps xmm9, xmm10");
     test_instr(&[0xc4, 0b000_00010, 0b0_1111_101, 0x0e, 0b11_001_010], "vtestps ymm9, ymm10");
+    test_invalid(&[0xc4, 0b000_00010, 0b1_1111_101, 0x0e, 0b11_001_010]);
     test_instr(&[0xc4, 0b000_00010, 0b0_1111_001, 0x0f, 0b11_001_010], "vtestpd xmm9, xmm10");
     test_instr(&[0xc4, 0b000_00010, 0b0_1111_101, 0x0f, 0b11_001_010], "vtestpd ymm9, ymm10");
+    test_invalid(&[0xc4, 0b000_00010, 0b1_1111_101, 0x0f, 0b11_001_010]);
 
     test_instr(&[0xc4, 0b000_00010, 0b0_1111_101, 0x16, 0b11_001_010], "vpermps ymm9, ymm0, ymm10");
     test_instr(&[0xc4, 0b000_00010, 0b0_1111_101, 0x16, 0b00_001_010], "vpermps ymm9, ymm0, ymmword [r10]");
@@ -1968,18 +1972,24 @@ fn test_vex() {
     test_invalid(&[0xc4, 0b000_00001, 0b1_0111_111, 0x10, 0b00_001_010]);
     test_instr(&[0xc4, 0b000_00001, 0b1_1111_001, 0x10, 0b00_001_010], "vmovupd xmm9, xmmword [r10]");
     test_instr(&[0xc4, 0b000_00001, 0b1_1111_101, 0x10, 0b00_001_010], "vmovupd ymm9, ymmword [r10]");
+    test_instr(&[0xc4, 0b000_00001, 0b1_1111_001, 0x11, 0b00_001_010], "vmovupd xmmword [r10], xmm9");
+    test_instr(&[0xc4, 0b000_00001, 0b1_1111_101, 0x11, 0b00_001_010], "vmovupd ymmword [r10], ymm9");
     test_instr(&[0xc4, 0b000_00001, 0b1_1111_011, 0x11, 0b00_001_010], "vmovsd qword [r10], xmm9");
     test_instr(&[0xc4, 0b000_00001, 0b1_1111_111, 0x11, 0b00_001_010], "vmovsd qword [r10], xmm9");
     test_invalid(&[0xc4, 0b000_00001, 0b1_0111_011, 0x11, 0b00_001_010]);
     test_invalid(&[0xc4, 0b000_00001, 0b1_0111_111, 0x11, 0b00_001_010]);
     test_instr(&[0xc4, 0b000_00001, 0b1_1111_001, 0x10, 0b00_001_010], "vmovupd xmm9, xmmword [r10]");
     test_instr(&[0xc4, 0b000_00001, 0b1_1111_101, 0x10, 0b00_001_010], "vmovupd ymm9, ymmword [r10]");
+    test_instr(&[0xc4, 0b000_00001, 0b1_1111_001, 0x11, 0b00_001_010], "vmovupd xmmword [r10], xmm9");
+    test_instr(&[0xc4, 0b000_00001, 0b1_1111_101, 0x11, 0b00_001_010], "vmovupd ymmword [r10], ymm9");
     test_instr(&[0xc4, 0b000_00001, 0b1_1111_010, 0x10, 0b00_001_010], "vmovss xmm9, dword [r10]");
     test_instr(&[0xc4, 0b000_00001, 0b1_1111_110, 0x10, 0b00_001_010], "vmovss xmm9, dword [r10]");
     test_invalid(&[0xc4, 0b000_00001, 0b1_0111_010, 0x10, 0b00_001_010]);
     test_invalid(&[0xc4, 0b000_00001, 0b1_0111_110, 0x10, 0b00_001_010]);
     test_instr(&[0xc4, 0b000_00001, 0b1_1111_000, 0x10, 0b00_001_010], "vmovups xmm9, xmmword [r10]");
     test_instr(&[0xc4, 0b000_00001, 0b1_1111_100, 0x10, 0b00_001_010], "vmovups ymm9, ymmword [r10]");
+    test_instr(&[0xc4, 0b000_00001, 0b1_1111_000, 0x11, 0b00_001_010], "vmovups xmmword [r10], xmm9");
+    test_instr(&[0xc4, 0b000_00001, 0b1_1111_100, 0x11, 0b00_001_010], "vmovups ymmword [r10], ymm9");
 
     test_instr(&[0xc4, 0b000_00001, 0b1_0111_011, 0x11, 0b11_001_010], "vmovsd xmm10, xmm8, xmm9");
     test_instr(&[0xc4, 0b000_00001, 0b1_0111_111, 0x11, 0b11_001_010], "vmovsd xmm10, xmm8, xmm9");
diff --git a/test/protected_mode/mod.rs b/test/protected_mode/mod.rs
index 800c5ec..c42ea73 100644
--- a/test/protected_mode/mod.rs
+++ b/test/protected_mode/mod.rs
@@ -1554,10 +1554,15 @@ fn test_vex() {
     test_instr(&[0xc4, 0b110_00010, 0b0_0111_101, 0x0c, 0b11_001_010], "vpermilps ymm1, ymm0, ymm2");
     test_instr(&[0xc4, 0b110_00010, 0b0_0111_001, 0x0d, 0b11_001_010], "vpermilpd xmm1, xmm0, xmm2");
     test_instr(&[0xc4, 0b110_00010, 0b0_0111_101, 0x0d, 0b11_001_010], "vpermilpd ymm1, ymm0, ymm2");
+    test_invalid(&[0xc4, 0b110_00010, 0b1_0111_001, 0x0d, 0b11_001_010]);
+    test_invalid(&[0xc4, 0b110_00010, 0b1_0111_101, 0x0d, 0b11_001_010]);
     test_instr(&[0xc4, 0b110_00010, 0b0_1111_001, 0x0e, 0b11_001_010], "vtestps xmm1, xmm2");
     test_instr(&[0xc4, 0b110_00010, 0b0_1111_101, 0x0e, 0b11_001_010], "vtestps ymm1, ymm2");
+    test_invalid(&[0xc4, 0b110_00010, 0b1_1111_101, 0x0e, 0b11_001_010]);
     test_instr(&[0xc4, 0b110_00010, 0b0_1111_001, 0x0f, 0b11_001_010], "vtestpd xmm1, xmm2");
     test_instr(&[0xc4, 0b110_00010, 0b0_1111_101, 0x0f, 0b11_001_010], "vtestpd ymm1, ymm2");
+    test_invalid(&[0xc4, 0b110_00010, 0b1_1111_101, 0x0f, 0b11_001_010]);
+
 
     test_instr(&[0xc4, 0b110_00010, 0b0_1111_101, 0x16, 0b11_001_010], "vpermps ymm1, ymm0, ymm2");
     test_instr(&[0xc4, 0b110_00010, 0b0_1111_101, 0x16, 0b00_001_010], "vpermps ymm1, ymm0, ymmword [edx]");
@@ -1773,18 +1778,24 @@ fn test_vex() {
     test_invalid(&[0xc4, 0b110_00001, 0b1_0111_111, 0x10, 0b00_001_010]);
     test_instr(&[0xc4, 0b110_00001, 0b1_1111_001, 0x10, 0b00_001_010], "vmovupd xmm1, xmmword [edx]");
     test_instr(&[0xc4, 0b110_00001, 0b1_1111_101, 0x10, 0b00_001_010], "vmovupd ymm1, ymmword [edx]");
+    test_instr(&[0xc4, 0b110_00001, 0b1_1111_001, 0x11, 0b00_001_010], "vmovupd xmmword [edx], xmm1");
+    test_instr(&[0xc4, 0b110_00001, 0b1_1111_101, 0x11, 0b00_001_010], "vmovupd ymmword [edx], ymm1");
     test_instr(&[0xc4, 0b110_00001, 0b1_1111_011, 0x11, 0b00_001_010], "vmovsd qword [edx], xmm1");
     test_instr(&[0xc4, 0b110_00001, 0b1_1111_111, 0x11, 0b00_001_010], "vmovsd qword [edx], xmm1");
     test_invalid(&[0xc4, 0b110_00001, 0b1_0111_011, 0x11, 0b00_001_010]);
     test_invalid(&[0xc4, 0b110_00001, 0b1_0111_111, 0x11, 0b00_001_010]);
     test_instr(&[0xc4, 0b110_00001, 0b1_1111_001, 0x10, 0b00_001_010], "vmovupd xmm1, xmmword [edx]");
     test_instr(&[0xc4, 0b110_00001, 0b1_1111_101, 0x10, 0b00_001_010], "vmovupd ymm1, ymmword [edx]");
+    test_instr(&[0xc4, 0b110_00001, 0b1_1111_001, 0x11, 0b00_001_010], "vmovupd xmmword [edx], xmm1");
+    test_instr(&[0xc4, 0b110_00001, 0b1_1111_101, 0x11, 0b00_001_010], "vmovupd ymmword [edx], ymm1");
     test_instr(&[0xc4, 0b110_00001, 0b1_1111_010, 0x10, 0b00_001_010], "vmovss xmm1, dword [edx]");
     test_instr(&[0xc4, 0b110_00001, 0b1_1111_110, 0x10, 0b00_001_010], "vmovss xmm1, dword [edx]");
     test_invalid(&[0xc4, 0b110_00001, 0b1_0111_010, 0x10, 0b00_001_010]);
     test_invalid(&[0xc4, 0b110_00001, 0b1_0111_110, 0x10, 0b00_001_010]);
     test_instr(&[0xc4, 0b110_00001, 0b1_1111_000, 0x10, 0b00_001_010], "vmovups xmm1, xmmword [edx]");
     test_instr(&[0xc4, 0b110_00001, 0b1_1111_100, 0x10, 0b00_001_010], "vmovups ymm1, ymmword [edx]");
+    test_instr(&[0xc4, 0b110_00001, 0b1_1111_000, 0x11, 0b00_001_010], "vmovups xmmword [edx], xmm1");
+    test_instr(&[0xc4, 0b110_00001, 0b1_1111_100, 0x11, 0b00_001_010], "vmovups ymmword [edx], ymm1");
 
     test_instr(&[0xc4, 0b110_00001, 0b1_0111_011, 0x11, 0b11_001_010], "vmovsd xmm2, xmm0, xmm1");
     test_instr(&[0xc4, 0b110_00001, 0b1_0111_111, 0x11, 0b11_001_010], "vmovsd xmm2, xmm0, xmm1");