more precision for vinsert/vextract/vblendv{ps,pd}

7 files changed, 106 insertions, 50 deletions

diff --git a/CHANGELOG b/CHANGELOG
index a2410ee..5072dde 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -23,7 +23,8 @@
 * monitor now reports a 1-byte memory access size. the monitored range in practice depends on CPUID.
 * fix incorrect operand order for VEX-encoded vmovupd opcode 0x11.
 * reject a few VEX-encoded instructions that are specific about allowed W-bits.
-  vcvtph2ps, vbroadcastf128, vmaskmovps, vmaskmovpd, vpermd, vpbroadcast{b,w,d,q}, among others
+  vcvtph2ps, vbroadcastf128, vmaskmovps, vmaskmovpd, vpermd,
+  vpbroadcast{b,w,d,q}, vinserti128, vextracti128, vblendv*, among others.
 * vpbroadcastb and vpbroadcastw should respect the L bit to select xmm/ymm vector width, but
   always decoded as ymm.
 * vmaskmovqdu now reports a memory access size for the implied write to ds:[rdi/edi/di].
diff --git a/src/long_mode/vex.rs b/src/long_mode/vex.rs
index 2f6054a..04e2589 100644
--- a/src/long_mode/vex.rs
+++ b/src/long_mode/vex.rs
@@ -2783,16 +2783,24 @@ fn read_vex_instruction<
                             VEXOperandCode::G_V_xmm_Ev_imm8
                         })
                     },
-                    0x38 => (Opcode::VINSERTI128, if L {
-                        VEXOperandCode::G_ymm_V_ymm_E_xmm_imm8
-                    } else {
+                    0x38 => if instruction.prefixes.vex_unchecked().w() {
                         return Err(DecodeError::InvalidOpcode);
-                    }),
-                    0x39 => (Opcode::VEXTRACTI128, if L {
-                        VEXOperandCode::E_xmm_G_ymm_imm8
                     } else {
+                        (Opcode::VINSERTI128, if L {
+                            VEXOperandCode::G_ymm_V_ymm_E_xmm_imm8
+                        } else {
+                            return Err(DecodeError::InvalidOpcode);
+                        })
+                    },
+                    0x39 => if instruction.prefixes.vex_unchecked().w() {
                         return Err(DecodeError::InvalidOpcode);
-                    }),
+                    } else {
+                        (Opcode::VEXTRACTI128, if L {
+                            VEXOperandCode::E_xmm_G_ymm_imm8
+                        } else {
+                            return Err(DecodeError::InvalidOpcode);
+                        })
+                    },
                     0x40 => (Opcode::VDPPS, VEXOperandCode::G_V_E_xyLmm_imm8),
                     0x41 => (Opcode::VDPPD, if L {
                         return Err(DecodeError::InvalidOpcode);
@@ -2813,16 +2821,24 @@ fn read_vex_instruction<
                     } else {
                         return Err(DecodeError::InvalidOpcode);
                     }),
-                    0x4A => (Opcode::VBLENDVPS, if L {
-                        VEXOperandCode::G_V_E_ymm_ymm4
+                    0x4A => if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
                     } else {
-                        VEXOperandCode::G_V_E_xmm_xmm4
-                    }),
-                    0x4B => (Opcode::VBLENDVPD, if L {
-                        VEXOperandCode::G_V_E_ymm_ymm4
+                        (Opcode::VBLENDVPS, if L {
+                            VEXOperandCode::G_V_E_ymm_ymm4
+                        } else {
+                            VEXOperandCode::G_V_E_xmm_xmm4
+                        })
+                    },
+                    0x4B => if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
                     } else {
-                        VEXOperandCode::G_V_E_xmm_xmm4
-                    }),
+                        (Opcode::VBLENDVPD, if L {
+                            VEXOperandCode::G_V_E_ymm_ymm4
+                        } else {
+                            VEXOperandCode::G_V_E_xmm_xmm4
+                        })
+                    },
                     0x4C => if instruction.prefixes.vex_unchecked().w() {
                         return Err(DecodeError::InvalidOpcode);
                     } else {
diff --git a/src/protected_mode/vex.rs b/src/protected_mode/vex.rs
index 5fa46b9..875f010 100644
--- a/src/protected_mode/vex.rs
+++ b/src/protected_mode/vex.rs
@@ -2642,16 +2642,24 @@ fn read_vex_instruction<
                             VEXOperandCode::G_V_xmm_Ev_imm8
                         })
                     },
-                    0x38 => (Opcode::VINSERTI128, if L {
-                        VEXOperandCode::G_ymm_V_ymm_E_xmm_imm8
-                    } else {
+                    0x38 => if instruction.prefixes.vex_unchecked().w() {
                         return Err(DecodeError::InvalidOpcode);
-                    }),
-                    0x39 => (Opcode::VEXTRACTI128, if L {
-                        VEXOperandCode::E_xmm_G_ymm_imm8
                     } else {
+                        (Opcode::VINSERTI128, if L {
+                            VEXOperandCode::G_ymm_V_ymm_E_xmm_imm8
+                        } else {
+                            return Err(DecodeError::InvalidOpcode);
+                        })
+                    },
+                    0x39 => if instruction.prefixes.vex_unchecked().w() {
                         return Err(DecodeError::InvalidOpcode);
-                    }),
+                    } else {
+                        (Opcode::VEXTRACTI128, if L {
+                            VEXOperandCode::E_xmm_G_ymm_imm8
+                        } else {
+                            return Err(DecodeError::InvalidOpcode);
+                        })
+                    },
                     0x40 => (Opcode::VDPPS, VEXOperandCode::G_V_E_xyLmm_imm8),
                     0x41 => (Opcode::VDPPD, if L {
                         return Err(DecodeError::InvalidOpcode);
@@ -2672,16 +2680,24 @@ fn read_vex_instruction<
                     } else {
                         return Err(DecodeError::InvalidOpcode);
                     }),
-                    0x4A => (Opcode::VBLENDVPS, if L {
-                        VEXOperandCode::G_V_E_ymm_ymm4
+                    0x4A => if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
                     } else {
-                        VEXOperandCode::G_V_E_xmm_xmm4
-                    }),
-                    0x4B => (Opcode::VBLENDVPD, if L {
-                        VEXOperandCode::G_V_E_ymm_ymm4
+                        (Opcode::VBLENDVPS, if L {
+                            VEXOperandCode::G_V_E_ymm_ymm4
+                        } else {
+                            VEXOperandCode::G_V_E_xmm_xmm4
+                        })
+                    },
+                    0x4B => if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
                     } else {
-                        VEXOperandCode::G_V_E_xmm_xmm4
-                    }),
+                        (Opcode::VBLENDVPD, if L {
+                            VEXOperandCode::G_V_E_ymm_ymm4
+                        } else {
+                            VEXOperandCode::G_V_E_xmm_xmm4
+                        })
+                    },
                     0x4C => if instruction.prefixes.vex_unchecked().w() {
                         return Err(DecodeError::InvalidOpcode);
                     } else {
diff --git a/src/real_mode/vex.rs b/src/real_mode/vex.rs
index 9651760..1c1470a 100644
--- a/src/real_mode/vex.rs
+++ b/src/real_mode/vex.rs
@@ -2642,16 +2642,24 @@ fn read_vex_instruction<
                             VEXOperandCode::G_V_xmm_Ev_imm8
                         })
                     },
-                    0x38 => (Opcode::VINSERTI128, if L {
-                        VEXOperandCode::G_ymm_V_ymm_E_xmm_imm8
-                    } else {
+                    0x38 => if instruction.prefixes.vex_unchecked().w() {
                         return Err(DecodeError::InvalidOpcode);
-                    }),
-                    0x39 => (Opcode::VEXTRACTI128, if L {
-                        VEXOperandCode::E_xmm_G_ymm_imm8
                     } else {
+                        (Opcode::VINSERTI128, if L {
+                            VEXOperandCode::G_ymm_V_ymm_E_xmm_imm8
+                        } else {
+                            return Err(DecodeError::InvalidOpcode);
+                        })
+                    },
+                    0x39 => if instruction.prefixes.vex_unchecked().w() {
                         return Err(DecodeError::InvalidOpcode);
-                    }),
+                    } else {
+                        (Opcode::VEXTRACTI128, if L {
+                            VEXOperandCode::E_xmm_G_ymm_imm8
+                        } else {
+                            return Err(DecodeError::InvalidOpcode);
+                        })
+                    },
                     0x40 => (Opcode::VDPPS, VEXOperandCode::G_V_E_xyLmm_imm8),
                     0x41 => (Opcode::VDPPD, if L {
                         return Err(DecodeError::InvalidOpcode);
@@ -2672,16 +2680,24 @@ fn read_vex_instruction<
                     } else {
                         return Err(DecodeError::InvalidOpcode);
                     }),
-                    0x4A => (Opcode::VBLENDVPS, if L {
-                        VEXOperandCode::G_V_E_ymm_ymm4
+                    0x4A => if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
                     } else {
-                        VEXOperandCode::G_V_E_xmm_xmm4
-                    }),
-                    0x4B => (Opcode::VBLENDVPD, if L {
-                        VEXOperandCode::G_V_E_ymm_ymm4
+                        (Opcode::VBLENDVPS, if L {
+                            VEXOperandCode::G_V_E_ymm_ymm4
+                        } else {
+                            VEXOperandCode::G_V_E_xmm_xmm4
+                        })
+                    },
+                    0x4B => if instruction.prefixes.vex_unchecked().w() {
+                        return Err(DecodeError::InvalidOpcode);
                     } else {
-                        VEXOperandCode::G_V_E_xmm_xmm4
-                    }),
+                        (Opcode::VBLENDVPD, if L {
+                            VEXOperandCode::G_V_E_ymm_ymm4
+                        } else {
+                            VEXOperandCode::G_V_E_xmm_xmm4
+                        })
+                    },
                     0x4C => if instruction.prefixes.vex_unchecked().w() {
                         return Err(DecodeError::InvalidOpcode);
                     } else {
diff --git a/test/long_mode/mod.rs b/test/long_mode/mod.rs
index 0b78dd2..f53170b 100644
--- a/test/long_mode/mod.rs
+++ b/test/long_mode/mod.rs
@@ -1672,9 +1672,11 @@ fn test_vex() {
     test_invalid(&[0xc4, 0b000_00011, 0b0_1111_001, 0x19, 0b11_001_010, 0x77]);
     test_invalid(&[0xc4, 0b000_00011, 0b1_1111_101, 0x19, 0b11_001_010, 0x77]);
 
-    test_invalid(&[0xc4, 0b000_00011, 0b1_0111_001, 0x18, 0b11_001_010, 0x77]);
+    test_invalid(&[0xc4, 0b000_00011, 0b1_0111_001, 0x38, 0b11_001_010, 0x77]);
+    test_invalid(&[0xc4, 0b000_00011, 0b0_0111_001, 0x38, 0b11_001_010, 0x77]);
     test_avx2(&[0xc4, 0b000_00011, 0b0_0111_101, 0x38, 0b11_001_010, 0x77], "vinserti128 ymm9, ymm8, xmm10, 0x77");
-    test_invalid(&[0xc4, 0b000_00011, 0b1_0111_101, 0x18, 0b11_001_010, 0x77]);
+    test_invalid(&[0xc4, 0b000_00011, 0b1_1111_101, 0x39, 0b11_001_010, 0x77]);
+    test_invalid(&[0xc4, 0b000_00011, 0b0_0111_101, 0x39, 0b11_001_010, 0x77]);
     test_avx2(&[0xc4, 0b000_00011, 0b0_1111_101, 0x39, 0b11_001_010, 0x77], "vextracti128 xmm10, ymm9, 0x77");
     test_invalid(&[0xc4, 0b000_00011, 0b0_1111_001, 0x19, 0b11_001_010, 0x77]);
     test_invalid(&[0xc4, 0b000_00011, 0b1_1111_101, 0x19, 0b11_001_010, 0x77]);
@@ -2676,9 +2678,11 @@ fn test_vex() {
     test_instr(&[0xc4, 0xc3, 0x75, 0x4a, 0x7c, 0x12, 0x05, 0x61], "vblendvps ymm7, ymm1, ymmword [r10 + rdx * 1 + 0x5], ymm6");
     test_instr(&[0xc4, 0xc3, 0x71, 0x4a, 0x7c, 0x12, 0x05, 0x61], "vblendvps xmm7, xmm1, xmmword [r10 + rdx * 1 + 0x5], xmm6");
     test_instr(&[0xc4, 0xc3, 0x71, 0x4a, 0xdc, 0x61], "vblendvps xmm3, xmm1, xmm12, xmm6");
+    test_invalid(&[0xc4, 0xc3, 0xf1, 0x4a, 0xdc, 0x61]);
     test_instr(&[0xc4, 0xc3, 0x75, 0x4b, 0x7c, 0x12, 0x05, 0x61], "vblendvpd ymm7, ymm1, ymmword [r10 + rdx * 1 + 0x5], ymm6");
     test_instr(&[0xc4, 0xc3, 0x71, 0x4b, 0x7c, 0x12, 0x05, 0x61], "vblendvpd xmm7, xmm1, xmmword [r10 + rdx * 1 + 0x5], xmm6");
     test_instr(&[0xc4, 0xc3, 0x71, 0x4b, 0xdc, 0x61], "vblendvpd xmm3, xmm1, xmm12, xmm6");
+    test_invalid(&[0xc4, 0xc3, 0xf1, 0x4b, 0xdc, 0x61]);
     test_instr(&[0xc4, 0xc3, 0x71, 0x4c, 0x7c, 0x12, 0x05, 0x61], "vpblendvb xmm7, xmm1, xmmword [r10 + rdx * 1 + 0x5], xmm6");
 
     test_instr(&[0xc5, 0xc9, 0xf1, 0x0f], "vpsllw xmm1, xmm6, xmmword [rdi]");
diff --git a/test/protected_mode/mod.rs b/test/protected_mode/mod.rs
index 87520cf..681184c 100644
--- a/test/protected_mode/mod.rs
+++ b/test/protected_mode/mod.rs
@@ -1475,9 +1475,11 @@ fn test_vex() {
     test_invalid(&[0xc4, 0b110_00011, 0b0_1111_001, 0x19, 0b11_001_010, 0x77]);
     test_invalid(&[0xc4, 0b110_00011, 0b1_1111_101, 0x19, 0b11_001_010, 0x77]);
 
-    test_invalid(&[0xc4, 0b110_00011, 0b1_0111_001, 0x18, 0b11_001_010, 0x77]);
+    test_invalid(&[0xc4, 0b110_00011, 0b1_0111_001, 0x38, 0b11_001_010, 0x77]);
+    test_invalid(&[0xc4, 0b110_00011, 0b0_0111_001, 0x38, 0b11_001_010, 0x77]);
     test_avx2(&[0xc4, 0b110_00011, 0b0_0111_101, 0x38, 0b11_001_010, 0x77], "vinserti128 ymm1, ymm0, xmm2, 0x77");
-    test_invalid(&[0xc4, 0b110_00011, 0b1_0111_101, 0x18, 0b11_001_010, 0x77]);
+    test_invalid(&[0xc4, 0b110_00011, 0b1_1111_101, 0x39, 0b11_001_010, 0x77]);
+    test_invalid(&[0xc4, 0b110_00011, 0b0_0111_101, 0x39, 0b11_001_010, 0x77]);
     test_avx2(&[0xc4, 0b110_00011, 0b0_1111_101, 0x39, 0b11_001_010, 0x77], "vextracti128 xmm2, ymm1, 0x77");
     test_invalid(&[0xc4, 0b110_00011, 0b0_1111_001, 0x19, 0b11_001_010, 0x77]);
     test_invalid(&[0xc4, 0b110_00011, 0b1_1111_101, 0x19, 0b11_001_010, 0x77]);
diff --git a/test/real_mode/mod.rs b/test/real_mode/mod.rs
index c81a8cd..72ccede 100644
--- a/test/real_mode/mod.rs
+++ b/test/real_mode/mod.rs
@@ -18291,6 +18291,7 @@ fn test_invalid_sequences() {
     test_invalid(&[0xc4, 0b110_00011, 0b1_1111_101, 0x02, 0b11_001_010, 0x77]);
     test_invalid(&[0xc4, 0b110_00011, 0b1_1111_101, 0x19, 0b11_001_010, 0x77]);
     test_invalid(&[0xc4, 0b110_00011, 0b1_1111_101, 0x1d, 0b11_001_010, 0x77]);
+    test_invalid(&[0xc4, 0b110_00011, 0b1_1111_101, 0x39, 0b11_001_010, 0x77]);
     test_invalid(&[0xc4, 0b110_00011, 0b1_1111_101, 0x46, 0b11_001_010, 0x77]);
     test_invalid(&[0xc4, 0b111_00010, 0b1_1111_001, 0x13, 0b11_001_010]);
     test_invalid(&[0xc4, 0xe2, 0xf9, 0x58, 0xc1]);