fix seam, user-ipi, {rd,wr}{fs,gs}base instructions decoding outside 64b mode

4 files changed, 106 insertions, 77 deletions

diff --git a/CHANGELOG b/CHANGELOG
index 01beece..95851c6 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -19,6 +19,7 @@
   broadcasted-to vector length.
 * fix incorrect index-vector-register size choice for vgatherdpd. the index register is xmm, not
   depends-on-L xmm/ymm.
+* fix SEAM and {rd,wr}{fs,gs}base instructions being decoded in 32-bit and 16-bit modes.
 
 testing instruction round-tripping through `masm` found a few bugs, which are also fixed in this release:
 
diff --git a/src/protected_mode/mod.rs b/src/protected_mode/mod.rs
index 9798f2f..54c9c0a 100644
--- a/src/protected_mode/mod.rs
+++ b/src/protected_mode/mod.rs
@@ -7383,10 +7383,12 @@ fn read_operands<
                         instruction.opcode = Opcode::VMXON;
                         instruction.operands[0] = read_E(words, instruction, modrm, bank, sink)?;
                         if instruction.operands[0] == OperandSpec::RegMMM {
-                            // invalid as `vmxon`, reg-form is `senduipi`
-                            instruction.opcode = Opcode::SENDUIPI;
-                            // and the operand is always a dword register
-                            instruction.regs[1].bank = RegisterBank::D;
+                            // from the SDM:
+                            // > The SENDUIPI instruction is not recognized in protected mode.
+                            // instruction.opcode = Opcode::SENDUIPI;
+                            instruction.operands[0] = OperandSpec::Nothing;
+                            instruction.operand_count = 0;
+                            return Err(DecodeError::InvalidOpcode);
                         } else {
                             instruction.mem_size = 8;
                         }
@@ -8141,13 +8143,19 @@ fn read_operands<
                                 instruction.opcode = Opcode::TDCALL;
                             }
                             0b101 => {
-                                instruction.opcode = Opcode::SEAMRET;
+                                // from intel-tdx-cpu-architectural-specification.pdf, not supported outside long mode:
+                                // > IF inSEAM==0 or ((IA32_EFER.LMA & CS.L) == 0) or [..]
+                                return Err(DecodeError::InvalidOpcode);
                             }
                             0b110 => {
-                                instruction.opcode = Opcode::SEAMOPS;
+                                // from intel-tdx-cpu-architectural-specification.pdf, not supported outside long mode:
+                                // > IF inSEAM==0 or ((IA32_EFER.LMA & CS.L) == 0) or [..]
+                                return Err(DecodeError::InvalidOpcode);
                             }
                             0b111 => {
-                                instruction.opcode = Opcode::SEAMCALL;
+                                // from intel-tdx-cpu-architectural-specification.pdf, not supported outside long mode:
+                                // > IF inSEAM==0 or ((IA32_EFER.LMA & CS.L) == 0) or [..]
+                                return Err(DecodeError::InvalidOpcode);
                             }
                             _ => {
                                 return Err(DecodeError::InvalidOpcode);
@@ -8349,9 +8357,12 @@ fn read_operands<
                     }
                     0b100 => {
                         if instruction.prefixes.rep() {
-                            instruction.opcode = Opcode::UIRET;
+                            // from the SDM:
+                            // > The UIRET instruction is not recognized in protected mode.
+                            // instruction.opcode = Opcode::UIRET;
                             instruction.operands[0] = OperandSpec::Nothing;
                             instruction.operand_count = 0;
+                            return Err(DecodeError::InvalidOpcode);
                         } else {
                             instruction.operands[0] = OperandSpec::Nothing;
                             instruction.operand_count = 0;
@@ -8360,9 +8371,12 @@ fn read_operands<
                     }
                     0b101 => {
                         if instruction.prefixes.rep() {
-                            instruction.opcode = Opcode::TESTUI;
+                            // from the SDM:
+                            // > The TESTUI instruction is not recognized in protected mode.
+                            // instruction.opcode = Opcode::TESTUI;
                             instruction.operands[0] = OperandSpec::Nothing;
                             instruction.operand_count = 0;
+                            return Err(DecodeError::InvalidOpcode);
                         } else {
                             instruction.operands[0] = OperandSpec::Nothing;
                             instruction.operand_count = 0;
@@ -8371,10 +8385,12 @@ fn read_operands<
                     }
                     0b110 => {
                         if instruction.prefixes.rep() {
-                            instruction.opcode = Opcode::CLUI;
+                            // from the SDM:
+                            // > The CLUI instruction is not recognized in protected mode.
+                            // instruction.opcode = Opcode::CLUI;
                             instruction.operands[0] = OperandSpec::Nothing;
                             instruction.operand_count = 0;
-                            return Ok(());
+                            return Err(DecodeError::InvalidOpcode);
                         } else if instruction.prefixes.operand_size() || instruction.prefixes.repnz() {
                             return Err(DecodeError::InvalidOpcode);
                         }
@@ -8384,10 +8400,12 @@ fn read_operands<
                     }
                     0b111 => {
                         if instruction.prefixes.rep() {
-                            instruction.opcode = Opcode::STUI;
+                            // from the SDM:
+                            // > The STUI instruction is not recognized in protected mode.
+                            // instruction.opcode = Opcode::STUI;
                             instruction.operands[0] = OperandSpec::Nothing;
                             instruction.operand_count = 0;
-                            return Ok(());
+                            return Err(DecodeError::InvalidOpcode);
                         } else if instruction.prefixes.operand_size() || instruction.prefixes.repnz() {
                             return Err(DecodeError::InvalidOpcode);
                         }
@@ -8571,29 +8589,24 @@ fn read_operands<
                 if (modrm & 0xc0) == 0xc0 {
                     match r {
                         0 => {
-                            instruction.opcode = Opcode::RDFSBASE;
-                            instruction.regs[1] = RegSpec::from_parts(m, RegisterBank::D);
-                            instruction.operands[0] = OperandSpec::RegMMM;
-                            instruction.operand_count = 1;
+                            // from the SDM:
+                            // > The RDFSBASE and RDGSBASE instructions are not recognized in protected mode.
+                            return Err(DecodeError::InvalidOpcode);
                         }
                         1 => {
-                            instruction.opcode = Opcode::RDGSBASE;
-                            instruction.regs[1] = RegSpec::from_parts(m, RegisterBank::D);
-                            instruction.operands[0] = OperandSpec::RegMMM;
-                            instruction.operand_count = 1;
-
+                            // from the SDM:
+                            // > The RDFSBASE and RDGSBASE instructions are not recognized in protected mode.
+                            return Err(DecodeError::InvalidOpcode);
                         }
                         2 => {
-                            instruction.opcode = Opcode::WRFSBASE;
-                            instruction.regs[1] = RegSpec::from_parts(m, RegisterBank::D);
-                            instruction.operands[0] = OperandSpec::RegMMM;
-                            instruction.operand_count = 1;
+                            // from the SDM:
+                            // > The RDFSBASE and RDGSBASE instructions are not recognized in protected mode.
+                            return Err(DecodeError::InvalidOpcode);
                         }
                         3 => {
-                            instruction.opcode = Opcode::WRGSBASE;
-                            instruction.regs[1] = RegSpec::from_parts(m, RegisterBank::D);
-                            instruction.operands[0] = OperandSpec::RegMMM;
-                            instruction.operand_count = 1;
+                            // from the SDM:
+                            // > The RDFSBASE and RDGSBASE instructions are not recognized in protected mode.
+                            return Err(DecodeError::InvalidOpcode);
                         }
                         5 => {
                             instruction.opcode = Opcode::INCSSP;
diff --git a/src/real_mode/mod.rs b/src/real_mode/mod.rs
index e66a4b8..6e7aaa4 100644
--- a/src/real_mode/mod.rs
+++ b/src/real_mode/mod.rs
@@ -7425,10 +7425,12 @@ fn read_operands<
                         instruction.opcode = Opcode::VMXON;
                         instruction.operands[0] = read_E(words, instruction, modrm, bank, sink)?;
                         if instruction.operands[0] == OperandSpec::RegMMM {
-                            // invalid as `vmxon`, reg-form is `senduipi`
-                            instruction.opcode = Opcode::SENDUIPI;
-                            // and the operand is always a dword register
-                            instruction.regs[1].bank = RegisterBank::D;
+                            // from the SDM:
+                            // > The SENDUIPI instruction is not recognized in real-address mode.
+                            // instruction.opcode = Opcode::SENDUIPI;
+                            instruction.operands[0] = OperandSpec::Nothing;
+                            instruction.operand_count = 0;
+                            return Err(DecodeError::InvalidOpcode);
                         } else {
                             instruction.mem_size = 8;
                         }
@@ -8189,13 +8191,19 @@ fn read_operands<
                                 instruction.opcode = Opcode::TDCALL;
                             }
                             0b101 => {
-                                instruction.opcode = Opcode::SEAMRET;
+                                // from intel-tdx-cpu-architectural-specification.pdf, not supported outside long mode:
+                                // > IF inSEAM==0 or ((IA32_EFER.LMA & CS.L) == 0) or [..]
+                                return Err(DecodeError::InvalidOpcode);
                             }
                             0b110 => {
-                                instruction.opcode = Opcode::SEAMOPS;
+                                // from intel-tdx-cpu-architectural-specification.pdf, not supported outside long mode:
+                                // > IF inSEAM==0 or ((IA32_EFER.LMA & CS.L) == 0) or [..]
+                                return Err(DecodeError::InvalidOpcode);
                             }
                             0b111 => {
-                                instruction.opcode = Opcode::SEAMCALL;
+                                // from intel-tdx-cpu-architectural-specification.pdf, not supported outside long mode:
+                                // > IF inSEAM==0 or ((IA32_EFER.LMA & CS.L) == 0) or [..]
+                                return Err(DecodeError::InvalidOpcode);
                             }
                             _ => {
                                 return Err(DecodeError::InvalidOpcode);
@@ -8394,9 +8402,12 @@ fn read_operands<
                     }
                     0b100 => {
                         if instruction.prefixes.rep() {
-                            instruction.opcode = Opcode::UIRET;
+                            // from the SDM:
+                            // > The UIRET instruction is not recognized in real-address mode.
+                            // instruction.opcode = Opcode::UIRET;
                             instruction.operands[0] = OperandSpec::Nothing;
                             instruction.operand_count = 0;
+                            return Err(DecodeError::InvalidOpcode);
                         } else {
                             instruction.operands[0] = OperandSpec::Nothing;
                             instruction.operand_count = 0;
@@ -8405,9 +8416,12 @@ fn read_operands<
                     }
                     0b101 => {
                         if instruction.prefixes.rep() {
-                            instruction.opcode = Opcode::TESTUI;
+                            // from the SDM:
+                            // > The UIRET instruction is not recognized in real-address mode.
+                            // instruction.opcode = Opcode::TESTUI;
                             instruction.operands[0] = OperandSpec::Nothing;
                             instruction.operand_count = 0;
+                            return Err(DecodeError::InvalidOpcode);
                         } else {
                             instruction.operands[0] = OperandSpec::Nothing;
                             instruction.operand_count = 0;
@@ -8416,10 +8430,12 @@ fn read_operands<
                     }
                     0b110 => {
                         if instruction.prefixes.rep() {
-                            instruction.opcode = Opcode::CLUI;
+                            // from the SDM:
+                            // > The CLUI instruction is not recognized in real-address mode.
+                            // instruction.opcode = Opcode::CLUI;
                             instruction.operands[0] = OperandSpec::Nothing;
                             instruction.operand_count = 0;
-                            return Ok(());
+                            return Err(DecodeError::InvalidOpcode);
                         } else if instruction.prefixes.operand_size() || instruction.prefixes.repnz() {
                             return Err(DecodeError::InvalidOpcode);
                         }
@@ -8429,10 +8445,12 @@ fn read_operands<
                     }
                     0b111 => {
                         if instruction.prefixes.rep() {
-                            instruction.opcode = Opcode::STUI;
+                            // from the SDM:
+                            // > The STUI instruction is not recognized in real-address mode.
+                            // instruction.opcode = Opcode::STUI;
                             instruction.operands[0] = OperandSpec::Nothing;
                             instruction.operand_count = 0;
-                            return Ok(());
+                            return Err(DecodeError::InvalidOpcode);
                         } else if instruction.prefixes.operand_size() || instruction.prefixes.repnz() {
                             return Err(DecodeError::InvalidOpcode);
                         }
@@ -8616,29 +8634,24 @@ fn read_operands<
                 if (modrm & 0xc0) == 0xc0 {
                     match r {
                         0 => {
-                            instruction.opcode = Opcode::RDFSBASE;
-                            instruction.regs[1] = RegSpec::from_parts(m, RegisterBank::D);
-                            instruction.operands[0] = OperandSpec::RegMMM;
-                            instruction.operand_count = 1;
+                            // from the SDM:
+                            // > The RDFSBASE and RDGSBASE instructions are not recognized in real-address mode.
+                            return Err(DecodeError::InvalidOpcode);
                         }
                         1 => {
-                            instruction.opcode = Opcode::RDGSBASE;
-                            instruction.regs[1] = RegSpec::from_parts(m, RegisterBank::D);
-                            instruction.operands[0] = OperandSpec::RegMMM;
-                            instruction.operand_count = 1;
-
+                            // from the SDM:
+                            // > The RDFSBASE and RDGSBASE instructions are not recognized in real-address mode.
+                            return Err(DecodeError::InvalidOpcode);
                         }
                         2 => {
-                            instruction.opcode = Opcode::WRFSBASE;
-                            instruction.regs[1] = RegSpec::from_parts(m, RegisterBank::D);
-                            instruction.operands[0] = OperandSpec::RegMMM;
-                            instruction.operand_count = 1;
+                            // from the SDM:
+                            // > The WRFSBASE and WRGSBASE instructions are not recognized in real-address mode.
+                            return Err(DecodeError::InvalidOpcode);
                         }
                         3 => {
-                            instruction.opcode = Opcode::WRGSBASE;
-                            instruction.regs[1] = RegSpec::from_parts(m, RegisterBank::D);
-                            instruction.operands[0] = OperandSpec::RegMMM;
-                            instruction.operand_count = 1;
+                            // from the SDM:
+                            // > The WRFSBASE and WRGSBASE instructions are not recognized in real-address mode.
+                            return Err(DecodeError::InvalidOpcode);
                         }
                         5 => {
                             instruction.opcode = Opcode::INCSSP;
diff --git a/test/protected_mode/mod.rs b/test/protected_mode/mod.rs
index 0d3dd07..be16635 100644
--- a/test/protected_mode/mod.rs
+++ b/test/protected_mode/mod.rs
@@ -2356,10 +2356,10 @@ mod misc {
         testcase!(&[0xf3, 0x0f, 0xae, 0x26], "ptwrite dword [esi]"),
         testcase!(&[0xf3, 0x0f, 0xae, 0xe6], "ptwrite esi"),
         testcase!(invalid: &[0x66, 0xf3, 0x0f, 0xae, 0xe6]),
-        testcase!(&[0xf3, 0x0f, 0xae, 0xc4], "rdfsbase esp"),
-        testcase!(&[0xf3, 0x0f, 0xae, 0xcc], "rdgsbase esp"),
-        testcase!(&[0xf3, 0x0f, 0xae, 0xd4], "wrfsbase esp"),
-        testcase!(&[0xf3, 0x0f, 0xae, 0xdc], "wrgsbase esp"),
+        testcase!(invalid: &[0xf3, 0x0f, 0xae, 0xc4]), // "rdfsbase esp", > The RDFSBASE and RDGSBASE instructions are not recognized in protected mode.
+        testcase!(invalid: &[0xf3, 0x0f, 0xae, 0xcc]), // "rdgsbase esp", > The RDFSBASE and RDGSBASE instructions are not recognized in protected mode.
+        testcase!(invalid: &[0xf3, 0x0f, 0xae, 0xd4]), // "wrfsbase esp", > The WRFSBASE and WRGSBASE instructions are not recognized in protected mode.
+        testcase!(invalid: &[0xf3, 0x0f, 0xae, 0xdc]), // "wrgsbase esp", > The WRFSBASE and WRGSBASE instructions are not recognized in protected mode.
         testcase!(&[0x66, 0x0f, 0xae, 0x3f], "clflushopt zmmword [edi]"), // or clflush without 66
         testcase!(invalid: &[0x66, 0x0f, 0xae, 0xff]),
         testcase!(&[0x66, 0x0f, 0xae, 0x37], "clwb zmmword [edi]"),
@@ -3362,10 +3362,10 @@ mod vex {
         testcase!(features { AVX: true } &[0xc5, 0xf8, 0x10, 0x01], "vmovups xmm0, xmmword [ecx]"),
     ];
 
-        #[test]
-        fn test() {
-            run_test(CASES);
-        }
+    #[test]
+    fn test() {
+        run_test(CASES);
+    }
 }
 
 mod strange_prefixing {
@@ -4376,16 +4376,18 @@ mod key_locker {
 
 // these uinter test cases come from llvm:
 // https://reviews.llvm.org/differential/changeset/?ref=2226860
+//
+// as it turns out, UINTR is not supported at all in protected or real mode.
 mod uintr {
     use crate::protected_mode::{TestCase, run_test};
 
     const CASES: &'static [TestCase] = &[
-        testcase!(&[0xf3, 0x0f, 0x01, 0xec], "uiret"),
-        testcase!(&[0xf3, 0x0f, 0x01, 0xed], "testui"),
-        testcase!(&[0xf3, 0x0f, 0x01, 0xee], "clui"),
-        testcase!(&[0xf3, 0x0f, 0x01, 0xef], "stui"),
-        testcase!(&[0xf3, 0x0f, 0xc7, 0xf0], "senduipi eax"),
-        testcase!(&[0xf3, 0x0f, 0xc7, 0xf2], "senduipi edx"),
+        testcase!(invalid: &[0xf3, 0x0f, 0x01, 0xec]), // "uiret"
+        testcase!(invalid: &[0xf3, 0x0f, 0x01, 0xed]), // "testui"
+        testcase!(invalid: &[0xf3, 0x0f, 0x01, 0xee]), // "clui"
+        testcase!(invalid: &[0xf3, 0x0f, 0x01, 0xef]), // "stui"
+        testcase!(invalid: &[0xf3, 0x0f, 0xc7, 0xf0]), // "senduipi eax"
+        testcase!(invalid: &[0xf3, 0x0f, 0xc7, 0xf2]), // "senduipi edx"
     ];
 
     #[test]
@@ -4429,9 +4431,9 @@ mod tdx {
 
     const CASES: &'static [TestCase] = &[
         testcase!(&[0x66, 0x0f, 0x01, 0xcc], "tdcall"),
-        testcase!(&[0x66, 0x0f, 0x01, 0xcd], "seamret"),
-        testcase!(&[0x66, 0x0f, 0x01, 0xce], "seamops"),
-        testcase!(&[0x66, 0x0f, 0x01, 0xcf], "seamcall"),
+        testcase!(invalid: &[0x66, 0x0f, 0x01, 0xcd]), // "seamret" is long-mode only
+        testcase!(invalid: &[0x66, 0x0f, 0x01, 0xce]), // "seamops" is long-mode only
+        testcase!(invalid: &[0x66, 0x0f, 0x01, 0xcf]), // "seamcall" is long-mode only
     ];
 
     #[test]