port long-mode decoder updates to protected-mode

2 files changed, 154 insertions, 25 deletions

diff --git a/test/protected_mode/display.rs b/test/protected_mode/display.rs
new file mode 100644
index 0000000..8426a0a
--- /dev/null
+++ b/test/protected_mode/display.rs
@@ -0,0 +1,131 @@
+use std::fmt::Write;
+
+use yaxpeax_arch::{AddressBase, Decoder, LengthedInstruction};
+use yaxpeax_x86::protected_mode::{DisplayStyle, InstDecoder};
+
+fn test_display(data: &[u8], expected: &'static str) {
+    test_display_under(&InstDecoder::default(), data, expected);
+}
+
+fn test_display_under(decoder: &InstDecoder, data: &[u8], expected: &'static str) {
+    let mut hex = String::new();
+    for b in data {
+        write!(hex, "{:02x}", b).unwrap();
+    }
+    match decoder.decode(data.into_iter().map(|x| *x)) {
+        Ok(instr) => {
+            let text = format!("{}", instr.display_with(DisplayStyle::C));
+            assert!(
+                text == expected,
+                "display error for {}:\n  decoded: {:?} under decoder {}\n displayed: {}\n expected: {}\n",
+                hex,
+                instr,
+                decoder,
+                text,
+                expected
+            );
+            // while we're at it, test that the instruction is as long, and no longer, than its
+            // input
+            assert_eq!((0u32.wrapping_offset(instr.len()).to_linear()) as usize, data.len(), "instruction length is incorrect, wanted instruction {}", expected);
+        },
+        Err(e) => {
+            assert!(false, "decode error ({}) for {} under decoder {}:\n  expected: {}\n", e, hex, decoder, expected);
+        }
+    }
+}
+
+// decided i do not like at&t syntax much at all. not going to write a formatter for it. some test
+// cases will live on in case someone else feels like adding one, or i get mad enough to do it.
+#[ignore]
+#[test]
+fn test_instructions_atnt() {
+    // just modrm
+    test_display(&[0x33, 0x08], "xor (%eax), %ecx");
+    test_display(&[0x33, 0x20], "xor (%eax), %esp");
+    test_display(&[0x33, 0x05, 0x78, 0x56, 0x34, 0x12], "xor (0x12345678), %eax");
+    test_display(&[0x33, 0x41, 0x23], "xor 0x23(%ecx), %eax");
+    test_display(&[0x33, 0x81, 0x23, 0x01, 0x65, 0x43], "xor %0x43650123, %eax");
+    test_display(&[0x33, 0xc1], "xor %ecx, %eax");
+
+    // sib
+    test_display(&[0x33, 0x04, 0x25, 0x11, 0x22, 0x33, 0x44], "xor (0x44332211), %eax");
+    test_display(&[0x41, 0x33, 0x04, 0x25, 0x11, 0x22, 0x33, 0x44], "xor (0x44332211), %eax");
+
+    test_display(&[0x33, 0x44, 0x65, 0x11], "xor 0x11(%r13), %eax");
+
+    test_display(&[0x33, 0x34, 0x25, 0x20, 0x30, 0x40, 0x50], "xor 0x50403020, %esi");
+
+    test_display(&[0x0f, 0xe7, 0x03], "movntq %mm0, (%ebx)");
+
+    test_display(&[0x0f, 0x7f, 0x0f], "movq %mm1, (%edi)");
+    test_display(&[0x0f, 0xc4, 0xc0, 0x14], "pinsrw $0x14, %eax, %mm0");
+
+    test_display(&[0x0f, 0xd1, 0x00], "psrlw (%eax), %mm0");
+    test_display(&[0x0f, 0xe5, 0x3d, 0xaa, 0xbb, 0xcc, 0x77], "pmulhw 0x77ccbbaa, %mm7");
+}
+
+#[test]
+fn test_instructions_c() {
+    // just modrm
+    test_display(&[0x33, 0x08], "ecx ^= [eax]");
+    test_display(&[0x33, 0x20], "esp ^= [eax]");
+    test_display(&[0x33, 0x05, 0x78, 0x56, 0x34, 0x12], "eax ^= [0x12345678]");
+    test_display(&[0x33, 0x41, 0x23], "eax ^= [ecx + 0x23]");
+    test_display(&[0x33, 0x81, 0x23, 0x01, 0x65, 0x43], "eax ^= [ecx + 0x43650123]");
+    test_display(&[0x33, 0xc1], "eax ^= ecx");
+
+    // sib
+    test_display(&[0x33, 0x04, 0x25, 0x11, 0x22, 0x33, 0x44], "eax ^= [0x44332211]");
+
+    test_display(&[0x33, 0x44, 0x65, 0x11], "eax ^= [ebp + 0x11]");
+
+    test_display(&[0x33, 0x34, 0x25, 0x20, 0x30, 0x40, 0x50], "esi ^= [0x50403020]");
+
+    test_display(&[0x0f, 0xe7, 0x03], "[ebx] = movntq(mm0)");
+
+    test_display(&[0x0f, 0x7f, 0x0f], "[edi] = movq(mm1)");
+    test_display(&[0x0f, 0xc4, 0xc0, 0x14], "mm0 = pinsrw(mm0, eax, 0x14)");
+
+    test_display(&[0x0f, 0xd1, 0x00], "mm0 = psrlw(mm0, [eax])");
+    test_display(&[0x0f, 0xe5, 0x3d, 0xaa, 0xbb, 0xcc, 0x77], "mm7 = pmulhw(mm7, [0x77ccbbaa])");
+
+    test_display(&[0xf3, 0xa5], "rep dword { es:[edi++] = ds:[esi++] }");
+    test_display(&[0xf3, 0x66, 0xa5], "rep word { es:[edi++] = ds:[esi++] }");
+    test_display(&[0xf3, 0xa4], "rep byte { es:[edi++] = ds:[esi++] }");
+
+    test_display(&[0xf6, 0xc2, 0x18], "eflags = flags(dl & 0x18)");
+    test_display(&[0xf6, 0xc2, 0x18], "eflags = flags(dl & 0x18)");
+    test_display(&[0x84, 0xc0], "eflags = flags(al & al)");
+    test_display(&[0x85, 0xc0], "eflags = flags(eax & eax)");
+    test_display(&[0x3a, 0xc0], "eflags = flags(al - al)");
+    test_display(&[0x3b, 0xc0], "eflags = flags(eax - eax)");
+
+    test_display(&[0x0f, 0xbc, 0xd3], "edx = lsb(ebx) (x86 bsf)");
+    test_display(&[0xf3, 0x0f, 0xbc, 0xd3], "edx = lsb(ebx)");
+    // test_display(&[0x41, 0x0f, 0xbc, 0xd3], "edx = lsb(ebx) (x86 bsf"); // for non-bm1
+    test_display(&[0x0f, 0xbd, 0xd3], "edx = msb(ebx)");
+    // test_display(&[0x41, 0x0f, 0xbc, 0xd3], "edx = lsb(ebx) (x86 bsr"); // for non-bm1
+    test_display(&[0xd2, 0xc0], "al = al rol cl");
+    test_display(&[0xd2, 0xc8], "al = al ror cl");
+    test_display(&[0xd2, 0xd0], "al = al rcl cl");
+    test_display(&[0xd2, 0xd8], "al = al rcr cl");
+    test_display(&[0xd2, 0xe0], "al = al << cl");
+    test_display(&[0xd2, 0xe8], "al = al >> cl");
+    test_display(&[0xd2, 0xf0], "al = al <<< cl");
+    test_display(&[0xd2, 0xf8], "al = al >>> cl");
+
+    test_display(&[0xc4, 0xc3, 0x7b, 0xf0, 0x01, 0x05], "eax = [ecx] ror 0x5 (x86 rorx)");
+    test_display(&[0xc4, 0xc2, 0xe3, 0xf7, 0x01], "eax = [ecx] >> ebx (x86 shrx)");
+    test_display(&[0xc4, 0xc2, 0xe1, 0xf7, 0x01], "eax = [ecx] << ebx (x86 shlx)");
+
+    test_display(&[0xd2, 0xe0], "al = al << cl");
+
+    test_display(&[0x66, 0x0f, 0xac, 0xcf, 0x11], "di = shrd(di, cx, 0x11)");
+    test_display(&[0x0f, 0xa5, 0xc9], "ecx = shld(ecx, ecx, cl)");
+
+    test_display(&[0x66, 0x0f, 0x38, 0xf6, 0x01], "eax += [ecx] + eflags.cf");
+
+    test_display(&[0xfe, 0x00], "byte [eax]++");
+    test_display(&[0x66, 0xff, 0x08], "word [eax]--");
+    test_display(&[0xff, 0x00], "dword [eax]++");
+}
diff --git a/test/protected_mode/mod.rs b/test/protected_mode/mod.rs
index 98fc8b3..a29b795 100644
--- a/test/protected_mode/mod.rs
+++ b/test/protected_mode/mod.rs
@@ -189,7 +189,6 @@ fn test_aesni() {
 
     test_instr(&[0x66, 0x0f, 0x38, 0xde, 0x0f], "aesdec xmm1, [edi]");
     test_instr(&[0x67, 0x66, 0x0f, 0x38, 0xde, 0x0f], "aesdec xmm1, [bx]");
-    test_invalid(&[0x66, 0x67, 0x0f, 0x38, 0xde, 0x0f]);
 
     test_instr(&[0x66, 0x0f, 0x38, 0xdf, 0x0f], "aesdeclast xmm1, [edi]");
     test_instr(&[0x67, 0x66, 0x0f, 0x38, 0xdf, 0x0f], "aesdeclast xmm1, [bx]");
@@ -377,7 +376,6 @@ fn test_sse2() {
     test_instr(&[0x66, 0x0f, 0xc4, 0x03, 0x08], "pinsrw xmm0, [ebx], 0x8");
 
 //    test_instr(&[0x66, 0x0f, 0xc5, 0xc3, 0x08], "pextrw eax, xmm3, 0x8");
-//    test_instr(&[0x66, 0x4f, 0x0f, 0xc5, 0xc3, 0x08], "pextrw eax, xmm11, 0x8");
 //    test_instr_invalid(&[0x66, 0x0f, 0xc5, 0x03, 0x08]);
 //    test_instr_invalid(&[0x66, 0x0f, 0xc5, 0x40, 0x08]);
 //    test_instr_invalid(&[0x66, 0x0f, 0xc5, 0x80, 0x08]);
@@ -1066,7 +1064,7 @@ fn test_prefixes() {
     test_display(&[0xf0, 0x31, 0x00], "lock xor [eax], eax");
     test_display(&[0xf0, 0x80, 0x30, 0x00], "lock xor [eax], 0x0");
     test_display(&[0xf0, 0x0f, 0xbb, 0x17], "lock btc [edi], edx");
-    test_display(&[0x66, 0x2e, 0xf2, 0xf0, 0x0f, 0xbb, 0x13], "lock btc [ebx], dx");
+    test_display(&[0x66, 0x2e, 0xf2, 0xf0, 0x0f, 0xbb, 0x13], "lock btc cs:[ebx], dx");
     test_invalid(&[0xf0, 0xc7, 0x00, 0x00, 0x00, 0x00]);
     test_display(&[0x0f, 0xc1, 0xcc], "xadd esp, ecx");
     test_display(&[0x66, 0x0f, 0xc1, 0xcc], "xadd sp, cx");
@@ -1447,7 +1445,7 @@ fn test_vex() {
     test_instr(&[0xc5, 0xe1, 0x55, 0x03], "vandnps xmm0, xmm3, [ebx]");
     test_instr(&[0xc5, 0xe0, 0x56, 0x03], "vorpd xmm0, xmm3, [ebx]");
     test_instr(&[0xc5, 0xe1, 0x56, 0x03], "vorps xmm0, xmm3, [ebx]");
-    test_instr(&[0xc4, 0xa2, 0x15, 0x3e, 0x14, 0xb9], "vpmaxuw ymm2, ymm13, [ecx + edi * 4]");
+    test_instr(&[0xc4, 0xe2, 0x15, 0x3e, 0x14, 0xb9], "vpmaxuw ymm2, ymm5, [ecx + edi * 4]");
 }
 
 #[test]
@@ -1494,7 +1492,7 @@ fn prefixed_0f() {
     test_display(&[0x0f, 0x1d, 0x20], "nop [eax]");
     test_display(&[0x0f, 0x1e, 0x20], "nop [eax]");
     test_display(&[0x0f, 0x1f, 0x20], "nop [eax]");
-    test_display(&[0x0f, 0x20, 0xc8], "mov eax, cr0");
+    test_display(&[0x0f, 0x20, 0xc0], "mov eax, cr0");
     test_invalid(&[0x0f, 0x20, 0xc8]);
     test_display(&[0x0f, 0x21, 0xc8], "mov eax, dr1");
     test_display(&[0x0f, 0x22, 0xc0], "mov cr0, eax");
@@ -1581,7 +1579,7 @@ fn prefixed_660f() {
 #[test]
 fn prefixed_f20f() {
     test_invalid(&[0xf2, 0x0f, 0x16, 0xcf]);
-    test_invalid(&[0x40, 0x66, 0xf2, 0x66, 0x4d, 0x0f, 0x16, 0xcf]);
+    test_invalid(&[0x66, 0xf2, 0x66, 0x0f, 0x16, 0xcf]);
 }
 
 #[test]
@@ -2119,27 +2117,27 @@ fn test_mishegos_finds() {
     // impossible instruction if operands could be read: lock is illegal here.
     // test_display(&[f06565f2640f16], "???");
 //    test_display(&[0x0f, 0x38, 0xf6, 0x8c, 0x98, 0x4d, 0x33, 0xf5, 0xd3, ], "wrssd");
-    test_display(&[0x26, 0x36, 0x0f, 0x0f, 0x70, 0xfb, 0x0c], "pi2fw mm6, [eax - 0x5]");
+    test_display(&[0x26, 0x36, 0x0f, 0x0f, 0x70, 0xfb, 0x0c], "pi2fw mm6, ss:[eax - 0x5]");
     test_display(&[0x0f, 0xc7, 0x0f], "cmpxchg8b [edi]");
-    test_display(&[0x66, 0x3e, 0x26, 0x2e, 0x2e, 0x0f, 0x38, 0x2a, 0x2b, ], "movntdqa xmm5, [ebx]");
-    test_display(&[0x66, 0x2e, 0x67, 0x0f, 0x3a, 0x0d, 0xb8, 0xf0, 0x2f, 0x7c, 0xf0, 0x63, ], "blendpd xmm7, [ax - 0xf83d010], 0x63");
-    test_display(&[0x66, 0x66, 0x64, 0x3e, 0x0f, 0x38, 0x23, 0x9d, 0x69, 0x0f, 0xa8, 0x2d, ], "pmovsxwd xmm3, fs:[ebp + 0x2da80f69]");
-    test_display(&[0x2e, 0x66, 0x26, 0x64, 0x0f, 0x3a, 0x21, 0x0b, 0xb1, ], "insertps xmm1, fs:[ecx], -0x4f");
-    test_display(&[0x66, 0x26, 0x0f, 0x3a, 0x42, 0x96, 0x74, 0x29, 0x96, 0xf9, 0x6a], "mpsadbw xmm10, [esi - 0x669d68c], 0x6a");
-    test_display(&[0x67, 0x26, 0x66, 0x65, 0x0f, 0x38, 0x3f, 0x9d, 0xcc, 0x03, 0xb3, 0xfa], "pmaxud xmm3, gs:[ebp - 0x54cfc34]");
-    test_display(&[0x36, 0x36, 0x2e, 0x0f, 0x38, 0xf9, 0x55, 0x3e, ], "movdiri [ebp + 0x3e], edx");
+    test_display(&[0x66, 0x3e, 0x26, 0x2e, 0x2e, 0x0f, 0x38, 0x2a, 0x2b], "movntdqa xmm5, cs:[ebx]");
+    test_display(&[0x66, 0x2e, 0x67, 0x0f, 0x3a, 0x0d, 0xb8, 0xf0, 0x2f, 0x7c], "blendpd xmm7, cs:[bx + si + 0x2ff0], 0x7c");
+    test_display(&[0x66, 0x66, 0x64, 0x3e, 0x0f, 0x38, 0x23, 0x9d, 0x69, 0x0f, 0xa8, 0x2d], "pmovsxwd xmm3, [ebp + 0x2da80f69]");
+    test_display(&[0x2e, 0x66, 0x26, 0x64, 0x0f, 0x3a, 0x21, 0x0b, 0xb1], "insertps xmm1, fs:[ebx], -0x4f");
+    test_display(&[0x66, 0x26, 0x0f, 0x3a, 0x42, 0x96, 0x74, 0x29, 0x96, 0xf9, 0x6a], "mpsadbw xmm2, es:[esi - 0x669d68c], 0x6a");
+    test_display(&[0x67, 0x26, 0x66, 0x65, 0x0f, 0x38, 0x3f, 0x9d, 0xcc, 0x03], "pmaxud xmm3, gs:[di + 0x3cc]");
+    test_display(&[0x36, 0x36, 0x2e, 0x0f, 0x38, 0xf9, 0x55, 0x3e], "movdiri cs:[ebp + 0x3e], edx");
     test_invalid(&[0x66, 0x2e, 0x64, 0x66, 0x0f, 0x38, 0xf8, 0xe2]);
     test_display(&[0x67, 0x66, 0x65, 0x3e, 0x0f, 0x6d, 0xd1], "punpckhqdq xmm2, xmm1");
-    test_display(&[0x2e, 0x66, 0x0f, 0x3a, 0x0d, 0x40, 0x2d, 0x57], "blendpd xmm0, [eax + 0x2d], 0x57");
-    test_display(&[0xf2, 0x3e, 0x26, 0x67, 0x0f, 0xf0, 0xa0, 0x1b, 0x5f, 0xcd, 0xd7], "lddqu xmm4, [ax - 0x2832a0e5]");
+    test_display(&[0x2e, 0x66, 0x0f, 0x3a, 0x0d, 0x40, 0x2d, 0x57], "blendpd xmm0, cs:[eax + 0x2d], 0x57");
+    test_display(&[0xf2, 0x3e, 0x26, 0x67, 0x0f, 0xf0, 0xa0, 0x1b, 0x5f], "lddqu xmm4, es:[bx + si + 0x5f1b]");
     test_display(&[0x2e, 0x3e, 0x66, 0x3e, 0x0f, 0x3a, 0x41, 0x30, 0x48], "dppd xmm6, [eax], 0x48");
 
-    test_display(&[0x2e, 0x36, 0x47, 0x0f, 0x18, 0xe7], "nop r15d");
+    test_display(&[0x2e, 0x36, 0x0f, 0x18, 0xe7], "nop edi");
     test_display(&[0x65, 0xf0, 0x87, 0x0f], "lock xchg gs:[edi], ecx");
     test_display(&[0x66, 0x0f, 0x3a, 0x44, 0x88, 0xb3, 0xad, 0x26, 0x35, 0x75], "pclmulqdq xmm1, [eax + 0x3526adb3], 0x75");
     test_display(&[0x0f, 0xff, 0x6b, 0xac], "ud0 ebp, [ebx - 0x54]");
 
-    test_display(&[0xf2, 0xf2, 0x2e, 0x36, 0x0f, 0x38, 0xf8, 0x83, 0x09, 0x1c, 0x9d, 0x3f], "enqcmd eax, [ebx + 0x3f9d1c09]");
+    test_display(&[0xf2, 0xf2, 0x2e, 0x36, 0x0f, 0x38, 0xf8, 0x83, 0x09, 0x1c, 0x9d, 0x3f], "enqcmd eax, ss:[ebx + 0x3f9d1c09]");
     test_display(&[0x3e, 0x64, 0xf3, 0x64, 0x0f, 0x38, 0xf8, 0x72, 0x54], "enqcmds esi, fs:[edx + 0x54]");
     test_invalid(&[0xf3, 0x0f, 0x38, 0xf8, 0xf3]);
 
@@ -2208,8 +2206,8 @@ fn test_3dnow() {
     test_display(&[0x0f, 0x0f, 0xe0, 0x8a], "pfnacc mm4, mm0");
     test_display(&[0x0f, 0x0f, 0x38, 0x8e], "pfpnacc mm7, [eax]");
     test_display(&[0x65, 0x67, 0x65, 0x65, 0x0f, 0x0e], "femms");
-    test_display(&[0x3e, 0xf3, 0x2e, 0xf2, 0x0f, 0x0f, 0x64, 0x93, 0x93, 0xa4], "pfmax mm4, [ebx + edx * 4 - 0x6d]");
-    test_display(&[0x26, 0x36, 0x0f, 0x0f, 0x70, 0xfb, 0x0c], "pi2fw mm6, [eax - 0x5]");
+    test_display(&[0x3e, 0xf3, 0x2e, 0xf2, 0x0f, 0x0f, 0x64, 0x93, 0x93, 0xa4], "pfmax mm4, cs:[ebx + edx * 4 - 0x6d]");
+    test_display(&[0x26, 0x36, 0x0f, 0x0f, 0x70, 0xfb, 0x0c], "pi2fw mm6, ss:[eax - 0x5]");
     test_display(&[0x66, 0x0f, 0x0f, 0xc6, 0xb7], "pmulhrw mm0, mm6");
     test_display(&[0x0f, 0x0f, 0xc6, 0xb7], "pmulhrw mm0, mm6");
 }
@@ -2217,8 +2215,8 @@ fn test_3dnow() {
 // first appeared in tremont
 #[test]
 fn test_direct_stores() {
-    test_display(&[0x36, 0x36, 0x2e, 0x0f, 0x38, 0xf9, 0x55, 0x3e, ], "movdiri [ebp + 0x3e], edx");
-    test_display(&[0x36, 0x26, 0x66, 0x0f, 0x38, 0xf8, 0xad, 0x0b, 0x08, 0x29, 0x07], "movdir64b ebp, [ebp + 0x729080b]");
+    test_display(&[0x36, 0x36, 0x2e, 0x0f, 0x38, 0xf9, 0x55, 0x3e, ], "movdiri cs:[ebp + 0x3e], edx");
+    test_invalid(&[0x36, 0x26, 0x66, 0x0f, 0x38, 0xf8, 0xad, 0x0b, 0x08, 0x29, 0x07]);
 }
 
 #[test]
@@ -2243,15 +2241,15 @@ fn test_uintr() {
 // started shipping in sapphire rapids
 #[test]
 fn test_enqcmd() {
-    test_display(&[0xf2, 0xf2, 0x2e, 0x36, 0x0f, 0x38, 0xf8, 0x83, 0x09, 0x1c, 0x9d, 0x3f], "enqcmd eax, [ebx + 0x3f9d1c09]");
+    test_display(&[0xf2, 0xf2, 0x2e, 0x36, 0x0f, 0x38, 0xf8, 0x83, 0x09, 0x1c, 0x9d, 0x3f], "enqcmd eax, ss:[ebx + 0x3f9d1c09]");
     test_display(&[0x3e, 0x64, 0xf3, 0x64, 0x0f, 0x38, 0xf8, 0x72, 0x54], "enqcmds esi, fs:[edx + 0x54]");
 }
 
 #[test]
 fn test_gfni() {
     test_display(&[0x3e, 0x64, 0x64, 0x66, 0x0f, 0x3a, 0xcf, 0xba, 0x13, 0x23, 0x04, 0xba, 0x6b], "gf2p8affineinvqb xmm7, fs:[edx - 0x45fbdced], 0x6b");
-    test_display(&[0x66, 0x36, 0x0f, 0x3a, 0xce, 0x8c, 0x56, 0x9e, 0x82, 0xd1, 0xbe, 0xad], "gf2p8affineqb xmm1, [esi + edx * 2 - 0x412e7d62], 0xad");
-    test_display(&[0x66, 0x0f, 0x38, 0xcf, 0x1c, 0x54], "gf2p8mulb xmm11, [esp + edx * 2]");
+    test_display(&[0x66, 0x36, 0x0f, 0x3a, 0xce, 0x8c, 0x56, 0x9e, 0x82, 0xd1, 0xbe, 0xad], "gf2p8affineqb xmm1, ss:[esi + edx * 2 - 0x412e7d62], 0xad");
+    test_display(&[0x66, 0x0f, 0x38, 0xcf, 0x1c, 0x54], "gf2p8mulb xmm3, [esp + edx * 2]");
 }
 
 #[test]