test that invalid RegSpec constructions panic as expected

in the process, fix 64-bit rex-byte limit, 32/16-bit mode mask reg limit

7 files changed, 176 insertions, 3 deletions

diff --git a/CHANGELOG b/CHANGELOG
index 73af131..3d65e60 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,8 @@
+## 1.1.3
+* fix reachable unsoundness via `RegSpec` helper functions
+  - helpers should only permit creating valid `RegSpec` structs, but three helpers would permit out-of-range register numbers
+  - when displaying an invalid `RegSpec`, for some out-of-range mask registers, the displayed register name could be chosen as arbitrary const data interpreted as a pointer/length pair
+
 ## 1.1.2
 * fix panic when evex instructions with compressed displacements are decoded in
   debug builds
diff --git a/src/long_mode/mod.rs b/src/long_mode/mod.rs
index 866eb8f..7e8dff7 100644
--- a/src/long_mode/mod.rs
+++ b/src/long_mode/mod.rs
@@ -232,7 +232,7 @@ impl RegSpec {
     /// construct a `RegSpec` for non-rex byte reg `num`
     #[inline]
     pub fn rb(num: u8) -> RegSpec {
-        if num >= 18 {
+        if num >= 16 {
             panic!("invalid x86 rex-byte reg {}", num);
         }
 
diff --git a/src/protected_mode/mod.rs b/src/protected_mode/mod.rs
index 232284d..9c03faf 100644
--- a/src/protected_mode/mod.rs
+++ b/src/protected_mode/mod.rs
@@ -179,7 +179,7 @@ impl RegSpec {
     /// construct a `RegSpec` for mask reg `num`
     #[inline]
     pub fn mask(num: u8) -> RegSpec {
-        if num >= 32 {
+        if num >= 8 {
             panic!("invalid x86 mask reg {}", num);
         }
 
diff --git a/src/real_mode/mod.rs b/src/real_mode/mod.rs
index 92c0592..74e1c37 100644
--- a/src/real_mode/mod.rs
+++ b/src/real_mode/mod.rs
@@ -179,7 +179,7 @@ impl RegSpec {
     /// construct a `RegSpec` for mask reg `num`
     #[inline]
     pub fn mask(num: u8) -> RegSpec {
-        if num >= 32 {
+        if num >= 8 {
             panic!("invalid x86 mask reg {}", num);
         }
 
diff --git a/test/long_mode/mod.rs b/test/long_mode/mod.rs
index d23df49..0bef224 100644
--- a/test/long_mode/mod.rs
+++ b/test/long_mode/mod.rs
@@ -3397,3 +3397,67 @@ fn from_reports() {
     test_display(&[0x62, 0xf2, 0x6d, 0xac, 0x00, 0x59, 0xa7], "vpshufb ymm3{k4}{z}, ymm2, ymmword [rcx - 0xb20]");
     test_display(&[0x62, 0xf2, 0xfd, 0x0f, 0x8a, 0x62, 0xf2], "vcompresspd xmmword [rdx - 0x70]{k7}, xmm4");
 }
+
+mod reg_masks {
+    use yaxpeax_x86::long_mode::RegSpec;
+
+    #[test]
+    #[should_panic]
+    fn invalid_mask_reg_panics() {
+        RegSpec::mask(8);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_qword_reg_panics() {
+        RegSpec::q(16);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_dword_reg_panics() {
+        RegSpec::d(16);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_word_reg_panics() {
+        RegSpec::w(16);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_byte_reg_panics() {
+        RegSpec::b(8);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_rex_byte_reg_panics() {
+        RegSpec::rb(16);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_x87_reg_panics() {
+        RegSpec::st(8);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_xmm_reg_panics() {
+        RegSpec::xmm(32);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_ymm_reg_panics() {
+        RegSpec::ymm(32);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_zmm_reg_panics() {
+        RegSpec::zmm(32);
+    }
+}
diff --git a/test/protected_mode/mod.rs b/test/protected_mode/mod.rs
index 1b0ca59..9f34019 100644
--- a/test/protected_mode/mod.rs
+++ b/test/protected_mode/mod.rs
@@ -3084,3 +3084,55 @@ fn from_reports() {
     test_display(&[0x62, 0xf2, 0x6d, 0xac, 0x00, 0x59, 0xa7], "vpshufb ymm3{k4}{z}, ymm2, ymmword [ecx - 0xb20]");
     test_display(&[0x62, 0xf2, 0xfd, 0x0f, 0x8a, 0x62, 0xf2], "vcompresspd xmmword [edx - 0x70]{k7}, xmm4");
 }
+
+mod reg_masks {
+    use yaxpeax_x86::protected_mode::RegSpec;
+
+    #[test]
+    #[should_panic]
+    fn invalid_mask_reg_panics() {
+        RegSpec::mask(8);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_dword_reg_panics() {
+        RegSpec::d(8);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_word_reg_panics() {
+        RegSpec::w(8);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_byte_reg_panics() {
+        RegSpec::b(8);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_x87_reg_panics() {
+        RegSpec::st(8);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_xmm_reg_panics() {
+        RegSpec::xmm(32);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_ymm_reg_panics() {
+        RegSpec::ymm(32);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_zmm_reg_panics() {
+        RegSpec::zmm(32);
+    }
+}
diff --git a/test/real_mode/mod.rs b/test/real_mode/mod.rs
index b2d6d8d..bf2ae8c 100644
--- a/test/real_mode/mod.rs
+++ b/test/real_mode/mod.rs
@@ -18369,3 +18369,55 @@ fn from_reports() {
     test_display(&[0x62, 0xf2, 0x6d, 0xac, 0x00, 0x59, 0xa7], "vpshufb ymm3{k4}{z}, ymm2, ymmword [bx + di - 0xb20]");
     test_display(&[0x62, 0xf2, 0xfd, 0x0f, 0x8a, 0x62, 0xf2], "vcompresspd xmmword [bp + si - 0x70]{k7}, xmm4");
 }
+
+mod reg_masks {
+    use yaxpeax_x86::real_mode::RegSpec;
+
+    #[test]
+    #[should_panic]
+    fn invalid_mask_reg_panics() {
+        RegSpec::mask(8);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_dword_reg_panics() {
+        RegSpec::d(8);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_word_reg_panics() {
+        RegSpec::w(8);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_byte_reg_panics() {
+        RegSpec::b(8);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_x87_reg_panics() {
+        RegSpec::st(8);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_xmm_reg_panics() {
+        RegSpec::xmm(32);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_ymm_reg_panics() {
+        RegSpec::ymm(32);
+    }
+
+    #[test]
+    #[should_panic]
+    fn invalid_zmm_reg_panics() {
+        RegSpec::zmm(32);
+    }
+}