protected mode memory sizes

also some long-mode cleanup in corresponding areas

1 files changed, 92 insertions, 2 deletions

diff --git a/src/protected_mode/vex.rs b/src/protected_mode/vex.rs
index 5c66654..d4104cd 100644
--- a/src/protected_mode/vex.rs
+++ b/src/protected_mode/vex.rs
@@ -328,6 +328,15 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
                     instruction.operand_count = 3;
                 },
                 other => {
+                    if instruction.vex_reg.num != 0 {
+                        instruction.opcode = Opcode::Invalid;
+                        return Err(DecodeError::InvalidOperand);
+                    }
+                    if instruction.opcode == Opcode::VMOVSS {
+                        instruction.mem_size = 4;
+                    } else {
+                        instruction.mem_size = 8;
+                    }
                     instruction.operands[1] = other;
                     instruction.operand_count = 2;
                 }
@@ -348,6 +357,15 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
                     instruction.operand_count = 3;
                 },
                 other => {
+                    if instruction.vex_reg.num != 0 {
+                        instruction.opcode = Opcode::Invalid;
+                        return Err(DecodeError::InvalidOperand);
+                    }
+                    if instruction.opcode == Opcode::VMOVSS {
+                        instruction.mem_size = 4;
+                    } else {
+                        instruction.mem_size = 8;
+                    }
                     instruction.operands[0] = other;
                     instruction.operand_count = 2;
                 }
@@ -369,6 +387,9 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
             instruction.operands[0] = mem_oper;
             instruction.operands[1] = OperandSpec::RegRRR;
             instruction.operands[2] = OperandSpec::ImmU8;
+            if mem_oper != OperandSpec::RegMMM {
+                instruction.mem_size = 1;
+            }
             instruction.operand_count = 3;
             instruction.imm = read_imm_unsigned(bytes, 1, length)?;
             Ok(())
@@ -611,8 +632,28 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
             instruction.operand_count = 3;
             Ok(())
         }
-        _op @ VEXOperandCode::G_V_M_xmm |
-        _op @ VEXOperandCode::G_V_E_xmm => {
+        VEXOperandCode::G_V_M_xmm => {
+            let modrm = read_modrm(bytes, length)?;
+            if modrm & 0xc0 == 0xc0 {
+                return Err(DecodeError::InvalidOperand);
+            }
+            instruction.modrm_rrr =
+                RegSpec::from_parts((modrm >> 3) & 7, RegisterBank::X);
+            let mem_oper = read_E_xmm(bytes, instruction, modrm, length)?;
+            instruction.operands[0] = OperandSpec::RegRRR;
+            instruction.operands[1] = OperandSpec::RegVex;
+            instruction.operands[2] = mem_oper;
+            if mem_oper != OperandSpec::RegMMM {
+                if instruction.opcode == Opcode::VMOVLPD || instruction.opcode == Opcode::VMOVHPD {
+                    instruction.mem_size = 8;
+                } else {
+                    instruction.mem_size = 16;
+                }
+            }
+            instruction.operand_count = 3;
+            Ok(())
+        }
+        VEXOperandCode::G_V_E_xmm => {
             let modrm = read_modrm(bytes, length)?;
             instruction.modrm_rrr =
                 RegSpec::from_parts((modrm >> 3) & 7, RegisterBank::X);
@@ -620,6 +661,15 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
             instruction.operands[0] = OperandSpec::RegRRR;
             instruction.operands[1] = OperandSpec::RegVex;
             instruction.operands[2] = mem_oper;
+            if mem_oper != OperandSpec::RegMMM {
+                if [Opcode::VSQRTSS, Opcode::VADDSS, Opcode::VMULSS, Opcode::VSUBSS, Opcode::VMINSS, Opcode::VDIVSS, Opcode::VMAXSS].contains(&instruction.opcode) {
+                    instruction.mem_size = 4;
+                } else if [Opcode::VSQRTSD, Opcode::VADDSD, Opcode::VMULSD, Opcode::VSUBSD, Opcode::VMINSD, Opcode::VDIVSD, Opcode::VMAXSD].contains(&instruction.opcode) {
+                    instruction.mem_size = 8;
+                } else {
+                    instruction.mem_size = 16;
+                }
+            }
             instruction.operand_count = 3;
             Ok(())
         }
@@ -672,6 +722,9 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
             instruction.operands[0] = mem_oper;
             instruction.operands[1] = OperandSpec::RegVex;
             instruction.operands[2] = OperandSpec::RegRRR;
+            if mem_oper != OperandSpec::RegMMM {
+                instruction.mem_size = 16;
+            }
             instruction.operand_count = 3;
             Ok(())
         }
@@ -685,6 +738,9 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
             instruction.operands[0] = OperandSpec::RegRRR;
             instruction.operands[1] = mem_oper;
             instruction.operands[2] = OperandSpec::RegVex;
+            if mem_oper != OperandSpec::RegMMM {
+                instruction.mem_size = 4;
+            }
             instruction.operand_count = 3;
             Ok(())
         }
@@ -698,6 +754,13 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
             instruction.operands[0] = OperandSpec::RegRRR;
             instruction.operands[1] = mem_oper;
             instruction.operands[2] = OperandSpec::RegVex;
+            if mem_oper != OperandSpec::RegMMM {
+                if instruction.opcode == Opcode::VPGATHERDD {
+                    instruction.mem_size = 4;
+                } else {
+                    instruction.mem_size = 8;
+                }
+            }
             instruction.operand_count = 3;
             Ok(())
         }
@@ -711,6 +774,9 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
             instruction.operands[0] = OperandSpec::RegRRR;
             instruction.operands[1] = OperandSpec::RegVex;
             instruction.operands[2] = mem_oper;
+            if mem_oper != OperandSpec::RegMMM {
+                instruction.mem_size = opwidth;
+            }
             instruction.operand_count = 3;
             Ok(())
         }
@@ -724,6 +790,9 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
             instruction.operands[0] = OperandSpec::RegRRR;
             instruction.operands[1] = mem_oper;
             instruction.operands[2] = OperandSpec::RegVex;
+            if mem_oper != OperandSpec::RegMMM {
+                instruction.mem_size = opwidth;
+            }
             instruction.operand_count = 3;
             Ok(())
         }
@@ -737,6 +806,9 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
             instruction.operands[1] = mem_oper;
             instruction.imm = read_imm_unsigned(bytes, 1, length)?;
             instruction.operands[2] = OperandSpec::ImmI8;
+            if mem_oper != OperandSpec::RegMMM {
+                instruction.mem_size = opwidth;
+            }
             instruction.operand_count = 3;
             Ok(())
         }
@@ -764,6 +836,9 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
             instruction.operands[0] = OperandSpec::RegVex;
             instruction.operands[1] = mem_oper;
             instruction.operand_count = 2;
+            if mem_oper != OperandSpec::RegMMM {
+                instruction.mem_size = opwidth;
+            }
             instruction.vex_reg.bank = bank;
             Ok(())
         }
@@ -776,6 +851,9 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
             instruction.operands[1] = mem_oper;
             instruction.imm = read_imm_unsigned(bytes, 1, length)?;
             instruction.operands[2] = OperandSpec::ImmU8;
+            if mem_oper != OperandSpec::RegMMM {
+                instruction.mem_size = 32;
+            }
             instruction.operand_count = 3;
             Ok(())
         }
@@ -790,6 +868,9 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
             instruction.operands[2] = mem_oper;
             instruction.imm = read_imm_unsigned(bytes, 1, length)? >> 4;
             instruction.operands[3] = OperandSpec::Reg4;
+            if mem_oper != OperandSpec::RegMMM {
+                instruction.mem_size = 32;
+            }
             instruction.operand_count = 4;
             Ok(())
         }
@@ -804,6 +885,9 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
             instruction.operands[2] = mem_oper;
             instruction.imm = read_imm_unsigned(bytes, 1, length)? >> 4;
             instruction.operands[3] = OperandSpec::Reg4;
+            if mem_oper != OperandSpec::RegMMM {
+                instruction.mem_size = 16;
+            }
             instruction.operand_count = 4;
             Ok(())
         }
@@ -816,6 +900,9 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
             instruction.operands[0] = OperandSpec::RegRRR;
             instruction.operands[1] = OperandSpec::RegVex;
             instruction.operands[2] = mem_oper;
+            if mem_oper != OperandSpec::RegMMM {
+                instruction.mem_size = 16;
+            }
             instruction.operand_count = 3;
             Ok(())
         }
@@ -831,6 +918,9 @@ fn read_vex_operands<T: Iterator<Item=u8>>(bytes: &mut T, instruction: &mut Inst
             instruction.operands[2] = mem_oper;
             instruction.imm = read_imm_unsigned(bytes, 1, length)?;
             instruction.operands[3] = OperandSpec::ImmI8;
+            if mem_oper != OperandSpec::RegMMM {
+                instruction.mem_size = 2;
+            }
             instruction.operand_count = 4;
             Ok(())