diff options
| -rw-r--r-- | CHANGELOG | 22 | ||||
| -rw-r--r-- | Cargo.toml | 2 | ||||
| -rw-r--r-- | README.md | 31 | ||||
| -rw-r--r-- | docs/0001-AnnotatingDecoder.md | 85 | ||||
| -rw-r--r-- | rust-toolchain | 1 | ||||
| -rw-r--r-- | src/annotation/mod.rs | 126 | ||||
| -rw-r--r-- | src/lib.rs | 16 | ||||
| -rw-r--r-- | src/reader.rs | 2 |
8 files changed, 281 insertions, 4 deletions
@@ -1,3 +1,25 @@ +## 0.3.0 + +TODO: Reader::next_n should return the number of items read as Err(ReadError::Incomplete(n)) if the buffer is exhausted +TODO: Reader::offset should return an AddressDiff<Address>, not a bare Address +TODO: impls of `fn one` and `fn zero` so downstream users don't have to import num_traits directly + +## 0.2.7 + +moved `AnnotatingDecoder` and its associated types to `annotation/`, for module-level documentation about that feature. + +yanked 0.2.6 because there was not yet a user of it other than myself, and it had this feature in the wrong location in the crate. + +## 0.2.6 + +added `AnnotatingDecoder` and associated traits `FieldDescription` and `DescriptionSink` for architectures to report meanings for bit ranges in decoded instructions. + +added `NullSink`, with an `impl<T> DescriptionSink<T> for NullSink` - `NullSink` can always be used to discard instruction annotations. this is mostly useful for shared annotating and non-annotating decode logic. + +added a `docs/` directory for `yaxpeax-arch`: trip reports for `yaxpeax-arch` design. if `yaxpeax` eventually grows an RFC process one day, these are the kind of changes that would get RFC'd. + +added `docs/0001-AnnotatingDecoder.md`, describing motivation and implementation notes of `AnnotatingDecoder`. + ## 0.2.5 added `yaxpeax-lc87` to the matrix @@ -7,7 +7,7 @@ keywords = ["disassembly", "disassembler"] license = "0BSD" name = "yaxpeax-arch" repository = "https://git.iximeow.net/yaxpeax-arch/" -version = "0.2.5" +version = "0.2.7" [dependencies] "num-traits" = { version = "0.2", default-features = false } @@ -5,6 +5,24 @@ shared traits for architecture definitions, instruction decoders, and related interfaces for instruction decoders from the yaxpeax project. +typically this crate is only interesting if you're writing code to operate on multiple architectures that all implement `yaxpeax-arch` traits. for example, [yaxpeax-dis](https://crates.io/crates/yaxpeax-dis) implements disassembly and display logic generic over the traits defined here, so adding a new decoder is usually only a one or two line addition. + +`yaxpeax-arch` has several crate features, which implementers are encouraged to also support: +* `std`: opt-in for `std`-specific support - in this crate, `std` enables a [`std::error::Error`](https://doc.rust-lang.org/std/error/trait.Error.html) requirement on `DecodeError`, allowing users to `?`-unwrap decode results. +* `colors`: enables (optional) [`crossterm`](https://docs.rs/crossterm/latest/crossterm/)-based ANSI colorization. default coloring rules are defined by [`ColorSettings`](https://docs.rs/yaxpeax-arch/latest/yaxpeax_arch/struct.ColorSettings.html), when enabled. +* `address-parse`: enable a requirement that `yaxpeax_arch::Address` be parsable from `&str`. this is useful for use cases that, for example, read addresses from humans. +* `use-serde`: enable [`serde`](https://docs.rs/serde/latest/serde/) serialization and deserialization bounds for types like `Address`. + +with all features disabled, `yaxpeax-arch`'s only direct dependency is `num-traits`, and is suitable for `#![no_std]` usage. + +### design + +`yaxpeax-arch` has backwards-incompatible changes from time to time, but there's not much to make incompatible. the main benefit of this crate is the [`Arch`](https://docs.rs/yaxpeax-arch/latest/yaxpeax_arch/trait.Arch.html) trait, for other libraries to build architecture-agnostic functionality. + +nontrivial additions to `yaxpeax-arch` should include some discussion summarized by an addition to the crate [`docs/`](https://github.com/iximeow/yaxpeax-arch/tree/no-gods-no-/docs). you may ask, "where does discussion happen?", and the answer currently is in my (iximeow's) head, or various discord/irc/discord/email conversations. if there's need in the future, `yaxpeax` may develop a more consistent process. + +`yaxpeax-arch` intends to support ad-hoc development of architecture support. maintainers of various architectures' crates may not want to implement all available interfaces to a complete level of detail, and must not be required to. incomplete implementations may be an issue for downstream users, but library quality is mediated by human conversation, not `yaxpeax-arch` interfaces. extensions to these fundamental definitions should be considerate of partial and incomplete implementations. + ### implementations there are numerous architectures for which decoders are implemented, at varying levels of completion. now and in the future, they will be enumerated here: @@ -37,6 +55,19 @@ there are numerous architectures for which decoders are implemented, at varying | `MOS 6502` | [yaxpeax-6502](https://github.com/cr1901/yaxpeax-6502) | ⚠️ | ❓ | ❓ | contributed by [@cr1901](https://www.twitter.com/cr1901) | | `lc87` | [yaxpeax-lc87](https://www.github.com/iximeow/yaxpeax-lc87) | 🥳 | ⚠️ | ❓ | | +#### feature support + +`yaxpeax-arch` defines a few typically-optional features that decoders can also implement, in addition to simple `(bytes) -> instruction` decoding. these are `yaxpeax-arch` traits (or collections thereof) which architectures implement, not crate features. + +`description_spans`: implementation of [`AnnotatingDecoder`](https://docs.rs/yaxpeax-arch/latest/yaxpeax_arch/trait.AnnotatingDecoder.html), to decode instructions with bit-level details of what incoming bitstreams mean. +`contextualize`: implementation of [`ShowContextual`](https://docs.rs/yaxpeax-arch/latest/yaxpeax_arch/trait.ShowContextual.html), to display instructions with user-defined information in place of default instruction data. typically expected to show label names instead of relative branch addresses. **i do not recommend implementing this trait**, it needs significant reconsideration. + +| architecture | `description_spans` | `contextualize` | +| ------------ | ------------------- | --------------- | +| `x86_64` | 🥳 | ❓ | +| `ia64` | ⚠️ | ❓ | +| `msp430` | 🥳 | ❓ | + ### mirrors the canonical copy of `yaxpeax-arch` is at [https://git.iximeow.net/yaxpeax-arch](https://git.iximeow.net/yaxpeax-arch). diff --git a/docs/0001-AnnotatingDecoder.md b/docs/0001-AnnotatingDecoder.md new file mode 100644 index 0000000..bea57c3 --- /dev/null +++ b/docs/0001-AnnotatingDecoder.md @@ -0,0 +1,85 @@ +## `DescriptionSink` + +most architectures' machine code packs interesting meanings into specific bit fields, and one of the more important tasks of the yaxpeax decoders is to unpack these into opcodes, operands, and other instruction data for later use. in the worst case, some architectures - typically interpreted bytecodes - do less bit-packing and simply map bytes to instructions. + +the yaxpeax decoders' primary role is to handle this unpacking into user code-friendly structs. i want decoders to be able to report the meaning of bitfields too, so user code can mark up bit streams. + +implementing this capability should (borderline-"must") not regress performance for decoders that do not use it. as a constraint, this is surprisingly restrictive! + +a. it rules out a parameter to [`Decoder::decode_into`](https://docs.rs/yaxpeax-arch/0.2.5/yaxpeax_arch/trait.Decoder.html#tymethod.decode_into): an ignored or unused parameter can still change how `decode_into` inlines. +b. it rules out extra state on `Decoder` impls: writing to an unread `Vec` is still extra work at decode time. + +decoders other than x86 are less performance-sensitive, so **light** regressions in performance may be tolerable. + +i would also like to: + +c. not require decoders implement this to participate in code analysis [`yaxpeax-core`](https://github.com/iximeow/yaxpeax-core/) provides. +d. re-use existing decode logic -- requiring myself and other decoder authors to write everything twice would be miserable. + +the point `c` suggests not adding this capability to existing traits. taken together, these constraints point towards a _new_ trait that _could_ be implemented as an independent copy of decode logic, like: + +```rust +trait AnnotatingDecoder<A: Arch + ?Sized> { + fn decode_with_annotation< + T: Reader<A::Address, A::Word>, + >(&mut self, inst: &mut A::Instruction, words: &mut T) -> Result<(), A::DecodeError>; +} +``` + +but for implementations, it's easiest to tack this onto an existing `Arch`'s `InstDecoder`. point `b` means no new state, so wherever details about a span of bits are recorded, it should be an additional `&mut` parameter. then, if that parameter is an impl of some `Sink` trait, `yaxpeax_arch` can provide a no-op implementation of the `Sink` and let call sites be eliminated for non-annotating decodes. + +taken together, this ends up adding three traits: + +```rust +pub trait DescriptionSink<Descriptor> { + fn record(&mut self, start: u32, end: u32, description: Descriptor); +} + +pub trait FieldDescription { + fn id(&self) -> u32; +} + +pub trait AnnotatingDecoder<A: Arch + ?Sized> { + type FieldDescription: FieldDescription + Clone + Display + PartialEq; + + fn decode_with_annotation< + T: Reader<A::Address, A::Word>, + S: DescriptionSink<Self::FieldDescription> + >(&self, inst: &mut A::Instruction, words: &mut T, sink: &mut S) -> Result<(), A::DecodeError>; +} +``` + +where `FieldDescription` lets callers that operate generically over spans do *something* with them. implementations can use `id` to tag descriptions that should be ordered together, regardless of the actual order the decoder reports them in. for some architectures, fields parsed later in decoding may influence the understanding of earlier fields, so reporting spans in `id`-order up front is an unreasonable burden. consider an x86 instruction, `660f6ec0` - the leading `66` is an operand-size override, but only after reading `0f6e` is it known that that prefix is changing the operands from `mm`/`dword` registers to `xmm`/`qword` registers. in fact this is only known _after_ reporting the opcode of `0f6e`, too. + +`start` and `end` are bit offsets where a `description` applies. `description`s can overlap in part, or in full. exact bit order is known only by the architecture being decoded; is the order `0-7,8-15,16-23,24-31`, `7-0,15-8,23-16,31-24`, or something else? i'm not sure trying to encode that in `yaxpeax-arch` traits is useful right now. `start` and `end` are `u32` because in my professional opinion, `u16` is cursed, `u8` isn't large enough, and `u32` is the next smallest size. `id()` returns a `u32` because i never want to think of `id` space constraints; even if `id` encoded a `major.minor`-style pair of ordering components, the most constrained layout would be `u16.u16` for at most 65536 values in major or minor. that's a big instruction. + +### implementation + +i've added WIP support for span reporting to `msp430`, `ia64`, and `x86` decoders. i extended `yaxpeax-dis` to [make pretty lines](https://twitter.com/iximeow/status/1423930207614889984). more could be said about that; `id`-order is expected to be, roughtly, the order an instruction is decoded. some instructions sets keep the "first" bits as the low-order bits, some others use the higher bits first. so respecting `id`-order necessarily means some instruction sets will have fields "backwards" and make lines extra confusing. + +decoders probably ought to indicate boundaries for significant parts of decoding, lest large instructions [like itanium](https://twitter.com/iximeow/status/1424092536071618561) be a nebulous mess. maybe `FieldDescription` could have an `is_separator()` to know when an element (and its bit range) indicates the end of part of an instruction? + +for the most part, things work great. `yaxpeax-x86` had a minor performance regression. tracking it down wasn't too bad: the first one was because `sink` is a fifth argument for a non-inlined function. at this point most ABIs start spilling to memory. so an unused `sink` caused an extra stack write. this was a measurable overhead. the second regression was again pretty simple looking at `disas-bench` builds: + +```sh +diff \ + ` # a typical non-formatting build, from cratesio yaxpeax-x86 1.0.4 ` \ + <(objdump -d bench-yaxpeax-no-fmt | grep -o ' .*long_mode.*>:') + ` # a non-formatting build, from the local patch of yaxpeax-x86 with annotation reported to a no-op sink ` \ + <(objdump -d bench-yaxpeax-no-fmt-no-annotation | grep -o ' .*long_mode.*>:') +``` + +the entire diff output: +```diff +> <_ZN11yaxpeax_x869long_mode8read_sib17hdc339ef7a182098aE>: +``` + +indeed, [`read_sib`](https://github.com/iximeow/yaxpeax-x86/blob/4371ed02ac30cb56ec4ddbf60c87e85c183d860b/src/long_mode/mod.rs#L5769-L5770) is not written as `inline(always)`, so it's possible this might not get inlined sometimes. since the only difference to `read_sib` is an extra parameter, for which all calls are no-ops that ignore arguments, i'm surprised to see the change, anyway. adding `#[inline(always)]` to `read_sib` returned `yaxpeax-x86` to "same-as-before" decode throughput. + +in the process, i found a slight optimization for `read_sib` that removed a few extra branches from the function. the scrutiny was good after all. + +### conclusion + +in summary, it works. it doesn't slow down callers that don't need spans of information. decoders can implement it optionally and at their leisure, without being ineligible for analysis-oriented libraries. + +this is almost certainly going to be in `yaxpeax-arch 0.2.6` with implementations trickling into decoders whenever it seems like fun. diff --git a/rust-toolchain b/rust-toolchain new file mode 100644 index 0000000..b7921ae --- /dev/null +++ b/rust-toolchain @@ -0,0 +1 @@ +1.54.0 diff --git a/src/annotation/mod.rs b/src/annotation/mod.rs new file mode 100644 index 0000000..0139cf3 --- /dev/null +++ b/src/annotation/mod.rs @@ -0,0 +1,126 @@ +//! traits (and convenient impls) for decoders that also produce descriptions of parsed bit fields. +//! +//! the design of this API is discussed in [`yaxpeax-arch` +//! documentation](https://github.com/iximeow/yaxpeax-arch/blob/no-gods-no-/docs/0001-AnnotatingDecoder.md#descriptionsink). +//! +//! ## usage +//! +//! [`AnnotatingDecoder::decode_with_annotation`] decodes an instruction much like +//! [`crate::Decoder::decode_into`], but also reports descriptions of bit fields to a provided +//! [`DescriptionSink`]. [`VecSink`] is likely the `DescriptionSink` of interest to retain fields; +//! decoders are not required to make any guarantees about the order of descriptions, either by the +//! description's associated [`FieldDescription::id`], or with respect to the bits a +//! `FieldDescription` is reported against. fields may be described by multiple `FieldDescription` +//! with matching `id` and `desc` -- this is to describe data in an instruction where +//! non-contiguous bits are taken together for a single detail. for these cases, the various +//! `FieldDescription` must compare equal, and users of `yaxpeax-arch` can rely on this equivalence +//! for grouping bit ranges. +//! +//! in a generic setting, there isn't much to do with a `FieldDescription` other than display it. a +//! typical use might look something like: +//! ``` +//! fn show_field_descriptions<A: Arch>(decoder: A::Decoder, buf: &[u8]) +//! where +//! A::Decoder: AnnotatingDecoder<A>, +//! A::Instruction: fmt::Display, for<'data> U8Reader<'data>: Reader<A::Address, A::Word>, +//! { +//! let mut inst = A::Instruction::default(); +//! let mut reader = U8Reader::new(buf); +//! let mut sink: VecSink<<A::Decoder as AnnotatingDecoder<A>>::FieldDescription> = VecSink::new(); +//! +//! decoder.decode_with_annotation(&mut inst, &mut reader, &mut sink).unwrap(); +//! +//! println!("decoded instruction {}", inst); +//! for (start, end, desc) in sink.records.iter() { +//! println(" bits [{}, {}]: {}", start, end, desc); +//! } +//! } +//! ``` +//! +//! note that the range `[start, end]` for a reported span is _inclusive_. the `end`-th bit of a +//! an instruction's bit stream is described by the description. +//! +//! ## implementation guidance +//! +//! the typical implementation pattern is that an architecture's `Decoder` implements [`crate::Decoder`] +//! _and_ [`AnnotatingDecoder`], then callers are free to choose which style of decoding they want. +//! [`NullSink`] has a blanket impl of [`DescriptionSink`] for all possible descriptions, and +//! discards reported field descriptions. `decode_with_annotation` with annotations reported to a +//! `NullSink` must be functionally identical to a call to `Decoder::decode_into`. +//! +//! the important points: +//! +//! * `AnnotatingDecoder` is an **optional** implementation for decoders. +//! * `FieldDescription` in general is oriented towards human-directed output, but implementations +//! can be as precise as they want. +//! * since bit/byte order varies from architecture to architecture, a field's `start` and `end` +//! are defined with some ordering from the corresponding decoder crate. crates should describe the +//! bit ordering they select, and where possible, the bit ordering they describe should match +//! relevant ISA mauals. +//! * `FieldDescription` that return true for [`FieldDescription::is_separator`] are an exception +//! to bit span inclusivity: for these descriptions, the bit range should be `[b, b]` where `b` is +//! the last bit before the boundary being delimited. unlike other descriptions, `is_separator` +//! descriptions describe the space between bits `b` and `b+1`. +//! * if a description is to cover multiple bit fields, the reported `FieldDescription` must +//! be identical on `id` and `desc` for all involved bit fields. + +use crate::{Arch, Reader}; + +use core::fmt::Display; + +/// implementors of `DescriptionSink` receive descriptions of an instruction's disassembly process +/// and relevant offsets in the bitstream being decoded. descriptions are archtecture-specific, and +/// architectures are expected to be able to turn the bit-level `start` and `width` values into a +/// meaningful description of bits in the original instruction stream. +pub trait DescriptionSink<Descriptor> { + /// inform this `DescriptionSink` of a `description` that was informed by bits `start` to + /// `end` from the start of an instruction's decoding. `start` and `end` are only relative the + /// instruction being decoded when this sink `DescriptionSink` provided, so they will have no + /// relation to the position in an underlying data stream used for past or future instructions. + fn record(&mut self, start: u32, end: u32, description: Descriptor); +} + +pub struct NullSink; + +impl<T> DescriptionSink<T> for NullSink { + fn record(&mut self, _start: u32, _end: u32, _description: T) { } +} + +#[cfg(feature = "std")] +pub struct VecSink<T: Clone + Display> { + pub records: std::vec::Vec<(u32, u32, T)> +} + +#[cfg(feature = "std")] +impl<T: Clone + Display> VecSink<T> { + pub fn new() -> Self { + VecSink { records: std::vec::Vec::new() } + } +} + +#[cfg(feature = "std")] +impl<T: Clone + Display> DescriptionSink<T> for VecSink<T> { + fn record(&mut self, start: u32, end: u32, description: T) { + self.records.push((start, end, description)); + } +} + +pub trait FieldDescription { + fn id(&self) -> u32; + fn is_separator(&self) -> bool; +} + +/// an interface to decode [`Arch::Instruction`] words from a reader of [`Arch::Word`]s, with the +/// decoder able to report descriptions of bits or fields in the instruction to a sink implementing +/// [`DescriptionSink`]. the sink may be [`NullSink`] to discard provided data. decoding with a +/// `NullSink` should behave identically to `Decoder::decode_into`. implementors are recommended to +/// implement `Decoder::decode_into` as a call to `AnnotatingDecoder::decode_with_annotation` if +/// implementing both traits. +pub trait AnnotatingDecoder<A: Arch + ?Sized> { + type FieldDescription: FieldDescription + Clone + Display + PartialEq; + + fn decode_with_annotation< + T: Reader<A::Address, A::Word>, + S: DescriptionSink<Self::FieldDescription> + >(&self, inst: &mut A::Instruction, words: &mut T, sink: &mut S) -> Result<(), A::DecodeError>; +} @@ -1,4 +1,5 @@ #![no_std] +#![doc = include_str!("../README.md")] use core::fmt::{self, Debug, Display}; use core::hash::Hash; @@ -15,6 +16,8 @@ pub use address::{AddressDisplayUsize, AddressDisplayU64, AddressDisplayU32, Add #[cfg(feature="address-parse")] pub use address::AddrParse; +pub mod annotation; + mod color; pub use color::{Colorize, NoColors, YaxColors}; @@ -124,17 +127,26 @@ impl DecodeError for StandardPartialDecoderError { } } +#[derive(Copy, Clone)] +struct NoDescription {} + +impl fmt::Display for NoDescription { + fn fmt(&self, _: &mut fmt::Formatter) -> fmt::Result { + Ok(()) + } +} + /// an interface to decode [`Arch::Instruction`] words from a reader of [`Arch::Word`]s. errors are /// the architecture-defined [`DecodeError`] implemention. pub trait Decoder<A: Arch + ?Sized> { - /// decode one instruction for this architecture from the [`yaxpeax_arch::Reader`] of this + /// decode one instruction for this architecture from the [`crate::Reader`] of this /// architecture's `Word`. fn decode<T: Reader<A::Address, A::Word>>(&self, words: &mut T) -> Result<A::Instruction, A::DecodeError> { let mut inst = A::Instruction::default(); self.decode_into(&mut inst, words).map(|_: ()| inst) } - /// decode one instruction for this architecture from the [`yaxpeax_arch::Reader`] of this + /// decode one instruction for this architecture from the [`crate::Reader`] of this /// architecture's `Word`, writing into the provided `inst`. /// /// SAFETY: diff --git a/src/reader.rs b/src/reader.rs index acb0146..b9514ae 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -13,7 +13,7 @@ pub enum ReadError { } /// a trait defining how `Item`-sized words are read at `Address`-positioned offsets into some -/// stream of data. for *most* uses, [`yaxpeax_arch::U8Reader`] probably is sufficient. when +/// stream of data. for *most* uses, [`crate::U8Reader`] probably is sufficient. when /// reading from data sources that aren't `&[u8]`, `Address` isn't a multiple of `u8`, or `Item` /// isn't a multiple of 8 bits, `U8Reader` won't be sufficient. pub trait Reader<Address, Item> { |