dis-x86-amd64.h@ 104338

Last change on this file since 104338 was 103512, checked in by vboxsync, 9 months ago
DIS: Added vex group 13 and redid some or the vex prefix decoding.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 18.7 KB

Line
1	/** @file
2	* DIS - The VirtualBox Disassembler.
3	*/
4
5	/*
6	* Copyright (C) 2006-2023 Oracle and/or its affiliates.
7	*
8	* This file is part of VirtualBox base platform packages, as
9	* available from https://www.virtualbox.org.
10	*
11	* This program is free software; you can redistribute it and/or
12	* modify it under the terms of the GNU General Public License
13	* as published by the Free Software Foundation, in version 3 of the
14	* License.
15	*
16	* This program is distributed in the hope that it will be useful, but
17	* WITHOUT ANY WARRANTY; without even the implied warranty of
18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19	* General Public License for more details.
20	*
21	* You should have received a copy of the GNU General Public License
22	* along with this program; if not, see <https://www.gnu.org/licenses>.
23	*
24	* The contents of this file may alternatively be used under the terms
25	* of the Common Development and Distribution License Version 1.0
26	* (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
27	* in the VirtualBox distribution, in which case the provisions of the
28	* CDDL are applicable instead of those of the GPL.
29	*
30	* You may elect to license modified versions of this file under the
31	* terms and conditions of either the GPL or the CDDL or both.
32	*
33	* SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
34	*/
35
36	#ifndef VBOX_INCLUDED_dis_x86_amd64_h
37	#define VBOX_INCLUDED_dis_x86_amd64_h
38	#ifndef RT_WITHOUT_PRAGMA_ONCE
39	# pragma once
40	#endif
41
42	#include <VBox/types.h>
43	#include <VBox/disopcode-x86-amd64.h>
44	#include <iprt/assert.h>
45
46
47	RT_C_DECLS_BEGIN
48
49	/** @addtogroup grp_dis VBox Disassembler
50	* @{ */
51
52	/** @name Prefix byte flags (DISSTATE::fPrefix).
53	* @{
54	*/
55	#define DISPREFIX_NONE UINT8_C(0x00)
56	/** non-default address size. */
57	#define DISPREFIX_ADDRSIZE UINT8_C(0x01)
58	/** non-default operand size. */
59	#define DISPREFIX_OPSIZE UINT8_C(0x02)
60	/** lock prefix. */
61	#define DISPREFIX_LOCK UINT8_C(0x04)
62	/** segment prefix. */
63	#define DISPREFIX_SEG UINT8_C(0x08)
64	/** rep(e) prefix (not a prefix, but we'll treat is as one). */
65	#define DISPREFIX_REP UINT8_C(0x10)
66	/** repne prefix (not a prefix, but we'll treat is as one). */
67	#define DISPREFIX_REPNE UINT8_C(0x20)
68	/** REX prefix (64 bits) */
69	#define DISPREFIX_REX UINT8_C(0x40)
70	/** VEX or EVEX prefix (64 bits) */
71	#define DISPREFIX_VEX UINT8_C(0x80)
72	/** @} */
73
74	/** @name VEX.Lvvvv prefix destination register flag.
75	* @todo r=bird: redo this. See comments elsewhere.
76	* @{
77	*/
78	#define VEX_LEN256 UINT8_C(0x01)
79	#define VEXREG_IS256B(x) ((x) & VEX_LEN256)
80	/* Convert second byte of VEX prefix to internal format */
81	#define VEX_2B2INT(x) ((((x) >> 2) & 0x1f))
82	#define VEX_HAS_REX_R(x) (!((x) & 0x80))
83	#define DISPREFIX_VEX_FLAG_W UINT8_C(0x01)
84	/** @} */
85
86	/** @name DISPREFIX_VEX_F_XXX - The bVexStuff definitions
87	* @note This is the same layout as byte2 in the VEX3 prefix.
88	* @{ */
89	/** The VEX prefix mask. */
90	#define DISPREFIX_VEX_F_PP_MASK UINT8_C(0x03)
91	/** No prefix. */
92	#define DISPREFIX_VEX_F_PP_NONE UINT8_C(0x00)
93	/** 66h/opsize prefix. */
94	#define DISPREFIX_VEX_F_PP_66 UINT8_C(0x01)
95	/** f3h/repe prefix. */
96	#define DISPREFIX_VEX_F_PP_F3 UINT8_C(0x02)
97	/** f2h/repne prefix. */
98	#define DISPREFIX_VEX_F_PP_F2 UINT8_C(0x03)
99	/** The VEX.L flag. */
100	#define DISPREFIX_VEX_F_L UINT8_C(0x04)
101	/** The VEX.VVVV mask. */
102	#define DISPREFIX_VEX_F_VVVV UINT8_C(0x78)
103	/** The VEX.VVVV shift count. */
104	#define DISPREFIX_VEX_F_VVVV_SHIFT 3
105	/** The VEX.VVVV mask shifted down to bit 0. */
106	#define DISPREFIX_VEX_F_VVVV_SMASK UINT8_C(0x0f)
107	/** The VEX.W flag. */
108	#define DISPREFIX_VEX_F_W UINT8_C(0x80)
109	/** @} */
110
111	/** @name 64 bits prefix byte flags (DISSTATE::fRexPrefix).
112	* Requires VBox/disopcode.h.
113	* @{
114	*/
115	#define DISPREFIX_REX_OP_2_FLAGS(a) (a - OP_PARM_REX_START)
116	/#define DISPREFIX_REX_FLAGS DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX) - 0, which is no flag /
117	#define DISPREFIX_REX_FLAGS_B DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_B)
118	#define DISPREFIX_REX_FLAGS_X DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_X)
119	#define DISPREFIX_REX_FLAGS_XB DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_XB)
120	#define DISPREFIX_REX_FLAGS_R DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_R)
121	#define DISPREFIX_REX_FLAGS_RB DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_RB)
122	#define DISPREFIX_REX_FLAGS_RX DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_RX)
123	#define DISPREFIX_REX_FLAGS_RXB DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_RXB)
124	#define DISPREFIX_REX_FLAGS_W DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_W)
125	#define DISPREFIX_REX_FLAGS_WB DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_WB)
126	#define DISPREFIX_REX_FLAGS_WX DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_WX)
127	#define DISPREFIX_REX_FLAGS_WXB DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_WXB)
128	#define DISPREFIX_REX_FLAGS_WR DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_WR)
129	#define DISPREFIX_REX_FLAGS_WRB DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_WRB)
130	#define DISPREFIX_REX_FLAGS_WRX DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_WRX)
131	#define DISPREFIX_REX_FLAGS_WRXB DISPREFIX_REX_OP_2_FLAGS(OP_PARM_REX_WRXB)
132	/** @} */
133	AssertCompile(RT_IS_POWER_OF_TWO(DISPREFIX_REX_FLAGS_B));
134	AssertCompile(RT_IS_POWER_OF_TWO(DISPREFIX_REX_FLAGS_X));
135	AssertCompile(RT_IS_POWER_OF_TWO(DISPREFIX_REX_FLAGS_W));
136	AssertCompile(RT_IS_POWER_OF_TWO(DISPREFIX_REX_FLAGS_R));
137
138
139	/** @name 64-bit general register indexes.
140	* This matches the AMD64 register encoding. It is found used in
141	* DISOPPARAM::Base.idxGenReg and DISOPPARAM::Index.idxGenReg.
142	* @note Safe to assume same values as the 16-bit and 32-bit general registers.
143	* @{
144	*/
145	#define DISGREG_RAX UINT8_C(0)
146	#define DISGREG_RCX UINT8_C(1)
147	#define DISGREG_RDX UINT8_C(2)
148	#define DISGREG_RBX UINT8_C(3)
149	#define DISGREG_RSP UINT8_C(4)
150	#define DISGREG_RBP UINT8_C(5)
151	#define DISGREG_RSI UINT8_C(6)
152	#define DISGREG_RDI UINT8_C(7)
153	#define DISGREG_R8 UINT8_C(8)
154	#define DISGREG_R9 UINT8_C(9)
155	#define DISGREG_R10 UINT8_C(10)
156	#define DISGREG_R11 UINT8_C(11)
157	#define DISGREG_R12 UINT8_C(12)
158	#define DISGREG_R13 UINT8_C(13)
159	#define DISGREG_R14 UINT8_C(14)
160	#define DISGREG_R15 UINT8_C(15)
161	/** @} */
162
163	/** @name 32-bit general register indexes.
164	* This matches the AMD64 register encoding. It is found used in
165	* DISOPPARAM::Base.idxGenReg and DISOPPARAM::Index.idxGenReg.
166	* @note Safe to assume same values as the 16-bit and 64-bit general registers.
167	* @{
168	*/
169	#define DISGREG_EAX UINT8_C(0)
170	#define DISGREG_ECX UINT8_C(1)
171	#define DISGREG_EDX UINT8_C(2)
172	#define DISGREG_EBX UINT8_C(3)
173	#define DISGREG_ESP UINT8_C(4)
174	#define DISGREG_EBP UINT8_C(5)
175	#define DISGREG_ESI UINT8_C(6)
176	#define DISGREG_EDI UINT8_C(7)
177	#define DISGREG_R8D UINT8_C(8)
178	#define DISGREG_R9D UINT8_C(9)
179	#define DISGREG_R10D UINT8_C(10)
180	#define DISGREG_R11D UINT8_C(11)
181	#define DISGREG_R12D UINT8_C(12)
182	#define DISGREG_R13D UINT8_C(13)
183	#define DISGREG_R14D UINT8_C(14)
184	#define DISGREG_R15D UINT8_C(15)
185	/** @} */
186
187	/** @name 16-bit general register indexes.
188	* This matches the AMD64 register encoding. It is found used in
189	* DISOPPARAM::Base.idxGenReg and DISOPPARAM::Index.idxGenReg.
190	* @note Safe to assume same values as the 32-bit and 64-bit general registers.
191	* @{
192	*/
193	#define DISGREG_AX UINT8_C(0)
194	#define DISGREG_CX UINT8_C(1)
195	#define DISGREG_DX UINT8_C(2)
196	#define DISGREG_BX UINT8_C(3)
197	#define DISGREG_SP UINT8_C(4)
198	#define DISGREG_BP UINT8_C(5)
199	#define DISGREG_SI UINT8_C(6)
200	#define DISGREG_DI UINT8_C(7)
201	#define DISGREG_R8W UINT8_C(8)
202	#define DISGREG_R9W UINT8_C(9)
203	#define DISGREG_R10W UINT8_C(10)
204	#define DISGREG_R11W UINT8_C(11)
205	#define DISGREG_R12W UINT8_C(12)
206	#define DISGREG_R13W UINT8_C(13)
207	#define DISGREG_R14W UINT8_C(14)
208	#define DISGREG_R15W UINT8_C(15)
209	/** @} */
210
211	/** @name 8-bit general register indexes.
212	* This mostly (?) matches the AMD64 register encoding. It is found used in
213	* DISOPPARAM::Base.idxGenReg and DISOPPARAM::Index.idxGenReg.
214	* @{
215	*/
216	#define DISGREG_AL UINT8_C(0)
217	#define DISGREG_CL UINT8_C(1)
218	#define DISGREG_DL UINT8_C(2)
219	#define DISGREG_BL UINT8_C(3)
220	#define DISGREG_AH UINT8_C(4)
221	#define DISGREG_CH UINT8_C(5)
222	#define DISGREG_DH UINT8_C(6)
223	#define DISGREG_BH UINT8_C(7)
224	#define DISGREG_R8B UINT8_C(8)
225	#define DISGREG_R9B UINT8_C(9)
226	#define DISGREG_R10B UINT8_C(10)
227	#define DISGREG_R11B UINT8_C(11)
228	#define DISGREG_R12B UINT8_C(12)
229	#define DISGREG_R13B UINT8_C(13)
230	#define DISGREG_R14B UINT8_C(14)
231	#define DISGREG_R15B UINT8_C(15)
232	#define DISGREG_SPL UINT8_C(16)
233	#define DISGREG_BPL UINT8_C(17)
234	#define DISGREG_SIL UINT8_C(18)
235	#define DISGREG_DIL UINT8_C(19)
236	/** @} */
237
238	/** @name Segment registerindexes.
239	* This matches the AMD64 register encoding. It is found used in
240	* DISOPPARAM::Base.idxSegReg.
241	* @{
242	*/
243	typedef enum
244	{
245	DISSELREG_ES = 0,
246	DISSELREG_CS = 1,
247	DISSELREG_SS = 2,
248	DISSELREG_DS = 3,
249	DISSELREG_FS = 4,
250	DISSELREG_GS = 5,
251	/** End of the valid register index values. */
252	DISSELREG_END,
253	/** The usual 32-bit paranoia. */
254	DIS_SEGREG_32BIT_HACK = 0x7fffffff
255	} DISSELREG;
256	/** @} */
257
258	/** @name FPU register indexes.
259	* This matches the AMD64 register encoding. It is found used in
260	* DISOPPARAM::Base.idxFpuReg.
261	* @{
262	*/
263	#define DISFPREG_ST0 UINT8_C(0)
264	#define DISFPREG_ST1 UINT8_C(1)
265	#define DISFPREG_ST2 UINT8_C(2)
266	#define DISFPREG_ST3 UINT8_C(3)
267	#define DISFPREG_ST4 UINT8_C(4)
268	#define DISFPREG_ST5 UINT8_C(5)
269	#define DISFPREG_ST6 UINT8_C(6)
270	#define DISFPREG_ST7 UINT8_C(7)
271	/** @} */
272
273	/** @name Control register indexes.
274	* This matches the AMD64 register encoding. It is found used in
275	* DISOPPARAM::Base.idxCtrlReg.
276	* @{
277	*/
278	#define DISCREG_CR0 UINT8_C(0)
279	#define DISCREG_CR1 UINT8_C(1)
280	#define DISCREG_CR2 UINT8_C(2)
281	#define DISCREG_CR3 UINT8_C(3)
282	#define DISCREG_CR4 UINT8_C(4)
283	#define DISCREG_CR8 UINT8_C(8)
284	/** @} */
285
286	/** @name Debug register indexes.
287	* This matches the AMD64 register encoding. It is found used in
288	* DISOPPARAM::Base.idxDbgReg.
289	* @{
290	*/
291	#define DISDREG_DR0 UINT8_C(0)
292	#define DISDREG_DR1 UINT8_C(1)
293	#define DISDREG_DR2 UINT8_C(2)
294	#define DISDREG_DR3 UINT8_C(3)
295	#define DISDREG_DR4 UINT8_C(4)
296	#define DISDREG_DR5 UINT8_C(5)
297	#define DISDREG_DR6 UINT8_C(6)
298	#define DISDREG_DR7 UINT8_C(7)
299	/** @} */
300
301	/** @name MMX register indexes.
302	* This matches the AMD64 register encoding. It is found used in
303	* DISOPPARAM::Base.idxMmxReg.
304	* @{
305	*/
306	#define DISMREG_MMX0 UINT8_C(0)
307	#define DISMREG_MMX1 UINT8_C(1)
308	#define DISMREG_MMX2 UINT8_C(2)
309	#define DISMREG_MMX3 UINT8_C(3)
310	#define DISMREG_MMX4 UINT8_C(4)
311	#define DISMREG_MMX5 UINT8_C(5)
312	#define DISMREG_MMX6 UINT8_C(6)
313	#define DISMREG_MMX7 UINT8_C(7)
314	/** @} */
315
316	/** @name SSE register indexes.
317	* This matches the AMD64 register encoding. It is found used in
318	* DISOPPARAM::Base.idxXmmReg.
319	* @{
320	*/
321	#define DISXREG_XMM0 UINT8_C(0)
322	#define DISXREG_XMM1 UINT8_C(1)
323	#define DISXREG_XMM2 UINT8_C(2)
324	#define DISXREG_XMM3 UINT8_C(3)
325	#define DISXREG_XMM4 UINT8_C(4)
326	#define DISXREG_XMM5 UINT8_C(5)
327	#define DISXREG_XMM6 UINT8_C(6)
328	#define DISXREG_XMM7 UINT8_C(7)
329	/** @} */
330
331
332	/**
333	* Opcode parameter (operand) details for x86/AMD64.
334	*/
335	typedef struct
336	{
337	/** Disposition. */
338	union
339	{
340	/** 64-bit displacement, applicable if DISUSE_DISPLACEMENT64 is set in fUse. */
341	int64_t i64;
342	uint64_t u64;
343	/** 32-bit displacement, applicable if DISUSE_DISPLACEMENT32 or
344	* DISUSE_RIPDISPLACEMENT32 is set in fUse. */
345	int32_t i32;
346	uint32_t u32;
347	/** 16-bit displacement, applicable if DISUSE_DISPLACEMENT16 is set in fUse. */
348	int32_t i16;
349	uint32_t u16;
350	/** 8-bit displacement, applicable if DISUSE_DISPLACEMENT8 is set in fUse. */
351	int32_t i8;
352	uint32_t u8;
353	} uDisp;
354	/** The base register from ModR/M or SIB, applicable if DISUSE_BASE is
355	* set in fUse. */
356	union
357	{
358	/** General register index (DISGREG_XXX), applicable if DISUSE_REG_GEN8,
359	* DISUSE_REG_GEN16, DISUSE_REG_GEN32 or DISUSE_REG_GEN64 is set in fUse. */
360	uint8_t idxGenReg;
361	/** FPU stack register index (DISFPREG_XXX), applicable if DISUSE_REG_FP is
362	* set in fUse. 1:1 indexes. */
363	uint8_t idxFpuReg;
364	/** MMX register index (DISMREG_XXX), applicable if DISUSE_REG_MMX is
365	* set in fUse. 1:1 indexes. */
366	uint8_t idxMmxReg;
367	/** SSE register index (DISXREG_XXX), applicable if DISUSE_REG_XMM is
368	* set in fUse. 1:1 indexes. */
369	uint8_t idxXmmReg;
370	/** SSE2 register index (DISYREG_XXX), applicable if DISUSE_REG_YMM is
371	* set in fUse. 1:1 indexes. */
372	uint8_t idxYmmReg;
373	/** Segment register index (DISSELREG_XXX), applicable if DISUSE_REG_SEG is
374	* set in fUse. */
375	uint8_t idxSegReg;
376	/** Test register, TR0-TR7, present on early IA32 CPUs, applicable if
377	* DISUSE_REG_TEST is set in fUse. No index defines for these. */
378	uint8_t idxTestReg;
379	/** Control register index (DISCREG_XXX), applicable if DISUSE_REG_CR is
380	* set in fUse. 1:1 indexes. */
381	uint8_t idxCtrlReg;
382	/** Debug register index (DISDREG_XXX), applicable if DISUSE_REG_DBG is
383	* set in fUse. 1:1 indexes. */
384	uint8_t idxDbgReg;
385	} Base;
386	/** The SIB index register meaning, applicable if DISUSE_INDEX is
387	* set in fUse. */
388	union
389	{
390	/** General register index (DISGREG_XXX), applicable if DISUSE_REG_GEN8,
391	* DISUSE_REG_GEN16, DISUSE_REG_GEN32 or DISUSE_REG_GEN64 is set in fUse. */
392	uint8_t idxGenReg;
393	/** XMM register index (DISXREG_XXX), applicable if DISUSE_REG_XMM
394	* is set in fUse. */
395	uint8_t idxXmmReg;
396	/** YMM register index (DISXREG_XXX), applicable if DISUSE_REG_YMM
397	* is set in fUse. */
398	uint8_t idxYmmReg;
399	} Index;
400	/** 2, 4 or 8, if DISUSE_SCALE is set in fUse. */
401	uint8_t uScale;
402	/** Parameter size. */
403	uint8_t cb;
404	/** Copy of the corresponding DISOPCODE::fParam1 / DISOPCODE::fParam2 /
405	* DISOPCODE::fParam3. */
406	uint32_t fParam;
407	} DIS_OP_PARAM_X86_T;
408	AssertCompileSize(DIS_OP_PARAM_X86_T, 16);
409	/** Pointer to opcode parameter. */
410	typedef DIS_OP_PARAM_X86_T *PDIS_OP_PARAM_X86_T;
411	/** Pointer to opcode parameter. */
412	typedef const DIS_OP_PARAM_X86_T *PCDIS_OP_PARAM_X86_T;
413
414
415	/** Parser callback.
416	* @remark no DECLCALLBACK() here because it's considered to be internal and
417	* there is no point in enforcing CDECL. */
418	typedef size_t FNDISPARSEX86(size_t offInstr, PCDISOPCODE pOp, PDISSTATE pDis, PDISOPPARAM pParam);
419	/** Pointer to a disassembler parser function. */
420	typedef FNDISPARSEX86 *PFNDISPARSEX86;
421	/** Pointer to a const disassembler parser function pointer. */
422	typedef PFNDISPARSEX86 const *PCPFNDISPARSEX86;
423
424	/**
425	* The x86/AMD64 specific disassembler state and result.
426	*/
427	typedef struct
428	{
429	/** SIB fields. */
430	union
431	{
432	/** Bitfield view */
433	struct
434	{
435	uint8_t Base;
436	uint8_t Index;
437	uint8_t Scale;
438	} Bits;
439	} SIB;
440	/** ModRM fields. */
441	union
442	{
443	/** Bitfield view */
444	struct
445	{
446	uint8_t Rm;
447	uint8_t Reg;
448	uint8_t Mod;
449	} Bits;
450	} ModRM;
451	/** The addressing mode (DISCPUMODE). */
452	uint8_t uAddrMode;
453	/** The operand mode (DISCPUMODE). */
454	uint8_t uOpMode;
455	/** Per instruction prefix settings. */
456	uint8_t fPrefix;
457	/** REX prefix value (64 bits only). */
458	uint8_t fRexPrefix;
459	/** Segment prefix value (DISSELREG). */
460	uint8_t idxSegPrefix;
461	/** Last prefix byte (for SSE2 extension tables). */
462	uint8_t bLastPrefix;
463	/** Last significant opcode byte of instruction. */
464	uint8_t bOpCode;
465	/** The size of the prefix bytes. */
466	uint8_t cbPrefix;
467	/** VEX presence flag, destination register and size
468	* @todo r=bird: There is no VEX presence flage here, just ~vvvv and L.
469	* @deprecated DO NOT USE. All info is available in bVexByte2 and
470	* fPrefix/DISPREFIX_VEX. */
471	uint8_t bVexDestReg;
472	/** DISPREFIX_VEX_F_XXX - This is set for both VEX prefixes. */
473	uint8_t bVexByte2;
474	/** Internal: instruction filter */
475	uint32_t fFilter;
476	/** SIB displacment. */
477	int32_t i32SibDisp;
478	/** Internal: pointer to disassembly function table */
479	PCPFNDISPARSEX86 pfnDisasmFnTable;
480	#if ARCH_BITS == 32
481	uint32_t uPtrPadding1;
482	#endif
483	} DIS_STATE_X86_T;
484	AssertCompileSize(DIS_STATE_X86_T, 32);
485
486
487
488	DISDECL(bool) DISFormatYasmIsOddEncoding(PDISSTATE pDis);
489
490	/** @} */
491
492	RT_C_DECLS_END
493
494	#endif /* !VBOX_INCLUDED_dis_x86_amd64_h */
495

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/include/VBox/dis-x86-amd64.h@ 104338

Download in other formats: