Disasm.cpp@ 15424

Last change on this file since 15424 was 13835, checked in by vboxsync, 16 years ago
s/VBOX_SUCCESS/RT_SUCCESS/g s/VBOX_FAILURE/RT_FAILURE/g - VBOX_SUCCESS and VBOX_FAILURE have NOT been retired (because old habbits die hard) just sligtly deprecated.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 15.8 KB

Line
1	/** @file
2	*
3	* VBox disassembler:
4	* Main
5	*/
6
7	/*
8	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
9	*
10	* This file is part of VirtualBox Open Source Edition (OSE), as
11	* available from http://www.virtualbox.org. This file is free software;
12	* you can redistribute it and/or modify it under the terms of the GNU
13	* General Public License (GPL) as published by the Free Software
14	* Foundation, in version 2 as it comes in the "COPYING" file of the
15	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
16	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
17	*
18	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
19	* Clara, CA 95054 USA or visit http://www.sun.com if you need
20	* additional information or have any questions.
21	*/
22
23
24	/*******************************************************************************
25	* Header Files *
26	*******************************************************************************/
27	#ifdef USING_VISUAL_STUDIO
28	# include <stdafx.h>
29	#endif
30	#include <VBox/dis.h>
31	#include <VBox/disopcode.h>
32	#include <VBox/err.h>
33	#include <iprt/assert.h>
34	#include <iprt/string.h>
35	#include "DisasmInternal.h"
36	#include "DisasmTables.h"
37
38
39	/**
40	* Disassembles a code block.
41	*
42	* @returns VBox error code
43	* @param pCpu Pointer to cpu structure which have DISCPUSTATE::mode
44	* set correctly.
45	* @param pvCodeBlock Pointer to the strunction to disassemble.
46	* @param cbMax Maximum number of bytes to disassemble.
47	* @param pcbSize Where to store the size of the instruction.
48	* NULL is allowed.
49	*
50	*
51	* @todo Define output callback.
52	* @todo Using signed integers as sizes is a bit odd. There are still
53	* some GCC warnings about mixing signed and unsigend integers.
54	* @todo Need to extend this interface to include a code address so we
55	* can dissassemble GC code. Perhaps a new function is better...
56	* @remark cbMax isn't respected as a boundry. DISInstr() will read beyond cbMax.
57	* This means *pcbSize >= cbMax sometimes.
58	*/
59	DISDECL(int) DISBlock(PDISCPUSTATE pCpu, RTUINTPTR pvCodeBlock, unsigned cbMax, unsigned *pSize)
60	{
61	unsigned i = 0;
62	char szOutput[256];
63
64	while (i < cbMax)
65	{
66	unsigned cbInstr;
67	int rc = DISInstr(pCpu, pvCodeBlock + i, 0, &cbInstr, szOutput);
68	if (RT_FAILURE(rc))
69	return rc;
70
71	i += cbInstr;
72	}
73
74	if (pSize)
75	*pSize = i;
76	return true;
77	}
78
79	/**
80	* Disassembles one instruction
81	*
82	* @returns VBox error code
83	* @param pCpu Pointer to cpu structure which have DISCPUSTATE::mode
84	* set correctly.
85	* @param pu8Instruction Pointer to the strunction to disassemble.
86	* @param u32EipOffset Offset to add to instruction address to get the real virtual address
87	* @param pcbSize Where to store the size of the instruction.
88	* NULL is allowed.
89	* @param pszOutput Storage for disassembled instruction
90	*
91	* @todo Define output callback.
92	*/
93	DISDECL(int) DISInstr(PDISCPUSTATE pCpu, RTUINTPTR pu8Instruction, unsigned u32EipOffset, unsigned *pcbSize,
94	char *pszOutput)
95	{
96	return DISInstrEx(pCpu, pu8Instruction, u32EipOffset, pcbSize, pszOutput, OPTYPE_ALL);
97	}
98
99	/**
100	* Disassembles one instruction; only fully disassembly an instruction if it matches the filter criteria
101	*
102	* @returns VBox error code
103	* @param pCpu Pointer to cpu structure which have DISCPUSTATE::mode
104	* set correctly.
105	* @param pu8Instruction Pointer to the strunction to disassemble.
106	* @param u32EipOffset Offset to add to instruction address to get the real virtual address
107	* @param pcbSize Where to store the size of the instruction.
108	* NULL is allowed.
109	* @param pszOutput Storage for disassembled instruction
110	* @param uFilter Instruction type filter
111	*
112	* @todo Define output callback.
113	*/
114	DISDECL(int) DISInstrEx(PDISCPUSTATE pCpu, RTUINTPTR pu8Instruction, unsigned u32EipOffset, unsigned *pcbSize,
115	char *pszOutput, unsigned uFilter)
116	{
117	unsigned i = 0, prefixbytes;
118	unsigned idx, inc;
119	const OPCODE *paOneByteMap;
120	#ifdef __L4ENV__
121	jmp_buf jumpbuffer;
122	#endif
123
124	//reset instruction settings
125	pCpu->prefix = PREFIX_NONE;
126	pCpu->enmPrefixSeg = DIS_SELREG_DS;
127	pCpu->ModRM.u = 0;
128	pCpu->SIB.u = 0;
129	pCpu->lastprefix = 0;
130	pCpu->param1.parval = 0;
131	pCpu->param2.parval = 0;
132	pCpu->param3.parval = 0;
133	pCpu->param1.szParam[0] = 0;
134	pCpu->param2.szParam[0] = 0;
135	pCpu->param3.szParam[0] = 0;
136	pCpu->param1.size = 0;
137	pCpu->param2.size = 0;
138	pCpu->param3.size = 0;
139	pCpu->param1.flags = 0;
140	pCpu->param2.flags = 0;
141	pCpu->param3.flags = 0;
142	pCpu->uFilter = uFilter;
143	pCpu->pfnDisasmFnTable = pfnFullDisasm;
144
145	if (pszOutput)
146	*pszOutput = '\0';
147
148	if (pCpu->mode == CPUMODE_64BIT)
149	{
150	paOneByteMap = g_aOneByteMapX64;
151	pCpu->addrmode = CPUMODE_64BIT;
152	pCpu->opmode = CPUMODE_32BIT;
153	}
154	else
155	{
156	paOneByteMap = g_aOneByteMapX86;
157	pCpu->addrmode = pCpu->mode;
158	pCpu->opmode = pCpu->mode;
159	}
160
161	prefixbytes = 0;
162	#ifndef __L4ENV__ /* Unfortunately, we have no exception handling in l4env */
163	try
164	#else
165	pCpu->pJumpBuffer = &jumpbuffer;
166	if (setjmp(jumpbuffer) == 0)
167	#endif
168	{
169	while(1)
170	{
171	uint8_t codebyte = DISReadByte(pCpu, pu8Instruction+i);
172	uint8_t opcode = paOneByteMap[codebyte].opcode;
173
174	/* Hardcoded assumption about OP_* values!! */
175	if (opcode <= OP_LAST_PREFIX)
176	{
177	/* The REX prefix must precede the opcode byte(s). Any other placement is ignored. */
178	if (opcode != OP_REX)
179	{
180	pCpu->lastprefix = opcode;
181	pCpu->prefix &= ~PREFIX_REX;
182	}
183
184	switch(opcode)
185	{
186	case OP_INVALID:
187	#if 0 //defined (DEBUG_Sander)
188	AssertMsgFailed(("Invalid opcode!!\n"));
189	#endif
190	return VERR_DIS_INVALID_OPCODE;
191
192	// segment override prefix byte
193	case OP_SEG:
194	pCpu->enmPrefixSeg = (DIS_SELREG)(paOneByteMap[codebyte].param1 - OP_PARM_REG_SEG_START);
195	/* Segment prefixes for CS, DS, ES and SS are ignored in long mode. */
196	if ( pCpu->mode != CPUMODE_64BIT
197	\|\| pCpu->enmPrefixSeg >= DIS_SELREG_FS)
198	{
199	pCpu->prefix \|= PREFIX_SEG;
200	}
201	i += sizeof(uint8_t);
202	prefixbytes++;
203	continue; //fetch the next byte
204
205	// lock prefix byte
206	case OP_LOCK:
207	pCpu->prefix \|= PREFIX_LOCK;
208	i += sizeof(uint8_t);
209	prefixbytes++;
210	continue; //fetch the next byte
211
212	// address size override prefix byte
213	case OP_ADDRSIZE:
214	pCpu->prefix \|= PREFIX_ADDRSIZE;
215	if (pCpu->mode == CPUMODE_16BIT)
216	pCpu->addrmode = CPUMODE_32BIT;
217	else
218	if (pCpu->mode == CPUMODE_32BIT)
219	pCpu->addrmode = CPUMODE_16BIT;
220	else
221	pCpu->addrmode = CPUMODE_32BIT; /* 64 bits */
222
223	i += sizeof(uint8_t);
224	prefixbytes++;
225	continue; //fetch the next byte
226
227	// operand size override prefix byte
228	case OP_OPSIZE:
229	pCpu->prefix \|= PREFIX_OPSIZE;
230	if (pCpu->mode == CPUMODE_16BIT)
231	pCpu->opmode = CPUMODE_32BIT;
232	else
233	pCpu->opmode = CPUMODE_16BIT; /* for 32 and 64 bits mode (there is no 32 bits operand size override prefix) */
234
235	i += sizeof(uint8_t);
236	prefixbytes++;
237	continue; //fetch the next byte
238
239	// rep and repne are not really prefixes, but we'll treat them as such
240	case OP_REPE:
241	pCpu->prefix \|= PREFIX_REP;
242	i += sizeof(uint8_t);
243	prefixbytes += sizeof(uint8_t);
244	continue; //fetch the next byte
245
246	case OP_REPNE:
247	pCpu->prefix \|= PREFIX_REPNE;
248	i += sizeof(uint8_t);
249	prefixbytes += sizeof(uint8_t);
250	continue; //fetch the next byte
251
252	case OP_REX:
253	Assert(pCpu->mode == CPUMODE_64BIT);
254	/* REX prefix byte */
255	pCpu->prefix \|= PREFIX_REX;
256	pCpu->prefix_rex = PREFIX_REX_OP_2_FLAGS(paOneByteMap[codebyte].param1);
257	i += sizeof(uint8_t);
258	prefixbytes += sizeof(uint8_t);
259
260	if (pCpu->prefix_rex & PREFIX_REX_FLAGS_W)
261	pCpu->opmode = CPUMODE_64BIT; /* overrides size prefix byte */
262	continue; //fetch the next byte
263	}
264	}
265
266	idx = i;
267	i += sizeof(uint8_t); //first opcode byte
268
269	pCpu->opcode = codebyte;
270	/* Prefix byte(s) is/are part of the instruction. */
271	pCpu->opaddr = pu8Instruction + idx + u32EipOffset - prefixbytes;
272
273	inc = ParseInstruction(pu8Instruction + i, &paOneByteMap[pCpu->opcode], pCpu);
274
275	pCpu->opsize = prefixbytes + inc + sizeof(uint8_t);
276
277	if(pszOutput) {
278	disasmSprintf(pszOutput, pu8Instruction+i-1-prefixbytes, pCpu, &pCpu->param1, &pCpu->param2, &pCpu->param3);
279	}
280
281	i += inc;
282	prefixbytes = 0;
283	break;
284	}
285	}
286	#ifndef __L4ENV__
287	catch(...)
288	#else
289	else /* setjmp has returned a non-zero value: an exception occured */
290	#endif
291	{
292	if (pcbSize)
293	*pcbSize = 0;
294	return VERR_DIS_GEN_FAILURE;
295	}
296
297	if (pcbSize)
298	*pcbSize = i;
299
300	if (pCpu->prefix & PREFIX_LOCK)
301	disValidateLockSequence(pCpu);
302
303	return VINF_SUCCESS;
304	}
305	//*****************************************************************************
306	//*****************************************************************************
307	char DbgBytesToString(PDISCPUSTATE pCpu, RTUINTPTR pBytes, int size, char pszOutput)
308	{
309	char szByte[4];
310	int len = strlen(pszOutput);
311	int i;
312
313	for(i = len; i < 40; i++)
314	{
315	strcat(pszOutput, " ");
316	}
317	strcat(pszOutput, " [");
318	for(i = 0; i < size; i++)
319	{
320	RTStrPrintf(szByte, sizeof(szByte), "%02X ", DISReadByte(pCpu, pBytes+i));
321	RTStrPrintf(&pszOutput[strlen(pszOutput)], 64, szByte);
322	}
323	len = strlen(pszOutput);
324	pszOutput[len - 1] = 0; //cut off last space
325
326	strcat(pszOutput, "]");
327	return pszOutput;
328	}
329	//*****************************************************************************
330	//*****************************************************************************
331	void disasmSprintf(char pszOutput, RTUINTPTR pu8Instruction, PDISCPUSTATE pCpu, OP_PARAMETER pParam1, OP_PARAMETER pParam2, OP_PARAMETER pParam3)
332	{
333	const char *lpszFormat = pCpu->pszOpcode;
334	int param = 1;
335
336	RTStrPrintf(pszOutput, 64, "%08X: ", (unsigned)pCpu->opaddr);
337	if(pCpu->prefix & PREFIX_LOCK)
338	{
339	RTStrPrintf(&pszOutput[strlen(pszOutput)], 64, "lock ");
340	}
341	if(pCpu->prefix & PREFIX_REP)
342	{
343	RTStrPrintf(&pszOutput[strlen(pszOutput)], 64, "rep(e) ");
344	}
345	else
346	if(pCpu->prefix & PREFIX_REPNE)
347	{
348	RTStrPrintf(&pszOutput[strlen(pszOutput)], 64, "repne ");
349	}
350
351	if(!strcmp("Invalid Opcode", lpszFormat))
352	{
353	RTStrPrintf(&pszOutput[strlen(pszOutput)], 64, "Invalid Opcode [%02X][%02X]", DISReadByte(pCpu, pu8Instruction), DISReadByte(pCpu, pu8Instruction+1) );
354	}
355	else
356	while(*lpszFormat)
357	{
358	switch(*lpszFormat)
359	{
360	case '%':
361	switch(*(lpszFormat+1))
362	{
363	case 'J': //Relative jump offset
364	{
365	int32_t disp;
366
367	AssertMsg(param == 1, ("Invalid branch parameter nr"));
368	if(pParam1->flags & USE_IMMEDIATE8_REL)
369	{
370	disp = (int32_t)(char)pParam1->parval;
371	}
372	else
373	if(pParam1->flags & USE_IMMEDIATE16_REL)
374	{
375	disp = (int32_t)(uint16_t)pParam1->parval;
376	}
377	else
378	if(pParam1->flags & USE_IMMEDIATE32_REL)
379	{
380	disp = (int32_t)pParam1->parval;
381	}
382	else
383	{
384	AssertMsgFailed(("Oops!\n"));
385	return;
386	}
387	uint32_t addr = (uint32_t)(pCpu->opaddr + pCpu->opsize) + disp;
388	RTStrPrintf(&pszOutput[strlen(pszOutput)], 64, "[%08X]", addr);
389	}
390
391	//no break;
392
393	case 'A': //direct address
394	case 'C': //control register
395	case 'D': //debug register
396	case 'E': //ModRM specifies parameter
397	case 'F': //Eflags register
398	case 'G': //ModRM selects general register
399	case 'I': //Immediate data
400	case 'M': //ModRM may only refer to memory
401	case 'O': //No ModRM byte
402	case 'P': //ModRM byte selects MMX register
403	case 'Q': //ModRM byte selects MMX register or memory address
404	case 'R': //ModRM byte may only refer to a general register
405	case 'S': //ModRM byte selects a segment register
406	case 'T': //ModRM byte selects a test register
407	case 'V': //ModRM byte selects an XMM/SSE register
408	case 'W': //ModRM byte selects an XMM/SSE register or a memory address
409	case 'X': //DS:SI
410	case 'Y': //ES:DI
411	switch(param)
412	{
413	case 1:
414	RTStrPrintf(&pszOutput[strlen(pszOutput)], 64, pParam1->szParam);
415	break;
416	case 2:
417	RTStrPrintf(&pszOutput[strlen(pszOutput)], 64, pParam2->szParam);
418	break;
419	case 3:
420	RTStrPrintf(&pszOutput[strlen(pszOutput)], 64, pParam3->szParam);
421	break;
422	}
423	break;
424
425	case 'e': //register based on operand size (e.g. %eAX)
426	if(pCpu->opmode == CPUMODE_32BIT)
427	RTStrPrintf(&pszOutput[strlen(pszOutput)], 64, "E");
428	if(pCpu->opmode == CPUMODE_64BIT)
429	RTStrPrintf(&pszOutput[strlen(pszOutput)], 64, "R");
430
431	RTStrPrintf(&pszOutput[strlen(pszOutput)], 64, "%c%c", lpszFormat[2], lpszFormat[3]);
432	break;
433
434	default:
435	AssertMsgFailed(("Oops!\n"));
436	break;
437	}
438
439	//Go to the next parameter in the format string
440	while(lpszFormat && lpszFormat != ',') lpszFormat++;
441	if(*lpszFormat == ',') lpszFormat--;
442
443	break;
444
445	case ',':
446	param++;
447	//no break
448
449	default:
450	RTStrPrintf(&pszOutput[strlen(pszOutput)], 64, "%c", *lpszFormat);
451	break;
452	}
453
454	if(*lpszFormat) lpszFormat++;
455	}
456	DbgBytesToString(pCpu, pu8Instruction, pCpu->opsize, pszOutput);
457	RTStrPrintf(&pszOutput[strlen(pszOutput)], 64, "\n");
458	}
459	//*****************************************************************************
460	//*****************************************************************************

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Disassembler/Disasm.cpp@ 15424

Download in other formats: