op_helper.c@ 80823

Last change on this file since 80823 was 80024, checked in by vboxsync, 6 years ago
VMM: Kicking out raw-mode (work in progress) - em config. bugref:9517
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 200.2 KB

Line
1	/*
2	* i386 helpers
3	*
4	* Copyright (c) 2003 Fabrice Bellard
5	*
6	* This library is free software; you can redistribute it and/or
7	* modify it under the terms of the GNU Lesser General Public
8	* License as published by the Free Software Foundation; either
9	* version 2 of the License, or (at your option) any later version.
10	*
11	* This library is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	* Lesser General Public License for more details.
15	*
16	* You should have received a copy of the GNU Lesser General Public
17	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
18	*/
19
20	/*
21	* Oracle LGPL Disclaimer: For the avoidance of doubt, except that if any license choice
22	* other than GPL or LGPL is available it will apply instead, Oracle elects to use only
23	* the Lesser General Public License version 2.1 (LGPLv2) at this time for any software where
24	* a choice of LGPL license versions is made available with the language indicating
25	* that LGPLv2 or any later version may be used, or where a choice of which version
26	* of the LGPL is applied is otherwise unspecified.
27	*/
28
29	#include "exec.h"
30	#include "exec-all.h"
31	#include "host-utils.h"
32	#include "ioport.h"
33
34	#ifdef VBOX
35	# include "qemu-common.h"
36	# include <math.h>
37	# include "tcg.h"
38	# include <VBox/err.h>
39	#endif /* VBOX */
40
41	//#define DEBUG_PCALL
42
43
44	#ifdef DEBUG_PCALL
45	# define LOG_PCALL(...) qemu_log_mask(CPU_LOG_PCALL, ## __VA_ARGS__)
46	# define LOG_PCALL_STATE(env) \
47	log_cpu_state_mask(CPU_LOG_PCALL, (env), X86_DUMP_CCOP)
48	#else
49	# define LOG_PCALL(...) do { } while (0)
50	# define LOG_PCALL_STATE(env) do { } while (0)
51	#endif
52
53
54	#if 0
55	#define raise_exception_err(a, b)\
56	do {\
57	qemu_log("raise_exception line=%d\n", __LINE__);\
58	(raise_exception_err)(a, b);\
59	} while (0)
60	#endif
61
62	static const uint8_t parity_table[256] = {
63	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
64	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
65	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
66	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
67	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
68	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
69	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
70	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
71	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
72	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
73	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
74	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
75	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
76	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
77	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
78	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
79	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
80	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
81	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
82	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
83	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
84	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
85	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
86	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
87	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
88	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
89	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
90	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
91	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
92	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
93	CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
94	0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
95	};
96
97	/* modulo 17 table */
98	static const uint8_t rclw_table[32] = {
99	0, 1, 2, 3, 4, 5, 6, 7,
100	8, 9,10,11,12,13,14,15,
101	16, 0, 1, 2, 3, 4, 5, 6,
102	7, 8, 9,10,11,12,13,14,
103	};
104
105	/* modulo 9 table */
106	static const uint8_t rclb_table[32] = {
107	0, 1, 2, 3, 4, 5, 6, 7,
108	8, 0, 1, 2, 3, 4, 5, 6,
109	7, 8, 0, 1, 2, 3, 4, 5,
110	6, 7, 8, 0, 1, 2, 3, 4,
111	};
112
113	static const CPU86_LDouble f15rk[7] =
114	{
115	0.00000000000000000000L,
116	1.00000000000000000000L,
117	3.14159265358979323851L, /pi/
118	0.30102999566398119523L, /lg2/
119	0.69314718055994530943L, /ln2/
120	1.44269504088896340739L, /l2e/
121	3.32192809488736234781L, /l2t/
122	};
123
124	/* broken thread support */
125
126	static spinlock_t global_cpu_lock = SPIN_LOCK_UNLOCKED;
127
128	void helper_lock(void)
129	{
130	spin_lock(&global_cpu_lock);
131	}
132
133	void helper_unlock(void)
134	{
135	spin_unlock(&global_cpu_lock);
136	}
137
138	void helper_write_eflags(target_ulong t0, uint32_t update_mask)
139	{
140	load_eflags(t0, update_mask);
141	}
142
143	target_ulong helper_read_eflags(void)
144	{
145	uint32_t eflags;
146	eflags = helper_cc_compute_all(CC_OP);
147	eflags \|= (DF & DF_MASK);
148	eflags \|= env->eflags & ~(VM_MASK \| RF_MASK);
149	return eflags;
150	}
151
152	#ifdef VBOX
153
154	void helper_write_eflags_vme(target_ulong t0)
155	{
156	unsigned int new_eflags = t0;
157
158	assert(env->eflags & (1<<VM_SHIFT));
159
160	/* if virtual interrupt pending and (virtual) interrupts will be enabled -> #GP */
161	/* if TF will be set -> #GP */
162	if ( ((new_eflags & IF_MASK) && (env->eflags & VIP_MASK))
163	\|\| (new_eflags & TF_MASK)) {
164	raise_exception(EXCP0D_GPF);
165	} else {
166	load_eflags(new_eflags,
167	(TF_MASK \| AC_MASK \| ID_MASK \| NT_MASK) & 0xffff);
168
169	if (new_eflags & IF_MASK) {
170	env->eflags \|= VIF_MASK;
171	} else {
172	env->eflags &= ~VIF_MASK;
173	}
174	}
175	}
176
177	target_ulong helper_read_eflags_vme(void)
178	{
179	uint32_t eflags;
180	eflags = helper_cc_compute_all(CC_OP);
181	eflags \|= (DF & DF_MASK);
182	eflags \|= env->eflags & ~(VM_MASK \| RF_MASK);
183	if (env->eflags & VIF_MASK)
184	eflags \|= IF_MASK;
185	else
186	eflags &= ~IF_MASK;
187
188	/* According to AMD manual, should be read with IOPL == 3 */
189	eflags \|= (3 << IOPL_SHIFT);
190
191	/* We only use helper_read_eflags_vme() in 16-bits mode */
192	return eflags & 0xffff;
193	}
194
195	void helper_dump_state()
196	{
197	LogRel(("CS:EIP=%08x:%08x, FLAGS=%08x\n", env->segs[R_CS].base, env->eip, env->eflags));
198	LogRel(("EAX=%08x\tECX=%08x\tEDX=%08x\tEBX=%08x\n",
199	(uint32_t)env->regs[R_EAX], (uint32_t)env->regs[R_ECX],
200	(uint32_t)env->regs[R_EDX], (uint32_t)env->regs[R_EBX]));
201	LogRel(("ESP=%08x\tEBP=%08x\tESI=%08x\tEDI=%08x\n",
202	(uint32_t)env->regs[R_ESP], (uint32_t)env->regs[R_EBP],
203	(uint32_t)env->regs[R_ESI], (uint32_t)env->regs[R_EDI]));
204	}
205
206	/**
207	* Updates e2 with the DESC_A_MASK, writes it to the descriptor table, and
208	* returns the updated e2.
209	*
210	* @returns e2 with A set.
211	* @param e2 The 2nd selector DWORD.
212	*/
213	static uint32_t set_segment_accessed(int selector, uint32_t e2)
214	{
215	SegmentCache *dt = selector & X86_SEL_LDT ? &env->ldt : &env->gdt;
216	target_ulong ptr = dt->base + (selector & X86_SEL_MASK);
217
218	e2 \|= DESC_A_MASK;
219	stl_kernel(ptr + 4, e2);
220	return e2;
221	}
222
223	#endif /* VBOX */
224
225	/* return non zero if error */
226	static inline int load_segment(uint32_t e1_ptr, uint32_t e2_ptr,
227	int selector)
228	{
229	SegmentCache *dt;
230	int index;
231	target_ulong ptr;
232
233	if (selector & 0x4)
234	dt = &env->ldt;
235	else
236	dt = &env->gdt;
237	index = selector & ~7;
238	if ((index + 7) > dt->limit)
239	return -1;
240	ptr = dt->base + index;
241	*e1_ptr = ldl_kernel(ptr);
242	*e2_ptr = ldl_kernel(ptr + 4);
243	return 0;
244	}
245
246	static inline unsigned int get_seg_limit(uint32_t e1, uint32_t e2)
247	{
248	unsigned int limit;
249	limit = (e1 & 0xffff) \| (e2 & 0x000f0000);
250	if (e2 & DESC_G_MASK)
251	limit = (limit << 12) \| 0xfff;
252	return limit;
253	}
254
255	static inline uint32_t get_seg_base(uint32_t e1, uint32_t e2)
256	{
257	return ((e1 >> 16) \| ((e2 & 0xff) << 16) \| (e2 & 0xff000000));
258	}
259
260	static inline void load_seg_cache_raw_dt(SegmentCache *sc, uint32_t e1, uint32_t e2)
261	{
262	sc->base = get_seg_base(e1, e2);
263	sc->limit = get_seg_limit(e1, e2);
264	#ifndef VBOX
265	sc->flags = e2;
266	#else
267	sc->flags = e2 & DESC_RAW_FLAG_BITS;
268	sc->newselector = 0;
269	sc->fVBoxFlags = CPUMSELREG_FLAGS_VALID;
270	#endif
271	}
272
273	/* init the segment cache in vm86 mode. */
274	static inline void load_seg_vm(int seg, int selector)
275	{
276	selector &= 0xffff;
277	#ifdef VBOX
278	/* flags must be 0xf3; expand-up read/write accessed data segment with DPL=3. (VT-x) */
279	unsigned flags = DESC_P_MASK \| DESC_S_MASK \| DESC_W_MASK \| DESC_A_MASK;
280	flags \|= (3 << DESC_DPL_SHIFT);
281
282	cpu_x86_load_seg_cache(env, seg, selector,
283	(selector << 4), 0xffff, flags);
284	#else /* VBOX */
285	cpu_x86_load_seg_cache(env, seg, selector,
286	(selector << 4), 0xffff, 0);
287	#endif /* VBOX */
288	}
289
290	static inline void get_ss_esp_from_tss(uint32_t *ss_ptr,
291	uint32_t *esp_ptr, int dpl)
292	{
293	#ifndef VBOX
294	int type, index, shift;
295	#else
296	unsigned int type, index, shift;
297	#endif
298
299	#if 0
300	{
301	int i;
302	printf("TR: base=%p limit=%x\n", env->tr.base, env->tr.limit);
303	for(i=0;i<env->tr.limit;i++) {
304	printf("%02x ", env->tr.base[i]);
305	if ((i & 7) == 7) printf("\n");
306	}
307	printf("\n");
308	}
309	#endif
310
311	if (!(env->tr.flags & DESC_P_MASK))
312	cpu_abort(env, "invalid tss");
313	type = (env->tr.flags >> DESC_TYPE_SHIFT) & 0xf;
314	if ((type & 7) != 3)
315	cpu_abort(env, "invalid tss type");
316	shift = type >> 3;
317	index = (dpl * 4 + 2) << shift;
318	if (index + (4 << shift) - 1 > env->tr.limit)
319	raise_exception_err(EXCP0A_TSS, env->tr.selector & 0xfffc);
320	if (shift == 0) {
321	*esp_ptr = lduw_kernel(env->tr.base + index);
322	*ss_ptr = lduw_kernel(env->tr.base + index + 2);
323	} else {
324	*esp_ptr = ldl_kernel(env->tr.base + index);
325	*ss_ptr = lduw_kernel(env->tr.base + index + 4);
326	}
327	}
328
329	/* XXX: merge with load_seg() */
330	static void tss_load_seg(int seg_reg, int selector)
331	{
332	uint32_t e1, e2;
333	int rpl, dpl, cpl;
334
335	#ifdef VBOX
336	e1 = e2 = 0; /* gcc warning? */
337	cpl = env->hflags & HF_CPL_MASK;
338	/* Trying to load a selector with CPL=1? */
339	if (cpl == 0 && (selector & 3) == 1 && (env->state & CPU_RAW_RING0))
340	{
341	Log(("RPL 1 -> sel %04X -> %04X (tss_load_seg)\n", selector, selector & 0xfffc));
342	selector = selector & 0xfffc;
343	}
344	#endif /* VBOX */
345
346	if ((selector & 0xfffc) != 0) {
347	if (load_segment(&e1, &e2, selector) != 0)
348	raise_exception_err(EXCP0A_TSS, selector & 0xfffc);
349	if (!(e2 & DESC_S_MASK))
350	raise_exception_err(EXCP0A_TSS, selector & 0xfffc);
351	rpl = selector & 3;
352	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
353	cpl = env->hflags & HF_CPL_MASK;
354	if (seg_reg == R_CS) {
355	if (!(e2 & DESC_CS_MASK))
356	raise_exception_err(EXCP0A_TSS, selector & 0xfffc);
357	/* XXX: is it correct ? */
358	if (dpl != rpl)
359	raise_exception_err(EXCP0A_TSS, selector & 0xfffc);
360	if ((e2 & DESC_C_MASK) && dpl > rpl)
361	raise_exception_err(EXCP0A_TSS, selector & 0xfffc);
362	} else if (seg_reg == R_SS) {
363	/* SS must be writable data */
364	if ((e2 & DESC_CS_MASK) \|\| !(e2 & DESC_W_MASK))
365	raise_exception_err(EXCP0A_TSS, selector & 0xfffc);
366	if (dpl != cpl \|\| dpl != rpl)
367	raise_exception_err(EXCP0A_TSS, selector & 0xfffc);
368	} else {
369	/* not readable code */
370	if ((e2 & DESC_CS_MASK) && !(e2 & DESC_R_MASK))
371	raise_exception_err(EXCP0A_TSS, selector & 0xfffc);
372	/* if data or non conforming code, checks the rights */
373	if (((e2 >> DESC_TYPE_SHIFT) & 0xf) < 12) {
374	if (dpl < cpl \|\| dpl < rpl)
375	raise_exception_err(EXCP0A_TSS, selector & 0xfffc);
376	}
377	}
378	if (!(e2 & DESC_P_MASK))
379	raise_exception_err(EXCP0B_NOSEG, selector & 0xfffc);
380	cpu_x86_load_seg_cache(env, seg_reg, selector,
381	get_seg_base(e1, e2),
382	get_seg_limit(e1, e2),
383	e2);
384	} else {
385	if (seg_reg == R_SS \|\| seg_reg == R_CS)
386	raise_exception_err(EXCP0A_TSS, selector & 0xfffc);
387	#ifdef VBOX
388	# if 0 /** @todo now we ignore loading 0 selectors, need to check what is correct once */
389	cpu_x86_load_seg_cache(env, seg_reg, selector,
390	0, 0, 0);
391	# endif
392	#endif /* VBOX */
393	}
394	}
395
396	#define SWITCH_TSS_JMP 0
397	#define SWITCH_TSS_IRET 1
398	#define SWITCH_TSS_CALL 2
399
400	/* XXX: restore CPU state in registers (PowerPC case) */
401	static void switch_tss(int tss_selector,
402	uint32_t e1, uint32_t e2, int source,
403	uint32_t next_eip)
404	{
405	int tss_limit, tss_limit_max, type, old_tss_limit_max, old_type, v1, v2, i;
406	target_ulong tss_base;
407	uint32_t new_regs[8], new_segs[6];
408	uint32_t new_eflags, new_eip, new_cr3, new_ldt, new_trap;
409	uint32_t old_eflags, eflags_mask;
410	SegmentCache *dt;
411	#ifndef VBOX
412	int index;
413	#else
414	unsigned int index;
415	#endif
416	target_ulong ptr;
417
418	type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
419	LOG_PCALL("switch_tss: sel=0x%04x type=%d src=%d\n", tss_selector, type, source);
420
421	/* if task gate, we read the TSS segment and we load it */
422	if (type == 5) {
423	if (!(e2 & DESC_P_MASK))
424	raise_exception_err(EXCP0B_NOSEG, tss_selector & 0xfffc);
425	tss_selector = e1 >> 16;
426	if (tss_selector & 4)
427	raise_exception_err(EXCP0A_TSS, tss_selector & 0xfffc);
428	if (load_segment(&e1, &e2, tss_selector) != 0)
429	raise_exception_err(EXCP0D_GPF, tss_selector & 0xfffc);
430	if (e2 & DESC_S_MASK)
431	raise_exception_err(EXCP0D_GPF, tss_selector & 0xfffc);
432	type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
433	if ((type & 7) != 1)
434	raise_exception_err(EXCP0D_GPF, tss_selector & 0xfffc);
435	}
436
437	if (!(e2 & DESC_P_MASK))
438	raise_exception_err(EXCP0B_NOSEG, tss_selector & 0xfffc);
439
440	if (type & 8)
441	tss_limit_max = 103;
442	else
443	tss_limit_max = 43;
444	tss_limit = get_seg_limit(e1, e2);
445	tss_base = get_seg_base(e1, e2);
446	if ((tss_selector & 4) != 0 \|\|
447	tss_limit < tss_limit_max)
448	raise_exception_err(EXCP0A_TSS, tss_selector & 0xfffc);
449	old_type = (env->tr.flags >> DESC_TYPE_SHIFT) & 0xf;
450	if (old_type & 8)
451	old_tss_limit_max = 103;
452	else
453	old_tss_limit_max = 43;
454
455	#ifndef VBOX /* The old TSS is written first... */
456	/* read all the registers from the new TSS */
457	if (type & 8) {
458	/* 32 bit */
459	new_cr3 = ldl_kernel(tss_base + 0x1c);
460	new_eip = ldl_kernel(tss_base + 0x20);
461	new_eflags = ldl_kernel(tss_base + 0x24);
462	for(i = 0; i < 8; i++)
463	new_regs[i] = ldl_kernel(tss_base + (0x28 + i * 4));
464	for(i = 0; i < 6; i++)
465	new_segs[i] = lduw_kernel(tss_base + (0x48 + i * 4));
466	new_ldt = lduw_kernel(tss_base + 0x60);
467	new_trap = ldl_kernel(tss_base + 0x64);
468	} else {
469	/* 16 bit */
470	new_cr3 = 0;
471	new_eip = lduw_kernel(tss_base + 0x0e);
472	new_eflags = lduw_kernel(tss_base + 0x10);
473	for(i = 0; i < 8; i++)
474	new_regs[i] = lduw_kernel(tss_base + (0x12 + i * 2)) \| 0xffff0000;
475	for(i = 0; i < 4; i++)
476	new_segs[i] = lduw_kernel(tss_base + (0x22 + i * 4));
477	new_ldt = lduw_kernel(tss_base + 0x2a);
478	new_segs[R_FS] = 0;
479	new_segs[R_GS] = 0;
480	new_trap = 0;
481	}
482	#endif
483
484	/* NOTE: we must avoid memory exceptions during the task switch,
485	so we make dummy accesses before */
486	/* XXX: it can still fail in some cases, so a bigger hack is
487	necessary to valid the TLB after having done the accesses */
488
489	v1 = ldub_kernel(env->tr.base);
490	v2 = ldub_kernel(env->tr.base + old_tss_limit_max);
491	stb_kernel(env->tr.base, v1);
492	stb_kernel(env->tr.base + old_tss_limit_max, v2);
493
494	/* clear busy bit (it is restartable) */
495	if (source == SWITCH_TSS_JMP \|\| source == SWITCH_TSS_IRET) {
496	target_ulong ptr;
497	uint32_t e2;
498	ptr = env->gdt.base + (env->tr.selector & ~7);
499	e2 = ldl_kernel(ptr + 4);
500	e2 &= ~DESC_TSS_BUSY_MASK;
501	stl_kernel(ptr + 4, e2);
502	}
503	old_eflags = compute_eflags();
504	if (source == SWITCH_TSS_IRET)
505	old_eflags &= ~NT_MASK;
506
507	/* save the current state in the old TSS */
508	if (type & 8) {
509	/* 32 bit */
510	stl_kernel(env->tr.base + 0x20, next_eip);
511	stl_kernel(env->tr.base + 0x24, old_eflags);
512	stl_kernel(env->tr.base + (0x28 + 0 * 4), EAX);
513	stl_kernel(env->tr.base + (0x28 + 1 * 4), ECX);
514	stl_kernel(env->tr.base + (0x28 + 2 * 4), EDX);
515	stl_kernel(env->tr.base + (0x28 + 3 * 4), EBX);
516	stl_kernel(env->tr.base + (0x28 + 4 * 4), ESP);
517	stl_kernel(env->tr.base + (0x28 + 5 * 4), EBP);
518	stl_kernel(env->tr.base + (0x28 + 6 * 4), ESI);
519	stl_kernel(env->tr.base + (0x28 + 7 * 4), EDI);
520	for(i = 0; i < 6; i++)
521	stw_kernel(env->tr.base + (0x48 + i * 4), env->segs[i].selector);
522	#if defined(VBOX) && defined(DEBUG)
523	printf("TSS 32 bits switch\n");
524	printf("Saving CS=%08X\n", env->segs[R_CS].selector);
525	#endif
526	} else {
527	/* 16 bit */
528	stw_kernel(env->tr.base + 0x0e, next_eip);
529	stw_kernel(env->tr.base + 0x10, old_eflags);
530	stw_kernel(env->tr.base + (0x12 + 0 * 2), EAX);
531	stw_kernel(env->tr.base + (0x12 + 1 * 2), ECX);
532	stw_kernel(env->tr.base + (0x12 + 2 * 2), EDX);
533	stw_kernel(env->tr.base + (0x12 + 3 * 2), EBX);
534	stw_kernel(env->tr.base + (0x12 + 4 * 2), ESP);
535	stw_kernel(env->tr.base + (0x12 + 5 * 2), EBP);
536	stw_kernel(env->tr.base + (0x12 + 6 * 2), ESI);
537	stw_kernel(env->tr.base + (0x12 + 7 * 2), EDI);
538	for(i = 0; i < 4; i++)
539	stw_kernel(env->tr.base + (0x22 + i * 2), env->segs[i].selector);
540	}
541
542	#ifdef VBOX
543	/* read all the registers from the new TSS - may be the same as the old one */
544	if (type & 8) {
545	/* 32 bit */
546	new_cr3 = ldl_kernel(tss_base + 0x1c);
547	new_eip = ldl_kernel(tss_base + 0x20);
548	new_eflags = ldl_kernel(tss_base + 0x24);
549	for(i = 0; i < 8; i++)
550	new_regs[i] = ldl_kernel(tss_base + (0x28 + i * 4));
551	for(i = 0; i < 6; i++)
552	new_segs[i] = lduw_kernel(tss_base + (0x48 + i * 4));
553	new_ldt = lduw_kernel(tss_base + 0x60);
554	new_trap = ldl_kernel(tss_base + 0x64);
555	} else {
556	/* 16 bit */
557	new_cr3 = 0;
558	new_eip = lduw_kernel(tss_base + 0x0e);
559	new_eflags = lduw_kernel(tss_base + 0x10);
560	for(i = 0; i < 8; i++)
561	new_regs[i] = lduw_kernel(tss_base + (0x12 + i * 2)) \| 0xffff0000;
562	for(i = 0; i < 4; i++)
563	new_segs[i] = lduw_kernel(tss_base + (0x22 + i * 2));
564	new_ldt = lduw_kernel(tss_base + 0x2a);
565	new_segs[R_FS] = 0;
566	new_segs[R_GS] = 0;
567	new_trap = 0;
568	}
569	#endif
570
571	/* now if an exception occurs, it will occurs in the next task
572	context */
573
574	if (source == SWITCH_TSS_CALL) {
575	stw_kernel(tss_base, env->tr.selector);
576	new_eflags \|= NT_MASK;
577	}
578
579	/* set busy bit */
580	if (source == SWITCH_TSS_JMP \|\| source == SWITCH_TSS_CALL) {
581	target_ulong ptr;
582	uint32_t e2;
583	ptr = env->gdt.base + (tss_selector & ~7);
584	e2 = ldl_kernel(ptr + 4);
585	e2 \|= DESC_TSS_BUSY_MASK;
586	stl_kernel(ptr + 4, e2);
587	}
588
589	/* set the new CPU state */
590	/* from this point, any exception which occurs can give problems */
591	env->cr[0] \|= CR0_TS_MASK;
592	env->hflags \|= HF_TS_MASK;
593	env->tr.selector = tss_selector;
594	env->tr.base = tss_base;
595	env->tr.limit = tss_limit;
596	#ifndef VBOX
597	env->tr.flags = e2 & ~DESC_TSS_BUSY_MASK;
598	#else
599	env->tr.flags = (e2 \| DESC_TSS_BUSY_MASK) & DESC_RAW_FLAG_BITS;
600	env->tr.fVBoxFlags = CPUMSELREG_FLAGS_VALID;
601	env->tr.newselector = 0;
602	#endif
603
604	if ((type & 8) && (env->cr[0] & CR0_PG_MASK)) {
605	cpu_x86_update_cr3(env, new_cr3);
606	}
607
608	/* load all registers without an exception, then reload them with
609	possible exception */
610	env->eip = new_eip;
611	eflags_mask = TF_MASK \| AC_MASK \| ID_MASK \|
612	IF_MASK \| IOPL_MASK \| VM_MASK \| RF_MASK \| NT_MASK;
613	if (!(type & 8))
614	eflags_mask &= 0xffff;
615	load_eflags(new_eflags, eflags_mask);
616	/* XXX: what to do in 16 bit case ? */
617	EAX = new_regs[0];
618	ECX = new_regs[1];
619	EDX = new_regs[2];
620	EBX = new_regs[3];
621	ESP = new_regs[4];
622	EBP = new_regs[5];
623	ESI = new_regs[6];
624	EDI = new_regs[7];
625	if (new_eflags & VM_MASK) {
626	for(i = 0; i < 6; i++)
627	load_seg_vm(i, new_segs[i]);
628	/* in vm86, CPL is always 3 */
629	cpu_x86_set_cpl(env, 3);
630	} else {
631	/* CPL is set the RPL of CS */
632	cpu_x86_set_cpl(env, new_segs[R_CS] & 3);
633	/* first just selectors as the rest may trigger exceptions */
634	for(i = 0; i < 6; i++)
635	cpu_x86_load_seg_cache(env, i, new_segs[i], 0, 0, 0);
636	}
637
638	env->ldt.selector = new_ldt & ~4;
639	env->ldt.base = 0;
640	env->ldt.limit = 0;
641	env->ldt.flags = 0;
642	#ifdef VBOX
643	env->ldt.flags = DESC_INTEL_UNUSABLE;
644	env->ldt.fVBoxFlags = CPUMSELREG_FLAGS_VALID;
645	env->ldt.newselector = 0;
646	#endif
647
648	/* load the LDT */
649	if (new_ldt & 4)
650	raise_exception_err(EXCP0A_TSS, new_ldt & 0xfffc);
651
652	if ((new_ldt & 0xfffc) != 0) {
653	dt = &env->gdt;
654	index = new_ldt & ~7;
655	if ((index + 7) > dt->limit)
656	raise_exception_err(EXCP0A_TSS, new_ldt & 0xfffc);
657	ptr = dt->base + index;
658	e1 = ldl_kernel(ptr);
659	e2 = ldl_kernel(ptr + 4);
660	if ((e2 & DESC_S_MASK) \|\| ((e2 >> DESC_TYPE_SHIFT) & 0xf) != 2)
661	raise_exception_err(EXCP0A_TSS, new_ldt & 0xfffc);
662	if (!(e2 & DESC_P_MASK))
663	raise_exception_err(EXCP0A_TSS, new_ldt & 0xfffc);
664	load_seg_cache_raw_dt(&env->ldt, e1, e2);
665	}
666
667	/* load the segments */
668	if (!(new_eflags & VM_MASK)) {
669	tss_load_seg(R_CS, new_segs[R_CS]);
670	tss_load_seg(R_SS, new_segs[R_SS]);
671	tss_load_seg(R_ES, new_segs[R_ES]);
672	tss_load_seg(R_DS, new_segs[R_DS]);
673	tss_load_seg(R_FS, new_segs[R_FS]);
674	tss_load_seg(R_GS, new_segs[R_GS]);
675	}
676
677	/* check that EIP is in the CS segment limits */
678	if (new_eip > env->segs[R_CS].limit) {
679	/* XXX: different exception if CALL ? */
680	raise_exception_err(EXCP0D_GPF, 0);
681	}
682
683	#ifndef CONFIG_USER_ONLY
684	/* reset local breakpoints */
685	if (env->dr[7] & 0x55) {
686	for (i = 0; i < 4; i++) {
687	if (hw_breakpoint_enabled(env->dr[7], i) == 0x1)
688	hw_breakpoint_remove(env, i);
689	}
690	env->dr[7] &= ~0x55;
691	}
692	#endif
693	}
694
695	/* check if Port I/O is allowed in TSS */
696	static inline void check_io(int addr, int size)
697	{
698	#ifndef VBOX
699	int io_offset, val, mask;
700	#else
701	int val, mask;
702	unsigned int io_offset;
703	#endif /* VBOX */
704
705	/* TSS must be a valid 32 bit one */
706	if (!(env->tr.flags & DESC_P_MASK) \|\|
707	((env->tr.flags >> DESC_TYPE_SHIFT) & 0xf) != 11 \|\|
708	env->tr.limit < 103)
709	goto fail;
710	io_offset = lduw_kernel(env->tr.base + 0x66);
711	io_offset += (addr >> 3);
712	/* Note: the check needs two bytes */
713	if ((io_offset + 1) > env->tr.limit)
714	goto fail;
715	val = lduw_kernel(env->tr.base + io_offset);
716	val >>= (addr & 7);
717	mask = (1 << size) - 1;
718	/* all bits must be zero to allow the I/O */
719	if ((val & mask) != 0) {
720	fail:
721	raise_exception_err(EXCP0D_GPF, 0);
722	}
723	}
724
725	#ifdef VBOX
726
727	/* Keep in sync with gen_check_external_event() */
728	void helper_check_external_event()
729	{
730	if ( (env->interrupt_request & ( CPU_INTERRUPT_EXTERNAL_FLUSH_TLB
731	\| CPU_INTERRUPT_EXTERNAL_EXIT
732	\| CPU_INTERRUPT_EXTERNAL_TIMER
733	\| CPU_INTERRUPT_EXTERNAL_DMA))
734	\|\| ( (env->interrupt_request & CPU_INTERRUPT_EXTERNAL_HARD)
735	&& (env->eflags & IF_MASK)
736	&& !(env->hflags & HF_INHIBIT_IRQ_MASK) ) )
737	{
738	helper_external_event();
739	}
740
741	}
742
743	void helper_sync_seg(uint32_t reg)
744	{
745	if (env->segs[reg].newselector)
746	sync_seg(env, reg, env->segs[reg].newselector);
747	}
748
749	#endif /* VBOX */
750
751	void helper_check_iob(uint32_t t0)
752	{
753	check_io(t0, 1);
754	}
755
756	void helper_check_iow(uint32_t t0)
757	{
758	check_io(t0, 2);
759	}
760
761	void helper_check_iol(uint32_t t0)
762	{
763	check_io(t0, 4);
764	}
765
766	void helper_outb(uint32_t port, uint32_t data)
767	{
768	#ifndef VBOX
769	cpu_outb(port, data & 0xff);
770	#else
771	cpu_outb(env, port, data & 0xff);
772	#endif
773	}
774
775	target_ulong helper_inb(uint32_t port)
776	{
777	#ifndef VBOX
778	return cpu_inb(port);
779	#else
780	return cpu_inb(env, port);
781	#endif
782	}
783
784	void helper_outw(uint32_t port, uint32_t data)
785	{
786	#ifndef VBOX
787	cpu_outw(port, data & 0xffff);
788	#else
789	cpu_outw(env, port, data & 0xffff);
790	#endif
791	}
792
793	target_ulong helper_inw(uint32_t port)
794	{
795	#ifndef VBOX
796	return cpu_inw(port);
797	#else
798	return cpu_inw(env, port);
799	#endif
800	}
801
802	void helper_outl(uint32_t port, uint32_t data)
803	{
804	#ifndef VBOX
805	cpu_outl(port, data);
806	#else
807	cpu_outl(env, port, data);
808	#endif
809	}
810
811	target_ulong helper_inl(uint32_t port)
812	{
813	#ifndef VBOX
814	return cpu_inl(port);
815	#else
816	return cpu_inl(env, port);
817	#endif
818	}
819
820	static inline unsigned int get_sp_mask(unsigned int e2)
821	{
822	if (e2 & DESC_B_MASK)
823	return 0xffffffff;
824	else
825	return 0xffff;
826	}
827
828	static int exeption_has_error_code(int intno)
829	{
830	switch(intno) {
831	case 8:
832	case 10:
833	case 11:
834	case 12:
835	case 13:
836	case 14:
837	case 17:
838	return 1;
839	}
840	return 0;
841	}
842
843	#ifdef TARGET_X86_64
844	#define SET_ESP(val, sp_mask)\
845	do {\
846	if ((sp_mask) == 0xffff)\
847	ESP = (ESP & ~0xffff) \| ((val) & 0xffff);\
848	else if ((sp_mask) == 0xffffffffLL)\
849	ESP = (uint32_t)(val);\
850	else\
851	ESP = (val);\
852	} while (0)
853	#else
854	#define SET_ESP(val, sp_mask) ESP = (ESP & ~(sp_mask)) \| ((val) & (sp_mask))
855	#endif
856
857	/* in 64-bit machines, this can overflow. So this segment addition macro
858	* can be used to trim the value to 32-bit whenever needed */
859	#define SEG_ADDL(ssp, sp, sp_mask) ((uint32_t)((ssp) + (sp & (sp_mask))))
860
861	/* XXX: add a is_user flag to have proper security support */
862	#define PUSHW(ssp, sp, sp_mask, val)\
863	{\
864	sp -= 2;\
865	stw_kernel((ssp) + (sp & (sp_mask)), (val));\
866	}
867
868	#define PUSHL(ssp, sp, sp_mask, val)\
869	{\
870	sp -= 4;\
871	stl_kernel(SEG_ADDL(ssp, sp, sp_mask), (uint32_t)(val));\
872	}
873
874	#define POPW(ssp, sp, sp_mask, val)\
875	{\
876	val = lduw_kernel((ssp) + (sp & (sp_mask)));\
877	sp += 2;\
878	}
879
880	#define POPL(ssp, sp, sp_mask, val)\
881	{\
882	val = (uint32_t)ldl_kernel(SEG_ADDL(ssp, sp, sp_mask));\
883	sp += 4;\
884	}
885
886	/* protected mode interrupt */
887	static void do_interrupt_protected(int intno, int is_int, int error_code,
888	unsigned int next_eip, int is_hw)
889	{
890	SegmentCache *dt;
891	target_ulong ptr, ssp;
892	int type, dpl, selector, ss_dpl, cpl;
893	int has_error_code, new_stack, shift;
894	uint32_t e1, e2, offset, ss = 0, esp, ss_e1 = 0, ss_e2 = 0;
895	uint32_t old_eip, sp_mask;
896
897	#ifdef VBOX
898	if (remR3NotifyTrap(env, intno, error_code, next_eip) != VINF_SUCCESS)
899	cpu_loop_exit();
900	#endif
901
902	has_error_code = 0;
903	if (!is_int && !is_hw)
904	has_error_code = exeption_has_error_code(intno);
905	if (is_int)
906	old_eip = next_eip;
907	else
908	old_eip = env->eip;
909
910	dt = &env->idt;
911	#ifndef VBOX
912	if (intno * 8 + 7 > dt->limit)
913	#else
914	if ((unsigned)intno * 8 + 7 > dt->limit)
915	#endif
916	raise_exception_err(EXCP0D_GPF, intno * 8 + 2);
917	ptr = dt->base + intno * 8;
918	e1 = ldl_kernel(ptr);
919	e2 = ldl_kernel(ptr + 4);
920	/* check gate type */
921	type = (e2 >> DESC_TYPE_SHIFT) & 0x1f;
922	switch(type) {
923	case 5: /* task gate */
924	#ifdef VBOX
925	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
926	cpl = env->hflags & HF_CPL_MASK;
927	/* check privilege if software int */
928	if (is_int && dpl < cpl)
929	raise_exception_err(EXCP0D_GPF, intno * 8 + 2);
930	#endif
931	/* must do that check here to return the correct error code */
932	if (!(e2 & DESC_P_MASK))
933	raise_exception_err(EXCP0B_NOSEG, intno * 8 + 2);
934	switch_tss(intno * 8, e1, e2, SWITCH_TSS_CALL, old_eip);
935	if (has_error_code) {
936	int type;
937	uint32_t mask;
938	/* push the error code */
939	type = (env->tr.flags >> DESC_TYPE_SHIFT) & 0xf;
940	shift = type >> 3;
941	if (env->segs[R_SS].flags & DESC_B_MASK)
942	mask = 0xffffffff;
943	else
944	mask = 0xffff;
945	esp = (ESP - (2 << shift)) & mask;
946	ssp = env->segs[R_SS].base + esp;
947	if (shift)
948	stl_kernel(ssp, error_code);
949	else
950	stw_kernel(ssp, error_code);
951	SET_ESP(esp, mask);
952	}
953	return;
954	case 6: /* 286 interrupt gate */
955	case 7: /* 286 trap gate */
956	case 14: /* 386 interrupt gate */
957	case 15: /* 386 trap gate */
958	break;
959	default:
960	raise_exception_err(EXCP0D_GPF, intno * 8 + 2);
961	break;
962	}
963	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
964	cpl = env->hflags & HF_CPL_MASK;
965	/* check privilege if software int */
966	if (is_int && dpl < cpl)
967	raise_exception_err(EXCP0D_GPF, intno * 8 + 2);
968	/* check valid bit */
969	if (!(e2 & DESC_P_MASK))
970	raise_exception_err(EXCP0B_NOSEG, intno * 8 + 2);
971	selector = e1 >> 16;
972	offset = (e2 & 0xffff0000) \| (e1 & 0x0000ffff);
973	if ((selector & 0xfffc) == 0)
974	raise_exception_err(EXCP0D_GPF, 0);
975
976	if (load_segment(&e1, &e2, selector) != 0)
977	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
978	#ifdef VBOX /** @todo figure out when this is done one day... */
979	if (!(e2 & DESC_A_MASK))
980	e2 = set_segment_accessed(selector, e2);
981	#endif
982	if (!(e2 & DESC_S_MASK) \|\| !(e2 & (DESC_CS_MASK)))
983	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
984	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
985	if (dpl > cpl)
986	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
987	if (!(e2 & DESC_P_MASK))
988	raise_exception_err(EXCP0B_NOSEG, selector & 0xfffc);
989	if (!(e2 & DESC_C_MASK) && dpl < cpl) {
990	/* to inner privilege */
991	get_ss_esp_from_tss(&ss, &esp, dpl);
992	if ((ss & 0xfffc) == 0)
993	raise_exception_err(EXCP0A_TSS, ss & 0xfffc);
994	if ((ss & 3) != dpl)
995	raise_exception_err(EXCP0A_TSS, ss & 0xfffc);
996	if (load_segment(&ss_e1, &ss_e2, ss) != 0)
997	raise_exception_err(EXCP0A_TSS, ss & 0xfffc);
998	#ifdef VBOX /** @todo figure out when this is done one day... */
999	if (!(ss_e2 & DESC_A_MASK))
1000	ss_e2 = set_segment_accessed(ss, ss_e2);
1001	#endif
1002	ss_dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
1003	if (ss_dpl != dpl)
1004	raise_exception_err(EXCP0A_TSS, ss & 0xfffc);
1005	if (!(ss_e2 & DESC_S_MASK) \|\|
1006	(ss_e2 & DESC_CS_MASK) \|\|
1007	!(ss_e2 & DESC_W_MASK))
1008	raise_exception_err(EXCP0A_TSS, ss & 0xfffc);
1009	if (!(ss_e2 & DESC_P_MASK))
1010	#ifdef VBOX /* See page 3-477 of 253666.pdf */
1011	raise_exception_err(EXCP0C_STACK, ss & 0xfffc);
1012	#else
1013	raise_exception_err(EXCP0A_TSS, ss & 0xfffc);
1014	#endif
1015	new_stack = 1;
1016	sp_mask = get_sp_mask(ss_e2);
1017	ssp = get_seg_base(ss_e1, ss_e2);
1018	#if defined(VBOX) && defined(DEBUG)
1019	printf("new stack %04X:%08X gate dpl=%d\n", ss, esp, dpl);
1020	#endif
1021	} else if ((e2 & DESC_C_MASK) \|\| dpl == cpl) {
1022	/* to same privilege */
1023	if (env->eflags & VM_MASK)
1024	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
1025	new_stack = 0;
1026	sp_mask = get_sp_mask(env->segs[R_SS].flags);
1027	ssp = env->segs[R_SS].base;
1028	esp = ESP;
1029	dpl = cpl;
1030	} else {
1031	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
1032	new_stack = 0; /* avoid warning */
1033	sp_mask = 0; /* avoid warning */
1034	ssp = 0; /* avoid warning */
1035	esp = 0; /* avoid warning */
1036	}
1037
1038	shift = type >> 3;
1039
1040	#if 0
1041	/* XXX: check that enough room is available */
1042	push_size = 6 + (new_stack << 2) + (has_error_code << 1);
1043	if (env->eflags & VM_MASK)
1044	push_size += 8;
1045	push_size <<= shift;
1046	#endif
1047	if (shift == 1) {
1048	if (new_stack) {
1049	if (env->eflags & VM_MASK) {
1050	PUSHL(ssp, esp, sp_mask, env->segs[R_GS].selector);
1051	PUSHL(ssp, esp, sp_mask, env->segs[R_FS].selector);
1052	PUSHL(ssp, esp, sp_mask, env->segs[R_DS].selector);
1053	PUSHL(ssp, esp, sp_mask, env->segs[R_ES].selector);
1054	}
1055	PUSHL(ssp, esp, sp_mask, env->segs[R_SS].selector);
1056	PUSHL(ssp, esp, sp_mask, ESP);
1057	}
1058	PUSHL(ssp, esp, sp_mask, compute_eflags());
1059	PUSHL(ssp, esp, sp_mask, env->segs[R_CS].selector);
1060	PUSHL(ssp, esp, sp_mask, old_eip);
1061	if (has_error_code) {
1062	PUSHL(ssp, esp, sp_mask, error_code);
1063	}
1064	} else {
1065	if (new_stack) {
1066	if (env->eflags & VM_MASK) {
1067	PUSHW(ssp, esp, sp_mask, env->segs[R_GS].selector);
1068	PUSHW(ssp, esp, sp_mask, env->segs[R_FS].selector);
1069	PUSHW(ssp, esp, sp_mask, env->segs[R_DS].selector);
1070	PUSHW(ssp, esp, sp_mask, env->segs[R_ES].selector);
1071	}
1072	PUSHW(ssp, esp, sp_mask, env->segs[R_SS].selector);
1073	PUSHW(ssp, esp, sp_mask, ESP);
1074	}
1075	PUSHW(ssp, esp, sp_mask, compute_eflags());
1076	PUSHW(ssp, esp, sp_mask, env->segs[R_CS].selector);
1077	PUSHW(ssp, esp, sp_mask, old_eip);
1078	if (has_error_code) {
1079	PUSHW(ssp, esp, sp_mask, error_code);
1080	}
1081	}
1082
1083	if (new_stack) {
1084	if (env->eflags & VM_MASK) {
1085	cpu_x86_load_seg_cache(env, R_ES, 0, 0, 0, 0);
1086	cpu_x86_load_seg_cache(env, R_DS, 0, 0, 0, 0);
1087	cpu_x86_load_seg_cache(env, R_FS, 0, 0, 0, 0);
1088	cpu_x86_load_seg_cache(env, R_GS, 0, 0, 0, 0);
1089	}
1090	ss = (ss & ~3) \| dpl;
1091	cpu_x86_load_seg_cache(env, R_SS, ss,
1092	ssp, get_seg_limit(ss_e1, ss_e2), ss_e2);
1093	}
1094	SET_ESP(esp, sp_mask);
1095
1096	selector = (selector & ~3) \| dpl;
1097	cpu_x86_load_seg_cache(env, R_CS, selector,
1098	get_seg_base(e1, e2),
1099	get_seg_limit(e1, e2),
1100	e2);
1101	cpu_x86_set_cpl(env, dpl);
1102	env->eip = offset;
1103
1104	/* interrupt gate clear IF mask */
1105	if ((type & 1) == 0) {
1106	env->eflags &= ~IF_MASK;
1107	}
1108	#ifndef VBOX
1109	env->eflags &= ~(TF_MASK \| VM_MASK \| RF_MASK \| NT_MASK);
1110	#else
1111	/*
1112	* We must clear VIP/VIF too on interrupt entry, as otherwise FreeBSD
1113	* gets confused by seemingly changed EFLAGS. See #3491 and
1114	* public bug #2341.
1115	*/
1116	env->eflags &= ~(TF_MASK \| VM_MASK \| RF_MASK \| NT_MASK \| VIF_MASK \| VIP_MASK);
1117	#endif
1118	}
1119
1120	#ifdef VBOX
1121
1122	/* check if VME interrupt redirection is enabled in TSS */
1123	DECLINLINE(bool) is_vme_irq_redirected(int intno)
1124	{
1125	unsigned int io_offset, intredir_offset;
1126	unsigned char val, mask;
1127
1128	/* TSS must be a valid 32 bit one */
1129	if (!(env->tr.flags & DESC_P_MASK) \|\|
1130	((env->tr.flags >> DESC_TYPE_SHIFT) & 0xf) != 11 \|\|
1131	env->tr.limit < 103)
1132	goto fail;
1133	io_offset = lduw_kernel(env->tr.base + 0x66);
1134	/* Make sure the io bitmap offset is valid; anything less than sizeof(VBOXTSS) means there's none. */
1135	if (io_offset < 0x68 + 0x20)
1136	io_offset = 0x68 + 0x20;
1137	/* the virtual interrupt redirection bitmap is located below the io bitmap */
1138	intredir_offset = io_offset - 0x20;
1139
1140	intredir_offset += (intno >> 3);
1141	if ((intredir_offset) > env->tr.limit)
1142	goto fail;
1143
1144	val = ldub_kernel(env->tr.base + intredir_offset);
1145	mask = 1 << (unsigned char)(intno & 7);
1146
1147	/* bit set means no redirection. */
1148	if ((val & mask) != 0) {
1149	return false;
1150	}
1151	return true;
1152
1153	fail:
1154	raise_exception_err(EXCP0D_GPF, 0);
1155	return true;
1156	}
1157
1158	/* V86 mode software interrupt with CR4.VME=1 */
1159	static void do_soft_interrupt_vme(int intno, int error_code, unsigned int next_eip)
1160	{
1161	target_ulong ptr, ssp;
1162	int selector;
1163	uint32_t offset, esp;
1164	uint32_t old_cs, old_eflags;
1165	uint32_t iopl;
1166
1167	iopl = ((env->eflags >> IOPL_SHIFT) & 3);
1168
1169	if (!is_vme_irq_redirected(intno))
1170	{
1171	if (iopl == 3)
1172	{
1173	do_interrupt_protected(intno, 1, error_code, next_eip, 0);
1174	return;
1175	}
1176	else
1177	raise_exception_err(EXCP0D_GPF, 0);
1178	}
1179
1180	/* virtual mode idt is at linear address 0 */
1181	ptr = 0 + intno * 4;
1182	offset = lduw_kernel(ptr);
1183	selector = lduw_kernel(ptr + 2);
1184	esp = ESP;
1185	ssp = env->segs[R_SS].base;
1186	old_cs = env->segs[R_CS].selector;
1187
1188	old_eflags = compute_eflags();
1189	if (iopl < 3)
1190	{
1191	/* copy VIF into IF and set IOPL to 3 */
1192	if (env->eflags & VIF_MASK)
1193	old_eflags \|= IF_MASK;
1194	else
1195	old_eflags &= ~IF_MASK;
1196
1197	old_eflags \|= (3 << IOPL_SHIFT);
1198	}
1199
1200	/* XXX: use SS segment size ? */
1201	PUSHW(ssp, esp, 0xffff, old_eflags);
1202	PUSHW(ssp, esp, 0xffff, old_cs);
1203	PUSHW(ssp, esp, 0xffff, next_eip);
1204
1205	/* update processor state */
1206	ESP = (ESP & ~0xffff) \| (esp & 0xffff);
1207	env->eip = offset;
1208	env->segs[R_CS].selector = selector;
1209	env->segs[R_CS].base = (selector << 4);
1210	env->eflags &= ~(TF_MASK \| RF_MASK);
1211
1212	if (iopl < 3)
1213	env->eflags &= ~VIF_MASK;
1214	else
1215	env->eflags &= ~IF_MASK;
1216	}
1217
1218	#endif /* VBOX */
1219
1220	#ifdef TARGET_X86_64
1221
1222	#define PUSHQ(sp, val)\
1223	{\
1224	sp -= 8;\
1225	stq_kernel(sp, (val));\
1226	}
1227
1228	#define POPQ(sp, val)\
1229	{\
1230	val = ldq_kernel(sp);\
1231	sp += 8;\
1232	}
1233
1234	static inline target_ulong get_rsp_from_tss(int level)
1235	{
1236	int index;
1237
1238	#if 0
1239	printf("TR: base=" TARGET_FMT_lx " limit=%x\n",
1240	env->tr.base, env->tr.limit);
1241	#endif
1242
1243	if (!(env->tr.flags & DESC_P_MASK))
1244	cpu_abort(env, "invalid tss");
1245	index = 8 * level + 4;
1246	if ((index + 7) > env->tr.limit)
1247	raise_exception_err(EXCP0A_TSS, env->tr.selector & 0xfffc);
1248	return ldq_kernel(env->tr.base + index);
1249	}
1250
1251	/* 64 bit interrupt */
1252	static void do_interrupt64(int intno, int is_int, int error_code,
1253	target_ulong next_eip, int is_hw)
1254	{
1255	SegmentCache *dt;
1256	target_ulong ptr;
1257	int type, dpl, selector, cpl, ist;
1258	int has_error_code, new_stack;
1259	uint32_t e1, e2, e3, ss;
1260	target_ulong old_eip, esp, offset;
1261
1262	#ifdef VBOX
1263	if (remR3NotifyTrap(env, intno, error_code, next_eip) != VINF_SUCCESS)
1264	cpu_loop_exit();
1265	#endif
1266
1267	has_error_code = 0;
1268	if (!is_int && !is_hw)
1269	has_error_code = exeption_has_error_code(intno);
1270	if (is_int)
1271	old_eip = next_eip;
1272	else
1273	old_eip = env->eip;
1274
1275	dt = &env->idt;
1276	if (intno * 16 + 15 > dt->limit)
1277	raise_exception_err(EXCP0D_GPF, intno * 16 + 2);
1278	ptr = dt->base + intno * 16;
1279	e1 = ldl_kernel(ptr);
1280	e2 = ldl_kernel(ptr + 4);
1281	e3 = ldl_kernel(ptr + 8);
1282	/* check gate type */
1283	type = (e2 >> DESC_TYPE_SHIFT) & 0x1f;
1284	switch(type) {
1285	case 14: /* 386 interrupt gate */
1286	case 15: /* 386 trap gate */
1287	break;
1288	default:
1289	raise_exception_err(EXCP0D_GPF, intno * 16 + 2);
1290	break;
1291	}
1292	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
1293	cpl = env->hflags & HF_CPL_MASK;
1294	/* check privilege if software int */
1295	if (is_int && dpl < cpl)
1296	raise_exception_err(EXCP0D_GPF, intno * 16 + 2);
1297	/* check valid bit */
1298	if (!(e2 & DESC_P_MASK))
1299	raise_exception_err(EXCP0B_NOSEG, intno * 16 + 2);
1300	selector = e1 >> 16;
1301	offset = ((target_ulong)e3 << 32) \| (e2 & 0xffff0000) \| (e1 & 0x0000ffff);
1302	ist = e2 & 7;
1303	if ((selector & 0xfffc) == 0)
1304	raise_exception_err(EXCP0D_GPF, 0);
1305
1306	if (load_segment(&e1, &e2, selector) != 0)
1307	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
1308	if (!(e2 & DESC_S_MASK) \|\| !(e2 & (DESC_CS_MASK)))
1309	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
1310	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
1311	if (dpl > cpl)
1312	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
1313	if (!(e2 & DESC_P_MASK))
1314	raise_exception_err(EXCP0B_NOSEG, selector & 0xfffc);
1315	if (!(e2 & DESC_L_MASK) \|\| (e2 & DESC_B_MASK))
1316	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
1317	if ((!(e2 & DESC_C_MASK) && dpl < cpl) \|\| ist != 0) {
1318	/* to inner privilege */
1319	if (ist != 0)
1320	esp = get_rsp_from_tss(ist + 3);
1321	else
1322	esp = get_rsp_from_tss(dpl);
1323	esp &= ~0xfLL; /* align stack */
1324	ss = 0;
1325	new_stack = 1;
1326	} else if ((e2 & DESC_C_MASK) \|\| dpl == cpl) {
1327	/* to same privilege */
1328	if (env->eflags & VM_MASK)
1329	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
1330	new_stack = 0;
1331	if (ist != 0)
1332	esp = get_rsp_from_tss(ist + 3);
1333	else
1334	esp = ESP;
1335	esp &= ~0xfLL; /* align stack */
1336	dpl = cpl;
1337	} else {
1338	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
1339	new_stack = 0; /* avoid warning */
1340	esp = 0; /* avoid warning */
1341	}
1342
1343	PUSHQ(esp, env->segs[R_SS].selector);
1344	PUSHQ(esp, ESP);
1345	PUSHQ(esp, compute_eflags());
1346	PUSHQ(esp, env->segs[R_CS].selector);
1347	PUSHQ(esp, old_eip);
1348	if (has_error_code) {
1349	PUSHQ(esp, error_code);
1350	}
1351
1352	if (new_stack) {
1353	ss = 0 \| dpl;
1354	#ifndef VBOX
1355	cpu_x86_load_seg_cache(env, R_SS, ss, 0, 0, 0);
1356	#else
1357	cpu_x86_load_seg_cache(env, R_SS, ss, 0, 0, dpl << DESC_DPL_SHIFT);
1358	#endif
1359	}
1360	ESP = esp;
1361
1362	selector = (selector & ~3) \| dpl;
1363	cpu_x86_load_seg_cache(env, R_CS, selector,
1364	get_seg_base(e1, e2),
1365	get_seg_limit(e1, e2),
1366	e2);
1367	cpu_x86_set_cpl(env, dpl);
1368	env->eip = offset;
1369
1370	/* interrupt gate clear IF mask */
1371	if ((type & 1) == 0) {
1372	env->eflags &= ~IF_MASK;
1373	}
1374	#ifndef VBOX
1375	env->eflags &= ~(TF_MASK \| VM_MASK \| RF_MASK \| NT_MASK);
1376	#else /* VBOX */
1377	/*
1378	* We must clear VIP/VIF too on interrupt entry, as otherwise FreeBSD
1379	* gets confused by seemingly changed EFLAGS. See #3491 and
1380	* public bug #2341.
1381	*/
1382	env->eflags &= ~(TF_MASK \| VM_MASK \| RF_MASK \| NT_MASK \| VIF_MASK \| VIP_MASK);
1383	#endif /* VBOX */
1384	}
1385	#endif
1386
1387	#ifdef TARGET_X86_64
1388	#if defined(CONFIG_USER_ONLY)
1389	void helper_syscall(int next_eip_addend)
1390	{
1391	env->exception_index = EXCP_SYSCALL;
1392	env->exception_next_eip = env->eip + next_eip_addend;
1393	cpu_loop_exit();
1394	}
1395	#else
1396	void helper_syscall(int next_eip_addend)
1397	{
1398	int selector;
1399
1400	if (!(env->efer & MSR_EFER_SCE)) {
1401	raise_exception_err(EXCP06_ILLOP, 0);
1402	}
1403	selector = (env->star >> 32) & 0xffff;
1404	if (env->hflags & HF_LMA_MASK) {
1405	int code64;
1406
1407	ECX = env->eip + next_eip_addend;
1408	env->regs[11] = compute_eflags();
1409
1410	code64 = env->hflags & HF_CS64_MASK;
1411
1412	cpu_x86_set_cpl(env, 0);
1413	cpu_x86_load_seg_cache(env, R_CS, selector & 0xfffc,
1414	0, 0xffffffff,
1415	DESC_G_MASK \| DESC_P_MASK \|
1416	DESC_S_MASK \|
1417	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK \| DESC_L_MASK);
1418	cpu_x86_load_seg_cache(env, R_SS, (selector + 8) & 0xfffc,
1419	0, 0xffffffff,
1420	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
1421	DESC_S_MASK \|
1422	DESC_W_MASK \| DESC_A_MASK);
1423	env->eflags &= ~env->fmask;
1424	load_eflags(env->eflags, 0);
1425	if (code64)
1426	env->eip = env->lstar;
1427	else
1428	env->eip = env->cstar;
1429	} else {
1430	ECX = (uint32_t)(env->eip + next_eip_addend);
1431
1432	cpu_x86_set_cpl(env, 0);
1433	cpu_x86_load_seg_cache(env, R_CS, selector & 0xfffc,
1434	0, 0xffffffff,
1435	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
1436	DESC_S_MASK \|
1437	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK);
1438	cpu_x86_load_seg_cache(env, R_SS, (selector + 8) & 0xfffc,
1439	0, 0xffffffff,
1440	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
1441	DESC_S_MASK \|
1442	DESC_W_MASK \| DESC_A_MASK);
1443	env->eflags &= ~(IF_MASK \| RF_MASK \| VM_MASK);
1444	env->eip = (uint32_t)env->star;
1445	}
1446	}
1447	#endif
1448	#endif
1449
1450	#ifdef TARGET_X86_64
1451	void helper_sysret(int dflag)
1452	{
1453	int cpl, selector;
1454
1455	if (!(env->efer & MSR_EFER_SCE)) {
1456	raise_exception_err(EXCP06_ILLOP, 0);
1457	}
1458	cpl = env->hflags & HF_CPL_MASK;
1459	if (!(env->cr[0] & CR0_PE_MASK) \|\| cpl != 0) {
1460	raise_exception_err(EXCP0D_GPF, 0);
1461	}
1462	selector = (env->star >> 48) & 0xffff;
1463	if (env->hflags & HF_LMA_MASK) {
1464	if (dflag == 2) {
1465	cpu_x86_load_seg_cache(env, R_CS, (selector + 16) \| 3,
1466	0, 0xffffffff,
1467	DESC_G_MASK \| DESC_P_MASK \|
1468	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
1469	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK \|
1470	DESC_L_MASK);
1471	env->eip = ECX;
1472	} else {
1473	cpu_x86_load_seg_cache(env, R_CS, selector \| 3,
1474	0, 0xffffffff,
1475	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
1476	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
1477	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK);
1478	env->eip = (uint32_t)ECX;
1479	}
1480	cpu_x86_load_seg_cache(env, R_SS, selector + 8,
1481	0, 0xffffffff,
1482	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
1483	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
1484	DESC_W_MASK \| DESC_A_MASK);
1485	load_eflags((uint32_t)(env->regs[11]), TF_MASK \| AC_MASK \| ID_MASK \|
1486	IF_MASK \| IOPL_MASK \| VM_MASK \| RF_MASK \| NT_MASK);
1487	cpu_x86_set_cpl(env, 3);
1488	} else {
1489	cpu_x86_load_seg_cache(env, R_CS, selector \| 3,
1490	0, 0xffffffff,
1491	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
1492	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
1493	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK);
1494	env->eip = (uint32_t)ECX;
1495	cpu_x86_load_seg_cache(env, R_SS, selector + 8,
1496	0, 0xffffffff,
1497	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
1498	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
1499	DESC_W_MASK \| DESC_A_MASK);
1500	env->eflags \|= IF_MASK;
1501	cpu_x86_set_cpl(env, 3);
1502	}
1503	}
1504	#endif
1505
1506	#ifdef VBOX
1507
1508	/**
1509	* Checks and processes external VMM events.
1510	* Called by op_check_external_event() when any of the flags is set and can be serviced.
1511	*/
1512	void helper_external_event(void)
1513	{
1514	# if defined(RT_OS_DARWIN) && defined(VBOX_STRICT)
1515	uintptr_t uSP;
1516	# ifdef RT_ARCH_AMD64
1517	__asm__ __volatile__("movq %%rsp, %0" : "=r" (uSP));
1518	# else
1519	__asm__ __volatile__("movl %%esp, %0" : "=r" (uSP));
1520	# endif
1521	AssertMsg(!(uSP & 15), ("xSP=%#p\n", uSP));
1522	# endif
1523	/* Keep in sync with flags checked by gen_check_external_event() */
1524	if (env->interrupt_request & CPU_INTERRUPT_EXTERNAL_HARD)
1525	{
1526	ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request,
1527	~CPU_INTERRUPT_EXTERNAL_HARD);
1528	cpu_interrupt(env, CPU_INTERRUPT_HARD);
1529	}
1530	if (env->interrupt_request & CPU_INTERRUPT_EXTERNAL_EXIT)
1531	{
1532	ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request,
1533	~CPU_INTERRUPT_EXTERNAL_EXIT);
1534	cpu_exit(env);
1535	}
1536	if (env->interrupt_request & CPU_INTERRUPT_EXTERNAL_DMA)
1537	{
1538	ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request,
1539	~CPU_INTERRUPT_EXTERNAL_DMA);
1540	remR3DmaRun(env);
1541	}
1542	if (env->interrupt_request & CPU_INTERRUPT_EXTERNAL_TIMER)
1543	{
1544	ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request,
1545	~CPU_INTERRUPT_EXTERNAL_TIMER);
1546	remR3TimersRun(env);
1547	}
1548	if (env->interrupt_request & CPU_INTERRUPT_EXTERNAL_FLUSH_TLB)
1549	{
1550	ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request,
1551	~CPU_INTERRUPT_EXTERNAL_HARD);
1552	cpu_interrupt(env, CPU_INTERRUPT_HARD);
1553	}
1554	}
1555
1556	/* helper for recording call instruction addresses for later scanning */
1557	void helper_record_call()
1558	{
1559	if ( !(env->state & CPU_RAW_RING0)
1560	&& (env->cr[0] & CR0_PG_MASK)
1561	&& !(env->eflags & X86_EFL_IF))
1562	remR3RecordCall(env);
1563	}
1564
1565	#endif /* VBOX */
1566
1567	/* real mode interrupt */
1568	static void do_interrupt_real(int intno, int is_int, int error_code,
1569	unsigned int next_eip)
1570	{
1571	SegmentCache *dt;
1572	target_ulong ptr, ssp;
1573	int selector;
1574	uint32_t offset, esp;
1575	uint32_t old_cs, old_eip;
1576
1577	/* real mode (simpler !) */
1578	dt = &env->idt;
1579	#ifndef VBOX
1580	if (intno * 4 + 3 > dt->limit)
1581	#else
1582	if ((unsigned)intno * 4 + 3 > dt->limit)
1583	#endif
1584	raise_exception_err(EXCP0D_GPF, intno * 8 + 2);
1585	ptr = dt->base + intno * 4;
1586	offset = lduw_kernel(ptr);
1587	selector = lduw_kernel(ptr + 2);
1588	esp = ESP;
1589	ssp = env->segs[R_SS].base;
1590	if (is_int)
1591	old_eip = next_eip;
1592	else
1593	old_eip = env->eip;
1594	old_cs = env->segs[R_CS].selector;
1595	/* XXX: use SS segment size ? */
1596	PUSHW(ssp, esp, 0xffff, compute_eflags());
1597	PUSHW(ssp, esp, 0xffff, old_cs);
1598	PUSHW(ssp, esp, 0xffff, old_eip);
1599
1600	/* update processor state */
1601	ESP = (ESP & ~0xffff) \| (esp & 0xffff);
1602	env->eip = offset;
1603	env->segs[R_CS].selector = selector;
1604	env->segs[R_CS].base = (selector << 4);
1605	env->eflags &= ~(IF_MASK \| TF_MASK \| AC_MASK \| RF_MASK);
1606	}
1607
1608	/* fake user mode interrupt */
1609	void do_interrupt_user(int intno, int is_int, int error_code,
1610	target_ulong next_eip)
1611	{
1612	SegmentCache *dt;
1613	target_ulong ptr;
1614	int dpl, cpl, shift;
1615	uint32_t e2;
1616
1617	dt = &env->idt;
1618	if (env->hflags & HF_LMA_MASK) {
1619	shift = 4;
1620	} else {
1621	shift = 3;
1622	}
1623	ptr = dt->base + (intno << shift);
1624	e2 = ldl_kernel(ptr + 4);
1625
1626	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
1627	cpl = env->hflags & HF_CPL_MASK;
1628	/* check privilege if software int */
1629	if (is_int && dpl < cpl)
1630	raise_exception_err(EXCP0D_GPF, (intno << shift) + 2);
1631
1632	/* Since we emulate only user space, we cannot do more than
1633	exiting the emulation with the suitable exception and error
1634	code */
1635	if (is_int)
1636	EIP = next_eip;
1637	}
1638
1639	#if !defined(CONFIG_USER_ONLY)
1640	static void handle_even_inj(int intno, int is_int, int error_code,
1641	int is_hw, int rm)
1642	{
1643	uint32_t event_inj = ldl_phys(env->vm_vmcb + offsetof(struct vmcb, control.event_inj));
1644	if (!(event_inj & SVM_EVTINJ_VALID)) {
1645	int type;
1646	if (is_int)
1647	type = SVM_EVTINJ_TYPE_SOFT;
1648	else
1649	type = SVM_EVTINJ_TYPE_EXEPT;
1650	event_inj = intno \| type \| SVM_EVTINJ_VALID;
1651	if (!rm && exeption_has_error_code(intno)) {
1652	event_inj \|= SVM_EVTINJ_VALID_ERR;
1653	stl_phys(env->vm_vmcb + offsetof(struct vmcb, control.event_inj_err), error_code);
1654	}
1655	stl_phys(env->vm_vmcb + offsetof(struct vmcb, control.event_inj), event_inj);
1656	}
1657	}
1658	#endif
1659
1660	/*
1661	* Begin execution of an interruption. is_int is TRUE if coming from
1662	* the int instruction. next_eip is the EIP value AFTER the interrupt
1663	* instruction. It is only relevant if is_int is TRUE.
1664	*/
1665	void do_interrupt(int intno, int is_int, int error_code,
1666	target_ulong next_eip, int is_hw)
1667	{
1668	if (qemu_loglevel_mask(CPU_LOG_INT)) {
1669	if ((env->cr[0] & CR0_PE_MASK)) {
1670	static int count;
1671	qemu_log("%6d: v=%02x e=%04x i=%d cpl=%d IP=%04x:" TARGET_FMT_lx " pc=" TARGET_FMT_lx " SP=%04x:" TARGET_FMT_lx,
1672	count, intno, error_code, is_int,
1673	env->hflags & HF_CPL_MASK,
1674	env->segs[R_CS].selector, EIP,
1675	(int)env->segs[R_CS].base + EIP,
1676	env->segs[R_SS].selector, ESP);
1677	if (intno == 0x0e) {
1678	qemu_log(" CR2=" TARGET_FMT_lx, env->cr[2]);
1679	} else {
1680	qemu_log(" EAX=" TARGET_FMT_lx, EAX);
1681	}
1682	qemu_log("\n");
1683	log_cpu_state(env, X86_DUMP_CCOP);
1684	#if 0
1685	{
1686	int i;
1687	uint8_t *ptr;
1688	qemu_log(" code=");
1689	ptr = env->segs[R_CS].base + env->eip;
1690	for(i = 0; i < 16; i++) {
1691	qemu_log(" %02x", ldub(ptr + i));
1692	}
1693	qemu_log("\n");
1694	}
1695	#endif
1696	count++;
1697	}
1698	}
1699	#ifdef VBOX
1700	if (RT_UNLIKELY(env->state & CPU_EMULATE_SINGLE_STEP)) {
1701	if (is_int) {
1702	RTLogPrintf("do_interrupt: %#04x err=%#x pc=%#RGv%s\n",
1703	intno, error_code, (RTGCPTR)env->eip, is_hw ? " hw" : "");
1704	} else {
1705	RTLogPrintf("do_interrupt: %#04x err=%#x pc=%#RGv next=%#RGv%s\n",
1706	intno, error_code, (RTGCPTR)env->eip, (RTGCPTR)next_eip, is_hw ? " hw" : "");
1707	}
1708	}
1709	#endif
1710	if (env->cr[0] & CR0_PE_MASK) {
1711	#if !defined(CONFIG_USER_ONLY)
1712	if (env->hflags & HF_SVMI_MASK)
1713	handle_even_inj(intno, is_int, error_code, is_hw, 0);
1714	#endif
1715	#ifdef TARGET_X86_64
1716	if (env->hflags & HF_LMA_MASK) {
1717	do_interrupt64(intno, is_int, error_code, next_eip, is_hw);
1718	} else
1719	#endif
1720	{
1721	#ifdef VBOX
1722	/* int xx , v86 code and VME enabled? /
1723	if ( (env->eflags & VM_MASK)
1724	&& (env->cr[4] & CR4_VME_MASK)
1725	&& is_int
1726	&& !is_hw
1727	&& env->eip + 1 != next_eip /* single byte int 3 goes straight to the protected mode handler */
1728	)
1729	do_soft_interrupt_vme(intno, error_code, next_eip);
1730	else
1731	#endif /* VBOX */
1732	do_interrupt_protected(intno, is_int, error_code, next_eip, is_hw);
1733	}
1734	} else {
1735	#if !defined(CONFIG_USER_ONLY)
1736	if (env->hflags & HF_SVMI_MASK)
1737	handle_even_inj(intno, is_int, error_code, is_hw, 1);
1738	#endif
1739	do_interrupt_real(intno, is_int, error_code, next_eip);
1740	}
1741
1742	#if !defined(CONFIG_USER_ONLY)
1743	if (env->hflags & HF_SVMI_MASK) {
1744	uint32_t event_inj = ldl_phys(env->vm_vmcb + offsetof(struct vmcb, control.event_inj));
1745	stl_phys(env->vm_vmcb + offsetof(struct vmcb, control.event_inj), event_inj & ~SVM_EVTINJ_VALID);
1746	}
1747	#endif
1748	}
1749
1750	/* This should come from sysemu.h - if we could include it here... */
1751	void qemu_system_reset_request(void);
1752
1753	/*
1754	* Check nested exceptions and change to double or triple fault if
1755	* needed. It should only be called, if this is not an interrupt.
1756	* Returns the new exception number.
1757	*/
1758	static int check_exception(int intno, int *error_code)
1759	{
1760	int first_contributory = env->old_exception == 0 \|\|
1761	(env->old_exception >= 10 &&
1762	env->old_exception <= 13);
1763	int second_contributory = intno == 0 \|\|
1764	(intno >= 10 && intno <= 13);
1765
1766	qemu_log_mask(CPU_LOG_INT, "check_exception old: 0x%x new 0x%x\n",
1767	env->old_exception, intno);
1768
1769	#if !defined(CONFIG_USER_ONLY)
1770	if (env->old_exception == EXCP08_DBLE) {
1771	if (env->hflags & HF_SVMI_MASK)
1772	helper_vmexit(SVM_EXIT_SHUTDOWN, 0); /* does not return */
1773
1774	qemu_log_mask(CPU_LOG_RESET, "Triple fault\n");
1775
1776	# ifndef VBOX
1777	qemu_system_reset_request();
1778	return EXCP_HLT;
1779	# else
1780	remR3RaiseRC(env->pVM, VINF_EM_TRIPLE_FAULT);
1781	return EXCP_RC;
1782	# endif
1783	}
1784	#endif
1785
1786	if ((first_contributory && second_contributory)
1787	\|\| (env->old_exception == EXCP0E_PAGE &&
1788	(second_contributory \|\| (intno == EXCP0E_PAGE)))) {
1789	intno = EXCP08_DBLE;
1790	*error_code = 0;
1791	}
1792
1793	if (second_contributory \|\| (intno == EXCP0E_PAGE) \|\|
1794	(intno == EXCP08_DBLE))
1795	env->old_exception = intno;
1796
1797	return intno;
1798	}
1799
1800	/*
1801	* Signal an interruption. It is executed in the main CPU loop.
1802	* is_int is TRUE if coming from the int instruction. next_eip is the
1803	* EIP value AFTER the interrupt instruction. It is only relevant if
1804	* is_int is TRUE.
1805	*/
1806	static void QEMU_NORETURN raise_interrupt(int intno, int is_int, int error_code,
1807	int next_eip_addend)
1808	{
1809	#if defined(VBOX) && defined(DEBUG)
1810	Log2(("raise_interrupt: %x %x %x %RGv\n", intno, is_int, error_code, (RTGCPTR)env->eip + next_eip_addend));
1811	#endif
1812	if (!is_int) {
1813	helper_svm_check_intercept_param(SVM_EXIT_EXCP_BASE + intno, error_code);
1814	intno = check_exception(intno, &error_code);
1815	} else {
1816	helper_svm_check_intercept_param(SVM_EXIT_SWINT, 0);
1817	}
1818
1819	env->exception_index = intno;
1820	env->error_code = error_code;
1821	env->exception_is_int = is_int;
1822	env->exception_next_eip = env->eip + next_eip_addend;
1823	cpu_loop_exit();
1824	}
1825
1826	/* shortcuts to generate exceptions */
1827
1828	void raise_exception_err(int exception_index, int error_code)
1829	{
1830	raise_interrupt(exception_index, 0, error_code, 0);
1831	}
1832
1833	void raise_exception(int exception_index)
1834	{
1835	raise_interrupt(exception_index, 0, 0, 0);
1836	}
1837
1838	void raise_exception_env(int exception_index, CPUState *nenv)
1839	{
1840	env = nenv;
1841	raise_exception(exception_index);
1842	}
1843	/* SMM support */
1844
1845	#if defined(CONFIG_USER_ONLY)
1846
1847	void do_smm_enter(void)
1848	{
1849	}
1850
1851	void helper_rsm(void)
1852	{
1853	}
1854
1855	#else
1856
1857	#ifdef TARGET_X86_64
1858	#define SMM_REVISION_ID 0x00020064
1859	#else
1860	#define SMM_REVISION_ID 0x00020000
1861	#endif
1862
1863	void do_smm_enter(void)
1864	{
1865	target_ulong sm_state;
1866	SegmentCache *dt;
1867	int i, offset;
1868
1869	qemu_log_mask(CPU_LOG_INT, "SMM: enter\n");
1870	log_cpu_state_mask(CPU_LOG_INT, env, X86_DUMP_CCOP);
1871
1872	env->hflags \|= HF_SMM_MASK;
1873	cpu_smm_update(env);
1874
1875	sm_state = env->smbase + 0x8000;
1876
1877	#ifdef TARGET_X86_64
1878	for(i = 0; i < 6; i++) {
1879	dt = &env->segs[i];
1880	offset = 0x7e00 + i * 16;
1881	stw_phys(sm_state + offset, dt->selector);
1882	stw_phys(sm_state + offset + 2, (dt->flags >> 8) & 0xf0ff);
1883	stl_phys(sm_state + offset + 4, dt->limit);
1884	stq_phys(sm_state + offset + 8, dt->base);
1885	}
1886
1887	stq_phys(sm_state + 0x7e68, env->gdt.base);
1888	stl_phys(sm_state + 0x7e64, env->gdt.limit);
1889
1890	stw_phys(sm_state + 0x7e70, env->ldt.selector);
1891	stq_phys(sm_state + 0x7e78, env->ldt.base);
1892	stl_phys(sm_state + 0x7e74, env->ldt.limit);
1893	stw_phys(sm_state + 0x7e72, (env->ldt.flags >> 8) & 0xf0ff);
1894
1895	stq_phys(sm_state + 0x7e88, env->idt.base);
1896	stl_phys(sm_state + 0x7e84, env->idt.limit);
1897
1898	stw_phys(sm_state + 0x7e90, env->tr.selector);
1899	stq_phys(sm_state + 0x7e98, env->tr.base);
1900	stl_phys(sm_state + 0x7e94, env->tr.limit);
1901	stw_phys(sm_state + 0x7e92, (env->tr.flags >> 8) & 0xf0ff);
1902
1903	stq_phys(sm_state + 0x7ed0, env->efer);
1904
1905	stq_phys(sm_state + 0x7ff8, EAX);
1906	stq_phys(sm_state + 0x7ff0, ECX);
1907	stq_phys(sm_state + 0x7fe8, EDX);
1908	stq_phys(sm_state + 0x7fe0, EBX);
1909	stq_phys(sm_state + 0x7fd8, ESP);
1910	stq_phys(sm_state + 0x7fd0, EBP);
1911	stq_phys(sm_state + 0x7fc8, ESI);
1912	stq_phys(sm_state + 0x7fc0, EDI);
1913	for(i = 8; i < 16; i++)
1914	stq_phys(sm_state + 0x7ff8 - i * 8, env->regs[i]);
1915	stq_phys(sm_state + 0x7f78, env->eip);
1916	stl_phys(sm_state + 0x7f70, compute_eflags());
1917	stl_phys(sm_state + 0x7f68, env->dr[6]);
1918	stl_phys(sm_state + 0x7f60, env->dr[7]);
1919
1920	stl_phys(sm_state + 0x7f48, env->cr[4]);
1921	stl_phys(sm_state + 0x7f50, env->cr[3]);
1922	stl_phys(sm_state + 0x7f58, env->cr[0]);
1923
1924	stl_phys(sm_state + 0x7efc, SMM_REVISION_ID);
1925	stl_phys(sm_state + 0x7f00, env->smbase);
1926	#else
1927	stl_phys(sm_state + 0x7ffc, env->cr[0]);
1928	stl_phys(sm_state + 0x7ff8, env->cr[3]);
1929	stl_phys(sm_state + 0x7ff4, compute_eflags());
1930	stl_phys(sm_state + 0x7ff0, env->eip);
1931	stl_phys(sm_state + 0x7fec, EDI);
1932	stl_phys(sm_state + 0x7fe8, ESI);
1933	stl_phys(sm_state + 0x7fe4, EBP);
1934	stl_phys(sm_state + 0x7fe0, ESP);
1935	stl_phys(sm_state + 0x7fdc, EBX);
1936	stl_phys(sm_state + 0x7fd8, EDX);
1937	stl_phys(sm_state + 0x7fd4, ECX);
1938	stl_phys(sm_state + 0x7fd0, EAX);
1939	stl_phys(sm_state + 0x7fcc, env->dr[6]);
1940	stl_phys(sm_state + 0x7fc8, env->dr[7]);
1941
1942	stl_phys(sm_state + 0x7fc4, env->tr.selector);
1943	stl_phys(sm_state + 0x7f64, env->tr.base);
1944	stl_phys(sm_state + 0x7f60, env->tr.limit);
1945	stl_phys(sm_state + 0x7f5c, (env->tr.flags >> 8) & 0xf0ff);
1946
1947	stl_phys(sm_state + 0x7fc0, env->ldt.selector);
1948	stl_phys(sm_state + 0x7f80, env->ldt.base);
1949	stl_phys(sm_state + 0x7f7c, env->ldt.limit);
1950	stl_phys(sm_state + 0x7f78, (env->ldt.flags >> 8) & 0xf0ff);
1951
1952	stl_phys(sm_state + 0x7f74, env->gdt.base);
1953	stl_phys(sm_state + 0x7f70, env->gdt.limit);
1954
1955	stl_phys(sm_state + 0x7f58, env->idt.base);
1956	stl_phys(sm_state + 0x7f54, env->idt.limit);
1957
1958	for(i = 0; i < 6; i++) {
1959	dt = &env->segs[i];
1960	if (i < 3)
1961	offset = 0x7f84 + i * 12;
1962	else
1963	offset = 0x7f2c + (i - 3) * 12;
1964	stl_phys(sm_state + 0x7fa8 + i * 4, dt->selector);
1965	stl_phys(sm_state + offset + 8, dt->base);
1966	stl_phys(sm_state + offset + 4, dt->limit);
1967	stl_phys(sm_state + offset, (dt->flags >> 8) & 0xf0ff);
1968	}
1969	stl_phys(sm_state + 0x7f14, env->cr[4]);
1970
1971	stl_phys(sm_state + 0x7efc, SMM_REVISION_ID);
1972	stl_phys(sm_state + 0x7ef8, env->smbase);
1973	#endif
1974	/* init SMM cpu state */
1975
1976	#ifdef TARGET_X86_64
1977	cpu_load_efer(env, 0);
1978	#endif
1979	load_eflags(0, ~(CC_O \| CC_S \| CC_Z \| CC_A \| CC_P \| CC_C \| DF_MASK));
1980	env->eip = 0x00008000;
1981	cpu_x86_load_seg_cache(env, R_CS, (env->smbase >> 4) & 0xffff, env->smbase,
1982	0xffffffff, 0);
1983	cpu_x86_load_seg_cache(env, R_DS, 0, 0, 0xffffffff, 0);
1984	cpu_x86_load_seg_cache(env, R_ES, 0, 0, 0xffffffff, 0);
1985	cpu_x86_load_seg_cache(env, R_SS, 0, 0, 0xffffffff, 0);
1986	cpu_x86_load_seg_cache(env, R_FS, 0, 0, 0xffffffff, 0);
1987	cpu_x86_load_seg_cache(env, R_GS, 0, 0, 0xffffffff, 0);
1988
1989	cpu_x86_update_cr0(env,
1990	env->cr[0] & ~(CR0_PE_MASK \| CR0_EM_MASK \| CR0_TS_MASK \| CR0_PG_MASK));
1991	cpu_x86_update_cr4(env, 0);
1992	env->dr[7] = 0x00000400;
1993	CC_OP = CC_OP_EFLAGS;
1994	}
1995
1996	void helper_rsm(void)
1997	{
1998	#ifdef VBOX
1999	cpu_abort(env, "helper_rsm");
2000	#else /* !VBOX */
2001	target_ulong sm_state;
2002	int i, offset;
2003	uint32_t val;
2004
2005	sm_state = env->smbase + 0x8000;
2006	#ifdef TARGET_X86_64
2007	cpu_load_efer(env, ldq_phys(sm_state + 0x7ed0));
2008
2009	for(i = 0; i < 6; i++) {
2010	offset = 0x7e00 + i * 16;
2011	cpu_x86_load_seg_cache(env, i,
2012	lduw_phys(sm_state + offset),
2013	ldq_phys(sm_state + offset + 8),
2014	ldl_phys(sm_state + offset + 4),
2015	(lduw_phys(sm_state + offset + 2) & 0xf0ff) << 8);
2016	}
2017
2018	env->gdt.base = ldq_phys(sm_state + 0x7e68);
2019	env->gdt.limit = ldl_phys(sm_state + 0x7e64);
2020
2021	env->ldt.selector = lduw_phys(sm_state + 0x7e70);
2022	env->ldt.base = ldq_phys(sm_state + 0x7e78);
2023	env->ldt.limit = ldl_phys(sm_state + 0x7e74);
2024	env->ldt.flags = (lduw_phys(sm_state + 0x7e72) & 0xf0ff) << 8;
2025	#ifdef VBOX
2026	env->ldt.fVBoxFlags = CPUMSELREG_FLAGS_VALID;
2027	env->ldt.newselector = 0;
2028	#endif
2029
2030	env->idt.base = ldq_phys(sm_state + 0x7e88);
2031	env->idt.limit = ldl_phys(sm_state + 0x7e84);
2032
2033	env->tr.selector = lduw_phys(sm_state + 0x7e90);
2034	env->tr.base = ldq_phys(sm_state + 0x7e98);
2035	env->tr.limit = ldl_phys(sm_state + 0x7e94);
2036	env->tr.flags = (lduw_phys(sm_state + 0x7e92) & 0xf0ff) << 8;
2037	#ifdef VBOX
2038	env->tr.fVBoxFlags = CPUMSELREG_FLAGS_VALID;
2039	env->tr.newselector = 0;
2040	#endif
2041
2042	EAX = ldq_phys(sm_state + 0x7ff8);
2043	ECX = ldq_phys(sm_state + 0x7ff0);
2044	EDX = ldq_phys(sm_state + 0x7fe8);
2045	EBX = ldq_phys(sm_state + 0x7fe0);
2046	ESP = ldq_phys(sm_state + 0x7fd8);
2047	EBP = ldq_phys(sm_state + 0x7fd0);
2048	ESI = ldq_phys(sm_state + 0x7fc8);
2049	EDI = ldq_phys(sm_state + 0x7fc0);
2050	for(i = 8; i < 16; i++)
2051	env->regs[i] = ldq_phys(sm_state + 0x7ff8 - i * 8);
2052	env->eip = ldq_phys(sm_state + 0x7f78);
2053	load_eflags(ldl_phys(sm_state + 0x7f70),
2054	~(CC_O \| CC_S \| CC_Z \| CC_A \| CC_P \| CC_C \| DF_MASK));
2055	env->dr[6] = ldl_phys(sm_state + 0x7f68);
2056	env->dr[7] = ldl_phys(sm_state + 0x7f60);
2057
2058	cpu_x86_update_cr4(env, ldl_phys(sm_state + 0x7f48));
2059	cpu_x86_update_cr3(env, ldl_phys(sm_state + 0x7f50));
2060	cpu_x86_update_cr0(env, ldl_phys(sm_state + 0x7f58));
2061
2062	val = ldl_phys(sm_state + 0x7efc); /* revision ID */
2063	if (val & 0x20000) {
2064	env->smbase = ldl_phys(sm_state + 0x7f00) & ~0x7fff;
2065	}
2066	#else
2067	cpu_x86_update_cr0(env, ldl_phys(sm_state + 0x7ffc));
2068	cpu_x86_update_cr3(env, ldl_phys(sm_state + 0x7ff8));
2069	load_eflags(ldl_phys(sm_state + 0x7ff4),
2070	~(CC_O \| CC_S \| CC_Z \| CC_A \| CC_P \| CC_C \| DF_MASK));
2071	env->eip = ldl_phys(sm_state + 0x7ff0);
2072	EDI = ldl_phys(sm_state + 0x7fec);
2073	ESI = ldl_phys(sm_state + 0x7fe8);
2074	EBP = ldl_phys(sm_state + 0x7fe4);
2075	ESP = ldl_phys(sm_state + 0x7fe0);
2076	EBX = ldl_phys(sm_state + 0x7fdc);
2077	EDX = ldl_phys(sm_state + 0x7fd8);
2078	ECX = ldl_phys(sm_state + 0x7fd4);
2079	EAX = ldl_phys(sm_state + 0x7fd0);
2080	env->dr[6] = ldl_phys(sm_state + 0x7fcc);
2081	env->dr[7] = ldl_phys(sm_state + 0x7fc8);
2082
2083	env->tr.selector = ldl_phys(sm_state + 0x7fc4) & 0xffff;
2084	env->tr.base = ldl_phys(sm_state + 0x7f64);
2085	env->tr.limit = ldl_phys(sm_state + 0x7f60);
2086	env->tr.flags = (ldl_phys(sm_state + 0x7f5c) & 0xf0ff) << 8;
2087	#ifdef VBOX
2088	env->tr.fVBoxFlags = CPUMSELREG_FLAGS_VALID;
2089	env->tr.newselector = 0;
2090	#endif
2091
2092	env->ldt.selector = ldl_phys(sm_state + 0x7fc0) & 0xffff;
2093	env->ldt.base = ldl_phys(sm_state + 0x7f80);
2094	env->ldt.limit = ldl_phys(sm_state + 0x7f7c);
2095	env->ldt.flags = (ldl_phys(sm_state + 0x7f78) & 0xf0ff) << 8;
2096	#ifdef VBOX
2097	env->ldt.fVBoxFlags = CPUMSELREG_FLAGS_VALID;
2098	env->ldt.newselector = 0;
2099	#endif
2100
2101	env->gdt.base = ldl_phys(sm_state + 0x7f74);
2102	env->gdt.limit = ldl_phys(sm_state + 0x7f70);
2103
2104	env->idt.base = ldl_phys(sm_state + 0x7f58);
2105	env->idt.limit = ldl_phys(sm_state + 0x7f54);
2106
2107	for(i = 0; i < 6; i++) {
2108	if (i < 3)
2109	offset = 0x7f84 + i * 12;
2110	else
2111	offset = 0x7f2c + (i - 3) * 12;
2112	cpu_x86_load_seg_cache(env, i,
2113	ldl_phys(sm_state + 0x7fa8 + i * 4) & 0xffff,
2114	ldl_phys(sm_state + offset + 8),
2115	ldl_phys(sm_state + offset + 4),
2116	(ldl_phys(sm_state + offset) & 0xf0ff) << 8);
2117	}
2118	cpu_x86_update_cr4(env, ldl_phys(sm_state + 0x7f14));
2119
2120	val = ldl_phys(sm_state + 0x7efc); /* revision ID */
2121	if (val & 0x20000) {
2122	env->smbase = ldl_phys(sm_state + 0x7ef8) & ~0x7fff;
2123	}
2124	#endif
2125	CC_OP = CC_OP_EFLAGS;
2126	env->hflags &= ~HF_SMM_MASK;
2127	cpu_smm_update(env);
2128
2129	qemu_log_mask(CPU_LOG_INT, "SMM: after RSM\n");
2130	log_cpu_state_mask(CPU_LOG_INT, env, X86_DUMP_CCOP);
2131	#endif /* !VBOX */
2132	}
2133
2134	#endif /* !CONFIG_USER_ONLY */
2135
2136
2137	/* division, flags are undefined */
2138
2139	void helper_divb_AL(target_ulong t0)
2140	{
2141	unsigned int num, den, q, r;
2142
2143	num = (EAX & 0xffff);
2144	den = (t0 & 0xff);
2145	if (den == 0) {
2146	raise_exception(EXCP00_DIVZ);
2147	}
2148	q = (num / den);
2149	if (q > 0xff)
2150	raise_exception(EXCP00_DIVZ);
2151	q &= 0xff;
2152	r = (num % den) & 0xff;
2153	EAX = (EAX & ~0xffff) \| (r << 8) \| q;
2154	}
2155
2156	void helper_idivb_AL(target_ulong t0)
2157	{
2158	int num, den, q, r;
2159
2160	num = (int16_t)EAX;
2161	den = (int8_t)t0;
2162	if (den == 0) {
2163	raise_exception(EXCP00_DIVZ);
2164	}
2165	q = (num / den);
2166	if (q != (int8_t)q)
2167	raise_exception(EXCP00_DIVZ);
2168	q &= 0xff;
2169	r = (num % den) & 0xff;
2170	EAX = (EAX & ~0xffff) \| (r << 8) \| q;
2171	}
2172
2173	void helper_divw_AX(target_ulong t0)
2174	{
2175	unsigned int num, den, q, r;
2176
2177	num = (EAX & 0xffff) \| ((EDX & 0xffff) << 16);
2178	den = (t0 & 0xffff);
2179	if (den == 0) {
2180	raise_exception(EXCP00_DIVZ);
2181	}
2182	q = (num / den);
2183	if (q > 0xffff)
2184	raise_exception(EXCP00_DIVZ);
2185	q &= 0xffff;
2186	r = (num % den) & 0xffff;
2187	EAX = (EAX & ~0xffff) \| q;
2188	EDX = (EDX & ~0xffff) \| r;
2189	}
2190
2191	void helper_idivw_AX(target_ulong t0)
2192	{
2193	int num, den, q, r;
2194
2195	num = (EAX & 0xffff) \| ((EDX & 0xffff) << 16);
2196	den = (int16_t)t0;
2197	if (den == 0) {
2198	raise_exception(EXCP00_DIVZ);
2199	}
2200	q = (num / den);
2201	if (q != (int16_t)q)
2202	raise_exception(EXCP00_DIVZ);
2203	q &= 0xffff;
2204	r = (num % den) & 0xffff;
2205	EAX = (EAX & ~0xffff) \| q;
2206	EDX = (EDX & ~0xffff) \| r;
2207	}
2208
2209	void helper_divl_EAX(target_ulong t0)
2210	{
2211	unsigned int den, r;
2212	uint64_t num, q;
2213
2214	num = ((uint32_t)EAX) \| ((uint64_t)((uint32_t)EDX) << 32);
2215	den = t0;
2216	if (den == 0) {
2217	raise_exception(EXCP00_DIVZ);
2218	}
2219	q = (num / den);
2220	r = (num % den);
2221	if (q > 0xffffffff)
2222	raise_exception(EXCP00_DIVZ);
2223	EAX = (uint32_t)q;
2224	EDX = (uint32_t)r;
2225	}
2226
2227	void helper_idivl_EAX(target_ulong t0)
2228	{
2229	int den, r;
2230	int64_t num, q;
2231
2232	num = ((uint32_t)EAX) \| ((uint64_t)((uint32_t)EDX) << 32);
2233	den = t0;
2234	if (den == 0) {
2235	raise_exception(EXCP00_DIVZ);
2236	}
2237	q = (num / den);
2238	r = (num % den);
2239	if (q != (int32_t)q)
2240	raise_exception(EXCP00_DIVZ);
2241	EAX = (uint32_t)q;
2242	EDX = (uint32_t)r;
2243	}
2244
2245	/* bcd */
2246
2247	/* XXX: exception */
2248	void helper_aam(int base)
2249	{
2250	int al, ah;
2251	al = EAX & 0xff;
2252	ah = al / base;
2253	al = al % base;
2254	EAX = (EAX & ~0xffff) \| al \| (ah << 8);
2255	CC_DST = al;
2256	}
2257
2258	void helper_aad(int base)
2259	{
2260	int al, ah;
2261	al = EAX & 0xff;
2262	ah = (EAX >> 8) & 0xff;
2263	al = ((ah * base) + al) & 0xff;
2264	EAX = (EAX & ~0xffff) \| al;
2265	CC_DST = al;
2266	}
2267
2268	void helper_aaa(void)
2269	{
2270	int icarry;
2271	int al, ah, af;
2272	int eflags;
2273
2274	eflags = helper_cc_compute_all(CC_OP);
2275	af = eflags & CC_A;
2276	al = EAX & 0xff;
2277	ah = (EAX >> 8) & 0xff;
2278
2279	icarry = (al > 0xf9);
2280	if (((al & 0x0f) > 9 ) \|\| af) {
2281	al = (al + 6) & 0x0f;
2282	ah = (ah + 1 + icarry) & 0xff;
2283	eflags \|= CC_C \| CC_A;
2284	} else {
2285	eflags &= ~(CC_C \| CC_A);
2286	al &= 0x0f;
2287	}
2288	EAX = (EAX & ~0xffff) \| al \| (ah << 8);
2289	CC_SRC = eflags;
2290	}
2291
2292	void helper_aas(void)
2293	{
2294	int icarry;
2295	int al, ah, af;
2296	int eflags;
2297
2298	eflags = helper_cc_compute_all(CC_OP);
2299	af = eflags & CC_A;
2300	al = EAX & 0xff;
2301	ah = (EAX >> 8) & 0xff;
2302
2303	icarry = (al < 6);
2304	if (((al & 0x0f) > 9 ) \|\| af) {
2305	al = (al - 6) & 0x0f;
2306	ah = (ah - 1 - icarry) & 0xff;
2307	eflags \|= CC_C \| CC_A;
2308	} else {
2309	eflags &= ~(CC_C \| CC_A);
2310	al &= 0x0f;
2311	}
2312	EAX = (EAX & ~0xffff) \| al \| (ah << 8);
2313	CC_SRC = eflags;
2314	}
2315
2316	void helper_daa(void)
2317	{
2318	int al, af, cf;
2319	int eflags;
2320
2321	eflags = helper_cc_compute_all(CC_OP);
2322	cf = eflags & CC_C;
2323	af = eflags & CC_A;
2324	al = EAX & 0xff;
2325
2326	eflags = 0;
2327	if (((al & 0x0f) > 9 ) \|\| af) {
2328	al = (al + 6) & 0xff;
2329	eflags \|= CC_A;
2330	}
2331	if ((al > 0x9f) \|\| cf) {
2332	al = (al + 0x60) & 0xff;
2333	eflags \|= CC_C;
2334	}
2335	EAX = (EAX & ~0xff) \| al;
2336	/* well, speed is not an issue here, so we compute the flags by hand */
2337	eflags \|= (al == 0) << 6; /* zf */
2338	eflags \|= parity_table[al]; /* pf */
2339	eflags \|= (al & 0x80); /* sf */
2340	CC_SRC = eflags;
2341	}
2342
2343	void helper_das(void)
2344	{
2345	int al, al1, af, cf;
2346	int eflags;
2347
2348	eflags = helper_cc_compute_all(CC_OP);
2349	cf = eflags & CC_C;
2350	af = eflags & CC_A;
2351	al = EAX & 0xff;
2352
2353	eflags = 0;
2354	al1 = al;
2355	if (((al & 0x0f) > 9 ) \|\| af) {
2356	eflags \|= CC_A;
2357	if (al < 6 \|\| cf)
2358	eflags \|= CC_C;
2359	al = (al - 6) & 0xff;
2360	}
2361	if ((al1 > 0x99) \|\| cf) {
2362	al = (al - 0x60) & 0xff;
2363	eflags \|= CC_C;
2364	}
2365	EAX = (EAX & ~0xff) \| al;
2366	/* well, speed is not an issue here, so we compute the flags by hand */
2367	eflags \|= (al == 0) << 6; /* zf */
2368	eflags \|= parity_table[al]; /* pf */
2369	eflags \|= (al & 0x80); /* sf */
2370	CC_SRC = eflags;
2371	}
2372
2373	void helper_into(int next_eip_addend)
2374	{
2375	int eflags;
2376	eflags = helper_cc_compute_all(CC_OP);
2377	if (eflags & CC_O) {
2378	raise_interrupt(EXCP04_INTO, 1, 0, next_eip_addend);
2379	}
2380	}
2381
2382	void helper_cmpxchg8b(target_ulong a0)
2383	{
2384	uint64_t d;
2385	int eflags;
2386
2387	eflags = helper_cc_compute_all(CC_OP);
2388	d = ldq(a0);
2389	if (d == (((uint64_t)EDX << 32) \| (uint32_t)EAX)) {
2390	stq(a0, ((uint64_t)ECX << 32) \| (uint32_t)EBX);
2391	eflags \|= CC_Z;
2392	} else {
2393	/* always do the store */
2394	stq(a0, d);
2395	EDX = (uint32_t)(d >> 32);
2396	EAX = (uint32_t)d;
2397	eflags &= ~CC_Z;
2398	}
2399	CC_SRC = eflags;
2400	}
2401
2402	#ifdef TARGET_X86_64
2403	void helper_cmpxchg16b(target_ulong a0)
2404	{
2405	uint64_t d0, d1;
2406	int eflags;
2407
2408	if ((a0 & 0xf) != 0)
2409	raise_exception(EXCP0D_GPF);
2410	eflags = helper_cc_compute_all(CC_OP);
2411	d0 = ldq(a0);
2412	d1 = ldq(a0 + 8);
2413	if (d0 == EAX && d1 == EDX) {
2414	stq(a0, EBX);
2415	stq(a0 + 8, ECX);
2416	eflags \|= CC_Z;
2417	} else {
2418	/* always do the store */
2419	stq(a0, d0);
2420	stq(a0 + 8, d1);
2421	EDX = d1;
2422	EAX = d0;
2423	eflags &= ~CC_Z;
2424	}
2425	CC_SRC = eflags;
2426	}
2427	#endif
2428
2429	void helper_single_step(void)
2430	{
2431	#ifndef CONFIG_USER_ONLY
2432	check_hw_breakpoints(env, 1);
2433	env->dr[6] \|= DR6_BS;
2434	#endif
2435	raise_exception(EXCP01_DB);
2436	}
2437
2438	void helper_cpuid(void)
2439	{
2440	uint32_t eax, ebx, ecx, edx;
2441
2442	helper_svm_check_intercept_param(SVM_EXIT_CPUID, 0);
2443
2444	cpu_x86_cpuid(env, (uint32_t)EAX, (uint32_t)ECX, &eax, &ebx, &ecx, &edx);
2445	EAX = eax;
2446	EBX = ebx;
2447	ECX = ecx;
2448	EDX = edx;
2449	}
2450
2451	void helper_enter_level(int level, int data32, target_ulong t1)
2452	{
2453	target_ulong ssp;
2454	uint32_t esp_mask, esp, ebp;
2455
2456	esp_mask = get_sp_mask(env->segs[R_SS].flags);
2457	ssp = env->segs[R_SS].base;
2458	ebp = EBP;
2459	esp = ESP;
2460	if (data32) {
2461	/* 32 bit */
2462	esp -= 4;
2463	while (--level) {
2464	esp -= 4;
2465	ebp -= 4;
2466	stl(ssp + (esp & esp_mask), ldl(ssp + (ebp & esp_mask)));
2467	}
2468	esp -= 4;
2469	stl(ssp + (esp & esp_mask), t1);
2470	} else {
2471	/* 16 bit */
2472	esp -= 2;
2473	while (--level) {
2474	esp -= 2;
2475	ebp -= 2;
2476	stw(ssp + (esp & esp_mask), lduw(ssp + (ebp & esp_mask)));
2477	}
2478	esp -= 2;
2479	stw(ssp + (esp & esp_mask), t1);
2480	}
2481	}
2482
2483	#ifdef TARGET_X86_64
2484	void helper_enter64_level(int level, int data64, target_ulong t1)
2485	{
2486	target_ulong esp, ebp;
2487	ebp = EBP;
2488	esp = ESP;
2489
2490	if (data64) {
2491	/* 64 bit */
2492	esp -= 8;
2493	while (--level) {
2494	esp -= 8;
2495	ebp -= 8;
2496	stq(esp, ldq(ebp));
2497	}
2498	esp -= 8;
2499	stq(esp, t1);
2500	} else {
2501	/* 16 bit */
2502	esp -= 2;
2503	while (--level) {
2504	esp -= 2;
2505	ebp -= 2;
2506	stw(esp, lduw(ebp));
2507	}
2508	esp -= 2;
2509	stw(esp, t1);
2510	}
2511	}
2512	#endif
2513
2514	void helper_lldt(int selector)
2515	{
2516	SegmentCache *dt;
2517	uint32_t e1, e2;
2518	#ifndef VBOX
2519	int index, entry_limit;
2520	#else
2521	unsigned int index, entry_limit;
2522	#endif
2523	target_ulong ptr;
2524
2525	#ifdef VBOX
2526	Log(("helper_lldt_T0: old ldtr=%RTsel {.base=%RGv, .limit=%RGv} new=%RTsel\n",
2527	(RTSEL)env->ldt.selector, (RTGCPTR)env->ldt.base, (RTGCPTR)env->ldt.limit, (RTSEL)(selector & 0xffff)));
2528	#endif
2529
2530	selector &= 0xffff;
2531	if ((selector & 0xfffc) == 0) {
2532	/* XXX: NULL selector case: invalid LDT */
2533	env->ldt.base = 0;
2534	env->ldt.limit = 0;
2535	#ifdef VBOX
2536	env->ldt.flags = DESC_INTEL_UNUSABLE;
2537	env->ldt.fVBoxFlags = CPUMSELREG_FLAGS_VALID;
2538	env->ldt.newselector = 0;
2539	#endif
2540	} else {
2541	if (selector & 0x4)
2542	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
2543	dt = &env->gdt;
2544	index = selector & ~7;
2545	#ifdef TARGET_X86_64
2546	if (env->hflags & HF_LMA_MASK)
2547	entry_limit = 15;
2548	else
2549	#endif
2550	entry_limit = 7;
2551	if ((index + entry_limit) > dt->limit)
2552	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
2553	ptr = dt->base + index;
2554	e1 = ldl_kernel(ptr);
2555	e2 = ldl_kernel(ptr + 4);
2556	if ((e2 & DESC_S_MASK) \|\| ((e2 >> DESC_TYPE_SHIFT) & 0xf) != 2)
2557	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
2558	if (!(e2 & DESC_P_MASK))
2559	raise_exception_err(EXCP0B_NOSEG, selector & 0xfffc);
2560	#ifdef TARGET_X86_64
2561	if (env->hflags & HF_LMA_MASK) {
2562	uint32_t e3;
2563	e3 = ldl_kernel(ptr + 8);
2564	load_seg_cache_raw_dt(&env->ldt, e1, e2);
2565	env->ldt.base \|= (target_ulong)e3 << 32;
2566	} else
2567	#endif
2568	{
2569	load_seg_cache_raw_dt(&env->ldt, e1, e2);
2570	}
2571	}
2572	env->ldt.selector = selector;
2573	#ifdef VBOX
2574	Log(("helper_lldt_T0: new ldtr=%RTsel {.base=%RGv, .limit=%RGv}\n",
2575	(RTSEL)env->ldt.selector, (RTGCPTR)env->ldt.base, (RTGCPTR)env->ldt.limit));
2576	#endif
2577	}
2578
2579	void helper_ltr(int selector)
2580	{
2581	SegmentCache *dt;
2582	uint32_t e1, e2;
2583	#ifndef VBOX
2584	int index, type, entry_limit;
2585	#else
2586	unsigned int index;
2587	int type, entry_limit;
2588	#endif
2589	target_ulong ptr;
2590
2591	#ifdef VBOX
2592	Log(("helper_ltr: pc=%RGv old tr=%RTsel {.base=%RGv, .limit=%RGv, .flags=%RX32} new=%RTsel\n",
2593	(RTGCPTR)env->eip, (RTSEL)env->tr.selector, (RTGCPTR)env->tr.base, (RTGCPTR)env->tr.limit,
2594	env->tr.flags, (RTSEL)(selector & 0xffff)));
2595	#endif
2596	selector &= 0xffff;
2597	if ((selector & 0xfffc) == 0) {
2598	/* NULL selector case: invalid TR */
2599	#ifdef VBOX
2600	raise_exception_err(EXCP0A_TSS, 0);
2601	#else
2602	env->tr.base = 0;
2603	env->tr.limit = 0;
2604	env->tr.flags = 0;
2605	#endif
2606	} else {
2607	if (selector & 0x4)
2608	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
2609	dt = &env->gdt;
2610	index = selector & ~7;
2611	#ifdef TARGET_X86_64
2612	if (env->hflags & HF_LMA_MASK)
2613	entry_limit = 15;
2614	else
2615	#endif
2616	entry_limit = 7;
2617	if ((index + entry_limit) > dt->limit)
2618	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
2619	ptr = dt->base + index;
2620	e1 = ldl_kernel(ptr);
2621	e2 = ldl_kernel(ptr + 4);
2622	type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
2623	if ((e2 & DESC_S_MASK) \|\|
2624	(type != 1 && type != 9))
2625	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
2626	if (!(e2 & DESC_P_MASK))
2627	raise_exception_err(EXCP0B_NOSEG, selector & 0xfffc);
2628	#ifdef TARGET_X86_64
2629	if (env->hflags & HF_LMA_MASK) {
2630	uint32_t e3, e4;
2631	e3 = ldl_kernel(ptr + 8);
2632	e4 = ldl_kernel(ptr + 12);
2633	if ((e4 >> DESC_TYPE_SHIFT) & 0xf)
2634	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
2635	load_seg_cache_raw_dt(&env->tr, e1, e2);
2636	env->tr.base \|= (target_ulong)e3 << 32;
2637	} else
2638	#endif
2639	{
2640	load_seg_cache_raw_dt(&env->tr, e1, e2);
2641	}
2642	env->tr.flags \|= DESC_TSS_BUSY_MASK;
2643	e2 \|= DESC_TSS_BUSY_MASK;
2644	stl_kernel(ptr + 4, e2);
2645	}
2646	env->tr.selector = selector;
2647	#ifdef VBOX
2648	Log(("helper_ltr: new tr=%RTsel {.base=%RGv, .limit=%RGv, .flags=%RX32} new=%RTsel\n",
2649	(RTSEL)env->tr.selector, (RTGCPTR)env->tr.base, (RTGCPTR)env->tr.limit,
2650	env->tr.flags, (RTSEL)(selector & 0xffff)));
2651	#endif
2652	}
2653
2654	/* only works if protected mode and not VM86. seg_reg must be != R_CS */
2655	void helper_load_seg(int seg_reg, int selector)
2656	{
2657	uint32_t e1, e2;
2658	int cpl, dpl, rpl;
2659	SegmentCache *dt;
2660	#ifndef VBOX
2661	int index;
2662	#else
2663	unsigned int index;
2664	#endif
2665	target_ulong ptr;
2666
2667	selector &= 0xffff;
2668	cpl = env->hflags & HF_CPL_MASK;
2669	#ifdef VBOX
2670
2671	/* Trying to load a selector with CPL=1? */
2672	if (cpl == 0 && (selector & 3) == 1 && (env->state & CPU_RAW_RING0))
2673	{
2674	Log(("RPL 1 -> sel %04X -> %04X (helper_load_seg)\n", selector, selector & 0xfffc));
2675	selector = selector & 0xfffc;
2676	}
2677	#endif /* VBOX */
2678	if ((selector & 0xfffc) == 0) {
2679	/* null selector case */
2680	#ifndef VBOX
2681	if (seg_reg == R_SS
2682	#ifdef TARGET_X86_64
2683	&& (!(env->hflags & HF_CS64_MASK) \|\| cpl == 3)
2684	#endif
2685	)
2686	raise_exception_err(EXCP0D_GPF, 0);
2687	cpu_x86_load_seg_cache(env, seg_reg, selector, 0, 0, 0);
2688	#else
2689	if (seg_reg == R_SS) {
2690	if (!(env->hflags & HF_CS64_MASK) \|\| cpl == 3)
2691	raise_exception_err(EXCP0D_GPF, 0);
2692	e2 = (cpl << DESC_DPL_SHIFT) \| DESC_INTEL_UNUSABLE;
2693	} else {
2694	e2 = DESC_INTEL_UNUSABLE;
2695	}
2696	cpu_x86_load_seg_cache_with_clean_flags(env, seg_reg, selector, 0, 0, e2);
2697	#endif
2698	} else {
2699
2700	if (selector & 0x4)
2701	dt = &env->ldt;
2702	else
2703	dt = &env->gdt;
2704	index = selector & ~7;
2705	if ((index + 7) > dt->limit)
2706	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
2707	ptr = dt->base + index;
2708	e1 = ldl_kernel(ptr);
2709	e2 = ldl_kernel(ptr + 4);
2710
2711	if (!(e2 & DESC_S_MASK))
2712	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
2713	rpl = selector & 3;
2714	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
2715	if (seg_reg == R_SS) {
2716	/* must be writable segment */
2717	if ((e2 & DESC_CS_MASK) \|\| !(e2 & DESC_W_MASK))
2718	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
2719	if (rpl != cpl \|\| dpl != cpl)
2720	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
2721	} else {
2722	/* must be readable segment */
2723	if ((e2 & (DESC_CS_MASK \| DESC_R_MASK)) == DESC_CS_MASK)
2724	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
2725
2726	if (!(e2 & DESC_CS_MASK) \|\| !(e2 & DESC_C_MASK)) {
2727	/* if not conforming code, test rights */
2728	if (dpl < cpl \|\| dpl < rpl)
2729	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
2730	}
2731	}
2732
2733	if (!(e2 & DESC_P_MASK)) {
2734	if (seg_reg == R_SS)
2735	raise_exception_err(EXCP0C_STACK, selector & 0xfffc);
2736	else
2737	raise_exception_err(EXCP0B_NOSEG, selector & 0xfffc);
2738	}
2739
2740	/* set the access bit if not already set */
2741	if (!(e2 & DESC_A_MASK)) {
2742	e2 \|= DESC_A_MASK;
2743	stl_kernel(ptr + 4, e2);
2744	}
2745
2746	cpu_x86_load_seg_cache(env, seg_reg, selector,
2747	get_seg_base(e1, e2),
2748	get_seg_limit(e1, e2),
2749	e2);
2750	#if 0
2751	qemu_log("load_seg: sel=0x%04x base=0x%08lx limit=0x%08lx flags=%08x\n",
2752	selector, (unsigned long)sc->base, sc->limit, sc->flags);
2753	#endif
2754	}
2755	}
2756
2757	/* protected mode jump */
2758	void helper_ljmp_protected(int new_cs, target_ulong new_eip,
2759	int next_eip_addend)
2760	{
2761	int gate_cs, type;
2762	uint32_t e1, e2, cpl, dpl, rpl, limit;
2763	target_ulong next_eip;
2764
2765	#ifdef VBOX /** @todo Why do we do this? */
2766	e1 = e2 = 0;
2767	#endif
2768	if ((new_cs & 0xfffc) == 0)
2769	raise_exception_err(EXCP0D_GPF, 0);
2770	if (load_segment(&e1, &e2, new_cs) != 0)
2771	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2772	cpl = env->hflags & HF_CPL_MASK;
2773	if (e2 & DESC_S_MASK) {
2774	if (!(e2 & DESC_CS_MASK))
2775	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2776	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
2777	if (e2 & DESC_C_MASK) {
2778	/* conforming code segment */
2779	if (dpl > cpl)
2780	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2781	} else {
2782	/* non conforming code segment */
2783	rpl = new_cs & 3;
2784	if (rpl > cpl)
2785	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2786	if (dpl != cpl)
2787	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2788	}
2789	if (!(e2 & DESC_P_MASK))
2790	raise_exception_err(EXCP0B_NOSEG, new_cs & 0xfffc);
2791	limit = get_seg_limit(e1, e2);
2792	if (new_eip > limit &&
2793	!(env->hflags & HF_LMA_MASK) && !(e2 & DESC_L_MASK))
2794	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2795	#ifdef VBOX
2796	if (!(e2 & DESC_A_MASK))
2797	e2 = set_segment_accessed(new_cs, e2);
2798	#endif
2799	cpu_x86_load_seg_cache(env, R_CS, (new_cs & 0xfffc) \| cpl,
2800	get_seg_base(e1, e2), limit, e2);
2801	EIP = new_eip;
2802	} else {
2803	/* jump to call or task gate */
2804	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
2805	rpl = new_cs & 3;
2806	cpl = env->hflags & HF_CPL_MASK;
2807	type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
2808	switch(type) {
2809	case 1: /* 286 TSS */
2810	case 9: /* 386 TSS */
2811	case 5: /* task gate */
2812	if (dpl < cpl \|\| dpl < rpl)
2813	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2814	next_eip = env->eip + next_eip_addend;
2815	switch_tss(new_cs, e1, e2, SWITCH_TSS_JMP, next_eip);
2816	CC_OP = CC_OP_EFLAGS;
2817	break;
2818	case 4: /* 286 call gate */
2819	case 12: /* 386 call gate */
2820	if ((dpl < cpl) \|\| (dpl < rpl))
2821	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2822	if (!(e2 & DESC_P_MASK))
2823	raise_exception_err(EXCP0B_NOSEG, new_cs & 0xfffc);
2824	gate_cs = e1 >> 16;
2825	new_eip = (e1 & 0xffff);
2826	if (type == 12)
2827	new_eip \|= (e2 & 0xffff0000);
2828	if (load_segment(&e1, &e2, gate_cs) != 0)
2829	raise_exception_err(EXCP0D_GPF, gate_cs & 0xfffc);
2830	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
2831	/* must be code segment */
2832	if (((e2 & (DESC_S_MASK \| DESC_CS_MASK)) !=
2833	(DESC_S_MASK \| DESC_CS_MASK)))
2834	raise_exception_err(EXCP0D_GPF, gate_cs & 0xfffc);
2835	if (((e2 & DESC_C_MASK) && (dpl > cpl)) \|\|
2836	(!(e2 & DESC_C_MASK) && (dpl != cpl)))
2837	raise_exception_err(EXCP0D_GPF, gate_cs & 0xfffc);
2838	if (!(e2 & DESC_P_MASK))
2839	#ifdef VBOX /* See page 3-514 of 253666.pdf */
2840	raise_exception_err(EXCP0B_NOSEG, gate_cs & 0xfffc);
2841	#else
2842	raise_exception_err(EXCP0D_GPF, gate_cs & 0xfffc);
2843	#endif
2844	limit = get_seg_limit(e1, e2);
2845	if (new_eip > limit)
2846	raise_exception_err(EXCP0D_GPF, 0);
2847	cpu_x86_load_seg_cache(env, R_CS, (gate_cs & 0xfffc) \| cpl,
2848	get_seg_base(e1, e2), limit, e2);
2849	EIP = new_eip;
2850	break;
2851	default:
2852	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2853	break;
2854	}
2855	}
2856	}
2857
2858	/* real mode call */
2859	void helper_lcall_real(int new_cs, target_ulong new_eip1,
2860	int shift, int next_eip)
2861	{
2862	int new_eip;
2863	uint32_t esp, esp_mask;
2864	target_ulong ssp;
2865
2866	new_eip = new_eip1;
2867	esp = ESP;
2868	esp_mask = get_sp_mask(env->segs[R_SS].flags);
2869	ssp = env->segs[R_SS].base;
2870	if (shift) {
2871	PUSHL(ssp, esp, esp_mask, env->segs[R_CS].selector);
2872	PUSHL(ssp, esp, esp_mask, next_eip);
2873	} else {
2874	PUSHW(ssp, esp, esp_mask, env->segs[R_CS].selector);
2875	PUSHW(ssp, esp, esp_mask, next_eip);
2876	}
2877
2878	SET_ESP(esp, esp_mask);
2879	env->eip = new_eip;
2880	env->segs[R_CS].selector = new_cs;
2881	env->segs[R_CS].base = (new_cs << 4);
2882	}
2883
2884	/* protected mode call */
2885	void helper_lcall_protected(int new_cs, target_ulong new_eip,
2886	int shift, int next_eip_addend)
2887	{
2888	int new_stack, i;
2889	uint32_t e1, e2, cpl, dpl, rpl, selector, offset, param_count;
2890	uint32_t ss = 0, ss_e1 = 0, ss_e2 = 0, sp, type, ss_dpl, sp_mask;
2891	uint32_t val, limit, old_sp_mask;
2892	target_ulong ssp, old_ssp, next_eip;
2893
2894	#ifdef VBOX /** @todo Why do we do this? */
2895	e1 = e2 = 0;
2896	#endif
2897	next_eip = env->eip + next_eip_addend;
2898	LOG_PCALL("lcall %04x:%08x s=%d\n", new_cs, (uint32_t)new_eip, shift);
2899	LOG_PCALL_STATE(env);
2900	if ((new_cs & 0xfffc) == 0)
2901	raise_exception_err(EXCP0D_GPF, 0);
2902	if (load_segment(&e1, &e2, new_cs) != 0)
2903	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2904	cpl = env->hflags & HF_CPL_MASK;
2905	LOG_PCALL("desc=%08x:%08x\n", e1, e2);
2906	if (e2 & DESC_S_MASK) {
2907	if (!(e2 & DESC_CS_MASK))
2908	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2909	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
2910	if (e2 & DESC_C_MASK) {
2911	/* conforming code segment */
2912	if (dpl > cpl)
2913	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2914	} else {
2915	/* non conforming code segment */
2916	rpl = new_cs & 3;
2917	if (rpl > cpl)
2918	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2919	if (dpl != cpl)
2920	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2921	}
2922	if (!(e2 & DESC_P_MASK))
2923	raise_exception_err(EXCP0B_NOSEG, new_cs & 0xfffc);
2924	#ifdef VBOX
2925	if (!(e2 & DESC_A_MASK))
2926	e2 = set_segment_accessed(new_cs, e2);
2927	#endif
2928
2929	#ifdef TARGET_X86_64
2930	/* XXX: check 16/32 bit cases in long mode */
2931	if (shift == 2) {
2932	target_ulong rsp;
2933	/* 64 bit case */
2934	rsp = ESP;
2935	PUSHQ(rsp, env->segs[R_CS].selector);
2936	PUSHQ(rsp, next_eip);
2937	/* from this point, not restartable */
2938	ESP = rsp;
2939	cpu_x86_load_seg_cache(env, R_CS, (new_cs & 0xfffc) \| cpl,
2940	get_seg_base(e1, e2),
2941	get_seg_limit(e1, e2), e2);
2942	EIP = new_eip;
2943	} else
2944	#endif
2945	{
2946	sp = ESP;
2947	sp_mask = get_sp_mask(env->segs[R_SS].flags);
2948	ssp = env->segs[R_SS].base;
2949	if (shift) {
2950	PUSHL(ssp, sp, sp_mask, env->segs[R_CS].selector);
2951	PUSHL(ssp, sp, sp_mask, next_eip);
2952	} else {
2953	PUSHW(ssp, sp, sp_mask, env->segs[R_CS].selector);
2954	PUSHW(ssp, sp, sp_mask, next_eip);
2955	}
2956
2957	limit = get_seg_limit(e1, e2);
2958	if (new_eip > limit)
2959	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2960	/* from this point, not restartable */
2961	SET_ESP(sp, sp_mask);
2962	cpu_x86_load_seg_cache(env, R_CS, (new_cs & 0xfffc) \| cpl,
2963	get_seg_base(e1, e2), limit, e2);
2964	EIP = new_eip;
2965	}
2966	} else {
2967	/* check gate type */
2968	type = (e2 >> DESC_TYPE_SHIFT) & 0x1f;
2969	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
2970	rpl = new_cs & 3;
2971	switch(type) {
2972	case 1: /* available 286 TSS */
2973	case 9: /* available 386 TSS */
2974	case 5: /* task gate */
2975	if (dpl < cpl \|\| dpl < rpl)
2976	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2977	switch_tss(new_cs, e1, e2, SWITCH_TSS_CALL, next_eip);
2978	CC_OP = CC_OP_EFLAGS;
2979	return;
2980	case 4: /* 286 call gate */
2981	case 12: /* 386 call gate */
2982	break;
2983	default:
2984	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2985	break;
2986	}
2987	shift = type >> 3;
2988
2989	if (dpl < cpl \|\| dpl < rpl)
2990	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
2991	/* check valid bit */
2992	if (!(e2 & DESC_P_MASK))
2993	raise_exception_err(EXCP0B_NOSEG, new_cs & 0xfffc);
2994	selector = e1 >> 16;
2995	offset = (e2 & 0xffff0000) \| (e1 & 0x0000ffff);
2996	param_count = e2 & 0x1f;
2997	if ((selector & 0xfffc) == 0)
2998	raise_exception_err(EXCP0D_GPF, 0);
2999
3000	if (load_segment(&e1, &e2, selector) != 0)
3001	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
3002	if (!(e2 & DESC_S_MASK) \|\| !(e2 & (DESC_CS_MASK)))
3003	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
3004	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
3005	if (dpl > cpl)
3006	raise_exception_err(EXCP0D_GPF, selector & 0xfffc);
3007	if (!(e2 & DESC_P_MASK))
3008	raise_exception_err(EXCP0B_NOSEG, selector & 0xfffc);
3009
3010	if (!(e2 & DESC_C_MASK) && dpl < cpl) {
3011	/* to inner privilege */
3012	get_ss_esp_from_tss(&ss, &sp, dpl);
3013	LOG_PCALL("new ss:esp=%04x:%08x param_count=%d ESP=" TARGET_FMT_lx "\n",
3014	ss, sp, param_count, ESP);
3015	if ((ss & 0xfffc) == 0)
3016	raise_exception_err(EXCP0A_TSS, ss & 0xfffc);
3017	if ((ss & 3) != dpl)
3018	raise_exception_err(EXCP0A_TSS, ss & 0xfffc);
3019	if (load_segment(&ss_e1, &ss_e2, ss) != 0)
3020	raise_exception_err(EXCP0A_TSS, ss & 0xfffc);
3021	ss_dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
3022	if (ss_dpl != dpl)
3023	raise_exception_err(EXCP0A_TSS, ss & 0xfffc);
3024	if (!(ss_e2 & DESC_S_MASK) \|\|
3025	(ss_e2 & DESC_CS_MASK) \|\|
3026	!(ss_e2 & DESC_W_MASK))
3027	raise_exception_err(EXCP0A_TSS, ss & 0xfffc);
3028	if (!(ss_e2 & DESC_P_MASK))
3029	#ifdef VBOX /* See page 3-99 of 253666.pdf */
3030	raise_exception_err(EXCP0C_STACK, ss & 0xfffc);
3031	#else
3032	raise_exception_err(EXCP0A_TSS, ss & 0xfffc);
3033	#endif
3034
3035	// push_size = ((param_count * 2) + 8) << shift;
3036
3037	old_sp_mask = get_sp_mask(env->segs[R_SS].flags);
3038	old_ssp = env->segs[R_SS].base;
3039
3040	sp_mask = get_sp_mask(ss_e2);
3041	ssp = get_seg_base(ss_e1, ss_e2);
3042	if (shift) {
3043	PUSHL(ssp, sp, sp_mask, env->segs[R_SS].selector);
3044	PUSHL(ssp, sp, sp_mask, ESP);
3045	for(i = param_count - 1; i >= 0; i--) {
3046	val = ldl_kernel(old_ssp + ((ESP + i * 4) & old_sp_mask));
3047	PUSHL(ssp, sp, sp_mask, val);
3048	}
3049	} else {
3050	PUSHW(ssp, sp, sp_mask, env->segs[R_SS].selector);
3051	PUSHW(ssp, sp, sp_mask, ESP);
3052	for(i = param_count - 1; i >= 0; i--) {
3053	val = lduw_kernel(old_ssp + ((ESP + i * 2) & old_sp_mask));
3054	PUSHW(ssp, sp, sp_mask, val);
3055	}
3056	}
3057	new_stack = 1;
3058	} else {
3059	/* to same privilege */
3060	sp = ESP;
3061	sp_mask = get_sp_mask(env->segs[R_SS].flags);
3062	ssp = env->segs[R_SS].base;
3063	// push_size = (4 << shift);
3064	new_stack = 0;
3065	}
3066
3067	if (shift) {
3068	PUSHL(ssp, sp, sp_mask, env->segs[R_CS].selector);
3069	PUSHL(ssp, sp, sp_mask, next_eip);
3070	} else {
3071	PUSHW(ssp, sp, sp_mask, env->segs[R_CS].selector);
3072	PUSHW(ssp, sp, sp_mask, next_eip);
3073	}
3074
3075	/* from this point, not restartable */
3076
3077	if (new_stack) {
3078	ss = (ss & ~3) \| dpl;
3079	cpu_x86_load_seg_cache(env, R_SS, ss,
3080	ssp,
3081	get_seg_limit(ss_e1, ss_e2),
3082	ss_e2);
3083	}
3084
3085	selector = (selector & ~3) \| dpl;
3086	cpu_x86_load_seg_cache(env, R_CS, selector,
3087	get_seg_base(e1, e2),
3088	get_seg_limit(e1, e2),
3089	e2);
3090	cpu_x86_set_cpl(env, dpl);
3091	SET_ESP(sp, sp_mask);
3092	EIP = offset;
3093	}
3094	}
3095
3096	/* real and vm86 mode iret */
3097	void helper_iret_real(int shift)
3098	{
3099	uint32_t sp, new_cs, new_eip, new_eflags, sp_mask;
3100	target_ulong ssp;
3101	int eflags_mask;
3102	#ifdef VBOX
3103	bool fVME = false;
3104
3105	remR3TrapClear(env->pVM);
3106	#endif /* VBOX */
3107
3108	sp_mask = 0xffff; /* XXXX: use SS segment size ? */
3109	sp = ESP;
3110	ssp = env->segs[R_SS].base;
3111	if (shift == 1) {
3112	/* 32 bits */
3113	POPL(ssp, sp, sp_mask, new_eip);
3114	POPL(ssp, sp, sp_mask, new_cs);
3115	new_cs &= 0xffff;
3116	POPL(ssp, sp, sp_mask, new_eflags);
3117	} else {
3118	/* 16 bits */
3119	POPW(ssp, sp, sp_mask, new_eip);
3120	POPW(ssp, sp, sp_mask, new_cs);
3121	POPW(ssp, sp, sp_mask, new_eflags);
3122	}
3123	#ifdef VBOX
3124	if ( (env->eflags & VM_MASK)
3125	&& ((env->eflags >> IOPL_SHIFT) & 3) != 3
3126	&& (env->cr[4] & CR4_VME_MASK)) /* implied or else we would fault earlier */
3127	{
3128	fVME = true;
3129	/* if virtual interrupt pending and (virtual) interrupts will be enabled -> #GP */
3130	/* if TF will be set -> #GP */
3131	if ( ((new_eflags & IF_MASK) && (env->eflags & VIP_MASK))
3132	\|\| (new_eflags & TF_MASK))
3133	raise_exception(EXCP0D_GPF);
3134	}
3135	#endif /* VBOX */
3136	ESP = (ESP & ~sp_mask) \| (sp & sp_mask);
3137	env->segs[R_CS].selector = new_cs;
3138	env->segs[R_CS].base = (new_cs << 4);
3139	env->eip = new_eip;
3140	#ifdef VBOX
3141	if (fVME)
3142	eflags_mask = TF_MASK \| AC_MASK \| ID_MASK \| RF_MASK \| NT_MASK;
3143	else
3144	#endif
3145	if (env->eflags & VM_MASK)
3146	eflags_mask = TF_MASK \| AC_MASK \| ID_MASK \| IF_MASK \| RF_MASK \| NT_MASK;
3147	else
3148	eflags_mask = TF_MASK \| AC_MASK \| ID_MASK \| IF_MASK \| IOPL_MASK \| RF_MASK \| NT_MASK;
3149	if (shift == 0)
3150	eflags_mask &= 0xffff;
3151	load_eflags(new_eflags, eflags_mask);
3152	env->hflags2 &= ~HF2_NMI_MASK;
3153	#ifdef VBOX
3154	if (fVME)
3155	{
3156	if (new_eflags & IF_MASK)
3157	env->eflags \|= VIF_MASK;
3158	else
3159	env->eflags &= ~VIF_MASK;
3160	}
3161	#endif /* VBOX */
3162	}
3163
3164	static inline void validate_seg(int seg_reg, int cpl)
3165	{
3166	int dpl;
3167	uint32_t e2;
3168
3169	/* XXX: on x86_64, we do not want to nullify FS and GS because
3170	they may still contain a valid base. I would be interested to
3171	know how a real x86_64 CPU behaves */
3172	if ((seg_reg == R_FS \|\| seg_reg == R_GS) &&
3173	(env->segs[seg_reg].selector & 0xfffc) == 0)
3174	return;
3175
3176	e2 = env->segs[seg_reg].flags;
3177	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
3178	if (!(e2 & DESC_CS_MASK) \|\| !(e2 & DESC_C_MASK)) {
3179	/* data or non conforming code segment */
3180	if (dpl < cpl) {
3181	cpu_x86_load_seg_cache(env, seg_reg, 0, 0, 0, 0);
3182	}
3183	}
3184	}
3185
3186	/* protected mode iret */
3187	static inline void helper_ret_protected(int shift, int is_iret, int addend)
3188	{
3189	uint32_t new_cs, new_eflags, new_ss;
3190	uint32_t new_es, new_ds, new_fs, new_gs;
3191	uint32_t e1, e2, ss_e1, ss_e2;
3192	int cpl, dpl, rpl, eflags_mask, iopl;
3193	target_ulong ssp, sp, new_eip, new_esp, sp_mask;
3194
3195	#ifdef VBOX /** @todo Why do we do this? */
3196	ss_e1 = ss_e2 = e1 = e2 = 0;
3197	#endif
3198
3199	#ifdef TARGET_X86_64
3200	if (shift == 2)
3201	sp_mask = -1;
3202	else
3203	#endif
3204	sp_mask = get_sp_mask(env->segs[R_SS].flags);
3205	sp = ESP;
3206	ssp = env->segs[R_SS].base;
3207	new_eflags = 0; /* avoid warning */
3208	#ifdef TARGET_X86_64
3209	if (shift == 2) {
3210	POPQ(sp, new_eip);
3211	POPQ(sp, new_cs);
3212	new_cs &= 0xffff;
3213	if (is_iret) {
3214	POPQ(sp, new_eflags);
3215	}
3216	} else
3217	#endif
3218	if (shift == 1) {
3219	/* 32 bits */
3220	POPL(ssp, sp, sp_mask, new_eip);
3221	POPL(ssp, sp, sp_mask, new_cs);
3222	new_cs &= 0xffff;
3223	if (is_iret) {
3224	POPL(ssp, sp, sp_mask, new_eflags);
3225	#define LOG_GROUP LOG_GROUP_REM
3226	#if defined(VBOX) && defined(DEBUG)
3227	Log(("iret: new CS %04X (old=%x)\n", new_cs, env->segs[R_CS].selector));
3228	Log(("iret: new EIP %08X\n", (uint32_t)new_eip));
3229	Log(("iret: new EFLAGS %08X\n", new_eflags));
3230	Log(("iret: EAX=%08x\n", (uint32_t)EAX));
3231	#endif
3232	if (new_eflags & VM_MASK)
3233	goto return_to_vm86;
3234	}
3235	} else {
3236	/* 16 bits */
3237	POPW(ssp, sp, sp_mask, new_eip);
3238	POPW(ssp, sp, sp_mask, new_cs);
3239	if (is_iret)
3240	POPW(ssp, sp, sp_mask, new_eflags);
3241	}
3242	LOG_PCALL("lret new %04x:" TARGET_FMT_lx " s=%d addend=0x%x\n",
3243	new_cs, new_eip, shift, addend);
3244	LOG_PCALL_STATE(env);
3245	if ((new_cs & 0xfffc) == 0)
3246	{
3247	#if defined(VBOX) && defined(DEBUG)
3248	Log(("new_cs & 0xfffc) == 0\n"));
3249	#endif
3250	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
3251	}
3252	if (load_segment(&e1, &e2, new_cs) != 0)
3253	{
3254	#if defined(VBOX) && defined(DEBUG)
3255	Log(("load_segment failed\n"));
3256	#endif
3257	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
3258	}
3259	if (!(e2 & DESC_S_MASK) \|\|
3260	!(e2 & DESC_CS_MASK))
3261	{
3262	#if defined(VBOX) && defined(DEBUG)
3263	Log(("e2 mask %08x\n", e2));
3264	#endif
3265	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
3266	}
3267	cpl = env->hflags & HF_CPL_MASK;
3268	rpl = new_cs & 3;
3269	if (rpl < cpl)
3270	{
3271	#if defined(VBOX) && defined(DEBUG)
3272	Log(("rpl < cpl (%d vs %d)\n", rpl, cpl));
3273	#endif
3274	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
3275	}
3276	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
3277
3278	if (e2 & DESC_C_MASK) {
3279	if (dpl > rpl)
3280	{
3281	#if defined(VBOX) && defined(DEBUG)
3282	Log(("dpl > rpl (%d vs %d)\n", dpl, rpl));
3283	#endif
3284	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
3285	}
3286	} else {
3287	if (dpl != rpl)
3288	{
3289	#if defined(VBOX) && defined(DEBUG)
3290	Log(("dpl != rpl (%d vs %d) e1=%x e2=%x\n", dpl, rpl, e1, e2));
3291	#endif
3292	raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
3293	}
3294	}
3295	if (!(e2 & DESC_P_MASK))
3296	{
3297	#if defined(VBOX) && defined(DEBUG)
3298	Log(("DESC_P_MASK e2=%08x\n", e2));
3299	#endif
3300	raise_exception_err(EXCP0B_NOSEG, new_cs & 0xfffc);
3301	}
3302
3303	sp += addend;
3304	if (rpl == cpl && (!(env->hflags & HF_CS64_MASK) \|\|
3305	((env->hflags & HF_CS64_MASK) && !is_iret))) {
3306	/* return to same privilege level */
3307	#ifdef VBOX
3308	if (!(e2 & DESC_A_MASK))
3309	e2 = set_segment_accessed(new_cs, e2);
3310	#endif
3311	cpu_x86_load_seg_cache(env, R_CS, new_cs,
3312	get_seg_base(e1, e2),
3313	get_seg_limit(e1, e2),
3314	e2);
3315	} else {
3316	/* return to different privilege level */
3317	#ifdef TARGET_X86_64
3318	if (shift == 2) {
3319	POPQ(sp, new_esp);
3320	POPQ(sp, new_ss);
3321	new_ss &= 0xffff;
3322	} else
3323	#endif
3324	if (shift == 1) {
3325	/* 32 bits */
3326	POPL(ssp, sp, sp_mask, new_esp);
3327	POPL(ssp, sp, sp_mask, new_ss);
3328	new_ss &= 0xffff;
3329	} else {
3330	/* 16 bits */
3331	POPW(ssp, sp, sp_mask, new_esp);
3332	POPW(ssp, sp, sp_mask, new_ss);
3333	}
3334	LOG_PCALL("new ss:esp=%04x:" TARGET_FMT_lx "\n",
3335	new_ss, new_esp);
3336	if ((new_ss & 0xfffc) == 0) {
3337	#ifdef TARGET_X86_64
3338	/* NULL ss is allowed in long mode if cpl != 3*/
3339	# ifndef VBOX
3340	/* XXX: test CS64 ? */
3341	if ((env->hflags & HF_LMA_MASK) && rpl != 3) {
3342	cpu_x86_load_seg_cache(env, R_SS, new_ss,
3343	0, 0xffffffff,
3344	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
3345	DESC_S_MASK \| (rpl << DESC_DPL_SHIFT) \|
3346	DESC_W_MASK \| DESC_A_MASK);
3347	ss_e2 = DESC_B_MASK; /* XXX: should not be needed ? */
3348	} else
3349	# else /* VBOX */
3350	if ((env->hflags & HF_LMA_MASK) && rpl != 3 && (e2 & DESC_L_MASK)) {
3351	if (!(e2 & DESC_A_MASK))
3352	e2 = set_segment_accessed(new_cs, e2);
3353	cpu_x86_load_seg_cache_with_clean_flags(env, R_SS, new_ss,
3354	0, 0xffffffff,
3355	DESC_INTEL_UNUSABLE \| (rpl << DESC_DPL_SHIFT) );
3356	ss_e2 = DESC_B_MASK; /* not really used */
3357	} else
3358	# endif
3359	#endif
3360	{
3361	#if defined(VBOX) && defined(DEBUG)
3362	Log(("NULL ss, rpl=%d\n", rpl));
3363	#endif
3364	raise_exception_err(EXCP0D_GPF, 0);
3365	}
3366	} else {
3367	if ((new_ss & 3) != rpl)
3368	{
3369	#if defined(VBOX) && defined(DEBUG)
3370	Log(("new_ss=%x != rpl=%d\n", new_ss, rpl));
3371	#endif
3372	raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
3373	}
3374	if (load_segment(&ss_e1, &ss_e2, new_ss) != 0)
3375	{
3376	#if defined(VBOX) && defined(DEBUG)
3377	Log(("new_ss=%x load error\n", new_ss));
3378	#endif
3379	raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
3380	}
3381	if (!(ss_e2 & DESC_S_MASK) \|\|
3382	(ss_e2 & DESC_CS_MASK) \|\|
3383	!(ss_e2 & DESC_W_MASK))
3384	{
3385	#if defined(VBOX) && defined(DEBUG)
3386	Log(("new_ss=%x ss_e2=%#x bad type\n", new_ss, ss_e2));
3387	#endif
3388	raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
3389	}
3390	dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
3391	if (dpl != rpl)
3392	{
3393	#if defined(VBOX) && defined(DEBUG)
3394	Log(("SS.dpl=%u != rpl=%u\n", dpl, rpl));
3395	#endif
3396	raise_exception_err(EXCP0D_GPF, new_ss & 0xfffc);
3397	}
3398	if (!(ss_e2 & DESC_P_MASK))
3399	{
3400	#if defined(VBOX) && defined(DEBUG)
3401	Log(("new_ss=%#x #NP\n", new_ss));
3402	#endif
3403	raise_exception_err(EXCP0B_NOSEG, new_ss & 0xfffc);
3404	}
3405	#ifdef VBOX
3406	if (!(e2 & DESC_A_MASK))
3407	e2 = set_segment_accessed(new_cs, e2);
3408	if (!(ss_e2 & DESC_A_MASK))
3409	ss_e2 = set_segment_accessed(new_ss, ss_e2);
3410	#endif
3411	cpu_x86_load_seg_cache(env, R_SS, new_ss,
3412	get_seg_base(ss_e1, ss_e2),
3413	get_seg_limit(ss_e1, ss_e2),
3414	ss_e2);
3415	}
3416
3417	cpu_x86_load_seg_cache(env, R_CS, new_cs,
3418	get_seg_base(e1, e2),
3419	get_seg_limit(e1, e2),
3420	e2);
3421	cpu_x86_set_cpl(env, rpl);
3422	sp = new_esp;
3423	#ifdef TARGET_X86_64
3424	if (env->hflags & HF_CS64_MASK)
3425	sp_mask = -1;
3426	else
3427	#endif
3428	sp_mask = get_sp_mask(ss_e2);
3429
3430	/* validate data segments */
3431	validate_seg(R_ES, rpl);
3432	validate_seg(R_DS, rpl);
3433	validate_seg(R_FS, rpl);
3434	validate_seg(R_GS, rpl);
3435
3436	sp += addend;
3437	}
3438	SET_ESP(sp, sp_mask);
3439	env->eip = new_eip;
3440	if (is_iret) {
3441	/* NOTE: 'cpl' is the _old_ CPL */
3442	eflags_mask = TF_MASK \| AC_MASK \| ID_MASK \| RF_MASK \| NT_MASK;
3443	if (cpl == 0)
3444	#ifdef VBOX
3445	eflags_mask \|= IOPL_MASK \| VIF_MASK \| VIP_MASK;
3446	#else
3447	eflags_mask \|= IOPL_MASK;
3448	#endif
3449	iopl = (env->eflags >> IOPL_SHIFT) & 3;
3450	if (cpl <= iopl)
3451	eflags_mask \|= IF_MASK;
3452	if (shift == 0)
3453	eflags_mask &= 0xffff;
3454	load_eflags(new_eflags, eflags_mask);
3455	}
3456	return;
3457
3458	return_to_vm86:
3459	POPL(ssp, sp, sp_mask, new_esp);
3460	POPL(ssp, sp, sp_mask, new_ss);
3461	POPL(ssp, sp, sp_mask, new_es);
3462	POPL(ssp, sp, sp_mask, new_ds);
3463	POPL(ssp, sp, sp_mask, new_fs);
3464	POPL(ssp, sp, sp_mask, new_gs);
3465
3466	/* modify processor state */
3467	load_eflags(new_eflags, TF_MASK \| AC_MASK \| ID_MASK \|
3468	IF_MASK \| IOPL_MASK \| VM_MASK \| NT_MASK \| VIF_MASK \| VIP_MASK);
3469	load_seg_vm(R_CS, new_cs & 0xffff);
3470	cpu_x86_set_cpl(env, 3);
3471	load_seg_vm(R_SS, new_ss & 0xffff);
3472	load_seg_vm(R_ES, new_es & 0xffff);
3473	load_seg_vm(R_DS, new_ds & 0xffff);
3474	load_seg_vm(R_FS, new_fs & 0xffff);
3475	load_seg_vm(R_GS, new_gs & 0xffff);
3476
3477	env->eip = new_eip & 0xffff;
3478	ESP = new_esp;
3479	}
3480
3481	void helper_iret_protected(int shift, int next_eip)
3482	{
3483	int tss_selector, type;
3484	uint32_t e1, e2;
3485
3486	#ifdef VBOX
3487	Log(("iret (shift=%d new_eip=%#x)\n", shift, next_eip));
3488	e1 = e2 = 0; /** @todo Why do we do this? */
3489	remR3TrapClear(env->pVM);
3490	#endif
3491
3492	/* specific case for TSS */
3493	if (env->eflags & NT_MASK) {
3494	#ifdef TARGET_X86_64
3495	if (env->hflags & HF_LMA_MASK)
3496	{
3497	#if defined(VBOX) && defined(DEBUG)
3498	Log(("eflags.NT=1 on iret in long mode\n"));
3499	#endif
3500	raise_exception_err(EXCP0D_GPF, 0);
3501	}
3502	#endif
3503	tss_selector = lduw_kernel(env->tr.base + 0);
3504	if (tss_selector & 4)
3505	raise_exception_err(EXCP0A_TSS, tss_selector & 0xfffc);
3506	if (load_segment(&e1, &e2, tss_selector) != 0)
3507	raise_exception_err(EXCP0A_TSS, tss_selector & 0xfffc);
3508	type = (e2 >> DESC_TYPE_SHIFT) & 0x17;
3509	/* NOTE: we check both segment and busy TSS */
3510	if (type != 3)
3511	raise_exception_err(EXCP0A_TSS, tss_selector & 0xfffc);
3512	switch_tss(tss_selector, e1, e2, SWITCH_TSS_IRET, next_eip);
3513	} else {
3514	helper_ret_protected(shift, 1, 0);
3515	}
3516	env->hflags2 &= ~HF2_NMI_MASK;
3517	}
3518
3519	void helper_lret_protected(int shift, int addend)
3520	{
3521	helper_ret_protected(shift, 0, addend);
3522	}
3523
3524	void helper_sysenter(void)
3525	{
3526	if (env->sysenter_cs == 0) {
3527	raise_exception_err(EXCP0D_GPF, 0);
3528	}
3529	env->eflags &= ~(VM_MASK \| IF_MASK \| RF_MASK);
3530	cpu_x86_set_cpl(env, 0);
3531
3532	#ifdef TARGET_X86_64
3533	if (env->hflags & HF_LMA_MASK) {
3534	cpu_x86_load_seg_cache(env, R_CS, env->sysenter_cs & 0xfffc,
3535	0, 0xffffffff,
3536	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
3537	DESC_S_MASK \|
3538	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK \| DESC_L_MASK);
3539	} else
3540	#endif
3541	{
3542	cpu_x86_load_seg_cache(env, R_CS, env->sysenter_cs & 0xfffc,
3543	0, 0xffffffff,
3544	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
3545	DESC_S_MASK \|
3546	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK);
3547	}
3548	cpu_x86_load_seg_cache(env, R_SS, (env->sysenter_cs + 8) & 0xfffc,
3549	0, 0xffffffff,
3550	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
3551	DESC_S_MASK \|
3552	DESC_W_MASK \| DESC_A_MASK);
3553	ESP = env->sysenter_esp;
3554	EIP = env->sysenter_eip;
3555	}
3556
3557	void helper_sysexit(int dflag)
3558	{
3559	int cpl;
3560
3561	cpl = env->hflags & HF_CPL_MASK;
3562	if (env->sysenter_cs == 0 \|\| cpl != 0) {
3563	raise_exception_err(EXCP0D_GPF, 0);
3564	}
3565	cpu_x86_set_cpl(env, 3);
3566	#ifdef TARGET_X86_64
3567	if (dflag == 2) {
3568	cpu_x86_load_seg_cache(env, R_CS, ((env->sysenter_cs + 32) & 0xfffc) \| 3,
3569	0, 0xffffffff,
3570	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
3571	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
3572	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK \| DESC_L_MASK);
3573	cpu_x86_load_seg_cache(env, R_SS, ((env->sysenter_cs + 40) & 0xfffc) \| 3,
3574	0, 0xffffffff,
3575	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
3576	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
3577	DESC_W_MASK \| DESC_A_MASK);
3578	} else
3579	#endif
3580	{
3581	cpu_x86_load_seg_cache(env, R_CS, ((env->sysenter_cs + 16) & 0xfffc) \| 3,
3582	0, 0xffffffff,
3583	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
3584	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
3585	DESC_CS_MASK \| DESC_R_MASK \| DESC_A_MASK);
3586	cpu_x86_load_seg_cache(env, R_SS, ((env->sysenter_cs + 24) & 0xfffc) \| 3,
3587	0, 0xffffffff,
3588	DESC_G_MASK \| DESC_B_MASK \| DESC_P_MASK \|
3589	DESC_S_MASK \| (3 << DESC_DPL_SHIFT) \|
3590	DESC_W_MASK \| DESC_A_MASK);
3591	}
3592	ESP = ECX;
3593	EIP = EDX;
3594	}
3595
3596	#if defined(CONFIG_USER_ONLY)
3597	target_ulong helper_read_crN(int reg)
3598	{
3599	return 0;
3600	}
3601
3602	void helper_write_crN(int reg, target_ulong t0)
3603	{
3604	}
3605
3606	void helper_movl_drN_T0(int reg, target_ulong t0)
3607	{
3608	}
3609	#else
3610	target_ulong helper_read_crN(int reg)
3611	{
3612	target_ulong val;
3613
3614	helper_svm_check_intercept_param(SVM_EXIT_READ_CR0 + reg, 0);
3615	switch(reg) {
3616	default:
3617	val = env->cr[reg];
3618	break;
3619	case 8:
3620	if (!(env->hflags2 & HF2_VINTR_MASK)) {
3621	#ifndef VBOX
3622	val = cpu_get_apic_tpr(env->apic_state);
3623	#else /* VBOX */
3624	val = cpu_get_apic_tpr(env);
3625	#endif /* VBOX */
3626	} else {
3627	val = env->v_tpr;
3628	}
3629	break;
3630	}
3631	return val;
3632	}
3633
3634	void helper_write_crN(int reg, target_ulong t0)
3635	{
3636	helper_svm_check_intercept_param(SVM_EXIT_WRITE_CR0 + reg, 0);
3637	switch(reg) {
3638	case 0:
3639	cpu_x86_update_cr0(env, t0);
3640	break;
3641	case 3:
3642	cpu_x86_update_cr3(env, t0);
3643	break;
3644	case 4:
3645	cpu_x86_update_cr4(env, t0);
3646	break;
3647	case 8:
3648	if (!(env->hflags2 & HF2_VINTR_MASK)) {
3649	#ifndef VBOX
3650	cpu_set_apic_tpr(env->apic_state, t0);
3651	#else /* VBOX */
3652	cpu_set_apic_tpr(env, t0);
3653	#endif /* VBOX */
3654	}
3655	env->v_tpr = t0 & 0x0f;
3656	break;
3657	default:
3658	env->cr[reg] = t0;
3659	break;
3660	}
3661	}
3662
3663	void helper_movl_drN_T0(int reg, target_ulong t0)
3664	{
3665	int i;
3666
3667	if (reg < 4) {
3668	hw_breakpoint_remove(env, reg);
3669	env->dr[reg] = t0;
3670	hw_breakpoint_insert(env, reg);
3671	# ifndef VBOX
3672	} else if (reg == 7) {
3673	# else
3674	} else if (reg == 7 \|\| reg == 5) { /* (DR5 is an alias for DR7.) */
3675	if (t0 & X86_DR7_MBZ_MASK)
3676	raise_exception_err(EXCP0D_GPF, 0);
3677	t0 \|= X86_DR7_RA1_MASK;
3678	t0 &= ~X86_DR7_RAZ_MASK;
3679	# endif
3680	for (i = 0; i < 4; i++)
3681	hw_breakpoint_remove(env, i);
3682	env->dr[7] = t0;
3683	for (i = 0; i < 4; i++)
3684	hw_breakpoint_insert(env, i);
3685	} else {
3686	# ifndef VBOX
3687	env->dr[reg] = t0;
3688	# else
3689	if (t0 & X86_DR6_MBZ_MASK)
3690	raise_exception_err(EXCP0D_GPF, 0);
3691	t0 \|= X86_DR6_RA1_MASK;
3692	t0 &= ~X86_DR6_RAZ_MASK;
3693	env->dr[6] = t0; /* (DR4 is an alias for DR6.) */
3694	# endif
3695	}
3696	}
3697	#endif
3698
3699	void helper_lmsw(target_ulong t0)
3700	{
3701	/* only 4 lower bits of CR0 are modified. PE cannot be set to zero
3702	if already set to one. */
3703	t0 = (env->cr[0] & ~0xe) \| (t0 & 0xf);
3704	helper_write_crN(0, t0);
3705	}
3706
3707	void helper_clts(void)
3708	{
3709	env->cr[0] &= ~CR0_TS_MASK;
3710	env->hflags &= ~HF_TS_MASK;
3711	}
3712
3713	void helper_invlpg(target_ulong addr)
3714	{
3715	helper_svm_check_intercept_param(SVM_EXIT_INVLPG, 0);
3716	tlb_flush_page(env, addr);
3717	}
3718
3719	void helper_rdtsc(void)
3720	{
3721	uint64_t val;
3722
3723	if ((env->cr[4] & CR4_TSD_MASK) && ((env->hflags & HF_CPL_MASK) != 0)) {
3724	raise_exception(EXCP0D_GPF);
3725	}
3726	helper_svm_check_intercept_param(SVM_EXIT_RDTSC, 0);
3727
3728	val = cpu_get_tsc(env) + env->tsc_offset;
3729	EAX = (uint32_t)(val);
3730	EDX = (uint32_t)(val >> 32);
3731	}
3732
3733	void helper_rdtscp(void)
3734	{
3735	helper_rdtsc();
3736	#ifndef VBOX
3737	ECX = (uint32_t)(env->tsc_aux);
3738	#else /* VBOX */
3739	uint64_t val;
3740	if (cpu_rdmsr(env, MSR_K8_TSC_AUX, &val) == 0)
3741	ECX = (uint32_t)(val);
3742	else
3743	ECX = 0;
3744	#endif /* VBOX */
3745	}
3746
3747	void helper_rdpmc(void)
3748	{
3749	#ifdef VBOX
3750	/* If X86_CR4_PCE is not set, then CPL must be zero. */
3751	if (!(env->cr[4] & CR4_PCE_MASK) && ((env->hflags & HF_CPL_MASK) != 0)) {
3752	raise_exception(EXCP0D_GPF);
3753	}
3754	/* Just return zero here; rather tricky to properly emulate this, especially as the specs are a mess. */
3755	EAX = 0;
3756	EDX = 0;
3757	#else /* !VBOX */
3758	if ((env->cr[4] & CR4_PCE_MASK) && ((env->hflags & HF_CPL_MASK) != 0)) {
3759	raise_exception(EXCP0D_GPF);
3760	}
3761	helper_svm_check_intercept_param(SVM_EXIT_RDPMC, 0);
3762
3763	/* currently unimplemented */
3764	raise_exception_err(EXCP06_ILLOP, 0);
3765	#endif /* !VBOX */
3766	}
3767
3768	#if defined(CONFIG_USER_ONLY)
3769	void helper_wrmsr(void)
3770	{
3771	}
3772
3773	void helper_rdmsr(void)
3774	{
3775	}
3776	#else
3777	void helper_wrmsr(void)
3778	{
3779	uint64_t val;
3780
3781	helper_svm_check_intercept_param(SVM_EXIT_MSR, 1);
3782
3783	val = ((uint32_t)EAX) \| ((uint64_t)((uint32_t)EDX) << 32);
3784
3785	switch((uint32_t)ECX) {
3786	case MSR_IA32_SYSENTER_CS:
3787	env->sysenter_cs = val & 0xffff;
3788	break;
3789	case MSR_IA32_SYSENTER_ESP:
3790	env->sysenter_esp = val;
3791	break;
3792	case MSR_IA32_SYSENTER_EIP:
3793	env->sysenter_eip = val;
3794	break;
3795	case MSR_IA32_APICBASE:
3796	# ifndef VBOX /* The CPUMSetGuestMsr call below does this now. */
3797	cpu_set_apic_base(env->apic_state, val);
3798	# endif
3799	break;
3800	case MSR_EFER:
3801	{
3802	uint64_t update_mask;
3803	update_mask = 0;
3804	if (env->cpuid_ext2_features & CPUID_EXT2_SYSCALL)
3805	update_mask \|= MSR_EFER_SCE;
3806	if (env->cpuid_ext2_features & CPUID_EXT2_LM)
3807	update_mask \|= MSR_EFER_LME;
3808	if (env->cpuid_ext2_features & CPUID_EXT2_FFXSR)
3809	update_mask \|= MSR_EFER_FFXSR;
3810	if (env->cpuid_ext2_features & CPUID_EXT2_NX)
3811	update_mask \|= MSR_EFER_NXE;
3812	if (env->cpuid_ext3_features & CPUID_EXT3_SVM)
3813	update_mask \|= MSR_EFER_SVME;
3814	if (env->cpuid_ext2_features & CPUID_EXT2_FFXSR)
3815	update_mask \|= MSR_EFER_FFXSR;
3816	cpu_load_efer(env, (env->efer & ~update_mask) \|
3817	(val & update_mask));
3818	}
3819	break;
3820	case MSR_STAR:
3821	env->star = val;
3822	break;
3823	case MSR_PAT:
3824	env->pat = val;
3825	break;
3826	case MSR_VM_HSAVE_PA:
3827	env->vm_hsave = val;
3828	break;
3829	#ifdef TARGET_X86_64
3830	case MSR_LSTAR:
3831	env->lstar = val;
3832	break;
3833	case MSR_CSTAR:
3834	env->cstar = val;
3835	break;
3836	case MSR_FMASK:
3837	env->fmask = val;
3838	break;
3839	case MSR_FSBASE:
3840	env->segs[R_FS].base = val;
3841	break;
3842	case MSR_GSBASE:
3843	env->segs[R_GS].base = val;
3844	break;
3845	case MSR_KERNELGSBASE:
3846	env->kernelgsbase = val;
3847	break;
3848	#endif
3849	# ifndef VBOX
3850	case MSR_MTRRphysBase(0):
3851	case MSR_MTRRphysBase(1):
3852	case MSR_MTRRphysBase(2):
3853	case MSR_MTRRphysBase(3):
3854	case MSR_MTRRphysBase(4):
3855	case MSR_MTRRphysBase(5):
3856	case MSR_MTRRphysBase(6):
3857	case MSR_MTRRphysBase(7):
3858	env->mtrr_var[((uint32_t)ECX - MSR_MTRRphysBase(0)) / 2].base = val;
3859	break;
3860	case MSR_MTRRphysMask(0):
3861	case MSR_MTRRphysMask(1):
3862	case MSR_MTRRphysMask(2):
3863	case MSR_MTRRphysMask(3):
3864	case MSR_MTRRphysMask(4):
3865	case MSR_MTRRphysMask(5):
3866	case MSR_MTRRphysMask(6):
3867	case MSR_MTRRphysMask(7):
3868	env->mtrr_var[((uint32_t)ECX - MSR_MTRRphysMask(0)) / 2].mask = val;
3869	break;
3870	case MSR_MTRRfix64K_00000:
3871	env->mtrr_fixed[(uint32_t)ECX - MSR_MTRRfix64K_00000] = val;
3872	break;
3873	case MSR_MTRRfix16K_80000:
3874	case MSR_MTRRfix16K_A0000:
3875	env->mtrr_fixed[(uint32_t)ECX - MSR_MTRRfix16K_80000 + 1] = val;
3876	break;
3877	case MSR_MTRRfix4K_C0000:
3878	case MSR_MTRRfix4K_C8000:
3879	case MSR_MTRRfix4K_D0000:
3880	case MSR_MTRRfix4K_D8000:
3881	case MSR_MTRRfix4K_E0000:
3882	case MSR_MTRRfix4K_E8000:
3883	case MSR_MTRRfix4K_F0000:
3884	case MSR_MTRRfix4K_F8000:
3885	env->mtrr_fixed[(uint32_t)ECX - MSR_MTRRfix4K_C0000 + 3] = val;
3886	break;
3887	case MSR_MTRRdefType:
3888	env->mtrr_deftype = val;
3889	break;
3890	case MSR_MCG_STATUS:
3891	env->mcg_status = val;
3892	break;
3893	case MSR_MCG_CTL:
3894	if ((env->mcg_cap & MCG_CTL_P)
3895	&& (val == 0 \|\| val == ~(uint64_t)0))
3896	env->mcg_ctl = val;
3897	break;
3898	case MSR_TSC_AUX:
3899	env->tsc_aux = val;
3900	break;
3901	# endif /* !VBOX */
3902	default:
3903	# ifndef VBOX
3904	if ((uint32_t)ECX >= MSR_MC0_CTL
3905	&& (uint32_t)ECX < MSR_MC0_CTL + (4 * env->mcg_cap & 0xff)) {
3906	uint32_t offset = (uint32_t)ECX - MSR_MC0_CTL;
3907	if ((offset & 0x3) != 0
3908	\|\| (val == 0 \|\| val == ~(uint64_t)0))
3909	env->mce_banks[offset] = val;
3910	break;
3911	}
3912	/* XXX: exception ? */
3913	# endif
3914	break;
3915	}
3916
3917	# ifdef VBOX
3918	/* call CPUM. */
3919	if (cpu_wrmsr(env, (uint32_t)ECX, val) != 0)
3920	{
3921	/** @todo be a brave man and raise a \#GP(0) here as we should... */
3922	}
3923	# endif
3924	}
3925
3926	void helper_rdmsr(void)
3927	{
3928	uint64_t val;
3929
3930	helper_svm_check_intercept_param(SVM_EXIT_MSR, 0);
3931
3932	switch((uint32_t)ECX) {
3933	case MSR_IA32_SYSENTER_CS:
3934	val = env->sysenter_cs;
3935	break;
3936	case MSR_IA32_SYSENTER_ESP:
3937	val = env->sysenter_esp;
3938	break;
3939	case MSR_IA32_SYSENTER_EIP:
3940	val = env->sysenter_eip;
3941	break;
3942	case MSR_IA32_APICBASE:
3943	#ifndef VBOX
3944	val = cpu_get_apic_base(env->apic_state);
3945	#else /* VBOX */
3946	val = cpu_get_apic_base(env);
3947	#endif /* VBOX */
3948	break;
3949	case MSR_EFER:
3950	val = env->efer;
3951	break;
3952	case MSR_STAR:
3953	val = env->star;
3954	break;
3955	case MSR_PAT:
3956	val = env->pat;
3957	break;
3958	case MSR_VM_HSAVE_PA:
3959	val = env->vm_hsave;
3960	break;
3961	# ifndef VBOX /* forward to CPUMQueryGuestMsr. */
3962	case MSR_IA32_PERF_STATUS:
3963	/* tsc_increment_by_tick */
3964	val = 1000ULL;
3965	/* CPU multiplier */
3966	val \|= (((uint64_t)4ULL) << 40);
3967	break;
3968	# endif /* !VBOX */
3969	#ifdef TARGET_X86_64
3970	case MSR_LSTAR:
3971	val = env->lstar;
3972	break;
3973	case MSR_CSTAR:
3974	val = env->cstar;
3975	break;
3976	case MSR_FMASK:
3977	val = env->fmask;
3978	break;
3979	case MSR_FSBASE:
3980	val = env->segs[R_FS].base;
3981	break;
3982	case MSR_GSBASE:
3983	val = env->segs[R_GS].base;
3984	break;
3985	case MSR_KERNELGSBASE:
3986	val = env->kernelgsbase;
3987	break;
3988	# ifndef VBOX
3989	case MSR_TSC_AUX:
3990	val = env->tsc_aux;
3991	break;
3992	# endif /!VBOX/
3993	#endif
3994	# ifndef VBOX
3995	case MSR_MTRRphysBase(0):
3996	case MSR_MTRRphysBase(1):
3997	case MSR_MTRRphysBase(2):
3998	case MSR_MTRRphysBase(3):
3999	case MSR_MTRRphysBase(4):
4000	case MSR_MTRRphysBase(5):
4001	case MSR_MTRRphysBase(6):
4002	case MSR_MTRRphysBase(7):
4003	val = env->mtrr_var[((uint32_t)ECX - MSR_MTRRphysBase(0)) / 2].base;
4004	break;
4005	case MSR_MTRRphysMask(0):
4006	case MSR_MTRRphysMask(1):
4007	case MSR_MTRRphysMask(2):
4008	case MSR_MTRRphysMask(3):
4009	case MSR_MTRRphysMask(4):
4010	case MSR_MTRRphysMask(5):
4011	case MSR_MTRRphysMask(6):
4012	case MSR_MTRRphysMask(7):
4013	val = env->mtrr_var[((uint32_t)ECX - MSR_MTRRphysMask(0)) / 2].mask;
4014	break;
4015	case MSR_MTRRfix64K_00000:
4016	val = env->mtrr_fixed[0];
4017	break;
4018	case MSR_MTRRfix16K_80000:
4019	case MSR_MTRRfix16K_A0000:
4020	val = env->mtrr_fixed[(uint32_t)ECX - MSR_MTRRfix16K_80000 + 1];
4021	break;
4022	case MSR_MTRRfix4K_C0000:
4023	case MSR_MTRRfix4K_C8000:
4024	case MSR_MTRRfix4K_D0000:
4025	case MSR_MTRRfix4K_D8000:
4026	case MSR_MTRRfix4K_E0000:
4027	case MSR_MTRRfix4K_E8000:
4028	case MSR_MTRRfix4K_F0000:
4029	case MSR_MTRRfix4K_F8000:
4030	val = env->mtrr_fixed[(uint32_t)ECX - MSR_MTRRfix4K_C0000 + 3];
4031	break;
4032	case MSR_MTRRdefType:
4033	val = env->mtrr_deftype;
4034	break;
4035	case MSR_MTRRcap:
4036	if (env->cpuid_features & CPUID_MTRR)
4037	val = MSR_MTRRcap_VCNT \| MSR_MTRRcap_FIXRANGE_SUPPORT \| MSR_MTRRcap_WC_SUPPORTED;
4038	else
4039	/* XXX: exception ? */
4040	val = 0;
4041	break;
4042	case MSR_MCG_CAP:
4043	val = env->mcg_cap;
4044	break;
4045	case MSR_MCG_CTL:
4046	if (env->mcg_cap & MCG_CTL_P)
4047	val = env->mcg_ctl;
4048	else
4049	val = 0;
4050	break;
4051	case MSR_MCG_STATUS:
4052	val = env->mcg_status;
4053	break;
4054	# endif /* !VBOX */
4055	default:
4056	# ifndef VBOX
4057	if ((uint32_t)ECX >= MSR_MC0_CTL
4058	&& (uint32_t)ECX < MSR_MC0_CTL + (4 * env->mcg_cap & 0xff)) {
4059	uint32_t offset = (uint32_t)ECX - MSR_MC0_CTL;
4060	val = env->mce_banks[offset];
4061	break;
4062	}
4063	/* XXX: exception ? */
4064	val = 0;
4065	# else /* VBOX */
4066	if (cpu_rdmsr(env, (uint32_t)ECX, &val) != 0)
4067	{
4068	/** @todo be a brave man and raise a \#GP(0) here as we should... */
4069	val = 0;
4070	}
4071	# endif /* VBOX */
4072	break;
4073	}
4074	EAX = (uint32_t)(val);
4075	EDX = (uint32_t)(val >> 32);
4076
4077	# ifdef VBOX_STRICT
4078	if ((uint32_t)ECX != MSR_IA32_TSC) {
4079	if (cpu_rdmsr(env, (uint32_t)ECX, &val) != 0)
4080	val = 0;
4081	AssertMsg(val == RT_MAKE_U64(EAX, EDX), ("idMsr=%#x val=%#llx eax:edx=%#llx\n", (uint32_t)ECX, val, RT_MAKE_U64(EAX, EDX)));
4082	}
4083	# endif
4084	}
4085	#endif
4086
4087	target_ulong helper_lsl(target_ulong selector1)
4088	{
4089	unsigned int limit;
4090	uint32_t e1, e2, eflags, selector;
4091	int rpl, dpl, cpl, type;
4092
4093	selector = selector1 & 0xffff;
4094	eflags = helper_cc_compute_all(CC_OP);
4095	if ((selector & 0xfffc) == 0)
4096	goto fail;
4097	if (load_segment(&e1, &e2, selector) != 0)
4098	goto fail;
4099	rpl = selector & 3;
4100	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
4101	cpl = env->hflags & HF_CPL_MASK;
4102	if (e2 & DESC_S_MASK) {
4103	if ((e2 & DESC_CS_MASK) && (e2 & DESC_C_MASK)) {
4104	/* conforming */
4105	} else {
4106	if (dpl < cpl \|\| dpl < rpl)
4107	goto fail;
4108	}
4109	} else {
4110	type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
4111	switch(type) {
4112	case 1:
4113	case 2:
4114	case 3:
4115	case 9:
4116	case 11:
4117	break;
4118	default:
4119	goto fail;
4120	}
4121	if (dpl < cpl \|\| dpl < rpl) {
4122	fail:
4123	CC_SRC = eflags & ~CC_Z;
4124	return 0;
4125	}
4126	}
4127	limit = get_seg_limit(e1, e2);
4128	CC_SRC = eflags \| CC_Z;
4129	return limit;
4130	}
4131
4132	target_ulong helper_lar(target_ulong selector1)
4133	{
4134	uint32_t e1, e2, eflags, selector;
4135	int rpl, dpl, cpl, type;
4136
4137	selector = selector1 & 0xffff;
4138	eflags = helper_cc_compute_all(CC_OP);
4139	if ((selector & 0xfffc) == 0)
4140	goto fail;
4141	if (load_segment(&e1, &e2, selector) != 0)
4142	goto fail;
4143	rpl = selector & 3;
4144	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
4145	cpl = env->hflags & HF_CPL_MASK;
4146	if (e2 & DESC_S_MASK) {
4147	if ((e2 & DESC_CS_MASK) && (e2 & DESC_C_MASK)) {
4148	/* conforming */
4149	} else {
4150	if (dpl < cpl \|\| dpl < rpl)
4151	goto fail;
4152	}
4153	} else {
4154	type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
4155	switch(type) {
4156	case 1:
4157	case 2:
4158	case 3:
4159	case 4:
4160	case 5:
4161	case 9:
4162	case 11:
4163	case 12:
4164	break;
4165	default:
4166	goto fail;
4167	}
4168	if (dpl < cpl \|\| dpl < rpl) {
4169	fail:
4170	CC_SRC = eflags & ~CC_Z;
4171	return 0;
4172	}
4173	}
4174	CC_SRC = eflags \| CC_Z;
4175	#ifdef VBOX /* AMD says 0x00ffff00, while intel says 0x00fxff00. Bochs and IEM does like AMD says (x=f). */
4176	return e2 & 0x00ffff00;
4177	#else
4178	return e2 & 0x00f0ff00;
4179	#endif
4180	}
4181
4182	void helper_verr(target_ulong selector1)
4183	{
4184	uint32_t e1, e2, eflags, selector;
4185	int rpl, dpl, cpl;
4186
4187	selector = selector1 & 0xffff;
4188	eflags = helper_cc_compute_all(CC_OP);
4189	if ((selector & 0xfffc) == 0)
4190	goto fail;
4191	if (load_segment(&e1, &e2, selector) != 0)
4192	goto fail;
4193	if (!(e2 & DESC_S_MASK))
4194	goto fail;
4195	rpl = selector & 3;
4196	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
4197	cpl = env->hflags & HF_CPL_MASK;
4198	if (e2 & DESC_CS_MASK) {
4199	if (!(e2 & DESC_R_MASK))
4200	goto fail;
4201	if (!(e2 & DESC_C_MASK)) {
4202	if (dpl < cpl \|\| dpl < rpl)
4203	goto fail;
4204	}
4205	} else {
4206	if (dpl < cpl \|\| dpl < rpl) {
4207	fail:
4208	CC_SRC = eflags & ~CC_Z;
4209	return;
4210	}
4211	}
4212	CC_SRC = eflags \| CC_Z;
4213	}
4214
4215	void helper_verw(target_ulong selector1)
4216	{
4217	uint32_t e1, e2, eflags, selector;
4218	int rpl, dpl, cpl;
4219
4220	selector = selector1 & 0xffff;
4221	eflags = helper_cc_compute_all(CC_OP);
4222	if ((selector & 0xfffc) == 0)
4223	goto fail;
4224	if (load_segment(&e1, &e2, selector) != 0)
4225	goto fail;
4226	if (!(e2 & DESC_S_MASK))
4227	goto fail;
4228	rpl = selector & 3;
4229	dpl = (e2 >> DESC_DPL_SHIFT) & 3;
4230	cpl = env->hflags & HF_CPL_MASK;
4231	if (e2 & DESC_CS_MASK) {
4232	goto fail;
4233	} else {
4234	if (dpl < cpl \|\| dpl < rpl)
4235	goto fail;
4236	if (!(e2 & DESC_W_MASK)) {
4237	fail:
4238	CC_SRC = eflags & ~CC_Z;
4239	return;
4240	}
4241	}
4242	CC_SRC = eflags \| CC_Z;
4243	}
4244
4245	/* x87 FPU helpers */
4246
4247	static void fpu_set_exception(int mask)
4248	{
4249	env->fpus \|= mask;
4250	if (env->fpus & (~env->fpuc & FPUC_EM))
4251	env->fpus \|= FPUS_SE \| FPUS_B;
4252	}
4253
4254	static inline CPU86_LDouble helper_fdiv(CPU86_LDouble a, CPU86_LDouble b)
4255	{
4256	if (b == 0.0)
4257	fpu_set_exception(FPUS_ZE);
4258	return a / b;
4259	}
4260
4261	static void fpu_raise_exception(void)
4262	{
4263	if (env->cr[0] & CR0_NE_MASK) {
4264	raise_exception(EXCP10_COPR);
4265	}
4266	#if !defined(CONFIG_USER_ONLY)
4267	else {
4268	cpu_set_ferr(env);
4269	}
4270	#endif
4271	}
4272
4273	void helper_flds_FT0(uint32_t val)
4274	{
4275	union {
4276	float32 f;
4277	uint32_t i;
4278	} u;
4279	u.i = val;
4280	FT0 = float32_to_floatx(u.f, &env->fp_status);
4281	}
4282
4283	void helper_fldl_FT0(uint64_t val)
4284	{
4285	union {
4286	float64 f;
4287	uint64_t i;
4288	} u;
4289	u.i = val;
4290	FT0 = float64_to_floatx(u.f, &env->fp_status);
4291	}
4292
4293	void helper_fildl_FT0(int32_t val)
4294	{
4295	FT0 = int32_to_floatx(val, &env->fp_status);
4296	}
4297
4298	void helper_flds_ST0(uint32_t val)
4299	{
4300	int new_fpstt;
4301	union {
4302	float32 f;
4303	uint32_t i;
4304	} u;
4305	new_fpstt = (env->fpstt - 1) & 7;
4306	u.i = val;
4307	env->fpregs[new_fpstt].d = float32_to_floatx(u.f, &env->fp_status);
4308	env->fpstt = new_fpstt;
4309	env->fptags[new_fpstt] = 0; /* validate stack entry */
4310	}
4311
4312	void helper_fldl_ST0(uint64_t val)
4313	{
4314	int new_fpstt;
4315	union {
4316	float64 f;
4317	uint64_t i;
4318	} u;
4319	new_fpstt = (env->fpstt - 1) & 7;
4320	u.i = val;
4321	env->fpregs[new_fpstt].d = float64_to_floatx(u.f, &env->fp_status);
4322	env->fpstt = new_fpstt;
4323	env->fptags[new_fpstt] = 0; /* validate stack entry */
4324	}
4325
4326	void helper_fildl_ST0(int32_t val)
4327	{
4328	int new_fpstt;
4329	new_fpstt = (env->fpstt - 1) & 7;
4330	env->fpregs[new_fpstt].d = int32_to_floatx(val, &env->fp_status);
4331	env->fpstt = new_fpstt;
4332	env->fptags[new_fpstt] = 0; /* validate stack entry */
4333	}
4334
4335	void helper_fildll_ST0(int64_t val)
4336	{
4337	int new_fpstt;
4338	new_fpstt = (env->fpstt - 1) & 7;
4339	env->fpregs[new_fpstt].d = int64_to_floatx(val, &env->fp_status);
4340	env->fpstt = new_fpstt;
4341	env->fptags[new_fpstt] = 0; /* validate stack entry */
4342	}
4343
4344	#ifndef VBOX
4345	uint32_t helper_fsts_ST0(void)
4346	#else
4347	RTCCUINTREG helper_fsts_ST0(void)
4348	#endif
4349	{
4350	union {
4351	float32 f;
4352	uint32_t i;
4353	} u;
4354	u.f = floatx_to_float32(ST0, &env->fp_status);
4355	return u.i;
4356	}
4357
4358	uint64_t helper_fstl_ST0(void)
4359	{
4360	union {
4361	float64 f;
4362	uint64_t i;
4363	} u;
4364	u.f = floatx_to_float64(ST0, &env->fp_status);
4365	return u.i;
4366	}
4367
4368	#ifndef VBOX
4369	int32_t helper_fist_ST0(void)
4370	#else
4371	RTCCINTREG helper_fist_ST0(void)
4372	#endif
4373	{
4374	int32_t val;
4375	val = floatx_to_int32(ST0, &env->fp_status);
4376	if (val != (int16_t)val)
4377	val = -32768;
4378	return val;
4379	}
4380
4381	#ifndef VBOX
4382	int32_t helper_fistl_ST0(void)
4383	#else
4384	RTCCINTREG helper_fistl_ST0(void)
4385	#endif
4386	{
4387	int32_t val;
4388	val = floatx_to_int32(ST0, &env->fp_status);
4389	return val;
4390	}
4391
4392	int64_t helper_fistll_ST0(void)
4393	{
4394	int64_t val;
4395	val = floatx_to_int64(ST0, &env->fp_status);
4396	return val;
4397	}
4398
4399	#ifndef VBOX
4400	int32_t helper_fistt_ST0(void)
4401	#else
4402	RTCCINTREG helper_fistt_ST0(void)
4403	#endif
4404	{
4405	int32_t val;
4406	val = floatx_to_int32_round_to_zero(ST0, &env->fp_status);
4407	if (val != (int16_t)val)
4408	val = -32768;
4409	return val;
4410	}
4411
4412	#ifndef VBOX
4413	int32_t helper_fisttl_ST0(void)
4414	#else
4415	RTCCINTREG helper_fisttl_ST0(void)
4416	#endif
4417	{
4418	int32_t val;
4419	val = floatx_to_int32_round_to_zero(ST0, &env->fp_status);
4420	return val;
4421	}
4422
4423	int64_t helper_fisttll_ST0(void)
4424	{
4425	int64_t val;
4426	val = floatx_to_int64_round_to_zero(ST0, &env->fp_status);
4427	return val;
4428	}
4429
4430	void helper_fldt_ST0(target_ulong ptr)
4431	{
4432	int new_fpstt;
4433	new_fpstt = (env->fpstt - 1) & 7;
4434	env->fpregs[new_fpstt].d = helper_fldt(ptr);
4435	env->fpstt = new_fpstt;
4436	env->fptags[new_fpstt] = 0; /* validate stack entry */
4437	}
4438
4439	void helper_fstt_ST0(target_ulong ptr)
4440	{
4441	helper_fstt(ST0, ptr);
4442	}
4443
4444	void helper_fpush(void)
4445	{
4446	fpush();
4447	}
4448
4449	void helper_fpop(void)
4450	{
4451	fpop();
4452	}
4453
4454	void helper_fdecstp(void)
4455	{
4456	env->fpstt = (env->fpstt - 1) & 7;
4457	env->fpus &= (~0x4700);
4458	}
4459
4460	void helper_fincstp(void)
4461	{
4462	env->fpstt = (env->fpstt + 1) & 7;
4463	env->fpus &= (~0x4700);
4464	}
4465
4466	/* FPU move */
4467
4468	void helper_ffree_STN(int st_index)
4469	{
4470	env->fptags[(env->fpstt + st_index) & 7] = 1;
4471	}
4472
4473	void helper_fmov_ST0_FT0(void)
4474	{
4475	ST0 = FT0;
4476	}
4477
4478	void helper_fmov_FT0_STN(int st_index)
4479	{
4480	FT0 = ST(st_index);
4481	}
4482
4483	void helper_fmov_ST0_STN(int st_index)
4484	{
4485	ST0 = ST(st_index);
4486	}
4487
4488	void helper_fmov_STN_ST0(int st_index)
4489	{
4490	ST(st_index) = ST0;
4491	}
4492
4493	void helper_fxchg_ST0_STN(int st_index)
4494	{
4495	CPU86_LDouble tmp;
4496	tmp = ST(st_index);
4497	ST(st_index) = ST0;
4498	ST0 = tmp;
4499	}
4500
4501	/* FPU operations */
4502
4503	static const int fcom_ccval[4] = {0x0100, 0x4000, 0x0000, 0x4500};
4504
4505	void helper_fcom_ST0_FT0(void)
4506	{
4507	int ret;
4508
4509	ret = floatx_compare(ST0, FT0, &env->fp_status);
4510	env->fpus = (env->fpus & ~0x4500) \| fcom_ccval[ret + 1];
4511	}
4512
4513	void helper_fucom_ST0_FT0(void)
4514	{
4515	int ret;
4516
4517	ret = floatx_compare_quiet(ST0, FT0, &env->fp_status);
4518	env->fpus = (env->fpus & ~0x4500) \| fcom_ccval[ret+ 1];
4519	}
4520
4521	static const int fcomi_ccval[4] = {CC_C, CC_Z, 0, CC_Z \| CC_P \| CC_C};
4522
4523	void helper_fcomi_ST0_FT0(void)
4524	{
4525	int eflags;
4526	int ret;
4527
4528	ret = floatx_compare(ST0, FT0, &env->fp_status);
4529	eflags = helper_cc_compute_all(CC_OP);
4530	eflags = (eflags & ~(CC_Z \| CC_P \| CC_C)) \| fcomi_ccval[ret + 1];
4531	CC_SRC = eflags;
4532	}
4533
4534	void helper_fucomi_ST0_FT0(void)
4535	{
4536	int eflags;
4537	int ret;
4538
4539	ret = floatx_compare_quiet(ST0, FT0, &env->fp_status);
4540	eflags = helper_cc_compute_all(CC_OP);
4541	eflags = (eflags & ~(CC_Z \| CC_P \| CC_C)) \| fcomi_ccval[ret + 1];
4542	CC_SRC = eflags;
4543	}
4544
4545	void helper_fadd_ST0_FT0(void)
4546	{
4547	ST0 += FT0;
4548	}
4549
4550	void helper_fmul_ST0_FT0(void)
4551	{
4552	ST0 *= FT0;
4553	}
4554
4555	void helper_fsub_ST0_FT0(void)
4556	{
4557	ST0 -= FT0;
4558	}
4559
4560	void helper_fsubr_ST0_FT0(void)
4561	{
4562	ST0 = FT0 - ST0;
4563	}
4564
4565	void helper_fdiv_ST0_FT0(void)
4566	{
4567	ST0 = helper_fdiv(ST0, FT0);
4568	}
4569
4570	void helper_fdivr_ST0_FT0(void)
4571	{
4572	ST0 = helper_fdiv(FT0, ST0);
4573	}
4574
4575	/* fp operations between STN and ST0 */
4576
4577	void helper_fadd_STN_ST0(int st_index)
4578	{
4579	ST(st_index) += ST0;
4580	}
4581
4582	void helper_fmul_STN_ST0(int st_index)
4583	{
4584	ST(st_index) *= ST0;
4585	}
4586
4587	void helper_fsub_STN_ST0(int st_index)
4588	{
4589	ST(st_index) -= ST0;
4590	}
4591
4592	void helper_fsubr_STN_ST0(int st_index)
4593	{
4594	CPU86_LDouble *p;
4595	p = &ST(st_index);
4596	p = ST0 - p;
4597	}
4598
4599	void helper_fdiv_STN_ST0(int st_index)
4600	{
4601	CPU86_LDouble *p;
4602	p = &ST(st_index);
4603	p = helper_fdiv(p, ST0);
4604	}
4605
4606	void helper_fdivr_STN_ST0(int st_index)
4607	{
4608	CPU86_LDouble *p;
4609	p = &ST(st_index);
4610	p = helper_fdiv(ST0, p);
4611	}
4612
4613	/* misc FPU operations */
4614	void helper_fchs_ST0(void)
4615	{
4616	ST0 = floatx_chs(ST0);
4617	}
4618
4619	void helper_fabs_ST0(void)
4620	{
4621	ST0 = floatx_abs(ST0);
4622	}
4623
4624	void helper_fld1_ST0(void)
4625	{
4626	ST0 = f15rk[1];
4627	}
4628
4629	void helper_fldl2t_ST0(void)
4630	{
4631	ST0 = f15rk[6];
4632	}
4633
4634	void helper_fldl2e_ST0(void)
4635	{
4636	ST0 = f15rk[5];
4637	}
4638
4639	void helper_fldpi_ST0(void)
4640	{
4641	ST0 = f15rk[2];
4642	}
4643
4644	void helper_fldlg2_ST0(void)
4645	{
4646	ST0 = f15rk[3];
4647	}
4648
4649	void helper_fldln2_ST0(void)
4650	{
4651	ST0 = f15rk[4];
4652	}
4653
4654	void helper_fldz_ST0(void)
4655	{
4656	ST0 = f15rk[0];
4657	}
4658
4659	void helper_fldz_FT0(void)
4660	{
4661	FT0 = f15rk[0];
4662	}
4663
4664	#ifndef VBOX
4665	uint32_t helper_fnstsw(void)
4666	#else
4667	RTCCUINTREG helper_fnstsw(void)
4668	#endif
4669	{
4670	return (env->fpus & ~0x3800) \| (env->fpstt & 0x7) << 11;
4671	}
4672
4673	#ifndef VBOX
4674	uint32_t helper_fnstcw(void)
4675	#else
4676	RTCCUINTREG helper_fnstcw(void)
4677	#endif
4678	{
4679	return env->fpuc;
4680	}
4681
4682	static void update_fp_status(void)
4683	{
4684	int rnd_type;
4685
4686	/* set rounding mode */
4687	switch(env->fpuc & RC_MASK) {
4688	default:
4689	case RC_NEAR:
4690	rnd_type = float_round_nearest_even;
4691	break;
4692	case RC_DOWN:
4693	rnd_type = float_round_down;
4694	break;
4695	case RC_UP:
4696	rnd_type = float_round_up;
4697	break;
4698	case RC_CHOP:
4699	rnd_type = float_round_to_zero;
4700	break;
4701	}
4702	set_float_rounding_mode(rnd_type, &env->fp_status);
4703	#ifdef FLOATX80
4704	switch((env->fpuc >> 8) & 3) {
4705	case 0:
4706	rnd_type = 32;
4707	break;
4708	case 2:
4709	rnd_type = 64;
4710	break;
4711	case 3:
4712	default:
4713	rnd_type = 80;
4714	break;
4715	}
4716	set_floatx80_rounding_precision(rnd_type, &env->fp_status);
4717	#endif
4718	}
4719
4720	void helper_fldcw(uint32_t val)
4721	{
4722	env->fpuc = val;
4723	update_fp_status();
4724	}
4725
4726	void helper_fclex(void)
4727	{
4728	env->fpus &= 0x7f00;
4729	}
4730
4731	void helper_fwait(void)
4732	{
4733	if (env->fpus & FPUS_SE)
4734	fpu_raise_exception();
4735	}
4736
4737	void helper_fninit(void)
4738	{
4739	env->fpus = 0;
4740	env->fpstt = 0;
4741	env->fpuc = 0x37f;
4742	env->fptags[0] = 1;
4743	env->fptags[1] = 1;
4744	env->fptags[2] = 1;
4745	env->fptags[3] = 1;
4746	env->fptags[4] = 1;
4747	env->fptags[5] = 1;
4748	env->fptags[6] = 1;
4749	env->fptags[7] = 1;
4750	}
4751
4752	/* BCD ops */
4753
4754	void helper_fbld_ST0(target_ulong ptr)
4755	{
4756	CPU86_LDouble tmp;
4757	uint64_t val;
4758	unsigned int v;
4759	int i;
4760
4761	val = 0;
4762	for(i = 8; i >= 0; i--) {
4763	v = ldub(ptr + i);
4764	val = (val * 100) + ((v >> 4) * 10) + (v & 0xf);
4765	}
4766	tmp = val;
4767	if (ldub(ptr + 9) & 0x80)
4768	tmp = -tmp;
4769	fpush();
4770	ST0 = tmp;
4771	}
4772
4773	void helper_fbst_ST0(target_ulong ptr)
4774	{
4775	int v;
4776	target_ulong mem_ref, mem_end;
4777	int64_t val;
4778
4779	val = floatx_to_int64(ST0, &env->fp_status);
4780	mem_ref = ptr;
4781	mem_end = mem_ref + 9;
4782	if (val < 0) {
4783	stb(mem_end, 0x80);
4784	val = -val;
4785	} else {
4786	stb(mem_end, 0x00);
4787	}
4788	while (mem_ref < mem_end) {
4789	if (val == 0)
4790	break;
4791	v = val % 100;
4792	val = val / 100;
4793	v = ((v / 10) << 4) \| (v % 10);
4794	stb(mem_ref++, v);
4795	}
4796	while (mem_ref < mem_end) {
4797	stb(mem_ref++, 0);
4798	}
4799	}
4800
4801	void helper_f2xm1(void)
4802	{
4803	ST0 = pow(2.0,ST0) - 1.0;
4804	}
4805
4806	void helper_fyl2x(void)
4807	{
4808	CPU86_LDouble fptemp;
4809
4810	fptemp = ST0;
4811	if (fptemp>0.0){
4812	fptemp = log(fptemp)/log(2.0); /* log2(ST) */
4813	ST1 *= fptemp;
4814	fpop();
4815	} else {
4816	env->fpus &= (~0x4700);
4817	env->fpus \|= 0x400;
4818	}
4819	}
4820
4821	void helper_fptan(void)
4822	{
4823	CPU86_LDouble fptemp;
4824
4825	fptemp = ST0;
4826	if((fptemp > MAXTAN)\|\|(fptemp < -MAXTAN)) {
4827	env->fpus \|= 0x400;
4828	} else {
4829	ST0 = tan(fptemp);
4830	fpush();
4831	ST0 = 1.0;
4832	env->fpus &= (~0x400); /* C2 <-- 0 */
4833	/* the above code is for \|arg\| < 2*52 only /
4834	}
4835	}
4836
4837	void helper_fpatan(void)
4838	{
4839	CPU86_LDouble fptemp, fpsrcop;
4840
4841	fpsrcop = ST1;
4842	fptemp = ST0;
4843	ST1 = atan2(fpsrcop,fptemp);
4844	fpop();
4845	}
4846
4847	void helper_fxtract(void)
4848	{
4849	CPU86_LDoubleU temp;
4850	unsigned int expdif;
4851
4852	temp.d = ST0;
4853	expdif = EXPD(temp) - EXPBIAS;
4854	/DP exponent bias/
4855	ST0 = expdif;
4856	fpush();
4857	BIASEXPONENT(temp);
4858	ST0 = temp.d;
4859	}
4860
4861	void helper_fprem1(void)
4862	{
4863	CPU86_LDouble dblq, fpsrcop, fptemp;
4864	CPU86_LDoubleU fpsrcop1, fptemp1;
4865	int expdif;
4866	signed long long int q;
4867
4868	#ifndef VBOX /* Unfortunately, we cannot handle isinf/isnan easily in wrapper */
4869	if (isinf(ST0) \|\| isnan(ST0) \|\| isnan(ST1) \|\| (ST1 == 0.0)) {
4870	#else
4871	if ((ST0 != ST0) \|\| (ST1 != ST1) \|\| (ST1 == 0.0)) {
4872	#endif
4873	ST0 = 0.0 / 0.0; /* NaN */
4874	env->fpus &= (~0x4700); /* (C3,C2,C1,C0) <-- 0000 */
4875	return;
4876	}
4877
4878	fpsrcop = ST0;
4879	fptemp = ST1;
4880	fpsrcop1.d = fpsrcop;
4881	fptemp1.d = fptemp;
4882	expdif = EXPD(fpsrcop1) - EXPD(fptemp1);
4883
4884	if (expdif < 0) {
4885	/* optimisation? taken from the AMD docs */
4886	env->fpus &= (~0x4700); /* (C3,C2,C1,C0) <-- 0000 */
4887	/* ST0 is unchanged */
4888	return;
4889	}
4890
4891	if (expdif < 53) {
4892	dblq = fpsrcop / fptemp;
4893	/* round dblq towards nearest integer */
4894	dblq = rint(dblq);
4895	ST0 = fpsrcop - fptemp * dblq;
4896
4897	/* convert dblq to q by truncating towards zero */
4898	if (dblq < 0.0)
4899	q = (signed long long int)(-dblq);
4900	else
4901	q = (signed long long int)dblq;
4902
4903	env->fpus &= (~0x4700); /* (C3,C2,C1,C0) <-- 0000 */
4904	/* (C0,C3,C1) <-- (q2,q1,q0) */
4905	env->fpus \|= (q & 0x4) << (8 - 2); /* (C0) <-- q2 */
4906	env->fpus \|= (q & 0x2) << (14 - 1); /* (C3) <-- q1 */
4907	env->fpus \|= (q & 0x1) << (9 - 0); /* (C1) <-- q0 */
4908	} else {
4909	env->fpus \|= 0x400; /* C2 <-- 1 */
4910	fptemp = pow(2.0, expdif - 50);
4911	fpsrcop = (ST0 / ST1) / fptemp;
4912	/* fpsrcop = integer obtained by chopping */
4913	fpsrcop = (fpsrcop < 0.0) ?
4914	-(floor(fabs(fpsrcop))) : floor(fpsrcop);
4915	ST0 -= (ST1 * fpsrcop * fptemp);
4916	}
4917	}
4918
4919	void helper_fprem(void)
4920	{
4921	CPU86_LDouble dblq, fpsrcop, fptemp;
4922	CPU86_LDoubleU fpsrcop1, fptemp1;
4923	int expdif;
4924	signed long long int q;
4925
4926	#ifndef VBOX /* Unfortunately, we cannot easily handle isinf/isnan in wrapper */
4927	if (isinf(ST0) \|\| isnan(ST0) \|\| isnan(ST1) \|\| (ST1 == 0.0)) {
4928	#else
4929	if ((ST0 != ST0) \|\| (ST1 != ST1) \|\| (ST1 == 0.0)) {
4930	#endif
4931	ST0 = 0.0 / 0.0; /* NaN */
4932	env->fpus &= (~0x4700); /* (C3,C2,C1,C0) <-- 0000 */
4933	return;
4934	}
4935
4936	fpsrcop = (CPU86_LDouble)ST0;
4937	fptemp = (CPU86_LDouble)ST1;
4938	fpsrcop1.d = fpsrcop;
4939	fptemp1.d = fptemp;
4940	expdif = EXPD(fpsrcop1) - EXPD(fptemp1);
4941
4942	if (expdif < 0) {
4943	/* optimisation? taken from the AMD docs */
4944	env->fpus &= (~0x4700); /* (C3,C2,C1,C0) <-- 0000 */
4945	/* ST0 is unchanged */
4946	return;
4947	}
4948
4949	if ( expdif < 53 ) {
4950	dblq = fpsrcop/ST0/ / fptemp/ST1/;
4951	/* round dblq towards zero */
4952	dblq = (dblq < 0.0) ? ceil(dblq) : floor(dblq);
4953	ST0 = fpsrcop/ST0/ - fptemp * dblq;
4954
4955	/* convert dblq to q by truncating towards zero */
4956	if (dblq < 0.0)
4957	q = (signed long long int)(-dblq);
4958	else
4959	q = (signed long long int)dblq;
4960
4961	env->fpus &= (~0x4700); /* (C3,C2,C1,C0) <-- 0000 */
4962	/* (C0,C3,C1) <-- (q2,q1,q0) */
4963	env->fpus \|= (q & 0x4) << (8 - 2); /* (C0) <-- q2 */
4964	env->fpus \|= (q & 0x2) << (14 - 1); /* (C3) <-- q1 */
4965	env->fpus \|= (q & 0x1) << (9 - 0); /* (C1) <-- q0 */
4966	} else {
4967	int N = 32 + (expdif % 32); /* as per AMD docs */
4968	env->fpus \|= 0x400; /* C2 <-- 1 */
4969	fptemp = pow(2.0, (double)(expdif - N));
4970	fpsrcop = (ST0 / ST1) / fptemp;
4971	/* fpsrcop = integer obtained by chopping */
4972	fpsrcop = (fpsrcop < 0.0) ?
4973	-(floor(fabs(fpsrcop))) : floor(fpsrcop);
4974	ST0 -= (ST1 * fpsrcop * fptemp);
4975	}
4976	}
4977
4978	void helper_fyl2xp1(void)
4979	{
4980	CPU86_LDouble fptemp;
4981
4982	fptemp = ST0;
4983	if ((fptemp+1.0)>0.0) {
4984	fptemp = log(fptemp+1.0) / log(2.0); /* log2(ST+1.0) */
4985	ST1 *= fptemp;
4986	fpop();
4987	} else {
4988	env->fpus &= (~0x4700);
4989	env->fpus \|= 0x400;
4990	}
4991	}
4992
4993	void helper_fsqrt(void)
4994	{
4995	CPU86_LDouble fptemp;
4996
4997	fptemp = ST0;
4998	if (fptemp<0.0) {
4999	env->fpus &= (~0x4700); /* (C3,C2,C1,C0) <-- 0000 */
5000	env->fpus \|= 0x400;
5001	}
5002	ST0 = sqrt(fptemp);
5003	}
5004
5005	void helper_fsincos(void)
5006	{
5007	CPU86_LDouble fptemp;
5008
5009	fptemp = ST0;
5010	if ((fptemp > MAXTAN)\|\|(fptemp < -MAXTAN)) {
5011	env->fpus \|= 0x400;
5012	} else {
5013	ST0 = sin(fptemp);
5014	fpush();
5015	ST0 = cos(fptemp);
5016	env->fpus &= (~0x400); /* C2 <-- 0 */
5017	/* the above code is for \|arg\| < 2*63 only /
5018	}
5019	}
5020
5021	void helper_frndint(void)
5022	{
5023	ST0 = floatx_round_to_int(ST0, &env->fp_status);
5024	}
5025
5026	void helper_fscale(void)
5027	{
5028	ST0 = ldexp (ST0, (int)(ST1));
5029	}
5030
5031	void helper_fsin(void)
5032	{
5033	CPU86_LDouble fptemp;
5034
5035	fptemp = ST0;
5036	if ((fptemp > MAXTAN)\|\|(fptemp < -MAXTAN)) {
5037	env->fpus \|= 0x400;
5038	} else {
5039	ST0 = sin(fptemp);
5040	env->fpus &= (~0x400); /* C2 <-- 0 */
5041	/* the above code is for \|arg\| < 2*53 only /
5042	}
5043	}
5044
5045	void helper_fcos(void)
5046	{
5047	CPU86_LDouble fptemp;
5048
5049	fptemp = ST0;
5050	if((fptemp > MAXTAN)\|\|(fptemp < -MAXTAN)) {
5051	env->fpus \|= 0x400;
5052	} else {
5053	ST0 = cos(fptemp);
5054	env->fpus &= (~0x400); /* C2 <-- 0 */
5055	/* the above code is for \|arg5 < 2*63 only /
5056	}
5057	}
5058
5059	void helper_fxam_ST0(void)
5060	{
5061	CPU86_LDoubleU temp;
5062	int expdif;
5063
5064	temp.d = ST0;
5065
5066	env->fpus &= (~0x4700); /* (C3,C2,C1,C0) <-- 0000 */
5067	if (SIGND(temp))
5068	env->fpus \|= 0x200; /* C1 <-- 1 */
5069
5070	/* XXX: test fptags too */
5071	expdif = EXPD(temp);
5072	if (expdif == MAXEXPD) {
5073	#ifdef USE_X86LDOUBLE
5074	if (MANTD(temp) == 0x8000000000000000ULL)
5075	#else
5076	if (MANTD(temp) == 0)
5077	#endif
5078	env->fpus \|= 0x500 /Infinity/;
5079	else
5080	env->fpus \|= 0x100 /NaN/;
5081	} else if (expdif == 0) {
5082	if (MANTD(temp) == 0)
5083	env->fpus \|= 0x4000 /Zero/;
5084	else
5085	env->fpus \|= 0x4400 /Denormal/;
5086	} else {
5087	env->fpus \|= 0x400;
5088	}
5089	}
5090
5091	void helper_fstenv(target_ulong ptr, int data32)
5092	{
5093	int fpus, fptag, exp, i;
5094	uint64_t mant;
5095	CPU86_LDoubleU tmp;
5096
5097	fpus = (env->fpus & ~0x3800) \| (env->fpstt & 0x7) << 11;
5098	fptag = 0;
5099	for (i=7; i>=0; i--) {
5100	fptag <<= 2;
5101	if (env->fptags[i]) {
5102	fptag \|= 3;
5103	} else {
5104	tmp.d = env->fpregs[i].d;
5105	exp = EXPD(tmp);
5106	mant = MANTD(tmp);
5107	if (exp == 0 && mant == 0) {
5108	/* zero */
5109	fptag \|= 1;
5110	} else if (exp == 0 \|\| exp == MAXEXPD
5111	#ifdef USE_X86LDOUBLE
5112	\|\| (mant & (1LL << 63)) == 0
5113	#endif
5114	) {
5115	/* NaNs, infinity, denormal */
5116	fptag \|= 2;
5117	}
5118	}
5119	}
5120	if (data32) {
5121	/* 32 bit */
5122	stl(ptr, env->fpuc);
5123	stl(ptr + 4, fpus);
5124	stl(ptr + 8, fptag);
5125	stl(ptr + 12, 0); /* fpip */
5126	stl(ptr + 16, 0); /* fpcs */
5127	stl(ptr + 20, 0); /* fpoo */
5128	stl(ptr + 24, 0); /* fpos */
5129	} else {
5130	/* 16 bit */
5131	stw(ptr, env->fpuc);
5132	stw(ptr + 2, fpus);
5133	stw(ptr + 4, fptag);
5134	stw(ptr + 6, 0);
5135	stw(ptr + 8, 0);
5136	stw(ptr + 10, 0);
5137	stw(ptr + 12, 0);
5138	}
5139	}
5140
5141	void helper_fldenv(target_ulong ptr, int data32)
5142	{
5143	int i, fpus, fptag;
5144
5145	if (data32) {
5146	env->fpuc = lduw(ptr);
5147	fpus = lduw(ptr + 4);
5148	fptag = lduw(ptr + 8);
5149	}
5150	else {
5151	env->fpuc = lduw(ptr);
5152	fpus = lduw(ptr + 2);
5153	fptag = lduw(ptr + 4);
5154	}
5155	env->fpstt = (fpus >> 11) & 7;
5156	env->fpus = fpus & ~0x3800;
5157	for(i = 0;i < 8; i++) {
5158	env->fptags[i] = ((fptag & 3) == 3);
5159	fptag >>= 2;
5160	}
5161	}
5162
5163	void helper_fsave(target_ulong ptr, int data32)
5164	{
5165	CPU86_LDouble tmp;
5166	int i;
5167
5168	helper_fstenv(ptr, data32);
5169
5170	ptr += (14 << data32);
5171	for(i = 0;i < 8; i++) {
5172	tmp = ST(i);
5173	helper_fstt(tmp, ptr);
5174	ptr += 10;
5175	}
5176
5177	/* fninit */
5178	env->fpus = 0;
5179	env->fpstt = 0;
5180	env->fpuc = 0x37f;
5181	env->fptags[0] = 1;
5182	env->fptags[1] = 1;
5183	env->fptags[2] = 1;
5184	env->fptags[3] = 1;
5185	env->fptags[4] = 1;
5186	env->fptags[5] = 1;
5187	env->fptags[6] = 1;
5188	env->fptags[7] = 1;
5189	}
5190
5191	void helper_frstor(target_ulong ptr, int data32)
5192	{
5193	CPU86_LDouble tmp;
5194	int i;
5195
5196	helper_fldenv(ptr, data32);
5197	ptr += (14 << data32);
5198
5199	for(i = 0;i < 8; i++) {
5200	tmp = helper_fldt(ptr);
5201	ST(i) = tmp;
5202	ptr += 10;
5203	}
5204	}
5205
5206	void helper_fxsave(target_ulong ptr, int data64)
5207	{
5208	int fpus, fptag, i, nb_xmm_regs;
5209	CPU86_LDouble tmp;
5210	target_ulong addr;
5211
5212	/* The operand must be 16 byte aligned */
5213	if (ptr & 0xf) {
5214	raise_exception(EXCP0D_GPF);
5215	}
5216
5217	fpus = (env->fpus & ~0x3800) \| (env->fpstt & 0x7) << 11;
5218	fptag = 0;
5219	for(i = 0; i < 8; i++) {
5220	fptag \|= (env->fptags[i] << i);
5221	}
5222	stw(ptr, env->fpuc);
5223	stw(ptr + 2, fpus);
5224	stw(ptr + 4, fptag ^ 0xff);
5225	#ifdef TARGET_X86_64
5226	if (data64) {
5227	stq(ptr + 0x08, 0); /* rip */
5228	stq(ptr + 0x10, 0); /* rdp */
5229	} else
5230	#endif
5231	{
5232	stl(ptr + 0x08, 0); /* eip */
5233	stl(ptr + 0x0c, 0); /* sel */
5234	stl(ptr + 0x10, 0); /* dp */
5235	stl(ptr + 0x14, 0); /* sel */
5236	}
5237
5238	addr = ptr + 0x20;
5239	for(i = 0;i < 8; i++) {
5240	tmp = ST(i);
5241	helper_fstt(tmp, addr);
5242	addr += 16;
5243	}
5244
5245	if (env->cr[4] & CR4_OSFXSR_MASK) {
5246	/* XXX: finish it */
5247	stl(ptr + 0x18, env->mxcsr); /* mxcsr */
5248	stl(ptr + 0x1c, 0x0000ffff); /* mxcsr_mask */
5249	if (env->hflags & HF_CS64_MASK)
5250	nb_xmm_regs = 16;
5251	else
5252	nb_xmm_regs = 8;
5253	addr = ptr + 0xa0;
5254	/* Fast FXSAVE leaves out the XMM registers */
5255	if (!(env->efer & MSR_EFER_FFXSR)
5256	\|\| (env->hflags & HF_CPL_MASK)
5257	\|\| !(env->hflags & HF_LMA_MASK)) {
5258	for(i = 0; i < nb_xmm_regs; i++) {
5259	stq(addr, env->xmm_regs[i].XMM_Q(0));
5260	stq(addr + 8, env->xmm_regs[i].XMM_Q(1));
5261	addr += 16;
5262	}
5263	}
5264	}
5265	}
5266
5267	void helper_fxrstor(target_ulong ptr, int data64)
5268	{
5269	int i, fpus, fptag, nb_xmm_regs;
5270	CPU86_LDouble tmp;
5271	target_ulong addr;
5272
5273	/* The operand must be 16 byte aligned */
5274	if (ptr & 0xf) {
5275	raise_exception(EXCP0D_GPF);
5276	}
5277
5278	env->fpuc = lduw(ptr);
5279	fpus = lduw(ptr + 2);
5280	fptag = lduw(ptr + 4);
5281	env->fpstt = (fpus >> 11) & 7;
5282	env->fpus = fpus & ~0x3800;
5283	fptag ^= 0xff;
5284	for(i = 0;i < 8; i++) {
5285	env->fptags[i] = ((fptag >> i) & 1);
5286	}
5287
5288	addr = ptr + 0x20;
5289	for(i = 0;i < 8; i++) {
5290	tmp = helper_fldt(addr);
5291	ST(i) = tmp;
5292	addr += 16;
5293	}
5294
5295	if (env->cr[4] & CR4_OSFXSR_MASK) {
5296	/* XXX: finish it */
5297	env->mxcsr = ldl(ptr + 0x18);
5298	//ldl(ptr + 0x1c);
5299	if (env->hflags & HF_CS64_MASK)
5300	nb_xmm_regs = 16;
5301	else
5302	nb_xmm_regs = 8;
5303	addr = ptr + 0xa0;
5304	/* Fast FXRESTORE leaves out the XMM registers */
5305	if (!(env->efer & MSR_EFER_FFXSR)
5306	\|\| (env->hflags & HF_CPL_MASK)
5307	\|\| !(env->hflags & HF_LMA_MASK)) {
5308	for(i = 0; i < nb_xmm_regs; i++) {
5309	#if !defined(VBOX) \|\| __GNUC__ < 4
5310	env->xmm_regs[i].XMM_Q(0) = ldq(addr);
5311	env->xmm_regs[i].XMM_Q(1) = ldq(addr + 8);
5312	#else /* VBOX + __GNUC__ >= 4: gcc 4.x compiler bug - it runs out of registers for the 64-bit value. */
5313	# if 1
5314	env->xmm_regs[i].XMM_L(0) = ldl(addr);
5315	env->xmm_regs[i].XMM_L(1) = ldl(addr + 4);
5316	env->xmm_regs[i].XMM_L(2) = ldl(addr + 8);
5317	env->xmm_regs[i].XMM_L(3) = ldl(addr + 12);
5318	# else
5319	/* this works fine on Mac OS X, gcc 4.0.1 */
5320	uint64_t u64 = ldq(addr);
5321	env->xmm_regs[i].XMM_Q(0);
5322	u64 = ldq(addr + 4);
5323	env->xmm_regs[i].XMM_Q(1) = u64;
5324	# endif
5325	#endif
5326	addr += 16;
5327	}
5328	}
5329	}
5330	}
5331
5332	#ifndef USE_X86LDOUBLE
5333
5334	void cpu_get_fp80(uint64_t pmant, uint16_t pexp, CPU86_LDouble f)
5335	{
5336	CPU86_LDoubleU temp;
5337	int e;
5338
5339	temp.d = f;
5340	/* mantissa */
5341	*pmant = (MANTD(temp) << 11) \| (1LL << 63);
5342	/* exponent + sign */
5343	e = EXPD(temp) - EXPBIAS + 16383;
5344	e \|= SIGND(temp) >> 16;
5345	*pexp = e;
5346	}
5347
5348	CPU86_LDouble cpu_set_fp80(uint64_t mant, uint16_t upper)
5349	{
5350	CPU86_LDoubleU temp;
5351	int e;
5352	uint64_t ll;
5353
5354	/* XXX: handle overflow ? */
5355	e = (upper & 0x7fff) - 16383 + EXPBIAS; /* exponent */
5356	e \|= (upper >> 4) & 0x800; /* sign */
5357	ll = (mant >> 11) & ((1LL << 52) - 1);
5358	#ifdef __arm__
5359	temp.l.upper = (e << 20) \| (ll >> 32);
5360	temp.l.lower = ll;
5361	#else
5362	temp.ll = ll \| ((uint64_t)e << 52);
5363	#endif
5364	return temp.d;
5365	}
5366
5367	#else
5368
5369	void cpu_get_fp80(uint64_t pmant, uint16_t pexp, CPU86_LDouble f)
5370	{
5371	CPU86_LDoubleU temp;
5372
5373	temp.d = f;
5374	*pmant = temp.l.lower;
5375	*pexp = temp.l.upper;
5376	}
5377
5378	CPU86_LDouble cpu_set_fp80(uint64_t mant, uint16_t upper)
5379	{
5380	CPU86_LDoubleU temp;
5381
5382	temp.l.upper = upper;
5383	temp.l.lower = mant;
5384	return temp.d;
5385	}
5386	#endif
5387
5388	#ifdef TARGET_X86_64
5389
5390	//#define DEBUG_MULDIV
5391
5392	static void add128(uint64_t plow, uint64_t phigh, uint64_t a, uint64_t b)
5393	{
5394	*plow += a;
5395	/* carry test */
5396	if (*plow < a)
5397	(*phigh)++;
5398	*phigh += b;
5399	}
5400
5401	static void neg128(uint64_t plow, uint64_t phigh)
5402	{
5403	plow = ~ plow;
5404	phigh = ~ phigh;
5405	add128(plow, phigh, 1, 0);
5406	}
5407
5408	/* return TRUE if overflow */
5409	static int div64(uint64_t plow, uint64_t phigh, uint64_t b)
5410	{
5411	uint64_t q, r, a1, a0;
5412	int i, qb, ab;
5413
5414	a0 = *plow;
5415	a1 = *phigh;
5416	if (a1 == 0) {
5417	q = a0 / b;
5418	r = a0 % b;
5419	*plow = q;
5420	*phigh = r;
5421	} else {
5422	if (a1 >= b)
5423	return 1;
5424	/* XXX: use a better algorithm */
5425	for(i = 0; i < 64; i++) {
5426	ab = a1 >> 63;
5427	a1 = (a1 << 1) \| (a0 >> 63);
5428	if (ab \|\| a1 >= b) {
5429	a1 -= b;
5430	qb = 1;
5431	} else {
5432	qb = 0;
5433	}
5434	a0 = (a0 << 1) \| qb;
5435	}
5436	#if defined(DEBUG_MULDIV)
5437	printf("div: 0x%016" PRIx64 "%016" PRIx64 " / 0x%016" PRIx64 ": q=0x%016" PRIx64 " r=0x%016" PRIx64 "\n",
5438	phigh, plow, b, a0, a1);
5439	#endif
5440	*plow = a0;
5441	*phigh = a1;
5442	}
5443	return 0;
5444	}
5445
5446	/* return TRUE if overflow */
5447	static int idiv64(uint64_t plow, uint64_t phigh, int64_t b)
5448	{
5449	int sa, sb;
5450	sa = ((int64_t)*phigh < 0);
5451	if (sa)
5452	neg128(plow, phigh);
5453	sb = (b < 0);
5454	if (sb)
5455	b = -b;
5456	if (div64(plow, phigh, b) != 0)
5457	return 1;
5458	if (sa ^ sb) {
5459	if (*plow > (1ULL << 63))
5460	return 1;
5461	plow = - plow;
5462	} else {
5463	if (*plow >= (1ULL << 63))
5464	return 1;
5465	}
5466	if (sa)
5467	phigh = - phigh;
5468	return 0;
5469	}
5470
5471	void helper_mulq_EAX_T0(target_ulong t0)
5472	{
5473	uint64_t r0, r1;
5474
5475	mulu64(&r0, &r1, EAX, t0);
5476	EAX = r0;
5477	EDX = r1;
5478	CC_DST = r0;
5479	CC_SRC = r1;
5480	}
5481
5482	void helper_imulq_EAX_T0(target_ulong t0)
5483	{
5484	uint64_t r0, r1;
5485
5486	muls64(&r0, &r1, EAX, t0);
5487	EAX = r0;
5488	EDX = r1;
5489	CC_DST = r0;
5490	CC_SRC = ((int64_t)r1 != ((int64_t)r0 >> 63));
5491	}
5492
5493	target_ulong helper_imulq_T0_T1(target_ulong t0, target_ulong t1)
5494	{
5495	uint64_t r0, r1;
5496
5497	muls64(&r0, &r1, t0, t1);
5498	CC_DST = r0;
5499	CC_SRC = ((int64_t)r1 != ((int64_t)r0 >> 63));
5500	return r0;
5501	}
5502
5503	void helper_divq_EAX(target_ulong t0)
5504	{
5505	uint64_t r0, r1;
5506	if (t0 == 0) {
5507	raise_exception(EXCP00_DIVZ);
5508	}
5509	r0 = EAX;
5510	r1 = EDX;
5511	if (div64(&r0, &r1, t0))
5512	raise_exception(EXCP00_DIVZ);
5513	EAX = r0;
5514	EDX = r1;
5515	}
5516
5517	void helper_idivq_EAX(target_ulong t0)
5518	{
5519	uint64_t r0, r1;
5520	if (t0 == 0) {
5521	raise_exception(EXCP00_DIVZ);
5522	}
5523	r0 = EAX;
5524	r1 = EDX;
5525	if (idiv64(&r0, &r1, t0))
5526	raise_exception(EXCP00_DIVZ);
5527	EAX = r0;
5528	EDX = r1;
5529	}
5530	#endif
5531
5532	static void do_hlt(void)
5533	{
5534	env->hflags &= ~HF_INHIBIT_IRQ_MASK; /* needed if sti is just before */
5535	env->halted = 1;
5536	env->exception_index = EXCP_HLT;
5537	cpu_loop_exit();
5538	}
5539
5540	void helper_hlt(int next_eip_addend)
5541	{
5542	helper_svm_check_intercept_param(SVM_EXIT_HLT, 0);
5543	EIP += next_eip_addend;
5544
5545	do_hlt();
5546	}
5547
5548	void helper_monitor(target_ulong ptr)
5549	{
5550	#ifdef VBOX
5551	if ((uint32_t)ECX > 1)
5552	raise_exception(EXCP0D_GPF);
5553	#else /* !VBOX */
5554	if ((uint32_t)ECX != 0)
5555	raise_exception(EXCP0D_GPF);
5556	#endif /* !VBOX */
5557	/* XXX: store address ? */
5558	helper_svm_check_intercept_param(SVM_EXIT_MONITOR, 0);
5559	}
5560
5561	void helper_mwait(int next_eip_addend)
5562	{
5563	if ((uint32_t)ECX != 0)
5564	raise_exception(EXCP0D_GPF);
5565	#ifdef VBOX
5566	helper_hlt(next_eip_addend);
5567	#else /* !VBOX */
5568	helper_svm_check_intercept_param(SVM_EXIT_MWAIT, 0);
5569	EIP += next_eip_addend;
5570
5571	/* XXX: not complete but not completely erroneous */
5572	if (env->cpu_index != 0 \|\| env->next_cpu != NULL) {
5573	/* more than one CPU: do not sleep because another CPU may
5574	wake this one */
5575	} else {
5576	do_hlt();
5577	}
5578	#endif /* !VBOX */
5579	}
5580
5581	void helper_debug(void)
5582	{
5583	env->exception_index = EXCP_DEBUG;
5584	cpu_loop_exit();
5585	}
5586
5587	void helper_reset_rf(void)
5588	{
5589	env->eflags &= ~RF_MASK;
5590	}
5591
5592	void helper_raise_interrupt(int intno, int next_eip_addend)
5593	{
5594	raise_interrupt(intno, 1, 0, next_eip_addend);
5595	}
5596
5597	void helper_raise_exception(int exception_index)
5598	{
5599	raise_exception(exception_index);
5600	}
5601
5602	void helper_cli(void)
5603	{
5604	env->eflags &= ~IF_MASK;
5605	}
5606
5607	void helper_sti(void)
5608	{
5609	env->eflags \|= IF_MASK;
5610	}
5611
5612	#ifdef VBOX
5613	void helper_cli_vme(void)
5614	{
5615	env->eflags &= ~VIF_MASK;
5616	}
5617
5618	void helper_sti_vme(void)
5619	{
5620	/* First check, then change eflags according to the AMD manual */
5621	if (env->eflags & VIP_MASK) {
5622	raise_exception(EXCP0D_GPF);
5623	}
5624	env->eflags \|= VIF_MASK;
5625	}
5626	#endif /* VBOX */
5627
5628	#if 0
5629	/* vm86plus instructions */
5630	void helper_cli_vm(void)
5631	{
5632	env->eflags &= ~VIF_MASK;
5633	}
5634
5635	void helper_sti_vm(void)
5636	{
5637	env->eflags \|= VIF_MASK;
5638	if (env->eflags & VIP_MASK) {
5639	raise_exception(EXCP0D_GPF);
5640	}
5641	}
5642	#endif
5643
5644	void helper_set_inhibit_irq(void)
5645	{
5646	env->hflags \|= HF_INHIBIT_IRQ_MASK;
5647	}
5648
5649	void helper_reset_inhibit_irq(void)
5650	{
5651	env->hflags &= ~HF_INHIBIT_IRQ_MASK;
5652	}
5653
5654	void helper_boundw(target_ulong a0, int v)
5655	{
5656	int low, high;
5657	low = ldsw(a0);
5658	high = ldsw(a0 + 2);
5659	v = (int16_t)v;
5660	if (v < low \|\| v > high) {
5661	raise_exception(EXCP05_BOUND);
5662	}
5663	}
5664
5665	void helper_boundl(target_ulong a0, int v)
5666	{
5667	int low, high;
5668	low = ldl(a0);
5669	high = ldl(a0 + 4);
5670	if (v < low \|\| v > high) {
5671	raise_exception(EXCP05_BOUND);
5672	}
5673	}
5674
5675	static float approx_rsqrt(float a)
5676	{
5677	return 1.0 / sqrt(a);
5678	}
5679
5680	static float approx_rcp(float a)
5681	{
5682	return 1.0 / a;
5683	}
5684
5685	#if !defined(CONFIG_USER_ONLY)
5686
5687	#define MMUSUFFIX _mmu
5688
5689	#define SHIFT 0
5690	#include "softmmu_template.h"
5691
5692	#define SHIFT 1
5693	#include "softmmu_template.h"
5694
5695	#define SHIFT 2
5696	#include "softmmu_template.h"
5697
5698	#define SHIFT 3
5699	#include "softmmu_template.h"
5700
5701	#endif
5702
5703	#if defined(VBOX) && defined(REM_PHYS_ADDR_IN_TLB)
5704	/* This code assumes real physical address always fit into host CPU reg,
5705	which is wrong in general, but true for our current use cases. */
5706	RTCCUINTREG REGPARM __ldb_vbox_phys(RTCCUINTREG addr)
5707	{
5708	return remR3PhysReadS8(addr);
5709	}
5710	RTCCUINTREG REGPARM __ldub_vbox_phys(RTCCUINTREG addr)
5711	{
5712	return remR3PhysReadU8(addr);
5713	}
5714	void REGPARM __stb_vbox_phys(RTCCUINTREG addr, RTCCUINTREG val)
5715	{
5716	remR3PhysWriteU8(addr, val);
5717	}
5718	RTCCUINTREG REGPARM __ldw_vbox_phys(RTCCUINTREG addr)
5719	{
5720	return remR3PhysReadS16(addr);
5721	}
5722	RTCCUINTREG REGPARM __lduw_vbox_phys(RTCCUINTREG addr)
5723	{
5724	return remR3PhysReadU16(addr);
5725	}
5726	void REGPARM __stw_vbox_phys(RTCCUINTREG addr, RTCCUINTREG val)
5727	{
5728	remR3PhysWriteU16(addr, val);
5729	}
5730	RTCCUINTREG REGPARM __ldl_vbox_phys(RTCCUINTREG addr)
5731	{
5732	return remR3PhysReadS32(addr);
5733	}
5734	RTCCUINTREG REGPARM __ldul_vbox_phys(RTCCUINTREG addr)
5735	{
5736	return remR3PhysReadU32(addr);
5737	}
5738	void REGPARM __stl_vbox_phys(RTCCUINTREG addr, RTCCUINTREG val)
5739	{
5740	remR3PhysWriteU32(addr, val);
5741	}
5742	uint64_t REGPARM __ldq_vbox_phys(RTCCUINTREG addr)
5743	{
5744	return remR3PhysReadU64(addr);
5745	}
5746	void REGPARM __stq_vbox_phys(RTCCUINTREG addr, uint64_t val)
5747	{
5748	remR3PhysWriteU64(addr, val);
5749	}
5750	#endif /* VBOX */
5751
5752	#if !defined(CONFIG_USER_ONLY)
5753	/* try to fill the TLB and return an exception if error. If retaddr is
5754	NULL, it means that the function was called in C code (i.e. not
5755	from generated code or from helper.c) */
5756	/* XXX: fix it to restore all registers */
5757	void tlb_fill(target_ulong addr, int is_write, int mmu_idx, void *retaddr)
5758	{
5759	TranslationBlock *tb;
5760	int ret;
5761	uintptr_t pc;
5762	CPUX86State *saved_env;
5763
5764	/* XXX: hack to restore env in all cases, even if not called from
5765	generated code */
5766	saved_env = env;
5767	env = cpu_single_env;
5768
5769	ret = cpu_x86_handle_mmu_fault(env, addr, is_write, mmu_idx, 1);
5770	if (ret) {
5771	if (retaddr) {
5772	/* now we have a real cpu fault */
5773	pc = (uintptr_t)retaddr;
5774	tb = tb_find_pc(pc);
5775	if (tb) {
5776	/* the PC is inside the translated code. It means that we have
5777	a virtual CPU fault */
5778	cpu_restore_state(tb, env, pc, NULL);
5779	}
5780	}
5781	raise_exception_err(env->exception_index, env->error_code);
5782	}
5783	env = saved_env;
5784	}
5785	#endif
5786
5787	#ifdef VBOX
5788
5789	/**
5790	* Correctly computes the eflags.
5791	* @returns eflags.
5792	* @param env1 CPU environment.
5793	*/
5794	uint32_t raw_compute_eflags(CPUX86State *env1)
5795	{
5796	CPUX86State *savedenv = env;
5797	uint32_t efl;
5798	env = env1;
5799	efl = compute_eflags();
5800	env = savedenv;
5801	return efl;
5802	}
5803
5804	/**
5805	* Reads byte from virtual address in guest memory area.
5806	* XXX: is it working for any addresses? swapped out pages?
5807	* @returns read data byte.
5808	* @param env1 CPU environment.
5809	* @param pvAddr GC Virtual address.
5810	*/
5811	uint8_t read_byte(CPUX86State *env1, target_ulong addr)
5812	{
5813	CPUX86State *savedenv = env;
5814	uint8_t u8;
5815	env = env1;
5816	u8 = ldub_kernel(addr);
5817	env = savedenv;
5818	return u8;
5819	}
5820
5821	/**
5822	* Reads byte from virtual address in guest memory area.
5823	* XXX: is it working for any addresses? swapped out pages?
5824	* @returns read data byte.
5825	* @param env1 CPU environment.
5826	* @param pvAddr GC Virtual address.
5827	*/
5828	uint16_t read_word(CPUX86State *env1, target_ulong addr)
5829	{
5830	CPUX86State *savedenv = env;
5831	uint16_t u16;
5832	env = env1;
5833	u16 = lduw_kernel(addr);
5834	env = savedenv;
5835	return u16;
5836	}
5837
5838	/**
5839	* Reads byte from virtual address in guest memory area.
5840	* XXX: is it working for any addresses? swapped out pages?
5841	* @returns read data byte.
5842	* @param env1 CPU environment.
5843	* @param pvAddr GC Virtual address.
5844	*/
5845	uint32_t read_dword(CPUX86State *env1, target_ulong addr)
5846	{
5847	CPUX86State *savedenv = env;
5848	uint32_t u32;
5849	env = env1;
5850	u32 = ldl_kernel(addr);
5851	env = savedenv;
5852	return u32;
5853	}
5854
5855	/**
5856	* Writes byte to virtual address in guest memory area.
5857	* XXX: is it working for any addresses? swapped out pages?
5858	* @returns read data byte.
5859	* @param env1 CPU environment.
5860	* @param pvAddr GC Virtual address.
5861	* @param val byte value
5862	*/
5863	void write_byte(CPUX86State *env1, target_ulong addr, uint8_t val)
5864	{
5865	CPUX86State *savedenv = env;
5866	env = env1;
5867	stb(addr, val);
5868	env = savedenv;
5869	}
5870
5871	void write_word(CPUX86State *env1, target_ulong addr, uint16_t val)
5872	{
5873	CPUX86State *savedenv = env;
5874	env = env1;
5875	stw(addr, val);
5876	env = savedenv;
5877	}
5878
5879	void write_dword(CPUX86State *env1, target_ulong addr, uint32_t val)
5880	{
5881	CPUX86State *savedenv = env;
5882	env = env1;
5883	stl(addr, val);
5884	env = savedenv;
5885	}
5886
5887	/**
5888	* Correctly loads selector into segment register with updating internal
5889	* qemu data/caches.
5890	* @param env1 CPU environment.
5891	* @param seg_reg Segment register.
5892	* @param selector Selector to load.
5893	*/
5894	void sync_seg(CPUX86State *env1, int seg_reg, int selector)
5895	{
5896	CPUX86State *savedenv = env;
5897	#ifdef FORCE_SEGMENT_SYNC
5898	jmp_buf old_buf;
5899	#endif
5900
5901	env = env1;
5902
5903	if ( env->eflags & X86_EFL_VM
5904	\|\| !(env->cr[0] & X86_CR0_PE))
5905	{
5906	load_seg_vm(seg_reg, selector);
5907
5908	env = savedenv;
5909
5910	/* Successful sync. */
5911	Assert(env1->segs[seg_reg].newselector == 0);
5912	}
5913	else
5914	{
5915	/* For some reasons, it works even w/o save/restore of the jump buffer, so as code is
5916	time critical - let's not do that */
5917	#ifdef FORCE_SEGMENT_SYNC
5918	memcpy(&old_buf, &env1->jmp_env, sizeof(old_buf));
5919	#endif
5920	if (setjmp(env1->jmp_env) == 0)
5921	{
5922	if (seg_reg == R_CS)
5923	{
5924	uint32_t e1, e2;
5925	e1 = e2 = 0;
5926	load_segment(&e1, &e2, selector);
5927	cpu_x86_load_seg_cache(env, R_CS, selector,
5928	get_seg_base(e1, e2),
5929	get_seg_limit(e1, e2),
5930	e2);
5931	}
5932	else
5933	helper_load_seg(seg_reg, selector);
5934	/* We used to use tss_load_seg(seg_reg, selector); which, for some reasons ignored
5935	loading 0 selectors, what, in order, lead to subtle problems like #3588 */
5936
5937	env = savedenv;
5938
5939	/* Successful sync. */
5940	Assert(env1->segs[seg_reg].newselector == 0);
5941	}
5942	else
5943	{
5944	env = savedenv;
5945
5946	/* Postpone sync until the guest uses the selector. */
5947	env1->segs[seg_reg].selector = selector; /* hidden values are now incorrect, but will be resynced when this register is accessed. */
5948	env1->segs[seg_reg].newselector = selector;
5949	Log(("sync_seg: out of sync seg_reg=%d selector=%#x\n", seg_reg, selector));
5950	env1->exception_index = -1;
5951	env1->error_code = 0;
5952	env1->old_exception = -1;
5953	}
5954	#ifdef FORCE_SEGMENT_SYNC
5955	memcpy(&env1->jmp_env, &old_buf, sizeof(old_buf));
5956	#endif
5957	}
5958
5959	}
5960
5961	DECLINLINE(void) tb_reset_jump(TranslationBlock *tb, int n)
5962	{
5963	tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
5964	}
5965
5966
5967	int emulate_single_instr(CPUX86State *env1)
5968	{
5969	TranslationBlock *tb;
5970	TranslationBlock *current;
5971	int flags;
5972	uint8_t *tc_ptr;
5973	target_ulong old_eip;
5974
5975	/* ensures env is loaded! */
5976	CPUX86State *savedenv = env;
5977	env = env1;
5978
5979	RAWEx_ProfileStart(env, STATS_EMULATE_SINGLE_INSTR);
5980
5981	current = env->current_tb;
5982	env->current_tb = NULL;
5983	flags = env->hflags \| (env->eflags & (IOPL_MASK \| TF_MASK \| VM_MASK));
5984
5985	/*
5986	* Translate only one instruction.
5987	*/
5988	ASMAtomicOrU32(&env->state, CPU_EMULATE_SINGLE_INSTR);
5989	tb = tb_gen_code(env, env->eip + env->segs[R_CS].base,
5990	env->segs[R_CS].base, flags, 0);
5991
5992	ASMAtomicAndU32(&env->state, ~CPU_EMULATE_SINGLE_INSTR);
5993
5994
5995	/* tb_link_phys: */
5996	tb->jmp_first = (TranslationBlock *)((intptr_t)tb \| 2);
5997	tb->jmp_next[0] = NULL;
5998	tb->jmp_next[1] = NULL;
5999	Assert(tb->jmp_next[0] == NULL);
6000	Assert(tb->jmp_next[1] == NULL);
6001	if (tb->tb_next_offset[0] != 0xffff)
6002	tb_reset_jump(tb, 0);
6003	if (tb->tb_next_offset[1] != 0xffff)
6004	tb_reset_jump(tb, 1);
6005
6006	/*
6007	* Execute it using emulation
6008	*/
6009	old_eip = env->eip;
6010	env->current_tb = tb;
6011
6012	/*
6013	* eip remains the same for repeated instructions; no idea why qemu doesn't do a jump inside the generated code
6014	* perhaps not a very safe hack
6015	*/
6016	while (old_eip == env->eip)
6017	{
6018	tc_ptr = tb->tc_ptr;
6019
6020	#if defined(VBOX) && defined(GCC_WITH_BUGGY_REGPARM)
6021	int fake_ret;
6022	tcg_qemu_tb_exec(tc_ptr, fake_ret);
6023	#else
6024	tcg_qemu_tb_exec(tc_ptr);
6025	#endif
6026
6027	/*
6028	* Exit once we detect an external interrupt and interrupts are enabled
6029	*/
6030	if ( (env->interrupt_request & (CPU_INTERRUPT_EXTERNAL_EXIT \| CPU_INTERRUPT_EXTERNAL_TIMER))
6031	\|\| ( (env->eflags & IF_MASK)
6032	&& !(env->hflags & HF_INHIBIT_IRQ_MASK)
6033	&& (env->interrupt_request & CPU_INTERRUPT_EXTERNAL_HARD) )
6034	)
6035	{
6036	break;
6037	}
6038	if (env->interrupt_request & CPU_INTERRUPT_EXTERNAL_FLUSH_TLB) {
6039	tlb_flush(env, true);
6040	}
6041	}
6042	env->current_tb = current;
6043
6044	tb_phys_invalidate(tb, -1);
6045	tb_free(tb);
6046	/*
6047	Assert(tb->tb_next_offset[0] == 0xffff);
6048	Assert(tb->tb_next_offset[1] == 0xffff);
6049	Assert(tb->tb_next[0] == 0xffff);
6050	Assert(tb->tb_next[1] == 0xffff);
6051	Assert(tb->jmp_next[0] == NULL);
6052	Assert(tb->jmp_next[1] == NULL);
6053	Assert(tb->jmp_first == NULL); */
6054
6055	RAWEx_ProfileStop(env, STATS_EMULATE_SINGLE_INSTR);
6056
6057	/*
6058	* Execute the next instruction when we encounter instruction fusing.
6059	*/
6060	if (env->hflags & HF_INHIBIT_IRQ_MASK)
6061	{
6062	Log(("REM: Emulating next instruction due to instruction fusing (HF_INHIBIT_IRQ_MASK) at %RGv\n", env->eip));
6063	env->hflags &= ~HF_INHIBIT_IRQ_MASK;
6064	emulate_single_instr(env);
6065	}
6066
6067	env = savedenv;
6068	return 0;
6069	}
6070
6071	/**
6072	* Correctly loads a new ldtr selector.
6073	*
6074	* @param env1 CPU environment.
6075	* @param selector Selector to load.
6076	*/
6077	void sync_ldtr(CPUX86State *env1, int selector)
6078	{
6079	CPUX86State *saved_env = env;
6080	if (setjmp(env1->jmp_env) == 0)
6081	{
6082	env = env1;
6083	helper_lldt(selector);
6084	env = saved_env;
6085	}
6086	else
6087	{
6088	env = saved_env;
6089	#ifdef VBOX_STRICT
6090	cpu_abort(env1, "sync_ldtr: selector=%#x\n", selector);
6091	#endif
6092	}
6093	}
6094
6095	int get_ss_esp_from_tss_raw(CPUX86State env1, uint32_t ss_ptr,
6096	uint32_t *esp_ptr, int dpl)
6097	{
6098	int type, index, shift;
6099
6100	CPUX86State *savedenv = env;
6101	env = env1;
6102
6103	if (!(env->tr.flags & DESC_P_MASK))
6104	cpu_abort(env, "invalid tss");
6105	type = (env->tr.flags >> DESC_TYPE_SHIFT) & 0xf;
6106	if ((type & 7) != 3)
6107	cpu_abort(env, "invalid tss type %d", type);
6108	shift = type >> 3;
6109	index = (dpl * 4 + 2) << shift;
6110	if (index + (4 << shift) - 1 > env->tr.limit)
6111	{
6112	env = savedenv;
6113	return 0;
6114	}
6115	//raise_exception_err(EXCP0A_TSS, env->tr.selector & 0xfffc);
6116
6117	if (shift == 0) {
6118	*esp_ptr = lduw_kernel(env->tr.base + index);
6119	*ss_ptr = lduw_kernel(env->tr.base + index + 2);
6120	} else {
6121	*esp_ptr = ldl_kernel(env->tr.base + index);
6122	*ss_ptr = lduw_kernel(env->tr.base + index + 4);
6123	}
6124
6125	env = savedenv;
6126	return 1;
6127	}
6128
6129	//*****************************************************************************
6130	// Needs to be at the bottom of the file (overriding macros)
6131
6132	static inline CPU86_LDouble helper_fldt_raw(uint8_t *ptr)
6133	{
6134	#ifdef USE_X86LDOUBLE
6135	CPU86_LDoubleU tmp;
6136	tmp.l.lower = (uint64_t const )ptr;
6137	tmp.l.upper = (uint16_t const )(ptr + 8);
6138	return tmp.d;
6139	#else
6140	# error "Busted FPU saving/restoring!"
6141	return (CPU86_LDouble )ptr;
6142	#endif
6143	}
6144
6145	static inline void helper_fstt_raw(CPU86_LDouble f, uint8_t *ptr)
6146	{
6147	#ifdef USE_X86LDOUBLE
6148	CPU86_LDoubleU tmp;
6149	tmp.d = f;
6150	(uint64_t )(ptr + 0) = tmp.l.lower;
6151	(uint16_t )(ptr + 8) = tmp.l.upper;
6152	(uint16_t )(ptr + 10) = 0;
6153	(uint32_t )(ptr + 12) = 0;
6154	AssertCompile(sizeof(long double) > 8);
6155	#else
6156	# error "Busted FPU saving/restoring!"
6157	(CPU86_LDouble )ptr = f;
6158	#endif
6159	}
6160
6161	#undef stw
6162	#undef stl
6163	#undef stq
6164	#define stw(a,b) (uint16_t )(a) = (uint16_t)(b)
6165	#define stl(a,b) (uint32_t )(a) = (uint32_t)(b)
6166	#define stq(a,b) (uint64_t )(a) = (uint64_t)(b)
6167
6168	//*****************************************************************************
6169	void restore_raw_fp_state(CPUX86State env, uint8_t ptr)
6170	{
6171	int fpus, fptag, i, nb_xmm_regs;
6172	CPU86_LDouble tmp;
6173	uint8_t *addr;
6174	int data64 = !!(env->hflags & HF_LMA_MASK);
6175
6176	if (env->cpuid_features & CPUID_FXSR)
6177	{
6178	fpus = (env->fpus & ~0x3800) \| (env->fpstt & 0x7) << 11;
6179	fptag = 0;
6180	for(i = 0; i < 8; i++) {
6181	fptag \|= (env->fptags[i] << i);
6182	}
6183	stw(ptr, env->fpuc);
6184	stw(ptr + 2, fpus);
6185	stw(ptr + 4, fptag ^ 0xff);
6186
6187	addr = ptr + 0x20;
6188	for(i = 0;i < 8; i++) {
6189	tmp = ST(i);
6190	helper_fstt_raw(tmp, addr);
6191	addr += 16;
6192	}
6193
6194	if (env->cr[4] & CR4_OSFXSR_MASK) {
6195	/* XXX: finish it */
6196	stl(ptr + 0x18, env->mxcsr); /* mxcsr */
6197	stl(ptr + 0x1c, 0x0000ffff); /* mxcsr_mask */
6198	nb_xmm_regs = 8 << data64;
6199	addr = ptr + 0xa0;
6200	for(i = 0; i < nb_xmm_regs; i++) {
6201	#if __GNUC__ < 4
6202	stq(addr, env->xmm_regs[i].XMM_Q(0));
6203	stq(addr + 8, env->xmm_regs[i].XMM_Q(1));
6204	#else /* VBOX + __GNUC__ >= 4: gcc 4.x compiler bug - it runs out of registers for the 64-bit value. */
6205	stl(addr, env->xmm_regs[i].XMM_L(0));
6206	stl(addr + 4, env->xmm_regs[i].XMM_L(1));
6207	stl(addr + 8, env->xmm_regs[i].XMM_L(2));
6208	stl(addr + 12, env->xmm_regs[i].XMM_L(3));
6209	#endif
6210	addr += 16;
6211	}
6212	}
6213	}
6214	else
6215	{
6216	PX86FPUSTATE fp = (PX86FPUSTATE)ptr;
6217	int fptag;
6218
6219	fp->FCW = env->fpuc;
6220	fp->FSW = (env->fpus & ~0x3800) \| (env->fpstt & 0x7) << 11;
6221	fptag = 0;
6222	for (i=7; i>=0; i--) {
6223	fptag <<= 2;
6224	if (env->fptags[i]) {
6225	fptag \|= 3;
6226	} else {
6227	/* the FPU automatically computes it */
6228	}
6229	}
6230	fp->FTW = fptag;
6231
6232	for(i = 0;i < 8; i++) {
6233	tmp = ST(i);
6234	helper_fstt_raw(tmp, &fp->regs[i].au8[0]);
6235	}
6236	}
6237	}
6238
6239	//*****************************************************************************
6240	#undef lduw
6241	#undef ldl
6242	#undef ldq
6243	#define lduw(a) (uint16_t )(a)
6244	#define ldl(a) (uint32_t )(a)
6245	#define ldq(a) (uint64_t )(a)
6246	//*****************************************************************************
6247	void save_raw_fp_state(CPUX86State env, uint8_t ptr)
6248	{
6249	int i, fpus, fptag, nb_xmm_regs;
6250	CPU86_LDouble tmp;
6251	uint8_t *addr;
6252	int data64 = !!(env->hflags & HF_LMA_MASK); /* don't use HF_CS64_MASK here as cs hasn't been synced when this function is called. */
6253
6254	if (env->cpuid_features & CPUID_FXSR)
6255	{
6256	env->fpuc = lduw(ptr);
6257	fpus = lduw(ptr + 2);
6258	fptag = lduw(ptr + 4);
6259	env->fpstt = (fpus >> 11) & 7;
6260	env->fpus = fpus & ~0x3800;
6261	fptag ^= 0xff;
6262	for(i = 0;i < 8; i++) {
6263	env->fptags[i] = ((fptag >> i) & 1);
6264	}
6265
6266	addr = ptr + 0x20;
6267	for(i = 0;i < 8; i++) {
6268	tmp = helper_fldt_raw(addr);
6269	ST(i) = tmp;
6270	addr += 16;
6271	}
6272
6273	if (env->cr[4] & CR4_OSFXSR_MASK) {
6274	/* XXX: finish it, endianness */
6275	env->mxcsr = ldl(ptr + 0x18);
6276	//ldl(ptr + 0x1c);
6277	nb_xmm_regs = 8 << data64;
6278	addr = ptr + 0xa0;
6279	for(i = 0; i < nb_xmm_regs; i++) {
6280	#if HC_ARCH_BITS == 32
6281	/* this is a workaround for http://gcc.gnu.org/bugzilla/show_bug.cgi?id=35135 */
6282	env->xmm_regs[i].XMM_L(0) = ldl(addr);
6283	env->xmm_regs[i].XMM_L(1) = ldl(addr + 4);
6284	env->xmm_regs[i].XMM_L(2) = ldl(addr + 8);
6285	env->xmm_regs[i].XMM_L(3) = ldl(addr + 12);
6286	#else
6287	env->xmm_regs[i].XMM_Q(0) = ldq(addr);
6288	env->xmm_regs[i].XMM_Q(1) = ldq(addr + 8);
6289	#endif
6290	addr += 16;
6291	}
6292	}
6293	}
6294	else
6295	{
6296	PX86FPUSTATE fp = (PX86FPUSTATE)ptr;
6297	int fptag, j;
6298
6299	env->fpuc = fp->FCW;
6300	env->fpstt = (fp->FSW >> 11) & 7;
6301	env->fpus = fp->FSW & ~0x3800;
6302	fptag = fp->FTW;
6303	for(i = 0;i < 8; i++) {
6304	env->fptags[i] = ((fptag & 3) == 3);
6305	fptag >>= 2;
6306	}
6307	j = env->fpstt;
6308	for(i = 0;i < 8; i++) {
6309	tmp = helper_fldt_raw(&fp->regs[i].au8[0]);
6310	ST(i) = tmp;
6311	}
6312	}
6313	}
6314	//*****************************************************************************
6315	//*****************************************************************************
6316
6317	#endif /* VBOX */
6318
6319	/* Secure Virtual Machine helpers */
6320
6321	#if defined(CONFIG_USER_ONLY)
6322
6323	void helper_vmrun(int aflag, int next_eip_addend)
6324	{
6325	}
6326	void helper_vmmcall(void)
6327	{
6328	}
6329	void helper_vmload(int aflag)
6330	{
6331	}
6332	void helper_vmsave(int aflag)
6333	{
6334	}
6335	void helper_stgi(void)
6336	{
6337	}
6338	void helper_clgi(void)
6339	{
6340	}
6341	void helper_skinit(void)
6342	{
6343	}
6344	void helper_invlpga(int aflag)
6345	{
6346	}
6347	void helper_vmexit(uint32_t exit_code, uint64_t exit_info_1)
6348	{
6349	}
6350	void helper_svm_check_intercept_param(uint32_t type, uint64_t param)
6351	{
6352	}
6353
6354	void helper_svm_check_io(uint32_t port, uint32_t param,
6355	uint32_t next_eip_addend)
6356	{
6357	}
6358	#else
6359
6360	static inline void svm_save_seg(target_phys_addr_t addr,
6361	const SegmentCache *sc)
6362	{
6363	stw_phys(addr + offsetof(struct vmcb_seg, selector),
6364	sc->selector);
6365	stq_phys(addr + offsetof(struct vmcb_seg, base),
6366	sc->base);
6367	stl_phys(addr + offsetof(struct vmcb_seg, limit),
6368	sc->limit);
6369	stw_phys(addr + offsetof(struct vmcb_seg, attrib),
6370	((sc->flags >> 8) & 0xff) \| ((sc->flags >> 12) & 0x0f00));
6371	}
6372
6373	static inline void svm_load_seg(target_phys_addr_t addr, SegmentCache *sc)
6374	{
6375	unsigned int flags;
6376
6377	sc->selector = lduw_phys(addr + offsetof(struct vmcb_seg, selector));
6378	sc->base = ldq_phys(addr + offsetof(struct vmcb_seg, base));
6379	sc->limit = ldl_phys(addr + offsetof(struct vmcb_seg, limit));
6380	flags = lduw_phys(addr + offsetof(struct vmcb_seg, attrib));
6381	sc->flags = ((flags & 0xff) << 8) \| ((flags & 0x0f00) << 12);
6382	}
6383
6384	static inline void svm_load_seg_cache(target_phys_addr_t addr,
6385	CPUState *env, int seg_reg)
6386	{
6387	SegmentCache sc1, *sc = &sc1;
6388	svm_load_seg(addr, sc);
6389	cpu_x86_load_seg_cache(env, seg_reg, sc->selector,
6390	sc->base, sc->limit, sc->flags);
6391	}
6392
6393	void helper_vmrun(int aflag, int next_eip_addend)
6394	{
6395	target_ulong addr;
6396	uint32_t event_inj;
6397	uint32_t int_ctl;
6398
6399	helper_svm_check_intercept_param(SVM_EXIT_VMRUN, 0);
6400
6401	if (aflag == 2)
6402	addr = EAX;
6403	else
6404	addr = (uint32_t)EAX;
6405
6406	qemu_log_mask(CPU_LOG_TB_IN_ASM, "vmrun! " TARGET_FMT_lx "\n", addr);
6407
6408	env->vm_vmcb = addr;
6409
6410	/* save the current CPU state in the hsave page */
6411	stq_phys(env->vm_hsave + offsetof(struct vmcb, save.gdtr.base), env->gdt.base);
6412	stl_phys(env->vm_hsave + offsetof(struct vmcb, save.gdtr.limit), env->gdt.limit);
6413
6414	stq_phys(env->vm_hsave + offsetof(struct vmcb, save.idtr.base), env->idt.base);
6415	stl_phys(env->vm_hsave + offsetof(struct vmcb, save.idtr.limit), env->idt.limit);
6416
6417	stq_phys(env->vm_hsave + offsetof(struct vmcb, save.cr0), env->cr[0]);
6418	stq_phys(env->vm_hsave + offsetof(struct vmcb, save.cr2), env->cr[2]);
6419	stq_phys(env->vm_hsave + offsetof(struct vmcb, save.cr3), env->cr[3]);
6420	stq_phys(env->vm_hsave + offsetof(struct vmcb, save.cr4), env->cr[4]);
6421	stq_phys(env->vm_hsave + offsetof(struct vmcb, save.dr6), env->dr[6]);
6422	stq_phys(env->vm_hsave + offsetof(struct vmcb, save.dr7), env->dr[7]);
6423
6424	stq_phys(env->vm_hsave + offsetof(struct vmcb, save.efer), env->efer);
6425	stq_phys(env->vm_hsave + offsetof(struct vmcb, save.rflags), compute_eflags());
6426
6427	svm_save_seg(env->vm_hsave + offsetof(struct vmcb, save.es),
6428	&env->segs[R_ES]);
6429	svm_save_seg(env->vm_hsave + offsetof(struct vmcb, save.cs),
6430	&env->segs[R_CS]);
6431	svm_save_seg(env->vm_hsave + offsetof(struct vmcb, save.ss),
6432	&env->segs[R_SS]);
6433	svm_save_seg(env->vm_hsave + offsetof(struct vmcb, save.ds),
6434	&env->segs[R_DS]);
6435
6436	stq_phys(env->vm_hsave + offsetof(struct vmcb, save.rip),
6437	EIP + next_eip_addend);
6438	stq_phys(env->vm_hsave + offsetof(struct vmcb, save.rsp), ESP);
6439	stq_phys(env->vm_hsave + offsetof(struct vmcb, save.rax), EAX);
6440
6441	/* load the interception bitmaps so we do not need to access the
6442	vmcb in svm mode */
6443	env->intercept = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, control.intercept));
6444	env->intercept_cr_read = lduw_phys(env->vm_vmcb + offsetof(struct vmcb, control.intercept_cr_read));
6445	env->intercept_cr_write = lduw_phys(env->vm_vmcb + offsetof(struct vmcb, control.intercept_cr_write));
6446	env->intercept_dr_read = lduw_phys(env->vm_vmcb + offsetof(struct vmcb, control.intercept_dr_read));
6447	env->intercept_dr_write = lduw_phys(env->vm_vmcb + offsetof(struct vmcb, control.intercept_dr_write));
6448	env->intercept_exceptions = ldl_phys(env->vm_vmcb + offsetof(struct vmcb, control.intercept_exceptions));
6449
6450	/* enable intercepts */
6451	env->hflags \|= HF_SVMI_MASK;
6452
6453	env->tsc_offset = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, control.tsc_offset));
6454
6455	env->gdt.base = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.gdtr.base));
6456	env->gdt.limit = ldl_phys(env->vm_vmcb + offsetof(struct vmcb, save.gdtr.limit));
6457
6458	env->idt.base = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.idtr.base));
6459	env->idt.limit = ldl_phys(env->vm_vmcb + offsetof(struct vmcb, save.idtr.limit));
6460
6461	/* clear exit_info_2 so we behave like the real hardware */
6462	stq_phys(env->vm_vmcb + offsetof(struct vmcb, control.exit_info_2), 0);
6463
6464	cpu_x86_update_cr0(env, ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.cr0)));
6465	cpu_x86_update_cr4(env, ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.cr4)));
6466	cpu_x86_update_cr3(env, ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.cr3)));
6467	env->cr[2] = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.cr2));
6468	int_ctl = ldl_phys(env->vm_vmcb + offsetof(struct vmcb, control.int_ctl));
6469	env->hflags2 &= ~(HF2_HIF_MASK \| HF2_VINTR_MASK);
6470	if (int_ctl & V_INTR_MASKING_MASK) {
6471	env->v_tpr = int_ctl & V_TPR_MASK;
6472	env->hflags2 \|= HF2_VINTR_MASK;
6473	if (env->eflags & IF_MASK)
6474	env->hflags2 \|= HF2_HIF_MASK;
6475	}
6476
6477	cpu_load_efer(env,
6478	ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.efer)));
6479	env->eflags = 0;
6480	load_eflags(ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.rflags)),
6481	~(CC_O \| CC_S \| CC_Z \| CC_A \| CC_P \| CC_C \| DF_MASK));
6482	CC_OP = CC_OP_EFLAGS;
6483
6484	svm_load_seg_cache(env->vm_vmcb + offsetof(struct vmcb, save.es),
6485	env, R_ES);
6486	svm_load_seg_cache(env->vm_vmcb + offsetof(struct vmcb, save.cs),
6487	env, R_CS);
6488	svm_load_seg_cache(env->vm_vmcb + offsetof(struct vmcb, save.ss),
6489	env, R_SS);
6490	svm_load_seg_cache(env->vm_vmcb + offsetof(struct vmcb, save.ds),
6491	env, R_DS);
6492
6493	EIP = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.rip));
6494	env->eip = EIP;
6495	ESP = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.rsp));
6496	EAX = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.rax));
6497	env->dr[7] = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.dr7));
6498	env->dr[6] = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.dr6));
6499	cpu_x86_set_cpl(env, ldub_phys(env->vm_vmcb + offsetof(struct vmcb, save.cpl)));
6500
6501	/* FIXME: guest state consistency checks */
6502
6503	switch(ldub_phys(env->vm_vmcb + offsetof(struct vmcb, control.tlb_ctl))) {
6504	case TLB_CONTROL_DO_NOTHING:
6505	break;
6506	case TLB_CONTROL_FLUSH_ALL_ASID:
6507	/* FIXME: this is not 100% correct but should work for now */
6508	tlb_flush(env, 1);
6509	break;
6510	}
6511
6512	env->hflags2 \|= HF2_GIF_MASK;
6513
6514	if (int_ctl & V_IRQ_MASK) {
6515	env->interrupt_request \|= CPU_INTERRUPT_VIRQ;
6516	}
6517
6518	/* maybe we need to inject an event */
6519	event_inj = ldl_phys(env->vm_vmcb + offsetof(struct vmcb, control.event_inj));
6520	if (event_inj & SVM_EVTINJ_VALID) {
6521	uint8_t vector = event_inj & SVM_EVTINJ_VEC_MASK;
6522	uint16_t valid_err = event_inj & SVM_EVTINJ_VALID_ERR;
6523	uint32_t event_inj_err = ldl_phys(env->vm_vmcb + offsetof(struct vmcb, control.event_inj_err));
6524
6525	qemu_log_mask(CPU_LOG_TB_IN_ASM, "Injecting(%#hx): ", valid_err);
6526	/* FIXME: need to implement valid_err */
6527	switch (event_inj & SVM_EVTINJ_TYPE_MASK) {
6528	case SVM_EVTINJ_TYPE_INTR:
6529	env->exception_index = vector;
6530	env->error_code = event_inj_err;
6531	env->exception_is_int = 0;
6532	env->exception_next_eip = -1;
6533	qemu_log_mask(CPU_LOG_TB_IN_ASM, "INTR");
6534	/* XXX: is it always correct ? */
6535	do_interrupt(vector, 0, 0, 0, 1);
6536	break;
6537	case SVM_EVTINJ_TYPE_NMI:
6538	env->exception_index = EXCP02_NMI;
6539	env->error_code = event_inj_err;
6540	env->exception_is_int = 0;
6541	env->exception_next_eip = EIP;
6542	qemu_log_mask(CPU_LOG_TB_IN_ASM, "NMI");
6543	cpu_loop_exit();
6544	break;
6545	case SVM_EVTINJ_TYPE_EXEPT:
6546	env->exception_index = vector;
6547	env->error_code = event_inj_err;
6548	env->exception_is_int = 0;
6549	env->exception_next_eip = -1;
6550	qemu_log_mask(CPU_LOG_TB_IN_ASM, "EXEPT");
6551	cpu_loop_exit();
6552	break;
6553	case SVM_EVTINJ_TYPE_SOFT:
6554	env->exception_index = vector;
6555	env->error_code = event_inj_err;
6556	env->exception_is_int = 1;
6557	env->exception_next_eip = EIP;
6558	qemu_log_mask(CPU_LOG_TB_IN_ASM, "SOFT");
6559	cpu_loop_exit();
6560	break;
6561	}
6562	qemu_log_mask(CPU_LOG_TB_IN_ASM, " %#x %#x\n", env->exception_index, env->error_code);
6563	}
6564	}
6565
6566	void helper_vmmcall(void)
6567	{
6568	helper_svm_check_intercept_param(SVM_EXIT_VMMCALL, 0);
6569	raise_exception(EXCP06_ILLOP);
6570	}
6571
6572	void helper_vmload(int aflag)
6573	{
6574	target_ulong addr;
6575	helper_svm_check_intercept_param(SVM_EXIT_VMLOAD, 0);
6576
6577	if (aflag == 2)
6578	addr = EAX;
6579	else
6580	addr = (uint32_t)EAX;
6581
6582	qemu_log_mask(CPU_LOG_TB_IN_ASM, "vmload! " TARGET_FMT_lx "\nFS: %016" PRIx64 " \| " TARGET_FMT_lx "\n",
6583	addr, ldq_phys(addr + offsetof(struct vmcb, save.fs.base)),
6584	env->segs[R_FS].base);
6585
6586	svm_load_seg_cache(addr + offsetof(struct vmcb, save.fs),
6587	env, R_FS);
6588	svm_load_seg_cache(addr + offsetof(struct vmcb, save.gs),
6589	env, R_GS);
6590	svm_load_seg(addr + offsetof(struct vmcb, save.tr),
6591	&env->tr);
6592	svm_load_seg(addr + offsetof(struct vmcb, save.ldtr),
6593	&env->ldt);
6594
6595	#ifdef TARGET_X86_64
6596	env->kernelgsbase = ldq_phys(addr + offsetof(struct vmcb, save.kernel_gs_base));
6597	env->lstar = ldq_phys(addr + offsetof(struct vmcb, save.lstar));
6598	env->cstar = ldq_phys(addr + offsetof(struct vmcb, save.cstar));
6599	env->fmask = ldq_phys(addr + offsetof(struct vmcb, save.sfmask));
6600	#endif
6601	env->star = ldq_phys(addr + offsetof(struct vmcb, save.star));
6602	env->sysenter_cs = ldq_phys(addr + offsetof(struct vmcb, save.sysenter_cs));
6603	env->sysenter_esp = ldq_phys(addr + offsetof(struct vmcb, save.sysenter_esp));
6604	env->sysenter_eip = ldq_phys(addr + offsetof(struct vmcb, save.sysenter_eip));
6605	}
6606
6607	void helper_vmsave(int aflag)
6608	{
6609	target_ulong addr;
6610	helper_svm_check_intercept_param(SVM_EXIT_VMSAVE, 0);
6611
6612	if (aflag == 2)
6613	addr = EAX;
6614	else
6615	addr = (uint32_t)EAX;
6616
6617	qemu_log_mask(CPU_LOG_TB_IN_ASM, "vmsave! " TARGET_FMT_lx "\nFS: %016" PRIx64 " \| " TARGET_FMT_lx "\n",
6618	addr, ldq_phys(addr + offsetof(struct vmcb, save.fs.base)),
6619	env->segs[R_FS].base);
6620
6621	svm_save_seg(addr + offsetof(struct vmcb, save.fs),
6622	&env->segs[R_FS]);
6623	svm_save_seg(addr + offsetof(struct vmcb, save.gs),
6624	&env->segs[R_GS]);
6625	svm_save_seg(addr + offsetof(struct vmcb, save.tr),
6626	&env->tr);
6627	svm_save_seg(addr + offsetof(struct vmcb, save.ldtr),
6628	&env->ldt);
6629
6630	#ifdef TARGET_X86_64
6631	stq_phys(addr + offsetof(struct vmcb, save.kernel_gs_base), env->kernelgsbase);
6632	stq_phys(addr + offsetof(struct vmcb, save.lstar), env->lstar);
6633	stq_phys(addr + offsetof(struct vmcb, save.cstar), env->cstar);
6634	stq_phys(addr + offsetof(struct vmcb, save.sfmask), env->fmask);
6635	#endif
6636	stq_phys(addr + offsetof(struct vmcb, save.star), env->star);
6637	stq_phys(addr + offsetof(struct vmcb, save.sysenter_cs), env->sysenter_cs);
6638	stq_phys(addr + offsetof(struct vmcb, save.sysenter_esp), env->sysenter_esp);
6639	stq_phys(addr + offsetof(struct vmcb, save.sysenter_eip), env->sysenter_eip);
6640	}
6641
6642	void helper_stgi(void)
6643	{
6644	helper_svm_check_intercept_param(SVM_EXIT_STGI, 0);
6645	env->hflags2 \|= HF2_GIF_MASK;
6646	}
6647
6648	void helper_clgi(void)
6649	{
6650	helper_svm_check_intercept_param(SVM_EXIT_CLGI, 0);
6651	env->hflags2 &= ~HF2_GIF_MASK;
6652	}
6653
6654	void helper_skinit(void)
6655	{
6656	helper_svm_check_intercept_param(SVM_EXIT_SKINIT, 0);
6657	/* XXX: not implemented */
6658	raise_exception(EXCP06_ILLOP);
6659	}
6660
6661	void helper_invlpga(int aflag)
6662	{
6663	target_ulong addr;
6664	helper_svm_check_intercept_param(SVM_EXIT_INVLPGA, 0);
6665
6666	if (aflag == 2)
6667	addr = EAX;
6668	else
6669	addr = (uint32_t)EAX;
6670
6671	/* XXX: could use the ASID to see if it is needed to do the
6672	flush */
6673	tlb_flush_page(env, addr);
6674	}
6675
6676	void helper_svm_check_intercept_param(uint32_t type, uint64_t param)
6677	{
6678	if (likely(!(env->hflags & HF_SVMI_MASK)))
6679	return;
6680	#ifndef VBOX
6681	switch(type) {
6682	case SVM_EXIT_READ_CR0 ... SVM_EXIT_READ_CR0 + 8:
6683	if (env->intercept_cr_read & (1 << (type - SVM_EXIT_READ_CR0))) {
6684	helper_vmexit(type, param);
6685	}
6686	break;
6687	case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR0 + 8:
6688	if (env->intercept_cr_write & (1 << (type - SVM_EXIT_WRITE_CR0))) {
6689	helper_vmexit(type, param);
6690	}
6691	break;
6692	case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR0 + 7:
6693	if (env->intercept_dr_read & (1 << (type - SVM_EXIT_READ_DR0))) {
6694	helper_vmexit(type, param);
6695	}
6696	break;
6697	case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR0 + 7:
6698	if (env->intercept_dr_write & (1 << (type - SVM_EXIT_WRITE_DR0))) {
6699	helper_vmexit(type, param);
6700	}
6701	break;
6702	case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 31:
6703	if (env->intercept_exceptions & (1 << (type - SVM_EXIT_EXCP_BASE))) {
6704	helper_vmexit(type, param);
6705	}
6706	break;
6707	case SVM_EXIT_MSR:
6708	if (env->intercept & (1ULL << (SVM_EXIT_MSR - SVM_EXIT_INTR))) {
6709	/* FIXME: this should be read in at vmrun (faster this way?) */
6710	uint64_t addr = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, control.msrpm_base_pa));
6711	uint32_t t0, t1;
6712	switch((uint32_t)ECX) {
6713	case 0 ... 0x1fff:
6714	t0 = (ECX * 2) % 8;
6715	t1 = ECX / 8;
6716	break;
6717	case 0xc0000000 ... 0xc0001fff:
6718	t0 = (8192 + ECX - 0xc0000000) * 2;
6719	t1 = (t0 / 8);
6720	t0 %= 8;
6721	break;
6722	case 0xc0010000 ... 0xc0011fff:
6723	t0 = (16384 + ECX - 0xc0010000) * 2;
6724	t1 = (t0 / 8);
6725	t0 %= 8;
6726	break;
6727	default:
6728	helper_vmexit(type, param);
6729	t0 = 0;
6730	t1 = 0;
6731	break;
6732	}
6733	if (ldub_phys(addr + t1) & ((1 << param) << t0))
6734	helper_vmexit(type, param);
6735	}
6736	break;
6737	default:
6738	if (env->intercept & (1ULL << (type - SVM_EXIT_INTR))) {
6739	helper_vmexit(type, param);
6740	}
6741	break;
6742	}
6743	#else /* VBOX */
6744	AssertMsgFailed(("We shouldn't be here, HM supported differently!"));
6745	#endif /* VBOX */
6746	}
6747
6748	void helper_svm_check_io(uint32_t port, uint32_t param,
6749	uint32_t next_eip_addend)
6750	{
6751	if (env->intercept & (1ULL << (SVM_EXIT_IOIO - SVM_EXIT_INTR))) {
6752	/* FIXME: this should be read in at vmrun (faster this way?) */
6753	uint64_t addr = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, control.iopm_base_pa));
6754	uint16_t mask = (1 << ((param >> 4) & 7)) - 1;
6755	if(lduw_phys(addr + port / 8) & (mask << (port & 7))) {
6756	/* next EIP */
6757	stq_phys(env->vm_vmcb + offsetof(struct vmcb, control.exit_info_2),
6758	env->eip + next_eip_addend);
6759	helper_vmexit(SVM_EXIT_IOIO, param \| (port << 16));
6760	}
6761	}
6762	}
6763
6764	/* Note: currently only 32 bits of exit_code are used */
6765	void helper_vmexit(uint32_t exit_code, uint64_t exit_info_1)
6766	{
6767	uint32_t int_ctl;
6768
6769	qemu_log_mask(CPU_LOG_TB_IN_ASM, "vmexit(%08x, %016" PRIx64 ", %016" PRIx64 ", " TARGET_FMT_lx ")!\n",
6770	exit_code, exit_info_1,
6771	ldq_phys(env->vm_vmcb + offsetof(struct vmcb, control.exit_info_2)),
6772	EIP);
6773
6774	if(env->hflags & HF_INHIBIT_IRQ_MASK) {
6775	stl_phys(env->vm_vmcb + offsetof(struct vmcb, control.int_state), SVM_INTERRUPT_SHADOW_MASK);
6776	env->hflags &= ~HF_INHIBIT_IRQ_MASK;
6777	} else {
6778	stl_phys(env->vm_vmcb + offsetof(struct vmcb, control.int_state), 0);
6779	}
6780
6781	/* Save the VM state in the vmcb */
6782	svm_save_seg(env->vm_vmcb + offsetof(struct vmcb, save.es),
6783	&env->segs[R_ES]);
6784	svm_save_seg(env->vm_vmcb + offsetof(struct vmcb, save.cs),
6785	&env->segs[R_CS]);
6786	svm_save_seg(env->vm_vmcb + offsetof(struct vmcb, save.ss),
6787	&env->segs[R_SS]);
6788	svm_save_seg(env->vm_vmcb + offsetof(struct vmcb, save.ds),
6789	&env->segs[R_DS]);
6790
6791	stq_phys(env->vm_vmcb + offsetof(struct vmcb, save.gdtr.base), env->gdt.base);
6792	stl_phys(env->vm_vmcb + offsetof(struct vmcb, save.gdtr.limit), env->gdt.limit);
6793
6794	stq_phys(env->vm_vmcb + offsetof(struct vmcb, save.idtr.base), env->idt.base);
6795	stl_phys(env->vm_vmcb + offsetof(struct vmcb, save.idtr.limit), env->idt.limit);
6796
6797	stq_phys(env->vm_vmcb + offsetof(struct vmcb, save.efer), env->efer);
6798	stq_phys(env->vm_vmcb + offsetof(struct vmcb, save.cr0), env->cr[0]);
6799	stq_phys(env->vm_vmcb + offsetof(struct vmcb, save.cr2), env->cr[2]);
6800	stq_phys(env->vm_vmcb + offsetof(struct vmcb, save.cr3), env->cr[3]);
6801	stq_phys(env->vm_vmcb + offsetof(struct vmcb, save.cr4), env->cr[4]);
6802
6803	int_ctl = ldl_phys(env->vm_vmcb + offsetof(struct vmcb, control.int_ctl));
6804	int_ctl &= ~(V_TPR_MASK \| V_IRQ_MASK);
6805	int_ctl \|= env->v_tpr & V_TPR_MASK;
6806	if (env->interrupt_request & CPU_INTERRUPT_VIRQ)
6807	int_ctl \|= V_IRQ_MASK;
6808	stl_phys(env->vm_vmcb + offsetof(struct vmcb, control.int_ctl), int_ctl);
6809
6810	stq_phys(env->vm_vmcb + offsetof(struct vmcb, save.rflags), compute_eflags());
6811	stq_phys(env->vm_vmcb + offsetof(struct vmcb, save.rip), env->eip);
6812	stq_phys(env->vm_vmcb + offsetof(struct vmcb, save.rsp), ESP);
6813	stq_phys(env->vm_vmcb + offsetof(struct vmcb, save.rax), EAX);
6814	stq_phys(env->vm_vmcb + offsetof(struct vmcb, save.dr7), env->dr[7]);
6815	stq_phys(env->vm_vmcb + offsetof(struct vmcb, save.dr6), env->dr[6]);
6816	stb_phys(env->vm_vmcb + offsetof(struct vmcb, save.cpl), env->hflags & HF_CPL_MASK);
6817
6818	/* Reload the host state from vm_hsave */
6819	env->hflags2 &= ~(HF2_HIF_MASK \| HF2_VINTR_MASK);
6820	env->hflags &= ~HF_SVMI_MASK;
6821	env->intercept = 0;
6822	env->intercept_exceptions = 0;
6823	env->interrupt_request &= ~CPU_INTERRUPT_VIRQ;
6824	env->tsc_offset = 0;
6825
6826	env->gdt.base = ldq_phys(env->vm_hsave + offsetof(struct vmcb, save.gdtr.base));
6827	env->gdt.limit = ldl_phys(env->vm_hsave + offsetof(struct vmcb, save.gdtr.limit));
6828
6829	env->idt.base = ldq_phys(env->vm_hsave + offsetof(struct vmcb, save.idtr.base));
6830	env->idt.limit = ldl_phys(env->vm_hsave + offsetof(struct vmcb, save.idtr.limit));
6831
6832	cpu_x86_update_cr0(env, ldq_phys(env->vm_hsave + offsetof(struct vmcb, save.cr0)) \| CR0_PE_MASK);
6833	cpu_x86_update_cr4(env, ldq_phys(env->vm_hsave + offsetof(struct vmcb, save.cr4)));
6834	cpu_x86_update_cr3(env, ldq_phys(env->vm_hsave + offsetof(struct vmcb, save.cr3)));
6835	/* we need to set the efer after the crs so the hidden flags get
6836	set properly */
6837	cpu_load_efer(env,
6838	ldq_phys(env->vm_hsave + offsetof(struct vmcb, save.efer)));
6839	env->eflags = 0;
6840	load_eflags(ldq_phys(env->vm_hsave + offsetof(struct vmcb, save.rflags)),
6841	~(CC_O \| CC_S \| CC_Z \| CC_A \| CC_P \| CC_C \| DF_MASK));
6842	CC_OP = CC_OP_EFLAGS;
6843
6844	svm_load_seg_cache(env->vm_hsave + offsetof(struct vmcb, save.es),
6845	env, R_ES);
6846	svm_load_seg_cache(env->vm_hsave + offsetof(struct vmcb, save.cs),
6847	env, R_CS);
6848	svm_load_seg_cache(env->vm_hsave + offsetof(struct vmcb, save.ss),
6849	env, R_SS);
6850	svm_load_seg_cache(env->vm_hsave + offsetof(struct vmcb, save.ds),
6851	env, R_DS);
6852
6853	EIP = ldq_phys(env->vm_hsave + offsetof(struct vmcb, save.rip));
6854	ESP = ldq_phys(env->vm_hsave + offsetof(struct vmcb, save.rsp));
6855	EAX = ldq_phys(env->vm_hsave + offsetof(struct vmcb, save.rax));
6856
6857	env->dr[6] = ldq_phys(env->vm_hsave + offsetof(struct vmcb, save.dr6));
6858	env->dr[7] = ldq_phys(env->vm_hsave + offsetof(struct vmcb, save.dr7));
6859
6860	/* other setups */
6861	cpu_x86_set_cpl(env, 0);
6862	stq_phys(env->vm_vmcb + offsetof(struct vmcb, control.exit_code), exit_code);
6863	stq_phys(env->vm_vmcb + offsetof(struct vmcb, control.exit_info_1), exit_info_1);
6864
6865	stl_phys(env->vm_vmcb + offsetof(struct vmcb, control.exit_int_info),
6866	ldl_phys(env->vm_vmcb + offsetof(struct vmcb, control.event_inj)));
6867	stl_phys(env->vm_vmcb + offsetof(struct vmcb, control.exit_int_info_err),
6868	ldl_phys(env->vm_vmcb + offsetof(struct vmcb, control.event_inj_err)));
6869	stl_phys(env->vm_vmcb + offsetof(struct vmcb, control.event_inj), 0);
6870
6871	env->hflags2 &= ~HF2_GIF_MASK;
6872	/* FIXME: Resets the current ASID register to zero (host ASID). */
6873
6874	/* Clears the V_IRQ and V_INTR_MASKING bits inside the processor. */
6875
6876	/* Clears the TSC_OFFSET inside the processor. */
6877
6878	/* If the host is in PAE mode, the processor reloads the host's PDPEs
6879	from the page table indicated the host's CR3. If the PDPEs contain
6880	illegal state, the processor causes a shutdown. */
6881
6882	/* Forces CR0.PE = 1, RFLAGS.VM = 0. */
6883	env->cr[0] \|= CR0_PE_MASK;
6884	env->eflags &= ~VM_MASK;
6885
6886	/* Disables all breakpoints in the host DR7 register. */
6887
6888	/* Checks the reloaded host state for consistency. */
6889
6890	/* If the host's rIP reloaded by #VMEXIT is outside the limit of the
6891	host's code segment or non-canonical (in the case of long mode), a
6892	#GP fault is delivered inside the host.) */
6893
6894	/* remove any pending exception */
6895	env->exception_index = -1;
6896	env->error_code = 0;
6897	env->old_exception = -1;
6898
6899	cpu_loop_exit();
6900	}
6901
6902	#endif
6903
6904	/* MMX/SSE */
6905	/* XXX: optimize by storing fptt and fptags in the static cpu state */
6906	void helper_enter_mmx(void)
6907	{
6908	env->fpstt = 0;
6909	(uint32_t )(env->fptags) = 0;
6910	(uint32_t )(env->fptags + 4) = 0;
6911	}
6912
6913	void helper_emms(void)
6914	{
6915	/* set to empty state */
6916	(uint32_t )(env->fptags) = 0x01010101;
6917	(uint32_t )(env->fptags + 4) = 0x01010101;
6918	}
6919
6920	/* XXX: suppress */
6921	void helper_movq(void d, void s)
6922	{
6923	(uint64_t )d = (uint64_t )s;
6924	}
6925
6926	#define SHIFT 0
6927	#include "ops_sse.h"
6928
6929	#define SHIFT 1
6930	#include "ops_sse.h"
6931
6932	#define SHIFT 0
6933	#include "helper_template.h"
6934	#undef SHIFT
6935
6936	#define SHIFT 1
6937	#include "helper_template.h"
6938	#undef SHIFT
6939
6940	#define SHIFT 2
6941	#include "helper_template.h"
6942	#undef SHIFT
6943
6944	#ifdef TARGET_X86_64
6945
6946	#define SHIFT 3
6947	#include "helper_template.h"
6948	#undef SHIFT
6949
6950	#endif
6951
6952	/* bit operations */
6953	target_ulong helper_bsf(target_ulong t0)
6954	{
6955	int count;
6956	target_ulong res;
6957
6958	res = t0;
6959	count = 0;
6960	while ((res & 1) == 0) {
6961	count++;
6962	res >>= 1;
6963	}
6964	return count;
6965	}
6966
6967	target_ulong helper_lzcnt(target_ulong t0, int wordsize)
6968	{
6969	int count;
6970	target_ulong res, mask;
6971
6972	if (wordsize > 0 && t0 == 0) {
6973	return wordsize;
6974	}
6975	res = t0;
6976	count = TARGET_LONG_BITS - 1;
6977	mask = (target_ulong)1 << (TARGET_LONG_BITS - 1);
6978	while ((res & mask) == 0) {
6979	count--;
6980	res <<= 1;
6981	}
6982	if (wordsize > 0) {
6983	return wordsize - 1 - count;
6984	}
6985	return count;
6986	}
6987
6988	target_ulong helper_bsr(target_ulong t0)
6989	{
6990	return helper_lzcnt(t0, 0);
6991	}
6992
6993	static int compute_all_eflags(void)
6994	{
6995	return CC_SRC;
6996	}
6997
6998	static int compute_c_eflags(void)
6999	{
7000	return CC_SRC & CC_C;
7001	}
7002
7003	uint32_t helper_cc_compute_all(int op)
7004	{
7005	switch (op) {
7006	default: /* should never happen */ return 0;
7007
7008	case CC_OP_EFLAGS: return compute_all_eflags();
7009
7010	case CC_OP_MULB: return compute_all_mulb();
7011	case CC_OP_MULW: return compute_all_mulw();
7012	case CC_OP_MULL: return compute_all_mull();
7013
7014	case CC_OP_ADDB: return compute_all_addb();
7015	case CC_OP_ADDW: return compute_all_addw();
7016	case CC_OP_ADDL: return compute_all_addl();
7017
7018	case CC_OP_ADCB: return compute_all_adcb();
7019	case CC_OP_ADCW: return compute_all_adcw();
7020	case CC_OP_ADCL: return compute_all_adcl();
7021
7022	case CC_OP_SUBB: return compute_all_subb();
7023	case CC_OP_SUBW: return compute_all_subw();
7024	case CC_OP_SUBL: return compute_all_subl();
7025
7026	case CC_OP_SBBB: return compute_all_sbbb();
7027	case CC_OP_SBBW: return compute_all_sbbw();
7028	case CC_OP_SBBL: return compute_all_sbbl();
7029
7030	case CC_OP_LOGICB: return compute_all_logicb();
7031	case CC_OP_LOGICW: return compute_all_logicw();
7032	case CC_OP_LOGICL: return compute_all_logicl();
7033
7034	case CC_OP_INCB: return compute_all_incb();
7035	case CC_OP_INCW: return compute_all_incw();
7036	case CC_OP_INCL: return compute_all_incl();
7037
7038	case CC_OP_DECB: return compute_all_decb();
7039	case CC_OP_DECW: return compute_all_decw();
7040	case CC_OP_DECL: return compute_all_decl();
7041
7042	case CC_OP_SHLB: return compute_all_shlb();
7043	case CC_OP_SHLW: return compute_all_shlw();
7044	case CC_OP_SHLL: return compute_all_shll();
7045
7046	case CC_OP_SARB: return compute_all_sarb();
7047	case CC_OP_SARW: return compute_all_sarw();
7048	case CC_OP_SARL: return compute_all_sarl();
7049
7050	#ifdef TARGET_X86_64
7051	case CC_OP_MULQ: return compute_all_mulq();
7052
7053	case CC_OP_ADDQ: return compute_all_addq();
7054
7055	case CC_OP_ADCQ: return compute_all_adcq();
7056
7057	case CC_OP_SUBQ: return compute_all_subq();
7058
7059	case CC_OP_SBBQ: return compute_all_sbbq();
7060
7061	case CC_OP_LOGICQ: return compute_all_logicq();
7062
7063	case CC_OP_INCQ: return compute_all_incq();
7064
7065	case CC_OP_DECQ: return compute_all_decq();
7066
7067	case CC_OP_SHLQ: return compute_all_shlq();
7068
7069	case CC_OP_SARQ: return compute_all_sarq();
7070	#endif
7071	}
7072	}
7073
7074	uint32_t helper_cc_compute_c(int op)
7075	{
7076	switch (op) {
7077	default: /* should never happen */ return 0;
7078
7079	case CC_OP_EFLAGS: return compute_c_eflags();
7080
7081	case CC_OP_MULB: return compute_c_mull();
7082	case CC_OP_MULW: return compute_c_mull();
7083	case CC_OP_MULL: return compute_c_mull();
7084
7085	case CC_OP_ADDB: return compute_c_addb();
7086	case CC_OP_ADDW: return compute_c_addw();
7087	case CC_OP_ADDL: return compute_c_addl();
7088
7089	case CC_OP_ADCB: return compute_c_adcb();
7090	case CC_OP_ADCW: return compute_c_adcw();
7091	case CC_OP_ADCL: return compute_c_adcl();
7092
7093	case CC_OP_SUBB: return compute_c_subb();
7094	case CC_OP_SUBW: return compute_c_subw();
7095	case CC_OP_SUBL: return compute_c_subl();
7096
7097	case CC_OP_SBBB: return compute_c_sbbb();
7098	case CC_OP_SBBW: return compute_c_sbbw();
7099	case CC_OP_SBBL: return compute_c_sbbl();
7100
7101	case CC_OP_LOGICB: return compute_c_logicb();
7102	case CC_OP_LOGICW: return compute_c_logicw();
7103	case CC_OP_LOGICL: return compute_c_logicl();
7104
7105	case CC_OP_INCB: return compute_c_incl();
7106	case CC_OP_INCW: return compute_c_incl();
7107	case CC_OP_INCL: return compute_c_incl();
7108
7109	case CC_OP_DECB: return compute_c_incl();
7110	case CC_OP_DECW: return compute_c_incl();
7111	case CC_OP_DECL: return compute_c_incl();
7112
7113	case CC_OP_SHLB: return compute_c_shlb();
7114	case CC_OP_SHLW: return compute_c_shlw();
7115	case CC_OP_SHLL: return compute_c_shll();
7116
7117	case CC_OP_SARB: return compute_c_sarl();
7118	case CC_OP_SARW: return compute_c_sarl();
7119	case CC_OP_SARL: return compute_c_sarl();
7120
7121	#ifdef TARGET_X86_64
7122	case CC_OP_MULQ: return compute_c_mull();
7123
7124	case CC_OP_ADDQ: return compute_c_addq();
7125
7126	case CC_OP_ADCQ: return compute_c_adcq();
7127
7128	case CC_OP_SUBQ: return compute_c_subq();
7129
7130	case CC_OP_SBBQ: return compute_c_sbbq();
7131
7132	case CC_OP_LOGICQ: return compute_c_logicq();
7133
7134	case CC_OP_INCQ: return compute_c_incl();
7135
7136	case CC_OP_DECQ: return compute_c_incl();
7137
7138	case CC_OP_SHLQ: return compute_c_shlq();
7139
7140	case CC_OP_SARQ: return compute_c_sarl();
7141	#endif
7142	}
7143	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/recompiler/target-i386/op_helper.c@ 80823

Download in other formats: