exec.c@ 36412

Last change on this file since 36412 was 36250, checked in by vboxsync, 14 years ago
REM: Missing pointer check.
Property svn:eol-style set to `native`
File size: 124.7 KB

Line
1	/*
2	* virtual page mapping and translated block handling
3	*
4	* Copyright (c) 2003 Fabrice Bellard
5	*
6	* This library is free software; you can redistribute it and/or
7	* modify it under the terms of the GNU Lesser General Public
8	* License as published by the Free Software Foundation; either
9	* version 2 of the License, or (at your option) any later version.
10	*
11	* This library is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	* Lesser General Public License for more details.
15	*
16	* You should have received a copy of the GNU Lesser General Public
17	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
18	*/
19
20	/*
21	* Oracle LGPL Disclaimer: For the avoidance of doubt, except that if any license choice
22	* other than GPL or LGPL is available it will apply instead, Oracle elects to use only
23	* the Lesser General Public License version 2.1 (LGPLv2) at this time for any software where
24	* a choice of LGPL license versions is made available with the language indicating
25	* that LGPLv2 or any later version may be used, or where a choice of which version
26	* of the LGPL is applied is otherwise unspecified.
27	*/
28
29	#include "config.h"
30	#ifndef VBOX
31	#ifdef _WIN32
32	#include <windows.h>
33	#else
34	#include <sys/types.h>
35	#include <sys/mman.h>
36	#endif
37	#include <stdlib.h>
38	#include <stdio.h>
39	#include <stdarg.h>
40	#include <string.h>
41	#include <errno.h>
42	#include <unistd.h>
43	#include <inttypes.h>
44	#else /* VBOX */
45	# include <stdlib.h>
46	# include <stdio.h>
47	# include <iprt/alloc.h>
48	# include <iprt/string.h>
49	# include <iprt/param.h>
50	# include <VBox/vmm/pgm.h> /* PGM_DYNAMIC_RAM_ALLOC */
51	#endif /* VBOX */
52
53	#include "cpu.h"
54	#include "exec-all.h"
55	#include "qemu-common.h"
56	#include "tcg.h"
57	#ifndef VBOX
58	#include "hw/hw.h"
59	#endif
60	#include "osdep.h"
61	#include "kvm.h"
62	#if defined(CONFIG_USER_ONLY)
63	#include <qemu.h>
64	#endif
65
66	//#define DEBUG_TB_INVALIDATE
67	//#define DEBUG_FLUSH
68	//#define DEBUG_TLB
69	//#define DEBUG_UNASSIGNED
70
71	/* make various TB consistency checks */
72	//#define DEBUG_TB_CHECK
73	//#define DEBUG_TLB_CHECK
74
75	//#define DEBUG_IOPORT
76	//#define DEBUG_SUBPAGE
77
78	#if !defined(CONFIG_USER_ONLY)
79	/* TB consistency checks only implemented for usermode emulation. */
80	#undef DEBUG_TB_CHECK
81	#endif
82
83	#define SMC_BITMAP_USE_THRESHOLD 10
84
85	#if defined(TARGET_SPARC64)
86	#define TARGET_PHYS_ADDR_SPACE_BITS 41
87	#elif defined(TARGET_SPARC)
88	#define TARGET_PHYS_ADDR_SPACE_BITS 36
89	#elif defined(TARGET_ALPHA)
90	#define TARGET_PHYS_ADDR_SPACE_BITS 42
91	#define TARGET_VIRT_ADDR_SPACE_BITS 42
92	#elif defined(TARGET_PPC64)
93	#define TARGET_PHYS_ADDR_SPACE_BITS 42
94	#elif defined(TARGET_X86_64) && !defined(CONFIG_KQEMU)
95	#define TARGET_PHYS_ADDR_SPACE_BITS 42
96	#elif defined(TARGET_I386) && !defined(CONFIG_KQEMU)
97	#define TARGET_PHYS_ADDR_SPACE_BITS 36
98	#else
99	/* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
100	#define TARGET_PHYS_ADDR_SPACE_BITS 32
101	#endif
102
103	static TranslationBlock *tbs;
104	int code_gen_max_blocks;
105	TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
106	static int nb_tbs;
107	/* any access to the tbs or the page table must use this lock */
108	spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
109
110	#ifndef VBOX
111	#if defined(__arm__) \|\| defined(__sparc_v9__)
112	/* The prologue must be reachable with a direct jump. ARM and Sparc64
113	have limited branch ranges (possibly also PPC) so place it in a
114	section close to code segment. */
115	#define code_gen_section \
116	__attribute__((__section__(".gen_code"))) \
117	__attribute__((aligned (32)))
118	#elif defined(_WIN32)
119	/* Maximum alignment for Win32 is 16. */
120	#define code_gen_section \
121	__attribute__((aligned (16)))
122	#else
123	#define code_gen_section \
124	__attribute__((aligned (32)))
125	#endif
126
127	uint8_t code_gen_prologue[1024] code_gen_section;
128	#else /* VBOX */
129	extern uint8_t* code_gen_prologue;
130	#endif /* VBOX */
131	static uint8_t *code_gen_buffer;
132	static unsigned long code_gen_buffer_size;
133	/* threshold to flush the translated code buffer */
134	static unsigned long code_gen_buffer_max_size;
135	uint8_t *code_gen_ptr;
136
137	#ifndef VBOX
138	#if !defined(CONFIG_USER_ONLY)
139	int phys_ram_fd;
140	uint8_t *phys_ram_dirty;
141	static int in_migration;
142
143	typedef struct RAMBlock {
144	uint8_t *host;
145	ram_addr_t offset;
146	ram_addr_t length;
147	struct RAMBlock *next;
148	} RAMBlock;
149
150	static RAMBlock *ram_blocks;
151	/* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)
152	then we can no longer assume contiguous ram offsets, and external uses
153	of this variable will break. */
154	ram_addr_t last_ram_offset;
155	#endif
156	#else /* VBOX */
157	/* we have memory ranges (the high PC-BIOS mapping) which
158	causes some pages to fall outside the dirty map here. */
159	RTGCPHYS phys_ram_dirty_size;
160	uint8_t *phys_ram_dirty;
161	#endif /* VBOX */
162
163	CPUState *first_cpu;
164	/* current CPU in the current thread. It is only valid inside
165	cpu_exec() */
166	CPUState *cpu_single_env;
167	/* 0 = Do not count executed instructions.
168	1 = Precise instruction counting.
169	2 = Adaptive rate instruction counting. */
170	int use_icount = 0;
171	/* Current instruction counter. While executing translated code this may
172	include some instructions that have not yet been executed. */
173	int64_t qemu_icount;
174
175	typedef struct PageDesc {
176	/* list of TBs intersecting this ram page */
177	TranslationBlock *first_tb;
178	/* in order to optimize self modifying code, we count the number
179	of lookups we do to a given page to use a bitmap */
180	unsigned int code_write_count;
181	uint8_t *code_bitmap;
182	#if defined(CONFIG_USER_ONLY)
183	unsigned long flags;
184	#endif
185	} PageDesc;
186
187	typedef struct PhysPageDesc {
188	/* offset in host memory of the page + io_index in the low bits */
189	ram_addr_t phys_offset;
190	ram_addr_t region_offset;
191	} PhysPageDesc;
192
193	#define L2_BITS 10
194	#if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
195	/* XXX: this is a temporary hack for alpha target.
196	* In the future, this is to be replaced by a multi-level table
197	* to actually be able to handle the complete 64 bits address space.
198	*/
199	#define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
200	#else
201	#define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
202	#endif
203	#ifdef VBOX
204	#define L0_BITS (TARGET_PHYS_ADDR_SPACE_BITS - 32)
205	#endif
206
207	#ifdef VBOX
208	#define L0_SIZE (1 << L0_BITS)
209	#endif
210	#define L1_SIZE (1 << L1_BITS)
211	#define L2_SIZE (1 << L2_BITS)
212
213	unsigned long qemu_real_host_page_size;
214	unsigned long qemu_host_page_bits;
215	unsigned long qemu_host_page_size;
216	unsigned long qemu_host_page_mask;
217
218	/* XXX: for system emulation, it could just be an array */
219	#ifndef VBOX
220	static PageDesc *l1_map[L1_SIZE];
221	static PhysPageDesc **l1_phys_map;
222	#else
223	static unsigned l0_map_max_used = 0;
224	static PageDesc **l0_map[L0_SIZE];
225	static void **l0_phys_map[L0_SIZE];
226	#endif
227
228	#if !defined(CONFIG_USER_ONLY)
229	static void io_mem_init(void);
230
231	/* io memory support */
232	CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
233	CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
234	void *io_mem_opaque[IO_MEM_NB_ENTRIES];
235	static char io_mem_used[IO_MEM_NB_ENTRIES];
236	static int io_mem_watch;
237	#endif
238
239	#ifndef VBOX
240	/* log support */
241	static const char *logfilename = "/tmp/qemu.log";
242	#endif /* !VBOX */
243	FILE *logfile;
244	int loglevel;
245	#ifndef VBOX
246	static int log_append = 0;
247	#endif
248
249	/* statistics */
250	#ifndef VBOX
251	static int tlb_flush_count;
252	static int tb_flush_count;
253	static int tb_phys_invalidate_count;
254	#else /* VBOX - Resettable U32 stats, see VBoxRecompiler.c. */
255	uint32_t tlb_flush_count;
256	uint32_t tb_flush_count;
257	uint32_t tb_phys_invalidate_count;
258	#endif /* VBOX */
259
260	#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
261	typedef struct subpage_t {
262	target_phys_addr_t base;
263	CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
264	CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
265	void *opaque[TARGET_PAGE_SIZE][2][4];
266	ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
267	} subpage_t;
268
269	#ifndef VBOX
270	#ifdef _WIN32
271	static void map_exec(void *addr, long size)
272	{
273	DWORD old_protect;
274	VirtualProtect(addr, size,
275	PAGE_EXECUTE_READWRITE, &old_protect);
276
277	}
278	#else
279	static void map_exec(void *addr, long size)
280	{
281	unsigned long start, end, page_size;
282
283	page_size = getpagesize();
284	start = (unsigned long)addr;
285	start &= ~(page_size - 1);
286
287	end = (unsigned long)addr + size;
288	end += page_size - 1;
289	end &= ~(page_size - 1);
290
291	mprotect((void *)start, end - start,
292	PROT_READ \| PROT_WRITE \| PROT_EXEC);
293	}
294	#endif
295	#else /* VBOX */
296	static void map_exec(void *addr, long size)
297	{
298	RTMemProtect(addr, size,
299	RTMEM_PROT_EXEC \| RTMEM_PROT_READ \| RTMEM_PROT_WRITE);
300	}
301	#endif /* VBOX */
302
303	static void page_init(void)
304	{
305	/* NOTE: we can always suppose that qemu_host_page_size >=
306	TARGET_PAGE_SIZE */
307	#ifdef VBOX
308	RTMemProtect(code_gen_buffer, sizeof(code_gen_buffer),
309	RTMEM_PROT_EXEC \| RTMEM_PROT_READ \| RTMEM_PROT_WRITE);
310	qemu_real_host_page_size = PAGE_SIZE;
311	#else /* !VBOX */
312	#ifdef _WIN32
313	{
314	SYSTEM_INFO system_info;
315
316	GetSystemInfo(&system_info);
317	qemu_real_host_page_size = system_info.dwPageSize;
318	}
319	#else
320	qemu_real_host_page_size = getpagesize();
321	#endif
322	#endif /* !VBOX */
323	if (qemu_host_page_size == 0)
324	qemu_host_page_size = qemu_real_host_page_size;
325	if (qemu_host_page_size < TARGET_PAGE_SIZE)
326	qemu_host_page_size = TARGET_PAGE_SIZE;
327	qemu_host_page_bits = 0;
328	#ifndef VBOX
329	while ((1 << qemu_host_page_bits) < qemu_host_page_size)
330	#else
331	while ((1 << qemu_host_page_bits) < (int)qemu_host_page_size)
332	#endif
333	qemu_host_page_bits++;
334	qemu_host_page_mask = ~(qemu_host_page_size - 1);
335	#ifndef VBOX
336	l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
337	memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
338	#endif
339
340	#ifdef VBOX
341	/* We use other means to set reserved bit on our pages */
342	#else /* !VBOX */
343	#if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
344	{
345	long long startaddr, endaddr;
346	FILE *f;
347	int n;
348
349	mmap_lock();
350	last_brk = (unsigned long)sbrk(0);
351	f = fopen("/proc/self/maps", "r");
352	if (f) {
353	do {
354	n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
355	if (n == 2) {
356	startaddr = MIN(startaddr,
357	(1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
358	endaddr = MIN(endaddr,
359	(1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
360	page_set_flags(startaddr & TARGET_PAGE_MASK,
361	TARGET_PAGE_ALIGN(endaddr),
362	PAGE_RESERVED);
363	}
364	} while (!feof(f));
365	fclose(f);
366	}
367	mmap_unlock();
368	}
369	#endif
370	#endif /* !VBOX */
371	}
372
373	static inline PageDesc **page_l1_map(target_ulong index)
374	{
375	#ifndef VBOX
376	#if TARGET_LONG_BITS > 32
377	/* Host memory outside guest VM. For 32-bit targets we have already
378	excluded high addresses. */
379	if (index > ((target_ulong)L2_SIZE * L1_SIZE))
380	return NULL;
381	#endif
382	return &l1_map[index >> L2_BITS];
383	#else /* VBOX */
384	PageDesc **l1_map;
385	AssertMsgReturn(index < (target_ulong)L2_SIZE * L1_SIZE * L0_SIZE,
386	("index=%RGp >= %RGp; L1_SIZE=%#x L2_SIZE=%#x L0_SIZE=%#x\n",
387	(RTGCPHYS)index, (RTGCPHYS)L2_SIZE * L1_SIZE, L1_SIZE, L2_SIZE, L0_SIZE),
388	NULL);
389	l1_map = l0_map[index >> (L1_BITS + L2_BITS)];
390	if (RT_UNLIKELY(!l1_map))
391	{
392	unsigned i0 = index >> (L1_BITS + L2_BITS);
393	l0_map[i0] = l1_map = qemu_mallocz(sizeof(PageDesc ) L1_SIZE);
394	if (RT_UNLIKELY(!l1_map))
395	return NULL;
396	if (i0 >= l0_map_max_used)
397	l0_map_max_used = i0 + 1;
398	}
399	return &l1_map[(index >> L2_BITS) & (L1_SIZE - 1)];
400	#endif /* VBOX */
401	}
402
403	static inline PageDesc *page_find_alloc(target_ulong index)
404	{
405	PageDesc *lp, p;
406	lp = page_l1_map(index);
407	if (!lp)
408	return NULL;
409
410	p = *lp;
411	if (!p) {
412	/* allocate if not found */
413	#if defined(CONFIG_USER_ONLY)
414	size_t len = sizeof(PageDesc) * L2_SIZE;
415	/* Don't use qemu_malloc because it may recurse. */
416	p = mmap(NULL, len, PROT_READ \| PROT_WRITE,
417	MAP_PRIVATE \| MAP_ANONYMOUS, -1, 0);
418	*lp = p;
419	if (h2g_valid(p)) {
420	unsigned long addr = h2g(p);
421	page_set_flags(addr & TARGET_PAGE_MASK,
422	TARGET_PAGE_ALIGN(addr + len),
423	PAGE_RESERVED);
424	}
425	#else
426	p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
427	*lp = p;
428	#endif
429	}
430	return p + (index & (L2_SIZE - 1));
431	}
432
433	static inline PageDesc *page_find(target_ulong index)
434	{
435	PageDesc *lp, p;
436	lp = page_l1_map(index);
437	if (!lp)
438	return NULL;
439
440	p = *lp;
441	if (!p) {
442	return NULL;
443	}
444	return p + (index & (L2_SIZE - 1));
445	}
446
447	static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
448	{
449	void lp, p;
450	PhysPageDesc *pd;
451
452	#ifndef VBOX
453	p = (void **)l1_phys_map;
454	#if TARGET_PHYS_ADDR_SPACE_BITS > 32
455
456	#if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
457	#error unsupported TARGET_PHYS_ADDR_SPACE_BITS
458	#endif
459	lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
460	p = *lp;
461	if (!p) {
462	/* allocate if not found */
463	if (!alloc)
464	return NULL;
465	p = qemu_vmalloc(sizeof(void ) L1_SIZE);
466	memset(p, 0, sizeof(void ) L1_SIZE);
467	*lp = p;
468	}
469	#endif
470	#else /* VBOX */
471	/* level 0 lookup and lazy allocation of level 1 map. */
472	if (RT_UNLIKELY(index >= (target_phys_addr_t)L2_SIZE * L1_SIZE * L0_SIZE))
473	return NULL;
474	p = l0_phys_map[index >> (L1_BITS + L2_BITS)];
475	if (RT_UNLIKELY(!p)) {
476	if (!alloc)
477	return NULL;
478	p = qemu_vmalloc(sizeof(void *) L1_SIZE);
479	memset(p, 0, sizeof(void *) L1_SIZE);
480	l0_phys_map[index >> (L1_BITS + L2_BITS)] = p;
481	}
482
483	/* level 1 lookup and lazy allocation of level 2 map. */
484	#endif /* VBOX */
485	lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
486	pd = *lp;
487	if (!pd) {
488	int i;
489	/* allocate if not found */
490	if (!alloc)
491	return NULL;
492	pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
493	*lp = pd;
494	for (i = 0; i < L2_SIZE; i++) {
495	pd[i].phys_offset = IO_MEM_UNASSIGNED;
496	pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
497	}
498	}
499	return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
500	}
501
502	static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
503	{
504	return phys_page_find_alloc(index, 0);
505	}
506
507	#if !defined(CONFIG_USER_ONLY)
508	static void tlb_protect_code(ram_addr_t ram_addr);
509	static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
510	target_ulong vaddr);
511	#define mmap_lock() do { } while(0)
512	#define mmap_unlock() do { } while(0)
513	#endif
514
515	#ifdef VBOX /* We don't need such huge codegen buffer size, as execute
516	most of the code in raw or hwacc mode. */
517	#define DEFAULT_CODE_GEN_BUFFER_SIZE (8 * 1024 * 1024)
518	#else /* !VBOX */
519	#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
520	#endif /* !VBOX */
521
522	#if defined(CONFIG_USER_ONLY)
523	/* Currently it is not recommended to allocate big chunks of data in
524	user mode. It will change when a dedicated libc will be used */
525	#define USE_STATIC_CODE_GEN_BUFFER
526	#endif
527
528	#if defined(VBOX) && defined(USE_STATIC_CODE_GEN_BUFFER)
529	# error "VBox allocates codegen buffer dynamically"
530	#endif
531
532	#ifdef USE_STATIC_CODE_GEN_BUFFER
533	static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
534	#endif
535
536	static void code_gen_alloc(unsigned long tb_size)
537	{
538	#ifdef USE_STATIC_CODE_GEN_BUFFER
539	code_gen_buffer = static_code_gen_buffer;
540	code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
541	map_exec(code_gen_buffer, code_gen_buffer_size);
542	#else
543	# ifdef VBOX
544	/* We cannot use phys_ram_size here, as it's 0 now,
545	* it only gets initialized once RAM registration callback
546	* (REMR3NotifyPhysRamRegister()) called.
547	*/
548	code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
549	# else /* !VBOX */
550	code_gen_buffer_size = tb_size;
551	if (code_gen_buffer_size == 0) {
552	#if defined(CONFIG_USER_ONLY)
553	/* in user mode, phys_ram_size is not meaningful */
554	code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
555	#else
556	/* XXX: needs ajustments */
557	code_gen_buffer_size = (unsigned long)(phys_ram_size / 4);
558	#endif
559	}
560	if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
561	code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
562	# endif /* !VBOX */
563	/* The code gen buffer location may have constraints depending on
564	the host cpu and OS */
565	# ifdef VBOX
566	code_gen_buffer = RTMemExecAlloc(code_gen_buffer_size);
567
568	if (!code_gen_buffer) {
569	LogRel(("REM: failed allocate codegen buffer %lld\n",
570	code_gen_buffer_size));
571	return;
572	}
573	# else /* !VBOX */
574	#if defined(__linux__)
575	{
576	int flags;
577	void *start = NULL;
578
579	flags = MAP_PRIVATE \| MAP_ANONYMOUS;
580	#if defined(__x86_64__)
581	flags \|= MAP_32BIT;
582	/* Cannot map more than that */
583	if (code_gen_buffer_size > (800 * 1024 * 1024))
584	code_gen_buffer_size = (800 * 1024 * 1024);
585	#elif defined(__sparc_v9__)
586	// Map the buffer below 2G, so we can use direct calls and branches
587	flags \|= MAP_FIXED;
588	start = (void *) 0x60000000UL;
589	if (code_gen_buffer_size > (512 * 1024 * 1024))
590	code_gen_buffer_size = (512 * 1024 * 1024);
591	#elif defined(__arm__)
592	/* Map the buffer below 32M, so we can use direct calls and branches */
593	flags \|= MAP_FIXED;
594	start = (void *) 0x01000000UL;
595	if (code_gen_buffer_size > 16 * 1024 * 1024)
596	code_gen_buffer_size = 16 * 1024 * 1024;
597	#endif
598	code_gen_buffer = mmap(start, code_gen_buffer_size,
599	PROT_WRITE \| PROT_READ \| PROT_EXEC,
600	flags, -1, 0);
601	if (code_gen_buffer == MAP_FAILED) {
602	fprintf(stderr, "Could not allocate dynamic translator buffer\n");
603	exit(1);
604	}
605	}
606	#elif defined(__FreeBSD__) \|\| defined(__DragonFly__)
607	{
608	int flags;
609	void *addr = NULL;
610	flags = MAP_PRIVATE \| MAP_ANONYMOUS;
611	#if defined(__x86_64__)
612	/* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
613	* 0x40000000 is free */
614	flags \|= MAP_FIXED;
615	addr = (void *)0x40000000;
616	/* Cannot map more than that */
617	if (code_gen_buffer_size > (800 * 1024 * 1024))
618	code_gen_buffer_size = (800 * 1024 * 1024);
619	#endif
620	code_gen_buffer = mmap(addr, code_gen_buffer_size,
621	PROT_WRITE \| PROT_READ \| PROT_EXEC,
622	flags, -1, 0);
623	if (code_gen_buffer == MAP_FAILED) {
624	fprintf(stderr, "Could not allocate dynamic translator buffer\n");
625	exit(1);
626	}
627	}
628	#else
629	code_gen_buffer = qemu_malloc(code_gen_buffer_size);
630	map_exec(code_gen_buffer, code_gen_buffer_size);
631	#endif
632	# endif /* !VBOX */
633	#endif /* !USE_STATIC_CODE_GEN_BUFFER */
634	#ifndef VBOX
635	map_exec(code_gen_prologue, sizeof(code_gen_prologue));
636	#else
637	map_exec(code_gen_prologue, _1K);
638	#endif
639	code_gen_buffer_max_size = code_gen_buffer_size -
640	code_gen_max_block_size();
641	code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
642	tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
643	}
644
645	/* Must be called before using the QEMU cpus. 'tb_size' is the size
646	(in bytes) allocated to the translation buffer. Zero means default
647	size. */
648	void cpu_exec_init_all(unsigned long tb_size)
649	{
650	cpu_gen_init();
651	code_gen_alloc(tb_size);
652	code_gen_ptr = code_gen_buffer;
653	page_init();
654	#if !defined(CONFIG_USER_ONLY)
655	io_mem_init();
656	#endif
657	}
658
659	#ifndef VBOX
660	#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
661
662	#define CPU_COMMON_SAVE_VERSION 1
663
664	static void cpu_common_save(QEMUFile f, void opaque)
665	{
666	CPUState *env = opaque;
667
668	cpu_synchronize_state(env, 0);
669
670	qemu_put_be32s(f, &env->halted);
671	qemu_put_be32s(f, &env->interrupt_request);
672	}
673
674	static int cpu_common_load(QEMUFile f, void opaque, int version_id)
675	{
676	CPUState *env = opaque;
677
678	if (version_id != CPU_COMMON_SAVE_VERSION)
679	return -EINVAL;
680
681	qemu_get_be32s(f, &env->halted);
682	qemu_get_be32s(f, &env->interrupt_request);
683	/* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
684	version_id is increased. */
685	env->interrupt_request &= ~0x01;
686	tlb_flush(env, 1);
687	cpu_synchronize_state(env, 1);
688
689	return 0;
690	}
691	#endif
692
693	CPUState *qemu_get_cpu(int cpu)
694	{
695	CPUState *env = first_cpu;
696
697	while (env) {
698	if (env->cpu_index == cpu)
699	break;
700	env = env->next_cpu;
701	}
702
703	return env;
704	}
705
706	#endif /* !VBOX */
707
708	void cpu_exec_init(CPUState *env)
709	{
710	CPUState **penv;
711	int cpu_index;
712
713	#if defined(CONFIG_USER_ONLY)
714	cpu_list_lock();
715	#endif
716	env->next_cpu = NULL;
717	penv = &first_cpu;
718	cpu_index = 0;
719	while (*penv != NULL) {
720	penv = &(*penv)->next_cpu;
721	cpu_index++;
722	}
723	env->cpu_index = cpu_index;
724	env->numa_node = 0;
725	TAILQ_INIT(&env->breakpoints);
726	TAILQ_INIT(&env->watchpoints);
727	*penv = env;
728	#ifndef VBOX
729	#if defined(CONFIG_USER_ONLY)
730	cpu_list_unlock();
731	#endif
732	#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
733	register_savevm("cpu_common", cpu_index, CPU_COMMON_SAVE_VERSION,
734	cpu_common_save, cpu_common_load, env);
735	register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
736	cpu_save, cpu_load, env);
737	#endif
738	#endif /* !VBOX */
739	}
740
741	static inline void invalidate_page_bitmap(PageDesc *p)
742	{
743	if (p->code_bitmap) {
744	qemu_free(p->code_bitmap);
745	p->code_bitmap = NULL;
746	}
747	p->code_write_count = 0;
748	}
749
750	/* set to NULL all the 'first_tb' fields in all PageDescs */
751	static void page_flush_tb(void)
752	{
753	int i, j;
754	PageDesc *p;
755	#ifdef VBOX
756	int k;
757	#endif
758
759	#ifdef VBOX
760	k = l0_map_max_used;
761	while (k-- > 0) {
762	PageDesc **l1_map = l0_map[k];
763	if (l1_map) {
764	#endif
765	for(i = 0; i < L1_SIZE; i++) {
766	p = l1_map[i];
767	if (p) {
768	for(j = 0; j < L2_SIZE; j++) {
769	p->first_tb = NULL;
770	invalidate_page_bitmap(p);
771	p++;
772	}
773	}
774	}
775	#ifdef VBOX
776	}
777	}
778	#endif
779	}
780
781	/* flush all the translation blocks */
782	/* XXX: tb_flush is currently not thread safe */
783	void tb_flush(CPUState *env1)
784	{
785	CPUState *env;
786	#ifdef VBOX
787	STAM_PROFILE_START(&env1->StatTbFlush, a);
788	#endif
789	#if defined(DEBUG_FLUSH)
790	printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
791	(unsigned long)(code_gen_ptr - code_gen_buffer),
792	nb_tbs, nb_tbs > 0 ?
793	((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
794	#endif
795	if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
796	cpu_abort(env1, "Internal error: code buffer overflow\n");
797
798	nb_tbs = 0;
799
800	for(env = first_cpu; env != NULL; env = env->next_cpu) {
801	memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
802	}
803
804	memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
805	page_flush_tb();
806
807	code_gen_ptr = code_gen_buffer;
808	/* XXX: flush processor icache at this point if cache flush is
809	expensive */
810	tb_flush_count++;
811	#ifdef VBOX
812	STAM_PROFILE_STOP(&env1->StatTbFlush, a);
813	#endif
814	}
815
816	#ifdef DEBUG_TB_CHECK
817
818	static void tb_invalidate_check(target_ulong address)
819	{
820	TranslationBlock *tb;
821	int i;
822	address &= TARGET_PAGE_MASK;
823	for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
824	for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
825	if (!(address + TARGET_PAGE_SIZE <= tb->pc \|\|
826	address >= tb->pc + tb->size)) {
827	printf("ERROR invalidate: address=" TARGET_FMT_lx
828	" PC=%08lx size=%04x\n",
829	address, (long)tb->pc, tb->size);
830	}
831	}
832	}
833	}
834
835	/* verify that all the pages have correct rights for code */
836	static void tb_page_check(void)
837	{
838	TranslationBlock *tb;
839	int i, flags1, flags2;
840
841	for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
842	for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
843	flags1 = page_get_flags(tb->pc);
844	flags2 = page_get_flags(tb->pc + tb->size - 1);
845	if ((flags1 & PAGE_WRITE) \|\| (flags2 & PAGE_WRITE)) {
846	printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
847	(long)tb->pc, tb->size, flags1, flags2);
848	}
849	}
850	}
851	}
852
853	#endif
854
855	/* invalidate one TB */
856	static inline void tb_remove(TranslationBlock *ptb, TranslationBlock tb,
857	int next_offset)
858	{
859	TranslationBlock *tb1;
860	for(;;) {
861	tb1 = *ptb;
862	if (tb1 == tb) {
863	ptb = (TranslationBlock *)((char )tb1 + next_offset);
864	break;
865	}
866	ptb = (TranslationBlock *)((char )tb1 + next_offset);
867	}
868	}
869
870	static inline void tb_page_remove(TranslationBlock *ptb, TranslationBlock tb)
871	{
872	TranslationBlock *tb1;
873	unsigned int n1;
874
875	for(;;) {
876	tb1 = *ptb;
877	n1 = (long)tb1 & 3;
878	tb1 = (TranslationBlock *)((long)tb1 & ~3);
879	if (tb1 == tb) {
880	*ptb = tb1->page_next[n1];
881	break;
882	}
883	ptb = &tb1->page_next[n1];
884	}
885	}
886
887	static inline void tb_jmp_remove(TranslationBlock *tb, int n)
888	{
889	TranslationBlock tb1, *ptb;
890	unsigned int n1;
891
892	ptb = &tb->jmp_next[n];
893	tb1 = *ptb;
894	if (tb1) {
895	/* find tb(n) in circular list */
896	for(;;) {
897	tb1 = *ptb;
898	n1 = (long)tb1 & 3;
899	tb1 = (TranslationBlock *)((long)tb1 & ~3);
900	if (n1 == n && tb1 == tb)
901	break;
902	if (n1 == 2) {
903	ptb = &tb1->jmp_first;
904	} else {
905	ptb = &tb1->jmp_next[n1];
906	}
907	}
908	/* now we can suppress tb(n) from the list */
909	*ptb = tb->jmp_next[n];
910
911	tb->jmp_next[n] = NULL;
912	}
913	}
914
915	/* reset the jump entry 'n' of a TB so that it is not chained to
916	another TB */
917	static inline void tb_reset_jump(TranslationBlock *tb, int n)
918	{
919	tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
920	}
921
922	void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
923	{
924	CPUState *env;
925	PageDesc *p;
926	unsigned int h, n1;
927	target_phys_addr_t phys_pc;
928	TranslationBlock tb1, tb2;
929
930	/* remove the TB from the hash list */
931	phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
932	h = tb_phys_hash_func(phys_pc);
933	tb_remove(&tb_phys_hash[h], tb,
934	offsetof(TranslationBlock, phys_hash_next));
935
936	/* remove the TB from the page list */
937	if (tb->page_addr[0] != page_addr) {
938	p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
939	tb_page_remove(&p->first_tb, tb);
940	invalidate_page_bitmap(p);
941	}
942	if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
943	p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
944	tb_page_remove(&p->first_tb, tb);
945	invalidate_page_bitmap(p);
946	}
947
948	tb_invalidated_flag = 1;
949
950	/* remove the TB from the hash list */
951	h = tb_jmp_cache_hash_func(tb->pc);
952	for(env = first_cpu; env != NULL; env = env->next_cpu) {
953	if (env->tb_jmp_cache[h] == tb)
954	env->tb_jmp_cache[h] = NULL;
955	}
956
957	/* suppress this TB from the two jump lists */
958	tb_jmp_remove(tb, 0);
959	tb_jmp_remove(tb, 1);
960
961	/* suppress any remaining jumps to this TB */
962	tb1 = tb->jmp_first;
963	for(;;) {
964	n1 = (long)tb1 & 3;
965	if (n1 == 2)
966	break;
967	tb1 = (TranslationBlock *)((long)tb1 & ~3);
968	tb2 = tb1->jmp_next[n1];
969	tb_reset_jump(tb1, n1);
970	tb1->jmp_next[n1] = NULL;
971	tb1 = tb2;
972	}
973	tb->jmp_first = (TranslationBlock )((long)tb \| 2); / fail safe */
974
975	tb_phys_invalidate_count++;
976	}
977
978	#ifdef VBOX
979
980	void tb_invalidate_virt(CPUState *env, uint32_t eip)
981	{
982	# if 1
983	tb_flush(env);
984	# else
985	uint8_t cs_base, pc;
986	unsigned int flags, h, phys_pc;
987	TranslationBlock tb, *ptb;
988
989	flags = env->hflags;
990	flags \|= (env->eflags & (IOPL_MASK \| TF_MASK \| VM_MASK));
991	cs_base = env->segs[R_CS].base;
992	pc = cs_base + eip;
993
994	tb = tb_find(&ptb, (unsigned long)pc, (unsigned long)cs_base,
995	flags);
996
997	if(tb)
998	{
999	# ifdef DEBUG
1000	printf("invalidating TB (%08X) at %08X\n", tb, eip);
1001	# endif
1002	tb_invalidate(tb);
1003	//Note: this will leak TBs, but the whole cache will be flushed
1004	// when it happens too often
1005	tb->pc = 0;
1006	tb->cs_base = 0;
1007	tb->flags = 0;
1008	}
1009	# endif
1010	}
1011
1012	# ifdef VBOX_STRICT
1013	/**
1014	* Gets the page offset.
1015	*/
1016	unsigned long get_phys_page_offset(target_ulong addr)
1017	{
1018	PhysPageDesc *p = phys_page_find(addr >> TARGET_PAGE_BITS);
1019	return p ? p->phys_offset : 0;
1020	}
1021	# endif /* VBOX_STRICT */
1022
1023	#endif /* VBOX */
1024
1025	static inline void set_bits(uint8_t *tab, int start, int len)
1026	{
1027	int end, mask, end1;
1028
1029	end = start + len;
1030	tab += start >> 3;
1031	mask = 0xff << (start & 7);
1032	if ((start & ~7) == (end & ~7)) {
1033	if (start < end) {
1034	mask &= ~(0xff << (end & 7));
1035	*tab \|= mask;
1036	}
1037	} else {
1038	*tab++ \|= mask;
1039	start = (start + 8) & ~7;
1040	end1 = end & ~7;
1041	while (start < end1) {
1042	*tab++ = 0xff;
1043	start += 8;
1044	}
1045	if (start < end) {
1046	mask = ~(0xff << (end & 7));
1047	*tab \|= mask;
1048	}
1049	}
1050	}
1051
1052	static void build_page_bitmap(PageDesc *p)
1053	{
1054	int n, tb_start, tb_end;
1055	TranslationBlock *tb;
1056
1057	p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
1058
1059	tb = p->first_tb;
1060	while (tb != NULL) {
1061	n = (long)tb & 3;
1062	tb = (TranslationBlock *)((long)tb & ~3);
1063	/* NOTE: this is subtle as a TB may span two physical pages */
1064	if (n == 0) {
1065	/* NOTE: tb_end may be after the end of the page, but
1066	it is not a problem */
1067	tb_start = tb->pc & ~TARGET_PAGE_MASK;
1068	tb_end = tb_start + tb->size;
1069	if (tb_end > TARGET_PAGE_SIZE)
1070	tb_end = TARGET_PAGE_SIZE;
1071	} else {
1072	tb_start = 0;
1073	tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1074	}
1075	set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
1076	tb = tb->page_next[n];
1077	}
1078	}
1079
1080	TranslationBlock tb_gen_code(CPUState env,
1081	target_ulong pc, target_ulong cs_base,
1082	int flags, int cflags)
1083	{
1084	TranslationBlock *tb;
1085	uint8_t *tc_ptr;
1086	target_ulong phys_pc, phys_page2, virt_page2;
1087	int code_gen_size;
1088
1089	phys_pc = get_phys_addr_code(env, pc);
1090	tb = tb_alloc(pc);
1091	if (!tb) {
1092	/* flush must be done */
1093	tb_flush(env);
1094	/* cannot fail at this point */
1095	tb = tb_alloc(pc);
1096	/* Don't forget to invalidate previous TB info. */
1097	tb_invalidated_flag = 1;
1098	}
1099	tc_ptr = code_gen_ptr;
1100	tb->tc_ptr = tc_ptr;
1101	tb->cs_base = cs_base;
1102	tb->flags = flags;
1103	tb->cflags = cflags;
1104	cpu_gen_code(env, tb, &code_gen_size);
1105	code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
1106
1107	/* check next page if needed */
1108	virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1109	phys_page2 = -1;
1110	if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1111	phys_page2 = get_phys_addr_code(env, virt_page2);
1112	}
1113	tb_link_phys(tb, phys_pc, phys_page2);
1114	return tb;
1115	}
1116
1117	/* invalidate all TBs which intersect with the target physical page
1118	starting in range [start;end[. NOTE: start and end must refer to
1119	the same physical page. 'is_cpu_write_access' should be true if called
1120	from a real cpu write access: the virtual CPU will exit the current
1121	TB if code is modified inside this TB. */
1122	void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
1123	int is_cpu_write_access)
1124	{
1125	TranslationBlock tb, tb_next, *saved_tb;
1126	CPUState *env = cpu_single_env;
1127	target_ulong tb_start, tb_end;
1128	PageDesc *p;
1129	int n;
1130	#ifdef TARGET_HAS_PRECISE_SMC
1131	int current_tb_not_found = is_cpu_write_access;
1132	TranslationBlock *current_tb = NULL;
1133	int current_tb_modified = 0;
1134	target_ulong current_pc = 0;
1135	target_ulong current_cs_base = 0;
1136	int current_flags = 0;
1137	#endif /* TARGET_HAS_PRECISE_SMC */
1138
1139	p = page_find(start >> TARGET_PAGE_BITS);
1140	if (!p)
1141	return;
1142	if (!p->code_bitmap &&
1143	++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1144	is_cpu_write_access) {
1145	/* build code bitmap */
1146	build_page_bitmap(p);
1147	}
1148
1149	/* we remove all the TBs in the range [start, end[ */
1150	/* XXX: see if in some cases it could be faster to invalidate all the code */
1151	tb = p->first_tb;
1152	while (tb != NULL) {
1153	n = (long)tb & 3;
1154	tb = (TranslationBlock *)((long)tb & ~3);
1155	tb_next = tb->page_next[n];
1156	/* NOTE: this is subtle as a TB may span two physical pages */
1157	if (n == 0) {
1158	/* NOTE: tb_end may be after the end of the page, but
1159	it is not a problem */
1160	tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1161	tb_end = tb_start + tb->size;
1162	} else {
1163	tb_start = tb->page_addr[1];
1164	tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1165	}
1166	if (!(tb_end <= start \|\| tb_start >= end)) {
1167	#ifdef TARGET_HAS_PRECISE_SMC
1168	if (current_tb_not_found) {
1169	current_tb_not_found = 0;
1170	current_tb = NULL;
1171	if (env->mem_io_pc) {
1172	/* now we have a real cpu fault */
1173	current_tb = tb_find_pc(env->mem_io_pc);
1174	}
1175	}
1176	if (current_tb == tb &&
1177	(current_tb->cflags & CF_COUNT_MASK) != 1) {
1178	/* If we are modifying the current TB, we must stop
1179	its execution. We could be more precise by checking
1180	that the modification is after the current PC, but it
1181	would require a specialized function to partially
1182	restore the CPU state */
1183
1184	current_tb_modified = 1;
1185	cpu_restore_state(current_tb, env,
1186	env->mem_io_pc, NULL);
1187	cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1188	&current_flags);
1189	}
1190	#endif /* TARGET_HAS_PRECISE_SMC */
1191	/* we need to do that to handle the case where a signal
1192	occurs while doing tb_phys_invalidate() */
1193	saved_tb = NULL;
1194	if (env) {
1195	saved_tb = env->current_tb;
1196	env->current_tb = NULL;
1197	}
1198	tb_phys_invalidate(tb, -1);
1199	if (env) {
1200	env->current_tb = saved_tb;
1201	if (env->interrupt_request && env->current_tb)
1202	cpu_interrupt(env, env->interrupt_request);
1203	}
1204	}
1205	tb = tb_next;
1206	}
1207	#if !defined(CONFIG_USER_ONLY)
1208	/* if no code remaining, no need to continue to use slow writes */
1209	if (!p->first_tb) {
1210	invalidate_page_bitmap(p);
1211	if (is_cpu_write_access) {
1212	tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1213	}
1214	}
1215	#endif
1216	#ifdef TARGET_HAS_PRECISE_SMC
1217	if (current_tb_modified) {
1218	/* we generate a block containing just the instruction
1219	modifying the memory. It will ensure that it cannot modify
1220	itself */
1221	env->current_tb = NULL;
1222	tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1223	cpu_resume_from_signal(env, NULL);
1224	}
1225	#endif
1226	}
1227
1228	/* len must be <= 8 and start must be a multiple of len */
1229	static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1230	{
1231	PageDesc *p;
1232	int offset, b;
1233	#if 0
1234	if (1) {
1235	qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1236	cpu_single_env->mem_io_vaddr, len,
1237	cpu_single_env->eip,
1238	cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1239	}
1240	#endif
1241	p = page_find(start >> TARGET_PAGE_BITS);
1242	if (!p)
1243	return;
1244	if (p->code_bitmap) {
1245	offset = start & ~TARGET_PAGE_MASK;
1246	b = p->code_bitmap[offset >> 3] >> (offset & 7);
1247	if (b & ((1 << len) - 1))
1248	goto do_invalidate;
1249	} else {
1250	do_invalidate:
1251	tb_invalidate_phys_page_range(start, start + len, 1);
1252	}
1253	}
1254
1255	#if !defined(CONFIG_SOFTMMU)
1256	static void tb_invalidate_phys_page(target_phys_addr_t addr,
1257	unsigned long pc, void *puc)
1258	{
1259	TranslationBlock *tb;
1260	PageDesc *p;
1261	int n;
1262	#ifdef TARGET_HAS_PRECISE_SMC
1263	TranslationBlock *current_tb = NULL;
1264	CPUState *env = cpu_single_env;
1265	int current_tb_modified = 0;
1266	target_ulong current_pc = 0;
1267	target_ulong current_cs_base = 0;
1268	int current_flags = 0;
1269	#endif
1270
1271	addr &= TARGET_PAGE_MASK;
1272	p = page_find(addr >> TARGET_PAGE_BITS);
1273	if (!p)
1274	return;
1275	tb = p->first_tb;
1276	#ifdef TARGET_HAS_PRECISE_SMC
1277	if (tb && pc != 0) {
1278	current_tb = tb_find_pc(pc);
1279	}
1280	#endif
1281	while (tb != NULL) {
1282	n = (long)tb & 3;
1283	tb = (TranslationBlock *)((long)tb & ~3);
1284	#ifdef TARGET_HAS_PRECISE_SMC
1285	if (current_tb == tb &&
1286	(current_tb->cflags & CF_COUNT_MASK) != 1) {
1287	/* If we are modifying the current TB, we must stop
1288	its execution. We could be more precise by checking
1289	that the modification is after the current PC, but it
1290	would require a specialized function to partially
1291	restore the CPU state */
1292
1293	current_tb_modified = 1;
1294	cpu_restore_state(current_tb, env, pc, puc);
1295	cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1296	&current_flags);
1297	}
1298	#endif /* TARGET_HAS_PRECISE_SMC */
1299	tb_phys_invalidate(tb, addr);
1300	tb = tb->page_next[n];
1301	}
1302	p->first_tb = NULL;
1303	#ifdef TARGET_HAS_PRECISE_SMC
1304	if (current_tb_modified) {
1305	/* we generate a block containing just the instruction
1306	modifying the memory. It will ensure that it cannot modify
1307	itself */
1308	env->current_tb = NULL;
1309	tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1310	cpu_resume_from_signal(env, puc);
1311	}
1312	#endif
1313	}
1314	#endif
1315
1316	/* add the tb in the target page and protect it if necessary */
1317	static inline void tb_alloc_page(TranslationBlock *tb,
1318	unsigned int n, target_ulong page_addr)
1319	{
1320	PageDesc *p;
1321	TranslationBlock *last_first_tb;
1322
1323	tb->page_addr[n] = page_addr;
1324	p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1325	tb->page_next[n] = p->first_tb;
1326	last_first_tb = p->first_tb;
1327	p->first_tb = (TranslationBlock *)((long)tb \| n);
1328	invalidate_page_bitmap(p);
1329
1330	#if defined(TARGET_HAS_SMC) \|\| 1
1331
1332	#if defined(CONFIG_USER_ONLY)
1333	if (p->flags & PAGE_WRITE) {
1334	target_ulong addr;
1335	PageDesc *p2;
1336	int prot;
1337
1338	/* force the host page as non writable (writes will have a
1339	page fault + mprotect overhead) */
1340	page_addr &= qemu_host_page_mask;
1341	prot = 0;
1342	for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1343	addr += TARGET_PAGE_SIZE) {
1344
1345	p2 = page_find (addr >> TARGET_PAGE_BITS);
1346	if (!p2)
1347	continue;
1348	prot \|= p2->flags;
1349	p2->flags &= ~PAGE_WRITE;
1350	page_get_flags(addr);
1351	}
1352	mprotect(g2h(page_addr), qemu_host_page_size,
1353	(prot & PAGE_BITS) & ~PAGE_WRITE);
1354	#ifdef DEBUG_TB_INVALIDATE
1355	printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1356	page_addr);
1357	#endif
1358	}
1359	#else
1360	/* if some code is already present, then the pages are already
1361	protected. So we handle the case where only the first TB is
1362	allocated in a physical page */
1363	if (!last_first_tb) {
1364	tlb_protect_code(page_addr);
1365	}
1366	#endif
1367
1368	#endif /* TARGET_HAS_SMC */
1369	}
1370
1371	/* Allocate a new translation block. Flush the translation buffer if
1372	too many translation blocks or too much generated code. */
1373	TranslationBlock *tb_alloc(target_ulong pc)
1374	{
1375	TranslationBlock *tb;
1376
1377	if (nb_tbs >= code_gen_max_blocks \|\|
1378	#ifndef VBOX
1379	(code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1380	#else
1381	(code_gen_ptr - code_gen_buffer) >= (int)code_gen_buffer_max_size)
1382	#endif
1383	return NULL;
1384	tb = &tbs[nb_tbs++];
1385	tb->pc = pc;
1386	tb->cflags = 0;
1387	return tb;
1388	}
1389
1390	void tb_free(TranslationBlock *tb)
1391	{
1392	/* In practice this is mostly used for single use temporary TB
1393	Ignore the hard cases and just back up if this TB happens to
1394	be the last one generated. */
1395	if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1396	code_gen_ptr = tb->tc_ptr;
1397	nb_tbs--;
1398	}
1399	}
1400
1401	/* add a new TB and link it to the physical page tables. phys_page2 is
1402	(-1) to indicate that only one page contains the TB. */
1403	void tb_link_phys(TranslationBlock *tb,
1404	target_ulong phys_pc, target_ulong phys_page2)
1405	{
1406	unsigned int h;
1407	TranslationBlock **ptb;
1408
1409	/* Grab the mmap lock to stop another thread invalidating this TB
1410	before we are done. */
1411	mmap_lock();
1412	/* add in the physical hash table */
1413	h = tb_phys_hash_func(phys_pc);
1414	ptb = &tb_phys_hash[h];
1415	tb->phys_hash_next = *ptb;
1416	*ptb = tb;
1417
1418	/* add in the page list */
1419	tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1420	if (phys_page2 != -1)
1421	tb_alloc_page(tb, 1, phys_page2);
1422	else
1423	tb->page_addr[1] = -1;
1424
1425	tb->jmp_first = (TranslationBlock *)((long)tb \| 2);
1426	tb->jmp_next[0] = NULL;
1427	tb->jmp_next[1] = NULL;
1428
1429	/* init original jump addresses */
1430	if (tb->tb_next_offset[0] != 0xffff)
1431	tb_reset_jump(tb, 0);
1432	if (tb->tb_next_offset[1] != 0xffff)
1433	tb_reset_jump(tb, 1);
1434
1435	#ifdef DEBUG_TB_CHECK
1436	tb_page_check();
1437	#endif
1438	mmap_unlock();
1439	}
1440
1441	/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1442	tb[1].tc_ptr. Return NULL if not found */
1443	TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1444	{
1445	int m_min, m_max, m;
1446	unsigned long v;
1447	TranslationBlock *tb;
1448
1449	if (nb_tbs <= 0)
1450	return NULL;
1451	if (tc_ptr < (unsigned long)code_gen_buffer \|\|
1452	tc_ptr >= (unsigned long)code_gen_ptr)
1453	return NULL;
1454	/* binary search (cf Knuth) */
1455	m_min = 0;
1456	m_max = nb_tbs - 1;
1457	while (m_min <= m_max) {
1458	m = (m_min + m_max) >> 1;
1459	tb = &tbs[m];
1460	v = (unsigned long)tb->tc_ptr;
1461	if (v == tc_ptr)
1462	return tb;
1463	else if (tc_ptr < v) {
1464	m_max = m - 1;
1465	} else {
1466	m_min = m + 1;
1467	}
1468	}
1469	return &tbs[m_max];
1470	}
1471
1472	static void tb_reset_jump_recursive(TranslationBlock *tb);
1473
1474	static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1475	{
1476	TranslationBlock tb1, tb_next, **ptb;
1477	unsigned int n1;
1478
1479	tb1 = tb->jmp_next[n];
1480	if (tb1 != NULL) {
1481	/* find head of list */
1482	for(;;) {
1483	n1 = (long)tb1 & 3;
1484	tb1 = (TranslationBlock *)((long)tb1 & ~3);
1485	if (n1 == 2)
1486	break;
1487	tb1 = tb1->jmp_next[n1];
1488	}
1489	/* we are now sure now that tb jumps to tb1 */
1490	tb_next = tb1;
1491
1492	/* remove tb from the jmp_first list */
1493	ptb = &tb_next->jmp_first;
1494	for(;;) {
1495	tb1 = *ptb;
1496	n1 = (long)tb1 & 3;
1497	tb1 = (TranslationBlock *)((long)tb1 & ~3);
1498	if (n1 == n && tb1 == tb)
1499	break;
1500	ptb = &tb1->jmp_next[n1];
1501	}
1502	*ptb = tb->jmp_next[n];
1503	tb->jmp_next[n] = NULL;
1504
1505	/* suppress the jump to next tb in generated code */
1506	tb_reset_jump(tb, n);
1507
1508	/* suppress jumps in the tb on which we could have jumped */
1509	tb_reset_jump_recursive(tb_next);
1510	}
1511	}
1512
1513	static void tb_reset_jump_recursive(TranslationBlock *tb)
1514	{
1515	tb_reset_jump_recursive2(tb, 0);
1516	tb_reset_jump_recursive2(tb, 1);
1517	}
1518
1519	#if defined(TARGET_HAS_ICE)
1520	static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1521	{
1522	target_phys_addr_t addr;
1523	target_ulong pd;
1524	ram_addr_t ram_addr;
1525	PhysPageDesc *p;
1526
1527	addr = cpu_get_phys_page_debug(env, pc);
1528	p = phys_page_find(addr >> TARGET_PAGE_BITS);
1529	if (!p) {
1530	pd = IO_MEM_UNASSIGNED;
1531	} else {
1532	pd = p->phys_offset;
1533	}
1534	ram_addr = (pd & TARGET_PAGE_MASK) \| (pc & ~TARGET_PAGE_MASK);
1535	tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1536	}
1537	#endif
1538
1539	/* Add a watchpoint. */
1540	int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1541	int flags, CPUWatchpoint **watchpoint)
1542	{
1543	target_ulong len_mask = ~(len - 1);
1544	CPUWatchpoint *wp;
1545
1546	/* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1547	if ((len != 1 && len != 2 && len != 4 && len != 8) \|\| (addr & ~len_mask)) {
1548	fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1549	TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1550	#ifndef VBOX
1551	return -EINVAL;
1552	#else
1553	return VERR_INVALID_PARAMETER;
1554	#endif
1555	}
1556	wp = qemu_malloc(sizeof(*wp));
1557
1558	wp->vaddr = addr;
1559	wp->len_mask = len_mask;
1560	wp->flags = flags;
1561
1562	/* keep all GDB-injected watchpoints in front */
1563	if (flags & BP_GDB)
1564	TAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1565	else
1566	TAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1567
1568	tlb_flush_page(env, addr);
1569
1570	if (watchpoint)
1571	*watchpoint = wp;
1572	return 0;
1573	}
1574
1575	/* Remove a specific watchpoint. */
1576	int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1577	int flags)
1578	{
1579	target_ulong len_mask = ~(len - 1);
1580	CPUWatchpoint *wp;
1581
1582	TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1583	if (addr == wp->vaddr && len_mask == wp->len_mask
1584	&& flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1585	cpu_watchpoint_remove_by_ref(env, wp);
1586	return 0;
1587	}
1588	}
1589	#ifndef VBOX
1590	return -ENOENT;
1591	#else
1592	return VERR_NOT_FOUND;
1593	#endif
1594	}
1595
1596	/* Remove a specific watchpoint by reference. */
1597	void cpu_watchpoint_remove_by_ref(CPUState env, CPUWatchpoint watchpoint)
1598	{
1599	TAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1600
1601	tlb_flush_page(env, watchpoint->vaddr);
1602
1603	qemu_free(watchpoint);
1604	}
1605
1606	/* Remove all matching watchpoints. */
1607	void cpu_watchpoint_remove_all(CPUState *env, int mask)
1608	{
1609	CPUWatchpoint wp, next;
1610
1611	TAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1612	if (wp->flags & mask)
1613	cpu_watchpoint_remove_by_ref(env, wp);
1614	}
1615	}
1616
1617	/* Add a breakpoint. */
1618	int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1619	CPUBreakpoint **breakpoint)
1620	{
1621	#if defined(TARGET_HAS_ICE)
1622	CPUBreakpoint *bp;
1623
1624	bp = qemu_malloc(sizeof(*bp));
1625
1626	bp->pc = pc;
1627	bp->flags = flags;
1628
1629	/* keep all GDB-injected breakpoints in front */
1630	if (flags & BP_GDB)
1631	TAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1632	else
1633	TAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1634
1635	breakpoint_invalidate(env, pc);
1636
1637	if (breakpoint)
1638	*breakpoint = bp;
1639	return 0;
1640	#else
1641	return -ENOSYS;
1642	#endif
1643	}
1644
1645	/* Remove a specific breakpoint. */
1646	int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1647	{
1648	#if defined(TARGET_HAS_ICE)
1649	CPUBreakpoint *bp;
1650
1651	TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1652	if (bp->pc == pc && bp->flags == flags) {
1653	cpu_breakpoint_remove_by_ref(env, bp);
1654	return 0;
1655	}
1656	}
1657	# ifndef VBOX
1658	return -ENOENT;
1659	# else
1660	return VERR_NOT_FOUND;
1661	# endif
1662	#else
1663	return -ENOSYS;
1664	#endif
1665	}
1666
1667	/* Remove a specific breakpoint by reference. */
1668	void cpu_breakpoint_remove_by_ref(CPUState env, CPUBreakpoint breakpoint)
1669	{
1670	#if defined(TARGET_HAS_ICE)
1671	TAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1672
1673	breakpoint_invalidate(env, breakpoint->pc);
1674
1675	qemu_free(breakpoint);
1676	#endif
1677	}
1678
1679	/* Remove all matching breakpoints. */
1680	void cpu_breakpoint_remove_all(CPUState *env, int mask)
1681	{
1682	#if defined(TARGET_HAS_ICE)
1683	CPUBreakpoint bp, next;
1684
1685	TAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1686	if (bp->flags & mask)
1687	cpu_breakpoint_remove_by_ref(env, bp);
1688	}
1689	#endif
1690	}
1691
1692	/* enable or disable single step mode. EXCP_DEBUG is returned by the
1693	CPU loop after each instruction */
1694	void cpu_single_step(CPUState *env, int enabled)
1695	{
1696	#if defined(TARGET_HAS_ICE)
1697	if (env->singlestep_enabled != enabled) {
1698	env->singlestep_enabled = enabled;
1699	if (kvm_enabled())
1700	kvm_update_guest_debug(env, 0);
1701	else {
1702	/* must flush all the translated code to avoid inconsistencies */
1703	/* XXX: only flush what is necessary */
1704	tb_flush(env);
1705	}
1706	}
1707	#endif
1708	}
1709
1710	#ifndef VBOX
1711
1712	/* enable or disable low levels log */
1713	void cpu_set_log(int log_flags)
1714	{
1715	loglevel = log_flags;
1716	if (loglevel && !logfile) {
1717	logfile = fopen(logfilename, log_append ? "a" : "w");
1718	if (!logfile) {
1719	perror(logfilename);
1720	_exit(1);
1721	}
1722	#if !defined(CONFIG_SOFTMMU)
1723	/* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1724	{
1725	static char logfile_buf[4096];
1726	setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1727	}
1728	#else
1729	setvbuf(logfile, NULL, _IOLBF, 0);
1730	#endif
1731	log_append = 1;
1732	}
1733	if (!loglevel && logfile) {
1734	fclose(logfile);
1735	logfile = NULL;
1736	}
1737	}
1738
1739	void cpu_set_log_filename(const char *filename)
1740	{
1741	logfilename = strdup(filename);
1742	if (logfile) {
1743	fclose(logfile);
1744	logfile = NULL;
1745	}
1746	cpu_set_log(loglevel);
1747	}
1748
1749	#endif /* !VBOX */
1750
1751	static void cpu_unlink_tb(CPUState *env)
1752	{
1753	#if defined(USE_NPTL)
1754	/* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1755	problem and hope the cpu will stop of its own accord. For userspace
1756	emulation this often isn't actually as bad as it sounds. Often
1757	signals are used primarily to interrupt blocking syscalls. */
1758	#else
1759	TranslationBlock *tb;
1760	static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1761
1762	tb = env->current_tb;
1763	/* if the cpu is currently executing code, we must unlink it and
1764	all the potentially executing TB */
1765	if (tb && !testandset(&interrupt_lock)) {
1766	env->current_tb = NULL;
1767	tb_reset_jump_recursive(tb);
1768	resetlock(&interrupt_lock);
1769	}
1770	#endif
1771	}
1772
1773	/* mask must never be zero, except for A20 change call */
1774	void cpu_interrupt(CPUState *env, int mask)
1775	{
1776	int old_mask;
1777
1778	old_mask = env->interrupt_request;
1779	#ifndef VBOX
1780	env->interrupt_request \|= mask;
1781	#else /* VBOX */
1782	VM_ASSERT_EMT(env->pVM);
1783	ASMAtomicOrS32((int32_t volatile *)&env->interrupt_request, mask);
1784	#endif /* VBOX */
1785
1786	#ifndef VBOX
1787	#ifndef CONFIG_USER_ONLY
1788	/*
1789	* If called from iothread context, wake the target cpu in
1790	* case its halted.
1791	*/
1792	if (!qemu_cpu_self(env)) {
1793	qemu_cpu_kick(env);
1794	return;
1795	}
1796	#endif
1797	#endif /* !VBOX */
1798
1799	if (use_icount) {
1800	env->icount_decr.u16.high = 0xffff;
1801	#ifndef CONFIG_USER_ONLY
1802	if (!can_do_io(env)
1803	&& (mask & ~old_mask) != 0) {
1804	cpu_abort(env, "Raised interrupt while not in I/O function");
1805	}
1806	#endif
1807	} else {
1808	cpu_unlink_tb(env);
1809	}
1810	}
1811
1812	void cpu_reset_interrupt(CPUState *env, int mask)
1813	{
1814	#ifdef VBOX
1815	/*
1816	* Note: the current implementation can be executed by another thread without problems; make sure this remains true
1817	* for future changes!
1818	*/
1819	ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request, ~mask);
1820	#else /* !VBOX */
1821	env->interrupt_request &= ~mask;
1822	#endif /* !VBOX */
1823	}
1824
1825	void cpu_exit(CPUState *env)
1826	{
1827	env->exit_request = 1;
1828	cpu_unlink_tb(env);
1829	}
1830
1831	#ifndef VBOX
1832	const CPULogItem cpu_log_items[] = {
1833	{ CPU_LOG_TB_OUT_ASM, "out_asm",
1834	"show generated host assembly code for each compiled TB" },
1835	{ CPU_LOG_TB_IN_ASM, "in_asm",
1836	"show target assembly code for each compiled TB" },
1837	{ CPU_LOG_TB_OP, "op",
1838	"show micro ops for each compiled TB" },
1839	{ CPU_LOG_TB_OP_OPT, "op_opt",
1840	"show micro ops "
1841	#ifdef TARGET_I386
1842	"before eflags optimization and "
1843	#endif
1844	"after liveness analysis" },
1845	{ CPU_LOG_INT, "int",
1846	"show interrupts/exceptions in short format" },
1847	{ CPU_LOG_EXEC, "exec",
1848	"show trace before each executed TB (lots of logs)" },
1849	{ CPU_LOG_TB_CPU, "cpu",
1850	"show CPU state before block translation" },
1851	#ifdef TARGET_I386
1852	{ CPU_LOG_PCALL, "pcall",
1853	"show protected mode far calls/returns/exceptions" },
1854	{ CPU_LOG_RESET, "cpu_reset",
1855	"show CPU state before CPU resets" },
1856	#endif
1857	#ifdef DEBUG_IOPORT
1858	{ CPU_LOG_IOPORT, "ioport",
1859	"show all i/o ports accesses" },
1860	#endif
1861	{ 0, NULL, NULL },
1862	};
1863
1864	static int cmp1(const char s1, int n, const char s2)
1865	{
1866	if (strlen(s2) != n)
1867	return 0;
1868	return memcmp(s1, s2, n) == 0;
1869	}
1870
1871	/* takes a comma separated list of log masks. Return 0 if error. */
1872	int cpu_str_to_log_mask(const char *str)
1873	{
1874	const CPULogItem *item;
1875	int mask;
1876	const char p, p1;
1877
1878	p = str;
1879	mask = 0;
1880	for(;;) {
1881	p1 = strchr(p, ',');
1882	if (!p1)
1883	p1 = p + strlen(p);
1884	if(cmp1(p,p1-p,"all")) {
1885	for(item = cpu_log_items; item->mask != 0; item++) {
1886	mask \|= item->mask;
1887	}
1888	} else {
1889	for(item = cpu_log_items; item->mask != 0; item++) {
1890	if (cmp1(p, p1 - p, item->name))
1891	goto found;
1892	}
1893	return 0;
1894	}
1895	found:
1896	mask \|= item->mask;
1897	if (*p1 != ',')
1898	break;
1899	p = p1 + 1;
1900	}
1901	return mask;
1902	}
1903	#endif /* !VBOX */
1904
1905	#ifndef VBOX /* VBOX: we have our own routine. */
1906	void cpu_abort(CPUState env, const char fmt, ...)
1907	{
1908	va_list ap;
1909	va_list ap2;
1910
1911	va_start(ap, fmt);
1912	va_copy(ap2, ap);
1913	fprintf(stderr, "qemu: fatal: ");
1914	vfprintf(stderr, fmt, ap);
1915	fprintf(stderr, "\n");
1916	#ifdef TARGET_I386
1917	cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU \| X86_DUMP_CCOP);
1918	#else
1919	cpu_dump_state(env, stderr, fprintf, 0);
1920	#endif
1921	if (qemu_log_enabled()) {
1922	qemu_log("qemu: fatal: ");
1923	qemu_log_vprintf(fmt, ap2);
1924	qemu_log("\n");
1925	#ifdef TARGET_I386
1926	log_cpu_state(env, X86_DUMP_FPU \| X86_DUMP_CCOP);
1927	#else
1928	log_cpu_state(env, 0);
1929	#endif
1930	qemu_log_flush();
1931	qemu_log_close();
1932	}
1933	va_end(ap2);
1934	va_end(ap);
1935	abort();
1936	}
1937	#endif /* !VBOX */
1938
1939	#ifndef VBOX
1940	CPUState cpu_copy(CPUState env)
1941	{
1942	CPUState *new_env = cpu_init(env->cpu_model_str);
1943	CPUState *next_cpu = new_env->next_cpu;
1944	int cpu_index = new_env->cpu_index;
1945	#if defined(TARGET_HAS_ICE)
1946	CPUBreakpoint *bp;
1947	CPUWatchpoint *wp;
1948	#endif
1949
1950	memcpy(new_env, env, sizeof(CPUState));
1951
1952	/* Preserve chaining and index. */
1953	new_env->next_cpu = next_cpu;
1954	new_env->cpu_index = cpu_index;
1955
1956	/* Clone all break/watchpoints.
1957	Note: Once we support ptrace with hw-debug register access, make sure
1958	BP_CPU break/watchpoints are handled correctly on clone. */
1959	TAILQ_INIT(&env->breakpoints);
1960	TAILQ_INIT(&env->watchpoints);
1961	#if defined(TARGET_HAS_ICE)
1962	TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1963	cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1964	}
1965	TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1966	cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1967	wp->flags, NULL);
1968	}
1969	#endif
1970
1971	return new_env;
1972	}
1973	#endif /* !VBOX */
1974
1975	#if !defined(CONFIG_USER_ONLY)
1976
1977	static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1978	{
1979	unsigned int i;
1980
1981	/* Discard jump cache entries for any tb which might potentially
1982	overlap the flushed page. */
1983	i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1984	memset (&env->tb_jmp_cache[i], 0,
1985	TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1986
1987	i = tb_jmp_cache_hash_page(addr);
1988	memset (&env->tb_jmp_cache[i], 0,
1989	TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1990
1991	#ifdef VBOX
1992	/* inform raw mode about TLB page flush */
1993	remR3FlushPage(env, addr);
1994	#endif /* VBOX */
1995	}
1996
1997	static CPUTLBEntry s_cputlb_empty_entry = {
1998	.addr_read = -1,
1999	.addr_write = -1,
2000	.addr_code = -1,
2001	.addend = -1,
2002	};
2003
2004	/* NOTE: if flush_global is true, also flush global entries (not
2005	implemented yet) */
2006	void tlb_flush(CPUState *env, int flush_global)
2007	{
2008	int i;
2009
2010	#if defined(DEBUG_TLB)
2011	printf("tlb_flush:\n");
2012	#endif
2013	/* must reset current TB so that interrupts cannot modify the
2014	links while we are modifying them */
2015	env->current_tb = NULL;
2016
2017	for(i = 0; i < CPU_TLB_SIZE; i++) {
2018	int mmu_idx;
2019	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2020	env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
2021	}
2022	}
2023
2024	memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
2025
2026	#ifdef CONFIG_KQEMU
2027	if (env->kqemu_enabled) {
2028	kqemu_flush(env, flush_global);
2029	}
2030	#endif
2031	#ifdef VBOX
2032	/* inform raw mode about TLB flush */
2033	remR3FlushTLB(env, flush_global);
2034	#endif
2035	tlb_flush_count++;
2036	}
2037
2038	static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
2039	{
2040	if (addr == (tlb_entry->addr_read &
2041	(TARGET_PAGE_MASK \| TLB_INVALID_MASK)) \|\|
2042	addr == (tlb_entry->addr_write &
2043	(TARGET_PAGE_MASK \| TLB_INVALID_MASK)) \|\|
2044	addr == (tlb_entry->addr_code &
2045	(TARGET_PAGE_MASK \| TLB_INVALID_MASK))) {
2046	*tlb_entry = s_cputlb_empty_entry;
2047	}
2048	}
2049
2050	void tlb_flush_page(CPUState *env, target_ulong addr)
2051	{
2052	int i;
2053	int mmu_idx;
2054
2055	#if defined(DEBUG_TLB)
2056	printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
2057	#endif
2058	/* must reset current TB so that interrupts cannot modify the
2059	links while we are modifying them */
2060	env->current_tb = NULL;
2061
2062	addr &= TARGET_PAGE_MASK;
2063	i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2064	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2065	tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
2066
2067	tlb_flush_jmp_cache(env, addr);
2068
2069	#ifdef CONFIG_KQEMU
2070	if (env->kqemu_enabled) {
2071	kqemu_flush_page(env, addr);
2072	}
2073	#endif
2074	}
2075
2076	/* update the TLBs so that writes to code in the virtual page 'addr'
2077	can be detected */
2078	static void tlb_protect_code(ram_addr_t ram_addr)
2079	{
2080	cpu_physical_memory_reset_dirty(ram_addr,
2081	ram_addr + TARGET_PAGE_SIZE,
2082	CODE_DIRTY_FLAG);
2083	#if defined(VBOX) && defined(REM_MONITOR_CODE_PAGES)
2084	/** @todo Retest this? This function has changed... */
2085	remR3ProtectCode(cpu_single_env, ram_addr);
2086	#endif
2087	}
2088
2089	/* update the TLB so that writes in physical page 'phys_addr' are no longer
2090	tested for self modifying code */
2091	static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
2092	target_ulong vaddr)
2093	{
2094	#ifdef VBOX
2095	if (RT_LIKELY((ram_addr >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
2096	#endif
2097	phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] \|= CODE_DIRTY_FLAG;
2098	}
2099
2100	static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
2101	unsigned long start, unsigned long length)
2102	{
2103	unsigned long addr;
2104
2105	#ifdef VBOX
2106	if (start & 3)
2107	return;
2108	#endif
2109	if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2110	addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
2111	if ((addr - start) < length) {
2112	tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) \| TLB_NOTDIRTY;
2113	}
2114	}
2115	}
2116
2117	/* Note: start and end must be within the same ram block. */
2118	void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
2119	int dirty_flags)
2120	{
2121	CPUState *env;
2122	unsigned long length, start1;
2123	int i, mask, len;
2124	uint8_t *p;
2125
2126	start &= TARGET_PAGE_MASK;
2127	end = TARGET_PAGE_ALIGN(end);
2128
2129	length = end - start;
2130	if (length == 0)
2131	return;
2132	len = length >> TARGET_PAGE_BITS;
2133	#ifdef CONFIG_KQEMU
2134	/* XXX: should not depend on cpu context */
2135	env = first_cpu;
2136	if (env->kqemu_enabled) {
2137	ram_addr_t addr;
2138	addr = start;
2139	for(i = 0; i < len; i++) {
2140	kqemu_set_notdirty(env, addr);
2141	addr += TARGET_PAGE_SIZE;
2142	}
2143	}
2144	#endif
2145	mask = ~dirty_flags;
2146	p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
2147	#ifdef VBOX
2148	if (RT_LIKELY((start >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
2149	#endif
2150	for(i = 0; i < len; i++)
2151	p[i] &= mask;
2152
2153	/* we modify the TLB cache so that the dirty bit will be set again
2154	when accessing the range */
2155	#if defined(VBOX) && defined(REM_PHYS_ADDR_IN_TLB)
2156	start1 = start;
2157	#elif !defined(VBOX)
2158	start1 = (unsigned long)qemu_get_ram_ptr(start);
2159	/* Chek that we don't span multiple blocks - this breaks the
2160	address comparisons below. */
2161	if ((unsigned long)qemu_get_ram_ptr(end - 1) - start1
2162	!= (end - 1) - start) {
2163	abort();
2164	}
2165	#else
2166	start1 = (unsigned long)remR3TlbGCPhys2Ptr(first_cpu, start, 1 /fWritable/); /** @todo page replacing (sharing or read only) may cause trouble, fix interface/whatever. */
2167	#endif
2168
2169	for(env = first_cpu; env != NULL; env = env->next_cpu) {
2170	int mmu_idx;
2171	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2172	for(i = 0; i < CPU_TLB_SIZE; i++)
2173	tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
2174	start1, length);
2175	}
2176	}
2177	}
2178
2179	#ifndef VBOX
2180	int cpu_physical_memory_set_dirty_tracking(int enable)
2181	{
2182	in_migration = enable;
2183	if (kvm_enabled()) {
2184	return kvm_set_migration_log(enable);
2185	}
2186	return 0;
2187	}
2188
2189	int cpu_physical_memory_get_dirty_tracking(void)
2190	{
2191	return in_migration;
2192	}
2193	#endif /* !VBOX */
2194
2195	int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
2196	target_phys_addr_t end_addr)
2197	{
2198	#ifndef VBOX
2199	int ret = 0;
2200
2201	if (kvm_enabled())
2202	ret = kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
2203	return ret;
2204	#else
2205	return 0;
2206	#endif
2207	}
2208
2209	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
2210	DECLINLINE(void) tlb_update_dirty(CPUTLBEntry *tlb_entry, target_phys_addr_t phys_addend)
2211	#else
2212	static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
2213	#endif
2214	{
2215	ram_addr_t ram_addr;
2216	void *p;
2217
2218	if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2219	#if defined(VBOX) && defined(REM_PHYS_ADDR_IN_TLB)
2220	ram_addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
2221	#elif !defined(VBOX)
2222	p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
2223	+ tlb_entry->addend);
2224	ram_addr = qemu_ram_addr_from_host(p);
2225	#else
2226	Assert(phys_addend != -1);
2227	ram_addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + phys_addend;
2228	#endif
2229	if (!cpu_physical_memory_is_dirty(ram_addr)) {
2230	tlb_entry->addr_write \|= TLB_NOTDIRTY;
2231	}
2232	}
2233	}
2234
2235	/* update the TLB according to the current state of the dirty bits */
2236	void cpu_tlb_update_dirty(CPUState *env)
2237	{
2238	int i;
2239	int mmu_idx;
2240	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2241	for(i = 0; i < CPU_TLB_SIZE; i++)
2242	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
2243	tlb_update_dirty(&env->tlb_table[mmu_idx][i], env->phys_addends[mmu_idx][i]);
2244	#else
2245	tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
2246	#endif
2247	}
2248	}
2249
2250	static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
2251	{
2252	if (tlb_entry->addr_write == (vaddr \| TLB_NOTDIRTY))
2253	tlb_entry->addr_write = vaddr;
2254	}
2255
2256	/* update the TLB corresponding to virtual page vaddr
2257	so that it is no longer dirty */
2258	static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2259	{
2260	int i;
2261	int mmu_idx;
2262
2263	vaddr &= TARGET_PAGE_MASK;
2264	i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2265	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2266	tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
2267	}
2268
2269	/* add a new TLB entry. At most one entry for a given virtual address
2270	is permitted. Return 0 if OK or 2 if the page could not be mapped
2271	(can only happen in non SOFTMMU mode for I/O pages or pages
2272	conflicting with the host address space). */
2273	int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2274	target_phys_addr_t paddr, int prot,
2275	int mmu_idx, int is_softmmu)
2276	{
2277	PhysPageDesc *p;
2278	unsigned long pd;
2279	unsigned int index;
2280	target_ulong address;
2281	target_ulong code_address;
2282	target_phys_addr_t addend;
2283	int ret;
2284	CPUTLBEntry *te;
2285	CPUWatchpoint *wp;
2286	target_phys_addr_t iotlb;
2287	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
2288	int read_mods = 0, write_mods = 0, code_mods = 0;
2289	#endif
2290
2291	p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2292	if (!p) {
2293	pd = IO_MEM_UNASSIGNED;
2294	} else {
2295	pd = p->phys_offset;
2296	}
2297	#if defined(DEBUG_TLB)
2298	printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
2299	vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
2300	#endif
2301
2302	ret = 0;
2303	address = vaddr;
2304	if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2305	/* IO memory case (romd handled later) */
2306	address \|= TLB_MMIO;
2307	}
2308	#if defined(VBOX) && defined(REM_PHYS_ADDR_IN_TLB)
2309	addend = pd & TARGET_PAGE_MASK;
2310	#elif !defined(VBOX)
2311	addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2312	#else
2313	/** @todo this is racing the phys_page_find call above since it may register
2314	* a new chunk of memory... */
2315	addend = (unsigned long)remR3TlbGCPhys2Ptr(env, pd & TARGET_PAGE_MASK, !!(prot & PAGE_WRITE));
2316	#endif
2317
2318	if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2319	/* Normal RAM. */
2320	iotlb = pd & TARGET_PAGE_MASK;
2321	if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2322	iotlb \|= IO_MEM_NOTDIRTY;
2323	else
2324	iotlb \|= IO_MEM_ROM;
2325	} else {
2326	/* IO handlers are currently passed a physical address.
2327	It would be nice to pass an offset from the base address
2328	of that region. This would avoid having to special case RAM,
2329	and avoid full address decoding in every device.
2330	We can't use the high bits of pd for this because
2331	IO_MEM_ROMD uses these as a ram address. */
2332	iotlb = (pd & ~TARGET_PAGE_MASK);
2333	#ifndef VBOX
2334	if (p) {
2335	#else
2336	if ( p
2337	&& p->phys_offset
2338	&& (pd & ~TARGET_PAGE_MASK) != env->pVM->rem.s.iMMIOMemType
2339	&& (pd & ~TARGET_PAGE_MASK) != env->pVM->rem.s.iHandlerMemType) {
2340	#endif
2341	iotlb += p->region_offset;
2342	} else {
2343	iotlb += paddr;
2344	}
2345	}
2346
2347	code_address = address;
2348	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
2349
2350	if (addend & 0x3)
2351	{
2352	if (addend & 0x2)
2353	{
2354	/* catch write */
2355	if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM)
2356	write_mods \|= TLB_MMIO;
2357	}
2358	else if (addend & 0x1)
2359	{
2360	/* catch all */
2361	if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM)
2362	{
2363	read_mods \|= TLB_MMIO;
2364	write_mods \|= TLB_MMIO;
2365	code_mods \|= TLB_MMIO;
2366	}
2367	}
2368	if ((iotlb & ~TARGET_PAGE_MASK) == 0)
2369	iotlb = env->pVM->rem.s.iHandlerMemType + paddr;
2370	addend &= ~(target_ulong)0x3;
2371	}
2372
2373	#endif
2374	/* Make accesses to pages with watchpoints go via the
2375	watchpoint trap routines. */
2376	TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2377	if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2378	iotlb = io_mem_watch + paddr;
2379	/* TODO: The memory case can be optimized by not trapping
2380	reads of pages with a write breakpoint. */
2381	address \|= TLB_MMIO;
2382	}
2383	}
2384
2385	index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2386	env->iotlb[mmu_idx][index] = iotlb - vaddr;
2387	te = &env->tlb_table[mmu_idx][index];
2388	te->addend = addend - vaddr;
2389	if (prot & PAGE_READ) {
2390	te->addr_read = address;
2391	} else {
2392	te->addr_read = -1;
2393	}
2394
2395	if (prot & PAGE_EXEC) {
2396	te->addr_code = code_address;
2397	} else {
2398	te->addr_code = -1;
2399	}
2400	if (prot & PAGE_WRITE) {
2401	if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM \|\|
2402	(pd & IO_MEM_ROMD)) {
2403	/* Write access calls the I/O callback. */
2404	te->addr_write = address \| TLB_MMIO;
2405	} else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2406	!cpu_physical_memory_is_dirty(pd)) {
2407	te->addr_write = address \| TLB_NOTDIRTY;
2408	} else {
2409	te->addr_write = address;
2410	}
2411	} else {
2412	te->addr_write = -1;
2413	}
2414
2415	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
2416	if (prot & PAGE_READ)
2417	te->addr_read \|= read_mods;
2418	if (prot & PAGE_EXEC)
2419	te->addr_code \|= code_mods;
2420	if (prot & PAGE_WRITE)
2421	te->addr_write \|= write_mods;
2422
2423	env->phys_addends[mmu_idx][index] = (pd & TARGET_PAGE_MASK)- vaddr;
2424	#endif
2425
2426	#ifdef VBOX
2427	/* inform raw mode about TLB page change */
2428	remR3FlushPage(env, vaddr);
2429	#endif
2430	return ret;
2431	}
2432
2433	#else
2434
2435	void tlb_flush(CPUState *env, int flush_global)
2436	{
2437	}
2438
2439	void tlb_flush_page(CPUState *env, target_ulong addr)
2440	{
2441	}
2442
2443	int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2444	target_phys_addr_t paddr, int prot,
2445	int mmu_idx, int is_softmmu)
2446	{
2447	return 0;
2448	}
2449
2450	#ifndef VBOX
2451
2452	/*
2453	* Walks guest process memory "regions" one by one
2454	* and calls callback function 'fn' for each region.
2455	*/
2456	int walk_memory_regions(void *priv,
2457	int (fn)(void , unsigned long, unsigned long, unsigned long))
2458	{
2459	unsigned long start, end;
2460	PageDesc *p = NULL;
2461	int i, j, prot, prot1;
2462	int rc = 0;
2463
2464	start = end = -1;
2465	prot = 0;
2466
2467	for (i = 0; i <= L1_SIZE; i++) {
2468	p = (i < L1_SIZE) ? l1_map[i] : NULL;
2469	for (j = 0; j < L2_SIZE; j++) {
2470	prot1 = (p == NULL) ? 0 : p[j].flags;
2471	/*
2472	* "region" is one continuous chunk of memory
2473	* that has same protection flags set.
2474	*/
2475	if (prot1 != prot) {
2476	end = (i << (32 - L1_BITS)) \| (j << TARGET_PAGE_BITS);
2477	if (start != -1) {
2478	rc = (*fn)(priv, start, end, prot);
2479	/* callback can stop iteration by returning != 0 */
2480	if (rc != 0)
2481	return (rc);
2482	}
2483	if (prot1 != 0)
2484	start = end;
2485	else
2486	start = -1;
2487	prot = prot1;
2488	}
2489	if (p == NULL)
2490	break;
2491	}
2492	}
2493	return (rc);
2494	}
2495
2496	static int dump_region(void *priv, unsigned long start,
2497	unsigned long end, unsigned long prot)
2498	{
2499	FILE f = (FILE )priv;
2500
2501	(void) fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2502	start, end, end - start,
2503	((prot & PAGE_READ) ? 'r' : '-'),
2504	((prot & PAGE_WRITE) ? 'w' : '-'),
2505	((prot & PAGE_EXEC) ? 'x' : '-'));
2506
2507	return (0);
2508	}
2509
2510	/* dump memory mappings */
2511	void page_dump(FILE *f)
2512	{
2513	(void) fprintf(f, "%-8s %-8s %-8s %s\n",
2514	"start", "end", "size", "prot");
2515	walk_memory_regions(f, dump_region);
2516	}
2517
2518	#endif /* !VBOX */
2519
2520	int page_get_flags(target_ulong address)
2521	{
2522	PageDesc *p;
2523
2524	p = page_find(address >> TARGET_PAGE_BITS);
2525	if (!p)
2526	return 0;
2527	return p->flags;
2528	}
2529
2530	/* modify the flags of a page and invalidate the code if
2531	necessary. The flag PAGE_WRITE_ORG is positioned automatically
2532	depending on PAGE_WRITE */
2533	void page_set_flags(target_ulong start, target_ulong end, int flags)
2534	{
2535	PageDesc *p;
2536	target_ulong addr;
2537
2538	/* mmap_lock should already be held. */
2539	start = start & TARGET_PAGE_MASK;
2540	end = TARGET_PAGE_ALIGN(end);
2541	if (flags & PAGE_WRITE)
2542	flags \|= PAGE_WRITE_ORG;
2543	#ifdef VBOX
2544	AssertMsgFailed(("We shouldn't be here, and if we should, we must have an env to do the proper locking!\n"));
2545	#endif
2546	for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2547	p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2548	/* We may be called for host regions that are outside guest
2549	address space. */
2550	if (!p)
2551	return;
2552	/* if the write protection is set, then we invalidate the code
2553	inside */
2554	if (!(p->flags & PAGE_WRITE) &&
2555	(flags & PAGE_WRITE) &&
2556	p->first_tb) {
2557	tb_invalidate_phys_page(addr, 0, NULL);
2558	}
2559	p->flags = flags;
2560	}
2561	}
2562
2563	int page_check_range(target_ulong start, target_ulong len, int flags)
2564	{
2565	PageDesc *p;
2566	target_ulong end;
2567	target_ulong addr;
2568
2569	if (start + len < start)
2570	/* we've wrapped around */
2571	return -1;
2572
2573	end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2574	start = start & TARGET_PAGE_MASK;
2575
2576	for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2577	p = page_find(addr >> TARGET_PAGE_BITS);
2578	if( !p )
2579	return -1;
2580	if( !(p->flags & PAGE_VALID) )
2581	return -1;
2582
2583	if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2584	return -1;
2585	if (flags & PAGE_WRITE) {
2586	if (!(p->flags & PAGE_WRITE_ORG))
2587	return -1;
2588	/* unprotect the page if it was put read-only because it
2589	contains translated code */
2590	if (!(p->flags & PAGE_WRITE)) {
2591	if (!page_unprotect(addr, 0, NULL))
2592	return -1;
2593	}
2594	return 0;
2595	}
2596	}
2597	return 0;
2598	}
2599
2600	/* called from signal handler: invalidate the code and unprotect the
2601	page. Return TRUE if the fault was successfully handled. */
2602	int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2603	{
2604	unsigned int page_index, prot, pindex;
2605	PageDesc p, p1;
2606	target_ulong host_start, host_end, addr;
2607
2608	/* Technically this isn't safe inside a signal handler. However we
2609	know this only ever happens in a synchronous SEGV handler, so in
2610	practice it seems to be ok. */
2611	mmap_lock();
2612
2613	host_start = address & qemu_host_page_mask;
2614	page_index = host_start >> TARGET_PAGE_BITS;
2615	p1 = page_find(page_index);
2616	if (!p1) {
2617	mmap_unlock();
2618	return 0;
2619	}
2620	host_end = host_start + qemu_host_page_size;
2621	p = p1;
2622	prot = 0;
2623	for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2624	prot \|= p->flags;
2625	p++;
2626	}
2627	/* if the page was really writable, then we change its
2628	protection back to writable */
2629	if (prot & PAGE_WRITE_ORG) {
2630	pindex = (address - host_start) >> TARGET_PAGE_BITS;
2631	if (!(p1[pindex].flags & PAGE_WRITE)) {
2632	mprotect((void *)g2h(host_start), qemu_host_page_size,
2633	(prot & PAGE_BITS) \| PAGE_WRITE);
2634	p1[pindex].flags \|= PAGE_WRITE;
2635	/* and since the content will be modified, we must invalidate
2636	the corresponding translated code. */
2637	tb_invalidate_phys_page(address, pc, puc);
2638	#ifdef DEBUG_TB_CHECK
2639	tb_invalidate_check(address);
2640	#endif
2641	mmap_unlock();
2642	return 1;
2643	}
2644	}
2645	mmap_unlock();
2646	return 0;
2647	}
2648
2649	static inline void tlb_set_dirty(CPUState *env,
2650	unsigned long addr, target_ulong vaddr)
2651	{
2652	}
2653	#endif /* defined(CONFIG_USER_ONLY) */
2654
2655	#if !defined(CONFIG_USER_ONLY)
2656
2657	static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2658	ram_addr_t memory, ram_addr_t region_offset);
2659	static void subpage_init (target_phys_addr_t base, ram_addr_t phys,
2660	ram_addr_t orig_memory, ram_addr_t region_offset);
2661	#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2662	need_subpage) \
2663	do { \
2664	if (addr > start_addr) \
2665	start_addr2 = 0; \
2666	else { \
2667	start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2668	if (start_addr2 > 0) \
2669	need_subpage = 1; \
2670	} \
2671	\
2672	if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2673	end_addr2 = TARGET_PAGE_SIZE - 1; \
2674	else { \
2675	end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2676	if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2677	need_subpage = 1; \
2678	} \
2679	} while (0)
2680
2681	/* register physical memory. 'size' must be a multiple of the target
2682	page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2683	io memory page. The address used when calling the IO function is
2684	the offset from the start of the region, plus region_offset. Both
2685	start_addr and region_offset are rounded down to a page boundary
2686	before calculating this offset. This should not be a problem unless
2687	the low bits of start_addr and region_offset differ. */
2688	void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2689	ram_addr_t size,
2690	ram_addr_t phys_offset,
2691	ram_addr_t region_offset)
2692	{
2693	target_phys_addr_t addr, end_addr;
2694	PhysPageDesc *p;
2695	CPUState *env;
2696	ram_addr_t orig_size = size;
2697	void *subpage;
2698
2699	#ifdef CONFIG_KQEMU
2700	/* XXX: should not depend on cpu context */
2701	env = first_cpu;
2702	if (env->kqemu_enabled) {
2703	kqemu_set_phys_mem(start_addr, size, phys_offset);
2704	}
2705	#endif
2706	if (kvm_enabled())
2707	kvm_set_phys_mem(start_addr, size, phys_offset);
2708
2709	if (phys_offset == IO_MEM_UNASSIGNED) {
2710	region_offset = start_addr;
2711	}
2712	region_offset &= TARGET_PAGE_MASK;
2713	size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2714	end_addr = start_addr + (target_phys_addr_t)size;
2715	for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2716	p = phys_page_find(addr >> TARGET_PAGE_BITS);
2717	if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2718	ram_addr_t orig_memory = p->phys_offset;
2719	target_phys_addr_t start_addr2, end_addr2;
2720	int need_subpage = 0;
2721
2722	CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2723	need_subpage);
2724	if (need_subpage \|\| phys_offset & IO_MEM_SUBWIDTH) {
2725	if (!(orig_memory & IO_MEM_SUBPAGE)) {
2726	subpage = subpage_init((addr & TARGET_PAGE_MASK),
2727	&p->phys_offset, orig_memory,
2728	p->region_offset);
2729	} else {
2730	subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2731	>> IO_MEM_SHIFT];
2732	}
2733	subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2734	region_offset);
2735	p->region_offset = 0;
2736	} else {
2737	p->phys_offset = phys_offset;
2738	if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM \|\|
2739	(phys_offset & IO_MEM_ROMD))
2740	phys_offset += TARGET_PAGE_SIZE;
2741	}
2742	} else {
2743	p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2744	p->phys_offset = phys_offset;
2745	p->region_offset = region_offset;
2746	if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM \|\|
2747	(phys_offset & IO_MEM_ROMD)) {
2748	phys_offset += TARGET_PAGE_SIZE;
2749	} else {
2750	target_phys_addr_t start_addr2, end_addr2;
2751	int need_subpage = 0;
2752
2753	CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2754	end_addr2, need_subpage);
2755
2756	if (need_subpage \|\| phys_offset & IO_MEM_SUBWIDTH) {
2757	subpage = subpage_init((addr & TARGET_PAGE_MASK),
2758	&p->phys_offset, IO_MEM_UNASSIGNED,
2759	addr & TARGET_PAGE_MASK);
2760	subpage_register(subpage, start_addr2, end_addr2,
2761	phys_offset, region_offset);
2762	p->region_offset = 0;
2763	}
2764	}
2765	}
2766	region_offset += TARGET_PAGE_SIZE;
2767	}
2768
2769	/* since each CPU stores ram addresses in its TLB cache, we must
2770	reset the modified entries */
2771	/* XXX: slow ! */
2772	for(env = first_cpu; env != NULL; env = env->next_cpu) {
2773	tlb_flush(env, 1);
2774	}
2775	}
2776
2777	/* XXX: temporary until new memory mapping API */
2778	ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2779	{
2780	PhysPageDesc *p;
2781
2782	p = phys_page_find(addr >> TARGET_PAGE_BITS);
2783	if (!p)
2784	return IO_MEM_UNASSIGNED;
2785	return p->phys_offset;
2786	}
2787
2788	#ifndef VBOX
2789	void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2790	{
2791	if (kvm_enabled())
2792	kvm_coalesce_mmio_region(addr, size);
2793	}
2794
2795	void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2796	{
2797	if (kvm_enabled())
2798	kvm_uncoalesce_mmio_region(addr, size);
2799	}
2800
2801	#ifdef CONFIG_KQEMU
2802	/* XXX: better than nothing */
2803	static ram_addr_t kqemu_ram_alloc(ram_addr_t size)
2804	{
2805	ram_addr_t addr;
2806	if ((last_ram_offset + size) > kqemu_phys_ram_size) {
2807	fprintf(stderr, "Not enough memory (requested_size = %" PRIu64 ", max memory = %" PRIu64 ")\n",
2808	(uint64_t)size, (uint64_t)kqemu_phys_ram_size);
2809	abort();
2810	}
2811	addr = last_ram_offset;
2812	last_ram_offset = TARGET_PAGE_ALIGN(last_ram_offset + size);
2813	return addr;
2814	}
2815	#endif
2816
2817	ram_addr_t qemu_ram_alloc(ram_addr_t size)
2818	{
2819	RAMBlock *new_block;
2820
2821	#ifdef CONFIG_KQEMU
2822	if (kqemu_phys_ram_base) {
2823	return kqemu_ram_alloc(size);
2824	}
2825	#endif
2826
2827	size = TARGET_PAGE_ALIGN(size);
2828	new_block = qemu_malloc(sizeof(*new_block));
2829
2830	new_block->host = qemu_vmalloc(size);
2831	new_block->offset = last_ram_offset;
2832	new_block->length = size;
2833
2834	new_block->next = ram_blocks;
2835	ram_blocks = new_block;
2836
2837	phys_ram_dirty = qemu_realloc(phys_ram_dirty,
2838	(last_ram_offset + size) >> TARGET_PAGE_BITS);
2839	memset(phys_ram_dirty + (last_ram_offset >> TARGET_PAGE_BITS),
2840	0xff, size >> TARGET_PAGE_BITS);
2841
2842	last_ram_offset += size;
2843
2844	if (kvm_enabled())
2845	kvm_setup_guest_memory(new_block->host, size);
2846
2847	return new_block->offset;
2848	}
2849
2850	void qemu_ram_free(ram_addr_t addr)
2851	{
2852	/* TODO: implement this. */
2853	}
2854
2855	/* Return a host pointer to ram allocated with qemu_ram_alloc.
2856	With the exception of the softmmu code in this file, this should
2857	only be used for local memory (e.g. video ram) that the device owns,
2858	and knows it isn't going to access beyond the end of the block.
2859
2860	It should not be used for general purpose DMA.
2861	Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2862	*/
2863	void *qemu_get_ram_ptr(ram_addr_t addr)
2864	{
2865	RAMBlock *prev;
2866	RAMBlock **prevp;
2867	RAMBlock *block;
2868
2869	#ifdef CONFIG_KQEMU
2870	if (kqemu_phys_ram_base) {
2871	return kqemu_phys_ram_base + addr;
2872	}
2873	#endif
2874
2875	prev = NULL;
2876	prevp = &ram_blocks;
2877	block = ram_blocks;
2878	while (block && (block->offset > addr
2879	\|\| block->offset + block->length <= addr)) {
2880	if (prev)
2881	prevp = &prev->next;
2882	prev = block;
2883	block = block->next;
2884	}
2885	if (!block) {
2886	fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
2887	abort();
2888	}
2889	/* Move this entry to to start of the list. */
2890	if (prev) {
2891	prev->next = block->next;
2892	block->next = *prevp;
2893	*prevp = block;
2894	}
2895	return block->host + (addr - block->offset);
2896	}
2897
2898	/* Some of the softmmu routines need to translate from a host pointer
2899	(typically a TLB entry) back to a ram offset. */
2900	ram_addr_t qemu_ram_addr_from_host(void *ptr)
2901	{
2902	RAMBlock *prev;
2903	RAMBlock **prevp;
2904	RAMBlock *block;
2905	uint8_t *host = ptr;
2906
2907	#ifdef CONFIG_KQEMU
2908	if (kqemu_phys_ram_base) {
2909	return host - kqemu_phys_ram_base;
2910	}
2911	#endif
2912
2913	prev = NULL;
2914	prevp = &ram_blocks;
2915	block = ram_blocks;
2916	while (block && (block->host > host
2917	\|\| block->host + block->length <= host)) {
2918	if (prev)
2919	prevp = &prev->next;
2920	prev = block;
2921	block = block->next;
2922	}
2923	if (!block) {
2924	fprintf(stderr, "Bad ram pointer %p\n", ptr);
2925	abort();
2926	}
2927	return block->offset + (host - block->host);
2928	}
2929
2930	#endif /* !VBOX */
2931
2932	static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2933	{
2934	#ifdef DEBUG_UNASSIGNED
2935	printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2936	#endif
2937	#if defined(TARGET_SPARC)
2938	do_unassigned_access(addr, 0, 0, 0, 1);
2939	#endif
2940	return 0;
2941	}
2942
2943	static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2944	{
2945	#ifdef DEBUG_UNASSIGNED
2946	printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2947	#endif
2948	#if defined(TARGET_SPARC)
2949	do_unassigned_access(addr, 0, 0, 0, 2);
2950	#endif
2951	return 0;
2952	}
2953
2954	static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2955	{
2956	#ifdef DEBUG_UNASSIGNED
2957	printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2958	#endif
2959	#if defined(TARGET_SPARC)
2960	do_unassigned_access(addr, 0, 0, 0, 4);
2961	#endif
2962	return 0;
2963	}
2964
2965	static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2966	{
2967	#ifdef DEBUG_UNASSIGNED
2968	printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2969	#endif
2970	#if defined(TARGET_SPARC)
2971	do_unassigned_access(addr, 1, 0, 0, 1);
2972	#endif
2973	}
2974
2975	static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2976	{
2977	#ifdef DEBUG_UNASSIGNED
2978	printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2979	#endif
2980	#if defined(TARGET_SPARC)
2981	do_unassigned_access(addr, 1, 0, 0, 2);
2982	#endif
2983	}
2984
2985	static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2986	{
2987	#ifdef DEBUG_UNASSIGNED
2988	printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2989	#endif
2990	#if defined(TARGET_SPARC)
2991	do_unassigned_access(addr, 1, 0, 0, 4);
2992	#endif
2993	}
2994
2995	static CPUReadMemoryFunc *unassigned_mem_read[3] = {
2996	unassigned_mem_readb,
2997	unassigned_mem_readw,
2998	unassigned_mem_readl,
2999	};
3000
3001	static CPUWriteMemoryFunc *unassigned_mem_write[3] = {
3002	unassigned_mem_writeb,
3003	unassigned_mem_writew,
3004	unassigned_mem_writel,
3005	};
3006
3007	static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
3008	uint32_t val)
3009	{
3010	int dirty_flags;
3011	#ifdef VBOX
3012	if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
3013	dirty_flags = 0xff;
3014	else
3015	#endif /* VBOX */
3016	dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
3017	if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3018	#if !defined(CONFIG_USER_ONLY)
3019	tb_invalidate_phys_page_fast(ram_addr, 1);
3020	# ifdef VBOX
3021	if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
3022	dirty_flags = 0xff;
3023	else
3024	# endif /* VBOX */
3025	dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
3026	#endif
3027	}
3028	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
3029	remR3PhysWriteU8(ram_addr, val);
3030	#else
3031	stb_p(qemu_get_ram_ptr(ram_addr), val);
3032	#endif
3033	#ifdef CONFIG_KQEMU
3034	if (cpu_single_env->kqemu_enabled &&
3035	(dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
3036	kqemu_modify_page(cpu_single_env, ram_addr);
3037	#endif
3038	dirty_flags \|= (0xff & ~CODE_DIRTY_FLAG);
3039	#ifdef VBOX
3040	if (RT_LIKELY((ram_addr >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
3041	#endif /* !VBOX */
3042	phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
3043	/* we remove the notdirty callback only if the code has been
3044	flushed */
3045	if (dirty_flags == 0xff)
3046	tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3047	}
3048
3049	static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
3050	uint32_t val)
3051	{
3052	int dirty_flags;
3053	#ifdef VBOX
3054	if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
3055	dirty_flags = 0xff;
3056	else
3057	#endif /* VBOX */
3058	dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
3059	if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3060	#if !defined(CONFIG_USER_ONLY)
3061	tb_invalidate_phys_page_fast(ram_addr, 2);
3062	# ifdef VBOX
3063	if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
3064	dirty_flags = 0xff;
3065	else
3066	# endif /* VBOX */
3067	dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
3068	#endif
3069	}
3070	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
3071	remR3PhysWriteU16(ram_addr, val);
3072	#else
3073	stw_p(qemu_get_ram_ptr(ram_addr), val);
3074	#endif
3075	#ifdef CONFIG_KQEMU
3076	if (cpu_single_env->kqemu_enabled &&
3077	(dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
3078	kqemu_modify_page(cpu_single_env, ram_addr);
3079	#endif
3080	dirty_flags \|= (0xff & ~CODE_DIRTY_FLAG);
3081	#ifdef VBOX
3082	if (RT_LIKELY((ram_addr >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
3083	#endif
3084	phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
3085	/* we remove the notdirty callback only if the code has been
3086	flushed */
3087	if (dirty_flags == 0xff)
3088	tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3089	}
3090
3091	static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
3092	uint32_t val)
3093	{
3094	int dirty_flags;
3095	#ifdef VBOX
3096	if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
3097	dirty_flags = 0xff;
3098	else
3099	#endif /* VBOX */
3100	dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
3101	if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3102	#if !defined(CONFIG_USER_ONLY)
3103	tb_invalidate_phys_page_fast(ram_addr, 4);
3104	# ifdef VBOX
3105	if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
3106	dirty_flags = 0xff;
3107	else
3108	# endif /* VBOX */
3109	dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
3110	#endif
3111	}
3112	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
3113	remR3PhysWriteU32(ram_addr, val);
3114	#else
3115	stl_p(qemu_get_ram_ptr(ram_addr), val);
3116	#endif
3117	#ifdef CONFIG_KQEMU
3118	if (cpu_single_env->kqemu_enabled &&
3119	(dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
3120	kqemu_modify_page(cpu_single_env, ram_addr);
3121	#endif
3122	dirty_flags \|= (0xff & ~CODE_DIRTY_FLAG);
3123	#ifdef VBOX
3124	if (RT_LIKELY((ram_addr >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
3125	#endif
3126	phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
3127	/* we remove the notdirty callback only if the code has been
3128	flushed */
3129	if (dirty_flags == 0xff)
3130	tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3131	}
3132
3133	static CPUReadMemoryFunc *error_mem_read[3] = {
3134	NULL, /* never used */
3135	NULL, /* never used */
3136	NULL, /* never used */
3137	};
3138
3139	static CPUWriteMemoryFunc *notdirty_mem_write[3] = {
3140	notdirty_mem_writeb,
3141	notdirty_mem_writew,
3142	notdirty_mem_writel,
3143	};
3144
3145	/* Generate a debug exception if a watchpoint has been hit. */
3146	static void check_watchpoint(int offset, int len_mask, int flags)
3147	{
3148	CPUState *env = cpu_single_env;
3149	target_ulong pc, cs_base;
3150	TranslationBlock *tb;
3151	target_ulong vaddr;
3152	CPUWatchpoint *wp;
3153	int cpu_flags;
3154
3155	if (env->watchpoint_hit) {
3156	/* We re-entered the check after replacing the TB. Now raise
3157	* the debug interrupt so that is will trigger after the
3158	* current instruction. */
3159	cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
3160	return;
3161	}
3162	vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
3163	TAILQ_FOREACH(wp, &env->watchpoints, entry) {
3164	if ((vaddr == (wp->vaddr & len_mask) \|\|
3165	(vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
3166	wp->flags \|= BP_WATCHPOINT_HIT;
3167	if (!env->watchpoint_hit) {
3168	env->watchpoint_hit = wp;
3169	tb = tb_find_pc(env->mem_io_pc);
3170	if (!tb) {
3171	cpu_abort(env, "check_watchpoint: could not find TB for "
3172	"pc=%p", (void *)env->mem_io_pc);
3173	}
3174	cpu_restore_state(tb, env, env->mem_io_pc, NULL);
3175	tb_phys_invalidate(tb, -1);
3176	if (wp->flags & BP_STOP_BEFORE_ACCESS) {
3177	env->exception_index = EXCP_DEBUG;
3178	} else {
3179	cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
3180	tb_gen_code(env, pc, cs_base, cpu_flags, 1);
3181	}
3182	cpu_resume_from_signal(env, NULL);
3183	}
3184	} else {
3185	wp->flags &= ~BP_WATCHPOINT_HIT;
3186	}
3187	}
3188	}
3189
3190	/* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3191	so these check for a hit then pass through to the normal out-of-line
3192	phys routines. */
3193	static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
3194	{
3195	check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
3196	return ldub_phys(addr);
3197	}
3198
3199	static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
3200	{
3201	check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
3202	return lduw_phys(addr);
3203	}
3204
3205	static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
3206	{
3207	check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
3208	return ldl_phys(addr);
3209	}
3210
3211	static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
3212	uint32_t val)
3213	{
3214	check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
3215	stb_phys(addr, val);
3216	}
3217
3218	static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
3219	uint32_t val)
3220	{
3221	check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
3222	stw_phys(addr, val);
3223	}
3224
3225	static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
3226	uint32_t val)
3227	{
3228	check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
3229	stl_phys(addr, val);
3230	}
3231
3232	static CPUReadMemoryFunc *watch_mem_read[3] = {
3233	watch_mem_readb,
3234	watch_mem_readw,
3235	watch_mem_readl,
3236	};
3237
3238	static CPUWriteMemoryFunc *watch_mem_write[3] = {
3239	watch_mem_writeb,
3240	watch_mem_writew,
3241	watch_mem_writel,
3242	};
3243
3244	static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
3245	unsigned int len)
3246	{
3247	uint32_t ret;
3248	unsigned int idx;
3249
3250	idx = SUBPAGE_IDX(addr);
3251	#if defined(DEBUG_SUBPAGE)
3252	printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
3253	mmio, len, addr, idx);
3254	#endif
3255	ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
3256	addr + mmio->region_offset[idx][0][len]);
3257
3258	return ret;
3259	}
3260
3261	static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
3262	uint32_t value, unsigned int len)
3263	{
3264	unsigned int idx;
3265
3266	idx = SUBPAGE_IDX(addr);
3267	#if defined(DEBUG_SUBPAGE)
3268	printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
3269	mmio, len, addr, idx, value);
3270	#endif
3271	(**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
3272	addr + mmio->region_offset[idx][1][len],
3273	value);
3274	}
3275
3276	static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
3277	{
3278	#if defined(DEBUG_SUBPAGE)
3279	printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
3280	#endif
3281
3282	return subpage_readlen(opaque, addr, 0);
3283	}
3284
3285	static void subpage_writeb (void *opaque, target_phys_addr_t addr,
3286	uint32_t value)
3287	{
3288	#if defined(DEBUG_SUBPAGE)
3289	printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
3290	#endif
3291	subpage_writelen(opaque, addr, value, 0);
3292	}
3293
3294	static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
3295	{
3296	#if defined(DEBUG_SUBPAGE)
3297	printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
3298	#endif
3299
3300	return subpage_readlen(opaque, addr, 1);
3301	}
3302
3303	static void subpage_writew (void *opaque, target_phys_addr_t addr,
3304	uint32_t value)
3305	{
3306	#if defined(DEBUG_SUBPAGE)
3307	printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
3308	#endif
3309	subpage_writelen(opaque, addr, value, 1);
3310	}
3311
3312	static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
3313	{
3314	#if defined(DEBUG_SUBPAGE)
3315	printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
3316	#endif
3317
3318	return subpage_readlen(opaque, addr, 2);
3319	}
3320
3321	static void subpage_writel (void *opaque,
3322	target_phys_addr_t addr, uint32_t value)
3323	{
3324	#if defined(DEBUG_SUBPAGE)
3325	printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
3326	#endif
3327	subpage_writelen(opaque, addr, value, 2);
3328	}
3329
3330	static CPUReadMemoryFunc *subpage_read[] = {
3331	&subpage_readb,
3332	&subpage_readw,
3333	&subpage_readl,
3334	};
3335
3336	static CPUWriteMemoryFunc *subpage_write[] = {
3337	&subpage_writeb,
3338	&subpage_writew,
3339	&subpage_writel,
3340	};
3341
3342	static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
3343	ram_addr_t memory, ram_addr_t region_offset)
3344	{
3345	int idx, eidx;
3346	unsigned int i;
3347
3348	if (start >= TARGET_PAGE_SIZE \|\| end >= TARGET_PAGE_SIZE)
3349	return -1;
3350	idx = SUBPAGE_IDX(start);
3351	eidx = SUBPAGE_IDX(end);
3352	#if defined(DEBUG_SUBPAGE)
3353	printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
3354	mmio, start, end, idx, eidx, memory);
3355	#endif
3356	memory >>= IO_MEM_SHIFT;
3357	for (; idx <= eidx; idx++) {
3358	for (i = 0; i < 4; i++) {
3359	if (io_mem_read[memory][i]) {
3360	mmio->mem_read[idx][i] = &io_mem_read[memory][i];
3361	mmio->opaque[idx][0][i] = io_mem_opaque[memory];
3362	mmio->region_offset[idx][0][i] = region_offset;
3363	}
3364	if (io_mem_write[memory][i]) {
3365	mmio->mem_write[idx][i] = &io_mem_write[memory][i];
3366	mmio->opaque[idx][1][i] = io_mem_opaque[memory];
3367	mmio->region_offset[idx][1][i] = region_offset;
3368	}
3369	}
3370	}
3371
3372	return 0;
3373	}
3374
3375	static void subpage_init (target_phys_addr_t base, ram_addr_t phys,
3376	ram_addr_t orig_memory, ram_addr_t region_offset)
3377	{
3378	subpage_t *mmio;
3379	int subpage_memory;
3380
3381	mmio = qemu_mallocz(sizeof(subpage_t));
3382
3383	mmio->base = base;
3384	subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio);
3385	#if defined(DEBUG_SUBPAGE)
3386	printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3387	mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3388	#endif
3389	*phys = subpage_memory \| IO_MEM_SUBPAGE;
3390	subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
3391	region_offset);
3392
3393	return mmio;
3394	}
3395
3396	static int get_free_io_mem_idx(void)
3397	{
3398	int i;
3399
3400	for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3401	if (!io_mem_used[i]) {
3402	io_mem_used[i] = 1;
3403	return i;
3404	}
3405
3406	return -1;
3407	}
3408
3409	/* mem_read and mem_write are arrays of functions containing the
3410	function to access byte (index 0), word (index 1) and dword (index
3411	2). Functions can be omitted with a NULL function pointer.
3412	If io_index is non zero, the corresponding io zone is
3413	modified. If it is zero, a new io zone is allocated. The return
3414	value can be used with cpu_register_physical_memory(). (-1) is
3415	returned if error. */
3416	static int cpu_register_io_memory_fixed(int io_index,
3417	CPUReadMemoryFunc **mem_read,
3418	CPUWriteMemoryFunc **mem_write,
3419	void *opaque)
3420	{
3421	int i, subwidth = 0;
3422
3423	if (io_index <= 0) {
3424	io_index = get_free_io_mem_idx();
3425	if (io_index == -1)
3426	return io_index;
3427	} else {
3428	io_index >>= IO_MEM_SHIFT;
3429	if (io_index >= IO_MEM_NB_ENTRIES)
3430	return -1;
3431	}
3432
3433	for(i = 0;i < 3; i++) {
3434	if (!mem_read[i] \|\| !mem_write[i])
3435	subwidth = IO_MEM_SUBWIDTH;
3436	io_mem_read[io_index][i] = mem_read[i];
3437	io_mem_write[io_index][i] = mem_write[i];
3438	}
3439	io_mem_opaque[io_index] = opaque;
3440	return (io_index << IO_MEM_SHIFT) \| subwidth;
3441	}
3442
3443	int cpu_register_io_memory(CPUReadMemoryFunc **mem_read,
3444	CPUWriteMemoryFunc **mem_write,
3445	void *opaque)
3446	{
3447	return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque);
3448	}
3449
3450	void cpu_unregister_io_memory(int io_table_address)
3451	{
3452	int i;
3453	int io_index = io_table_address >> IO_MEM_SHIFT;
3454
3455	for (i=0;i < 3; i++) {
3456	io_mem_read[io_index][i] = unassigned_mem_read[i];
3457	io_mem_write[io_index][i] = unassigned_mem_write[i];
3458	}
3459	io_mem_opaque[io_index] = NULL;
3460	io_mem_used[io_index] = 0;
3461	}
3462
3463	static void io_mem_init(void)
3464	{
3465	int i;
3466
3467	cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read, unassigned_mem_write, NULL);
3468	cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read, unassigned_mem_write, NULL);
3469	cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read, notdirty_mem_write, NULL);
3470	for (i=0; i<5; i++)
3471	io_mem_used[i] = 1;
3472
3473	io_mem_watch = cpu_register_io_memory(watch_mem_read,
3474	watch_mem_write, NULL);
3475	#ifdef CONFIG_KQEMU
3476	if (kqemu_phys_ram_base) {
3477	/* alloc dirty bits array */
3478	phys_ram_dirty = qemu_vmalloc(kqemu_phys_ram_size >> TARGET_PAGE_BITS);
3479	memset(phys_ram_dirty, 0xff, kqemu_phys_ram_size >> TARGET_PAGE_BITS);
3480	}
3481	#endif
3482	}
3483
3484	#endif /* !defined(CONFIG_USER_ONLY) */
3485
3486	/* physical memory access (slow version, mainly for debug) */
3487	#if defined(CONFIG_USER_ONLY)
3488	void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3489	int len, int is_write)
3490	{
3491	int l, flags;
3492	target_ulong page;
3493	void * p;
3494
3495	while (len > 0) {
3496	page = addr & TARGET_PAGE_MASK;
3497	l = (page + TARGET_PAGE_SIZE) - addr;
3498	if (l > len)
3499	l = len;
3500	flags = page_get_flags(page);
3501	if (!(flags & PAGE_VALID))
3502	return;
3503	if (is_write) {
3504	if (!(flags & PAGE_WRITE))
3505	return;
3506	/* XXX: this code should not depend on lock_user */
3507	if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3508	/* FIXME - should this return an error rather than just fail? */
3509	return;
3510	memcpy(p, buf, l);
3511	unlock_user(p, addr, l);
3512	} else {
3513	if (!(flags & PAGE_READ))
3514	return;
3515	/* XXX: this code should not depend on lock_user */
3516	if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3517	/* FIXME - should this return an error rather than just fail? */
3518	return;
3519	memcpy(buf, p, l);
3520	unlock_user(p, addr, 0);
3521	}
3522	len -= l;
3523	buf += l;
3524	addr += l;
3525	}
3526	}
3527
3528	#else
3529	void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3530	int len, int is_write)
3531	{
3532	int l, io_index;
3533	uint8_t *ptr;
3534	uint32_t val;
3535	target_phys_addr_t page;
3536	unsigned long pd;
3537	PhysPageDesc *p;
3538
3539	while (len > 0) {
3540	page = addr & TARGET_PAGE_MASK;
3541	l = (page + TARGET_PAGE_SIZE) - addr;
3542	if (l > len)
3543	l = len;
3544	p = phys_page_find(page >> TARGET_PAGE_BITS);
3545	if (!p) {
3546	pd = IO_MEM_UNASSIGNED;
3547	} else {
3548	pd = p->phys_offset;
3549	}
3550
3551	if (is_write) {
3552	if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3553	target_phys_addr_t addr1 = addr;
3554	io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3555	if (p)
3556	addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3557	/* XXX: could force cpu_single_env to NULL to avoid
3558	potential bugs */
3559	if (l >= 4 && ((addr1 & 3) == 0)) {
3560	/* 32 bit write access */
3561	#if !defined(VBOX) \|\| !defined(REM_PHYS_ADDR_IN_TLB)
3562	val = ldl_p(buf);
3563	#else
3564	val = (const uint32_t )buf;
3565	#endif
3566	io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3567	l = 4;
3568	} else if (l >= 2 && ((addr1 & 1) == 0)) {
3569	/* 16 bit write access */
3570	#if !defined(VBOX) \|\| !defined(REM_PHYS_ADDR_IN_TLB)
3571	val = lduw_p(buf);
3572	#else
3573	val = (const uint16_t )buf;
3574	#endif
3575	io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3576	l = 2;
3577	} else {
3578	/* 8 bit write access */
3579	#if !defined(VBOX) \|\| !defined(REM_PHYS_ADDR_IN_TLB)
3580	val = ldub_p(buf);
3581	#else
3582	val = (const uint8_t )buf;
3583	#endif
3584	io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3585	l = 1;
3586	}
3587	} else {
3588	unsigned long addr1;
3589	addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3590	/* RAM case */
3591	#ifdef VBOX
3592	remR3PhysWrite(addr1, buf, l); NOREF(ptr);
3593	#else
3594	ptr = qemu_get_ram_ptr(addr1);
3595	memcpy(ptr, buf, l);
3596	#endif
3597	if (!cpu_physical_memory_is_dirty(addr1)) {
3598	/* invalidate code */
3599	tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3600	/* set dirty bit */
3601	#ifdef VBOX
3602	if (RT_LIKELY((addr1 >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
3603	#endif
3604	phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] \|=
3605	(0xff & ~CODE_DIRTY_FLAG);
3606	}
3607	}
3608	} else {
3609	if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3610	!(pd & IO_MEM_ROMD)) {
3611	target_phys_addr_t addr1 = addr;
3612	/* I/O case */
3613	io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3614	if (p)
3615	addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3616	if (l >= 4 && ((addr1 & 3) == 0)) {
3617	/* 32 bit read access */
3618	val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3619	#if !defined(VBOX) \|\| !defined(REM_PHYS_ADDR_IN_TLB)
3620	stl_p(buf, val);
3621	#else
3622	(uint32_t )buf = val;
3623	#endif
3624	l = 4;
3625	} else if (l >= 2 && ((addr1 & 1) == 0)) {
3626	/* 16 bit read access */
3627	val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3628	#if !defined(VBOX) \|\| !defined(REM_PHYS_ADDR_IN_TLB)
3629	stw_p(buf, val);
3630	#else
3631	(uint16_t )buf = val;
3632	#endif
3633	l = 2;
3634	} else {
3635	/* 8 bit read access */
3636	val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3637	#if !defined(VBOX) \|\| !defined(REM_PHYS_ADDR_IN_TLB)
3638	stb_p(buf, val);
3639	#else
3640	(uint8_t )buf = val;
3641	#endif
3642	l = 1;
3643	}
3644	} else {
3645	/* RAM case */
3646	#ifdef VBOX
3647	remR3PhysRead((pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK), buf, l); NOREF(ptr);
3648	#else
3649	ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3650	(addr & ~TARGET_PAGE_MASK);
3651	memcpy(buf, ptr, l);
3652	#endif
3653	}
3654	}
3655	len -= l;
3656	buf += l;
3657	addr += l;
3658	}
3659	}
3660
3661	#ifndef VBOX
3662
3663	/* used for ROM loading : can write in RAM and ROM */
3664	void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3665	const uint8_t *buf, int len)
3666	{
3667	int l;
3668	uint8_t *ptr;
3669	target_phys_addr_t page;
3670	unsigned long pd;
3671	PhysPageDesc *p;
3672
3673	while (len > 0) {
3674	page = addr & TARGET_PAGE_MASK;
3675	l = (page + TARGET_PAGE_SIZE) - addr;
3676	if (l > len)
3677	l = len;
3678	p = phys_page_find(page >> TARGET_PAGE_BITS);
3679	if (!p) {
3680	pd = IO_MEM_UNASSIGNED;
3681	} else {
3682	pd = p->phys_offset;
3683	}
3684
3685	if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3686	(pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3687	!(pd & IO_MEM_ROMD)) {
3688	/* do nothing */
3689	} else {
3690	unsigned long addr1;
3691	addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3692	/* ROM/RAM case */
3693	ptr = qemu_get_ram_ptr(addr1);
3694	memcpy(ptr, buf, l);
3695	}
3696	len -= l;
3697	buf += l;
3698	addr += l;
3699	}
3700	}
3701
3702	typedef struct {
3703	void *buffer;
3704	target_phys_addr_t addr;
3705	target_phys_addr_t len;
3706	} BounceBuffer;
3707
3708	static BounceBuffer bounce;
3709
3710	typedef struct MapClient {
3711	void *opaque;
3712	void (callback)(void opaque);
3713	LIST_ENTRY(MapClient) link;
3714	} MapClient;
3715
3716	static LIST_HEAD(map_client_list, MapClient) map_client_list
3717	= LIST_HEAD_INITIALIZER(map_client_list);
3718
3719	void cpu_register_map_client(void opaque, void (callback)(void opaque))
3720	{
3721	MapClient client = qemu_malloc(sizeof(client));
3722
3723	client->opaque = opaque;
3724	client->callback = callback;
3725	LIST_INSERT_HEAD(&map_client_list, client, link);
3726	return client;
3727	}
3728
3729	void cpu_unregister_map_client(void *_client)
3730	{
3731	MapClient client = (MapClient )_client;
3732
3733	LIST_REMOVE(client, link);
3734	qemu_free(client);
3735	}
3736
3737	static void cpu_notify_map_clients(void)
3738	{
3739	MapClient *client;
3740
3741	while (!LIST_EMPTY(&map_client_list)) {
3742	client = LIST_FIRST(&map_client_list);
3743	client->callback(client->opaque);
3744	cpu_unregister_map_client(client);
3745	}
3746	}
3747
3748	/* Map a physical memory region into a host virtual address.
3749	* May map a subset of the requested range, given by and returned in *plen.
3750	* May return NULL if resources needed to perform the mapping are exhausted.
3751	* Use only for reads OR writes - not for read-modify-write operations.
3752	* Use cpu_register_map_client() to know when retrying the map operation is
3753	* likely to succeed.
3754	*/
3755	void *cpu_physical_memory_map(target_phys_addr_t addr,
3756	target_phys_addr_t *plen,
3757	int is_write)
3758	{
3759	target_phys_addr_t len = *plen;
3760	target_phys_addr_t done = 0;
3761	int l;
3762	uint8_t *ret = NULL;
3763	uint8_t *ptr;
3764	target_phys_addr_t page;
3765	unsigned long pd;
3766	PhysPageDesc *p;
3767	unsigned long addr1;
3768
3769	while (len > 0) {
3770	page = addr & TARGET_PAGE_MASK;
3771	l = (page + TARGET_PAGE_SIZE) - addr;
3772	if (l > len)
3773	l = len;
3774	p = phys_page_find(page >> TARGET_PAGE_BITS);
3775	if (!p) {
3776	pd = IO_MEM_UNASSIGNED;
3777	} else {
3778	pd = p->phys_offset;
3779	}
3780
3781	if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3782	if (done \|\| bounce.buffer) {
3783	break;
3784	}
3785	bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3786	bounce.addr = addr;
3787	bounce.len = l;
3788	if (!is_write) {
3789	cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3790	}
3791	ptr = bounce.buffer;
3792	} else {
3793	addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3794	ptr = qemu_get_ram_ptr(addr1);
3795	}
3796	if (!done) {
3797	ret = ptr;
3798	} else if (ret + done != ptr) {
3799	break;
3800	}
3801
3802	len -= l;
3803	addr += l;
3804	done += l;
3805	}
3806	*plen = done;
3807	return ret;
3808	}
3809
3810	/* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3811	* Will also mark the memory as dirty if is_write == 1. access_len gives
3812	* the amount of memory that was actually read or written by the caller.
3813	*/
3814	void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3815	int is_write, target_phys_addr_t access_len)
3816	{
3817	if (buffer != bounce.buffer) {
3818	if (is_write) {
3819	ram_addr_t addr1 = qemu_ram_addr_from_host(buffer);
3820	while (access_len) {
3821	unsigned l;
3822	l = TARGET_PAGE_SIZE;
3823	if (l > access_len)
3824	l = access_len;
3825	if (!cpu_physical_memory_is_dirty(addr1)) {
3826	/* invalidate code */
3827	tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3828	/* set dirty bit */
3829	phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] \|=
3830	(0xff & ~CODE_DIRTY_FLAG);
3831	}
3832	addr1 += l;
3833	access_len -= l;
3834	}
3835	}
3836	return;
3837	}
3838	if (is_write) {
3839	cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3840	}
3841	qemu_free(bounce.buffer);
3842	bounce.buffer = NULL;
3843	cpu_notify_map_clients();
3844	}
3845
3846	#endif /* !VBOX */
3847
3848	/* warning: addr must be aligned */
3849	uint32_t ldl_phys(target_phys_addr_t addr)
3850	{
3851	int io_index;
3852	uint8_t *ptr;
3853	uint32_t val;
3854	unsigned long pd;
3855	PhysPageDesc *p;
3856
3857	p = phys_page_find(addr >> TARGET_PAGE_BITS);
3858	if (!p) {
3859	pd = IO_MEM_UNASSIGNED;
3860	} else {
3861	pd = p->phys_offset;
3862	}
3863
3864	if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3865	!(pd & IO_MEM_ROMD)) {
3866	/* I/O case */
3867	io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3868	if (p)
3869	addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3870	val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3871	} else {
3872	/* RAM case */
3873	#ifndef VBOX
3874	ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3875	(addr & ~TARGET_PAGE_MASK);
3876	val = ldl_p(ptr);
3877	#else
3878	val = remR3PhysReadU32((pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK)); NOREF(ptr);
3879	#endif
3880	}
3881	return val;
3882	}
3883
3884	/* warning: addr must be aligned */
3885	uint64_t ldq_phys(target_phys_addr_t addr)
3886	{
3887	int io_index;
3888	uint8_t *ptr;
3889	uint64_t val;
3890	unsigned long pd;
3891	PhysPageDesc *p;
3892
3893	p = phys_page_find(addr >> TARGET_PAGE_BITS);
3894	if (!p) {
3895	pd = IO_MEM_UNASSIGNED;
3896	} else {
3897	pd = p->phys_offset;
3898	}
3899
3900	if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3901	!(pd & IO_MEM_ROMD)) {
3902	/* I/O case */
3903	io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3904	if (p)
3905	addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3906	#ifdef TARGET_WORDS_BIGENDIAN
3907	val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3908	val \|= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3909	#else
3910	val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3911	val \|= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3912	#endif
3913	} else {
3914	/* RAM case */
3915	#ifndef VBOX
3916	ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3917	(addr & ~TARGET_PAGE_MASK);
3918	val = ldq_p(ptr);
3919	#else
3920	val = remR3PhysReadU64((pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK)); NOREF(ptr);
3921	#endif
3922	}
3923	return val;
3924	}
3925
3926	/* XXX: optimize */
3927	uint32_t ldub_phys(target_phys_addr_t addr)
3928	{
3929	uint8_t val;
3930	cpu_physical_memory_read(addr, &val, 1);
3931	return val;
3932	}
3933
3934	/* XXX: optimize */
3935	uint32_t lduw_phys(target_phys_addr_t addr)
3936	{
3937	uint16_t val;
3938	cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3939	return tswap16(val);
3940	}
3941
3942	/* warning: addr must be aligned. The ram page is not masked as dirty
3943	and the code inside is not invalidated. It is useful if the dirty
3944	bits are used to track modified PTEs */
3945	void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3946	{
3947	int io_index;
3948	uint8_t *ptr;
3949	unsigned long pd;
3950	PhysPageDesc *p;
3951
3952	p = phys_page_find(addr >> TARGET_PAGE_BITS);
3953	if (!p) {
3954	pd = IO_MEM_UNASSIGNED;
3955	} else {
3956	pd = p->phys_offset;
3957	}
3958
3959	if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3960	io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3961	if (p)
3962	addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3963	io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3964	} else {
3965	#ifndef VBOX
3966	unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3967	ptr = qemu_get_ram_ptr(addr1);
3968	stl_p(ptr, val);
3969	#else
3970	remR3PhysWriteU32((pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK), val); NOREF(ptr);
3971	#endif
3972
3973	#ifndef VBOX
3974	if (unlikely(in_migration)) {
3975	if (!cpu_physical_memory_is_dirty(addr1)) {
3976	/* invalidate code */
3977	tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3978	/* set dirty bit */
3979	phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] \|=
3980	(0xff & ~CODE_DIRTY_FLAG);
3981	}
3982	}
3983	#endif /* !VBOX */
3984	}
3985	}
3986
3987	void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3988	{
3989	int io_index;
3990	uint8_t *ptr;
3991	unsigned long pd;
3992	PhysPageDesc *p;
3993
3994	p = phys_page_find(addr >> TARGET_PAGE_BITS);
3995	if (!p) {
3996	pd = IO_MEM_UNASSIGNED;
3997	} else {
3998	pd = p->phys_offset;
3999	}
4000
4001	if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4002	io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4003	if (p)
4004	addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4005	#ifdef TARGET_WORDS_BIGENDIAN
4006	io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
4007	io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
4008	#else
4009	io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4010	io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
4011	#endif
4012	} else {
4013	#ifndef VBOX
4014	ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4015	(addr & ~TARGET_PAGE_MASK);
4016	stq_p(ptr, val);
4017	#else
4018	remR3PhysWriteU64((pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK), val); NOREF(ptr);
4019	#endif
4020	}
4021	}
4022
4023	/* warning: addr must be aligned */
4024	void stl_phys(target_phys_addr_t addr, uint32_t val)
4025	{
4026	int io_index;
4027	uint8_t *ptr;
4028	unsigned long pd;
4029	PhysPageDesc *p;
4030
4031	p = phys_page_find(addr >> TARGET_PAGE_BITS);
4032	if (!p) {
4033	pd = IO_MEM_UNASSIGNED;
4034	} else {
4035	pd = p->phys_offset;
4036	}
4037
4038	if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4039	io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4040	if (p)
4041	addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4042	io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4043	} else {
4044	unsigned long addr1;
4045	addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4046	/* RAM case */
4047	#ifndef VBOX
4048	ptr = qemu_get_ram_ptr(addr1);
4049	stl_p(ptr, val);
4050	#else
4051	remR3PhysWriteU32((pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK), val); NOREF(ptr);
4052	#endif
4053	if (!cpu_physical_memory_is_dirty(addr1)) {
4054	/* invalidate code */
4055	tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4056	/* set dirty bit */
4057	#ifdef VBOX
4058	if (RT_LIKELY((addr1 >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
4059	#endif
4060	phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] \|=
4061	(0xff & ~CODE_DIRTY_FLAG);
4062	}
4063	}
4064	}
4065
4066	/* XXX: optimize */
4067	void stb_phys(target_phys_addr_t addr, uint32_t val)
4068	{
4069	uint8_t v = val;
4070	cpu_physical_memory_write(addr, &v, 1);
4071	}
4072
4073	/* XXX: optimize */
4074	void stw_phys(target_phys_addr_t addr, uint32_t val)
4075	{
4076	uint16_t v = tswap16(val);
4077	cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
4078	}
4079
4080	/* XXX: optimize */
4081	void stq_phys(target_phys_addr_t addr, uint64_t val)
4082	{
4083	val = tswap64(val);
4084	cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
4085	}
4086
4087	#endif
4088
4089	#ifndef VBOX
4090	/* virtual memory access for debug (includes writing to ROM) */
4091	int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
4092	uint8_t *buf, int len, int is_write)
4093	{
4094	int l;
4095	target_phys_addr_t phys_addr;
4096	target_ulong page;
4097
4098	while (len > 0) {
4099	page = addr & TARGET_PAGE_MASK;
4100	phys_addr = cpu_get_phys_page_debug(env, page);
4101	/* if no physical page mapped, return an error */
4102	if (phys_addr == -1)
4103	return -1;
4104	l = (page + TARGET_PAGE_SIZE) - addr;
4105	if (l > len)
4106	l = len;
4107	phys_addr += (addr & ~TARGET_PAGE_MASK);
4108	#if !defined(CONFIG_USER_ONLY)
4109	if (is_write)
4110	cpu_physical_memory_write_rom(phys_addr, buf, l);
4111	else
4112	#endif
4113	cpu_physical_memory_rw(phys_addr, buf, l, is_write);
4114	len -= l;
4115	buf += l;
4116	addr += l;
4117	}
4118	return 0;
4119	}
4120	#endif /* !VBOX */
4121
4122	/* in deterministic execution mode, instructions doing device I/Os
4123	must be at the end of the TB */
4124	void cpu_io_recompile(CPUState env, void retaddr)
4125	{
4126	TranslationBlock *tb;
4127	uint32_t n, cflags;
4128	target_ulong pc, cs_base;
4129	uint64_t flags;
4130
4131	tb = tb_find_pc((unsigned long)retaddr);
4132	if (!tb) {
4133	cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
4134	retaddr);
4135	}
4136	n = env->icount_decr.u16.low + tb->icount;
4137	cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
4138	/* Calculate how many instructions had been executed before the fault
4139	occurred. */
4140	n = n - env->icount_decr.u16.low;
4141	/* Generate a new TB ending on the I/O insn. */
4142	n++;
4143	/* On MIPS and SH, delay slot instructions can only be restarted if
4144	they were already the first instruction in the TB. If this is not
4145	the first instruction in a TB then re-execute the preceding
4146	branch. */
4147	#if defined(TARGET_MIPS)
4148	if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
4149	env->active_tc.PC -= 4;
4150	env->icount_decr.u16.low++;
4151	env->hflags &= ~MIPS_HFLAG_BMASK;
4152	}
4153	#elif defined(TARGET_SH4)
4154	if ((env->flags & ((DELAY_SLOT \| DELAY_SLOT_CONDITIONAL))) != 0
4155	&& n > 1) {
4156	env->pc -= 2;
4157	env->icount_decr.u16.low++;
4158	env->flags &= ~(DELAY_SLOT \| DELAY_SLOT_CONDITIONAL);
4159	}
4160	#endif
4161	/* This should never happen. */
4162	if (n > CF_COUNT_MASK)
4163	cpu_abort(env, "TB too big during recompile");
4164
4165	cflags = n \| CF_LAST_IO;
4166	pc = tb->pc;
4167	cs_base = tb->cs_base;
4168	flags = tb->flags;
4169	tb_phys_invalidate(tb, -1);
4170	/* FIXME: In theory this could raise an exception. In practice
4171	we have already translated the block once so it's probably ok. */
4172	tb_gen_code(env, pc, cs_base, flags, cflags);
4173	/* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4174	the first in the TB) then we end up generating a whole new TB and
4175	repeating the fault, which is horribly inefficient.
4176	Better would be to execute just this insn uncached, or generate a
4177	second new TB. */
4178	cpu_resume_from_signal(env, NULL);
4179	}
4180
4181	#ifndef VBOX
4182	void dump_exec_info(FILE *f,
4183	int (cpu_fprintf)(FILE f, const char *fmt, ...))
4184	{
4185	int i, target_code_size, max_target_code_size;
4186	int direct_jmp_count, direct_jmp2_count, cross_page;
4187	TranslationBlock *tb;
4188
4189	target_code_size = 0;
4190	max_target_code_size = 0;
4191	cross_page = 0;
4192	direct_jmp_count = 0;
4193	direct_jmp2_count = 0;
4194	for(i = 0; i < nb_tbs; i++) {
4195	tb = &tbs[i];
4196	target_code_size += tb->size;
4197	if (tb->size > max_target_code_size)
4198	max_target_code_size = tb->size;
4199	if (tb->page_addr[1] != -1)
4200	cross_page++;
4201	if (tb->tb_next_offset[0] != 0xffff) {
4202	direct_jmp_count++;
4203	if (tb->tb_next_offset[1] != 0xffff) {
4204	direct_jmp2_count++;
4205	}
4206	}
4207	}
4208	/* XXX: avoid using doubles ? */
4209	cpu_fprintf(f, "Translation buffer state:\n");
4210	cpu_fprintf(f, "gen code size %ld/%ld\n",
4211	code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
4212	cpu_fprintf(f, "TB count %d/%d\n",
4213	nb_tbs, code_gen_max_blocks);
4214	cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
4215	nb_tbs ? target_code_size / nb_tbs : 0,
4216	max_target_code_size);
4217	cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
4218	nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
4219	target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
4220	cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
4221	cross_page,
4222	nb_tbs ? (cross_page * 100) / nb_tbs : 0);
4223	cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4224	direct_jmp_count,
4225	nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
4226	direct_jmp2_count,
4227	nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
4228	cpu_fprintf(f, "\nStatistics:\n");
4229	cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
4230	cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
4231	cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
4232	tcg_dump_info(f, cpu_fprintf);
4233	}
4234	#endif /* !VBOX */
4235
4236	#if !defined(CONFIG_USER_ONLY)
4237
4238	#define MMUSUFFIX _cmmu
4239	#define GETPC() NULL
4240	#define env cpu_single_env
4241	#define SOFTMMU_CODE_ACCESS
4242
4243	#define SHIFT 0
4244	#include "softmmu_template.h"
4245
4246	#define SHIFT 1
4247	#include "softmmu_template.h"
4248
4249	#define SHIFT 2
4250	#include "softmmu_template.h"
4251
4252	#define SHIFT 3
4253	#include "softmmu_template.h"
4254
4255	#undef env
4256
4257	#endif

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/recompiler/exec.c@ 36412

Download in other formats: