PGMAllPhys.cpp@ 15348

Last change on this file since 15348 was 15344, checked in by vboxsync, 16 years ago
#3202: Optimizations of the dynamic page mapping code (ring-0). Do lots of the stuff inline, using the set as a 2st level cache and not releasing it for each inner VT-x iteration.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 84.2 KB

Line
1	/* $Id: PGMAllPhys.cpp 15344 2008-12-12 00:13:56Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22	/*******************************************************************************
23	* Defined Constants And Macros *
24	*******************************************************************************/
25	/** @def PGM_IGNORE_RAM_FLAGS_RESERVED
26	* Don't respect the MM_RAM_FLAGS_RESERVED flag when converting to HC addresses.
27	*
28	* Since this flag is currently incorrectly kept set for ROM regions we will
29	* have to ignore it for now so we don't break stuff.
30	*
31	* @todo this has been fixed now I believe, remove this hack.
32	*/
33	#define PGM_IGNORE_RAM_FLAGS_RESERVED
34
35
36	/*******************************************************************************
37	* Header Files *
38	*******************************************************************************/
39	#define LOG_GROUP LOG_GROUP_PGM_PHYS
40	#include <VBox/pgm.h>
41	#include <VBox/trpm.h>
42	#include <VBox/vmm.h>
43	#include <VBox/iom.h>
44	#include <VBox/em.h>
45	#include <VBox/rem.h>
46	#include "PGMInternal.h"
47	#include <VBox/vm.h>
48	#include <VBox/param.h>
49	#include <VBox/err.h>
50	#include <iprt/assert.h>
51	#include <iprt/string.h>
52	#include <iprt/asm.h>
53	#include <VBox/log.h>
54	#ifdef IN_RING3
55	# include <iprt/thread.h>
56	#endif
57
58
59
60	#ifndef IN_RING3
61
62	/**
63	* \#PF Handler callback for Guest ROM range write access.
64	* We simply ignore the writes or fall back to the recompiler if we don't support the instruction.
65	*
66	* @returns VBox status code (appropritate for trap handling and GC return).
67	* @param pVM VM Handle.
68	* @param uErrorCode CPU Error code.
69	* @param pRegFrame Trap register frame.
70	* @param pvFault The fault address (cr2).
71	* @param GCPhysFault The GC physical address corresponding to pvFault.
72	* @param pvUser User argument. Pointer to the ROM range structure.
73	*/
74	VMMDECL(int) pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, void pvFault, RTGCPHYS GCPhysFault, void pvUser)
75	{
76	int rc;
77	#ifdef VBOX_WITH_NEW_PHYS_CODE
78	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
79	uint32_t iPage = GCPhysFault - pRom->GCPhys;
80	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
81	switch (pRom->aPages[iPage].enmProt)
82	{
83	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
84	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
85	{
86	#endif
87	/*
88	* If it's a simple instruction which doesn't change the cpu state
89	* we will simply skip it. Otherwise we'll have to defer it to REM.
90	*/
91	uint32_t cbOp;
92	DISCPUSTATE Cpu;
93	rc = EMInterpretDisasOne(pVM, pRegFrame, &Cpu, &cbOp);
94	if ( RT_SUCCESS(rc)
95	&& Cpu.mode == CPUMODE_32BIT /** @todo why does this matter? */
96	&& !(Cpu.prefix & (PREFIX_REPNE \| PREFIX_REP \| PREFIX_SEG)))
97	{
98	switch (Cpu.opcode)
99	{
100	/** @todo Find other instructions we can safely skip, possibly
101	* adding this kind of detection to DIS or EM. */
102	case OP_MOV:
103	pRegFrame->rip += cbOp;
104	STAM_COUNTER_INC(&pVM->pgm.s.StatRZGuestROMWriteHandled);
105	return VINF_SUCCESS;
106	}
107	}
108	else if (RT_UNLIKELY(rc == VERR_INTERNAL_ERROR))
109	return rc;
110	#ifdef VBOX_WITH_NEW_PHYS_CODE
111	break;
112	}
113
114	case PGMROMPROT_READ_RAM_WRITE_RAM:
115	rc = PGMHandlerPhysicalPageTempOff(pVM, pRom->GCPhys, GCPhysFault & X86_PTE_PG_MASK);
116	AssertRC(rc);
117	case PGMROMPROT_READ_ROM_WRITE_RAM:
118	/* Handle it in ring-3 because it's way easier there. */
119	break;
120
121	default:
122	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhysFault=%RGp\n",
123	pRom->aPages[iPage].enmProt, iPage, GCPhysFault),
124	VERR_INTERNAL_ERROR);
125	}
126	#endif
127
128	STAM_COUNTER_INC(&pVM->pgm.s.StatRZGuestROMWriteUnhandled);
129	return VINF_EM_RAW_EMULATE_INSTR;
130	}
131
132	#endif /* IN_RING3 */
133
134	/**
135	* Checks if Address Gate 20 is enabled or not.
136	*
137	* @returns true if enabled.
138	* @returns false if disabled.
139	* @param pVM VM handle.
140	*/
141	VMMDECL(bool) PGMPhysIsA20Enabled(PVM pVM)
142	{
143	LogFlow(("PGMPhysIsA20Enabled %d\n", pVM->pgm.s.fA20Enabled));
144	return !!pVM->pgm.s.fA20Enabled ; /* stupid MS compiler doesn't trust me. */
145	}
146
147
148	/**
149	* Validates a GC physical address.
150	*
151	* @returns true if valid.
152	* @returns false if invalid.
153	* @param pVM The VM handle.
154	* @param GCPhys The physical address to validate.
155	*/
156	VMMDECL(bool) PGMPhysIsGCPhysValid(PVM pVM, RTGCPHYS GCPhys)
157	{
158	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
159	return pPage != NULL;
160	}
161
162
163	/**
164	* Checks if a GC physical address is a normal page,
165	* i.e. not ROM, MMIO or reserved.
166	*
167	* @returns true if normal.
168	* @returns false if invalid, ROM, MMIO or reserved page.
169	* @param pVM The VM handle.
170	* @param GCPhys The physical address to check.
171	*/
172	VMMDECL(bool) PGMPhysIsGCPhysNormal(PVM pVM, RTGCPHYS GCPhys)
173	{
174	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
175	return pPage
176	&& !(pPage->HCPhys & (MM_RAM_FLAGS_MMIO \| MM_RAM_FLAGS_ROM \| MM_RAM_FLAGS_RESERVED \| MM_RAM_FLAGS_MMIO2));
177	}
178
179
180	/**
181	* Converts a GC physical address to a HC physical address.
182	*
183	* @returns VINF_SUCCESS on success.
184	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
185	* page but has no physical backing.
186	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
187	* GC physical address.
188	*
189	* @param pVM The VM handle.
190	* @param GCPhys The GC physical address to convert.
191	* @param pHCPhys Where to store the HC physical address on success.
192	*/
193	VMMDECL(int) PGMPhysGCPhys2HCPhys(PVM pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
194	{
195	PPGMPAGE pPage;
196	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
197	if (RT_FAILURE(rc))
198	return rc;
199
200	#ifndef PGM_IGNORE_RAM_FLAGS_RESERVED
201	if (RT_UNLIKELY(pPage->HCPhys & MM_RAM_FLAGS_RESERVED)) /** @todo PAGE FLAGS */
202	return VERR_PGM_PHYS_PAGE_RESERVED;
203	#endif
204
205	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK);
206	return VINF_SUCCESS;
207	}
208
209
210	/**
211	* Invalidates the GC page mapping TLB.
212	*
213	* @param pVM The VM handle.
214	*/
215	VMMDECL(void) PGMPhysInvalidatePageGCMapTLB(PVM pVM)
216	{
217	/* later */
218	NOREF(pVM);
219	}
220
221
222	/**
223	* Invalidates the ring-0 page mapping TLB.
224	*
225	* @param pVM The VM handle.
226	*/
227	VMMDECL(void) PGMPhysInvalidatePageR0MapTLB(PVM pVM)
228	{
229	PGMPhysInvalidatePageR3MapTLB(pVM);
230	}
231
232
233	/**
234	* Invalidates the ring-3 page mapping TLB.
235	*
236	* @param pVM The VM handle.
237	*/
238	VMMDECL(void) PGMPhysInvalidatePageR3MapTLB(PVM pVM)
239	{
240	pgmLock(pVM);
241	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
242	{
243	pVM->pgm.s.PhysTlbHC.aEntries[i].GCPhys = NIL_RTGCPHYS;
244	pVM->pgm.s.PhysTlbHC.aEntries[i].pPage = 0;
245	pVM->pgm.s.PhysTlbHC.aEntries[i].pMap = 0;
246	pVM->pgm.s.PhysTlbHC.aEntries[i].pv = 0;
247	}
248	pgmUnlock(pVM);
249	}
250
251
252	/**
253	* Frees the specified RAM page.
254	*
255	* This is used by ballooning and remapping MMIO2.
256	*
257	* @param pVM Pointer to the shared VM structure.
258	* @param pPage Pointer to the page structure.
259	* @param GCPhys The guest physical address of the page, if applicable.
260	*/
261	void pgmPhysFreePage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
262	{
263	AssertFatal(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
264
265	/** @todo implement this... */
266	AssertFatalFailed();
267	}
268
269
270	/**
271	* Makes sure that there is at least one handy page ready for use.
272	*
273	* This will also take the appropriate actions when reaching water-marks.
274	*
275	* @returns The following VBox status codes.
276	* @retval VINF_SUCCESS on success.
277	* @retval VERR_EM_NO_MEMORY if we're really out of memory.
278	*
279	* @param pVM The VM handle.
280	*
281	* @remarks Must be called from within the PGM critical section. It may
282	* nip back to ring-3/0 in some cases.
283	*/
284	static int pgmPhysEnsureHandyPage(PVM pVM)
285	{
286	/** @remarks
287	* low-water mark logic for R0 & GC:
288	* - 75%: Set FF.
289	* - 50%: Force return to ring-3 ASAP.
290	*
291	* For ring-3 there is a little problem wrt to the recompiler, so:
292	* - 75%: Set FF.
293	* - 50%: Try allocate pages; on failure we'll force REM to quite ASAP.
294	*
295	* The basic idea is that we should be able to get out of any situation with
296	* only 50% of handy pages remaining.
297	*
298	* At the moment we'll not adjust the number of handy pages relative to the
299	* actual VM RAM committment, that's too much work for now.
300	*/
301	Assert(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages));
302	if ( !pVM->pgm.s.cHandyPages
303	#ifdef IN_RING3
304	\|\| pVM->pgm.s.cHandyPages - 1 <= RT_ELEMENTS(pVM->pgm.s.aHandyPages) / 2 /* 50% */
305	#endif
306	)
307	{
308	Log(("PGM: cHandyPages=%u out of %u -> allocate more\n", pVM->pgm.s.cHandyPages - 1 <= RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
309	#ifdef IN_RING3
310	int rc = SUPCallVMMR0Ex(pVM->pVMR0, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
311	#elif defined(IN_RING0)
312	/** @todo call PGMR0PhysAllocateHandyPages directly - need to make sure we can call kernel code first and deal with the seeding fallback. */
313	int rc = VMMR0CallHost(pVM, VMMCALLHOST_PGM_ALLOCATE_HANDY_PAGES, 0);
314	#else
315	int rc = VMMGCCallHost(pVM, VMMCALLHOST_PGM_ALLOCATE_HANDY_PAGES, 0);
316	#endif
317	if (RT_UNLIKELY(rc != VINF_SUCCESS))
318	{
319	Assert(rc == VINF_EM_NO_MEMORY);
320	if (!pVM->pgm.s.cHandyPages)
321	{
322	LogRel(("PGM: no more handy pages!\n"));
323	return VERR_EM_NO_MEMORY;
324	}
325	Assert(VM_FF_ISSET(pVM, VM_FF_PGM_NEED_HANDY_PAGES));
326	#ifdef IN_RING3
327	REMR3NotifyFF(pVM);
328	#else
329	VM_FF_SET(pVM, VM_FF_TO_R3);
330	#endif
331	}
332	Assert(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages));
333	}
334	else if (pVM->pgm.s.cHandyPages - 1 <= (RT_ELEMENTS(pVM->pgm.s.aHandyPages) / 4) * 3) /* 75% */
335	{
336	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
337	#ifndef IN_RING3
338	if (pVM->pgm.s.cHandyPages - 1 <= RT_ELEMENTS(pVM->pgm.s.aHandyPages) / 2)
339	{
340	Log(("PGM: VM_FF_TO_R3 - cHandyPages=%u out of %u\n", pVM->pgm.s.cHandyPages - 1 <= RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
341	VM_FF_SET(pVM, VM_FF_TO_R3);
342	}
343	#endif
344	}
345
346	return VINF_SUCCESS;
347	}
348
349
350	/**
351	* Replace a zero or shared page with new page that we can write to.
352	*
353	* @returns The following VBox status codes.
354	* @retval VINF_SUCCESS on success, pPage is modified.
355	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
356	*
357	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
358	*
359	* @param pVM The VM address.
360	* @param pPage The physical page tracking structure. This will
361	* be modified on success.
362	* @param GCPhys The address of the page.
363	*
364	* @remarks Must be called from within the PGM critical section. It may
365	* nip back to ring-3/0 in some cases.
366	*
367	* @remarks This function shouldn't really fail, however if it does
368	* it probably means we've screwed up the size of the amount
369	* and/or the low-water mark of handy pages. Or, that some
370	* device I/O is causing a lot of pages to be allocated while
371	* while the host is in a low-memory condition.
372	*/
373	int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
374	{
375	/*
376	* Ensure that we've got a page handy, take it and use it.
377	*/
378	int rc = pgmPhysEnsureHandyPage(pVM);
379	if (RT_FAILURE(rc))
380	{
381	Assert(rc == VERR_EM_NO_MEMORY);
382	return rc;
383	}
384	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%d %RGp\n", PGM_PAGE_GET_STATE(pPage), GCPhys));
385	Assert(!PGM_PAGE_IS_RESERVED(pPage));
386	Assert(!PGM_PAGE_IS_MMIO(pPage));
387
388	uint32_t iHandyPage = --pVM->pgm.s.cHandyPages;
389	Assert(iHandyPage < RT_ELEMENTS(pVM->pgm.s.aHandyPages));
390	Assert(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys != NIL_RTHCPHYS);
391	Assert(!(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
392	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idPage != NIL_GMM_PAGEID);
393	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
394
395	/*
396	* There are one or two action to be taken the next time we allocate handy pages:
397	* - Tell the GMM (global memory manager) what the page is being used for.
398	* (Speeds up replacement operations - sharing and defragmenting.)
399	* - If the current backing is shared, it must be freed.
400	*/
401	const RTHCPHYS HCPhys = pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys;
402	pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys = GCPhys;
403
404	if (PGM_PAGE_IS_SHARED(pPage))
405	{
406	pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage = PGM_PAGE_GET_PAGEID(pPage);
407	Assert(PGM_PAGE_GET_PAGEID(pPage) != NIL_GMM_PAGEID);
408	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
409
410	Log2(("PGM: Replaced shared page %#x at %RGp with %#x / %RHp\n", PGM_PAGE_GET_PAGEID(pPage),
411	GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
412	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PageReplaceShared));
413	pVM->pgm.s.cSharedPages--;
414	/** @todo err.. what about copying the page content? */
415	}
416	else
417	{
418	Log2(("PGM: Replaced zero page %RGp with %#x / %RHp\n", GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
419	STAM_COUNTER_INC(&pVM->pgm.s.StatRZPageReplaceZero);
420	pVM->pgm.s.cZeroPages--;
421	/** @todo verify that the handy page is zero! */
422	}
423
424	/*
425	* Do the PGMPAGE modifications.
426	*/
427	pVM->pgm.s.cPrivatePages++;
428	PGM_PAGE_SET_HCPHYS(pPage, HCPhys);
429	PGM_PAGE_SET_PAGEID(pPage, pVM->pgm.s.aHandyPages[iHandyPage].idPage);
430	PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
431
432	return VINF_SUCCESS;
433	}
434
435
436	/**
437	* Deal with pages that are not writable, i.e. not in the ALLOCATED state.
438	*
439	* @returns VBox status code.
440	* @retval VINF_SUCCESS on success.
441	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
442	*
443	* @param pVM The VM address.
444	* @param pPage The physical page tracking structure.
445	* @param GCPhys The address of the page.
446	*
447	* @remarks Called from within the PGM critical section.
448	*/
449	int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
450	{
451	switch (PGM_PAGE_GET_STATE(pPage))
452	{
453	case PGM_PAGE_STATE_WRITE_MONITORED:
454	PGM_PAGE_SET_WRITTEN_TO(pPage);
455	PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
456	/* fall thru */
457	default: /* to shut up GCC */
458	case PGM_PAGE_STATE_ALLOCATED:
459	return VINF_SUCCESS;
460
461	/*
462	* Zero pages can be dummy pages for MMIO or reserved memory,
463	* so we need to check the flags before joining cause with
464	* shared page replacement.
465	*/
466	case PGM_PAGE_STATE_ZERO:
467	if ( PGM_PAGE_IS_MMIO(pPage)
468	\|\| PGM_PAGE_IS_RESERVED(pPage))
469	return VERR_PGM_PHYS_PAGE_RESERVED;
470	/* fall thru */
471	case PGM_PAGE_STATE_SHARED:
472	return pgmPhysAllocPage(pVM, pPage, GCPhys);
473	}
474	}
475
476
477	/**
478	* Maps a page into the current virtual address space so it can be accessed.
479	*
480	* @returns VBox status code.
481	* @retval VINF_SUCCESS on success.
482	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
483	*
484	* @param pVM The VM address.
485	* @param pPage The physical page tracking structure.
486	* @param GCPhys The address of the page.
487	* @param ppMap Where to store the address of the mapping tracking structure.
488	* @param ppv Where to store the mapping address of the page. The page
489	* offset is masked off!
490	*
491	* @remarks Called from within the PGM critical section.
492	*/
493	int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv)
494	{
495	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
496	/*
497	* Just some sketchy GC/R0-darwin code.
498	*/
499	*ppMap = NULL;
500	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
501	Assert(HCPhys != pVM->pgm.s.HCPhysZeroPg);
502	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
503	return pgmR0DynMapHCPageInlined(&pVM->pgm.s, HCPhys, ppv);
504	# else
505	return PGMDynMapHCPage(pVM, HCPhys, ppv);
506	# endif
507
508	#else /* IN_RING3 \|\| IN_RING0 */
509
510	/*
511	* Find/make Chunk TLB entry for the mapping chunk.
512	*/
513	PPGMCHUNKR3MAP pMap;
514	const uint32_t idChunk = PGM_PAGE_GET_CHUNKID(pPage);
515	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
516	if (pTlbe->idChunk == idChunk)
517	{
518	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,ChunkR3MapTlbHits));
519	pMap = pTlbe->pChunk;
520	}
521	else if (idChunk != NIL_GMM_CHUNKID)
522	{
523	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
524
525	/*
526	* Find the chunk, map it if necessary.
527	*/
528	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
529	if (!pMap)
530	{
531	#ifdef IN_RING0
532	int rc = VMMR0CallHost(pVM, VMMCALLHOST_PGM_MAP_CHUNK, idChunk);
533	AssertRCReturn(rc, rc);
534	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
535	Assert(pMap);
536	#else
537	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
538	if (RT_FAILURE(rc))
539	return rc;
540	#endif
541	}
542
543	/*
544	* Enter it into the Chunk TLB.
545	*/
546	pTlbe->idChunk = idChunk;
547	pTlbe->pChunk = pMap;
548	pMap->iAge = 0;
549	}
550	else
551	{
552	Assert(PGM_PAGE_IS_ZERO(pPage));
553	*ppv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
554	*ppMap = NULL;
555	return VINF_SUCCESS;
556	}
557
558	ppv = (uint8_t )pMap->pv + (PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) << PAGE_SHIFT);
559	*ppMap = pMap;
560	return VINF_SUCCESS;
561	#endif /* IN_RING3 */
562	}
563
564
565	#if !defined(IN_RC) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
566	/**
567	* Load a guest page into the ring-3 physical TLB.
568	*
569	* @returns VBox status code.
570	* @retval VINF_SUCCESS on success
571	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
572	* @param pPGM The PGM instance pointer.
573	* @param GCPhys The guest physical address in question.
574	*/
575	int pgmPhysPageLoadIntoTlb(PPGM pPGM, RTGCPHYS GCPhys)
576	{
577	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbMisses));
578
579	/*
580	* Find the ram range.
581	* 99.8% of requests are expected to be in the first range.
582	*/
583	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
584	RTGCPHYS off = GCPhys - pRam->GCPhys;
585	if (RT_UNLIKELY(off >= pRam->cb))
586	{
587	do
588	{
589	pRam = pRam->CTX_SUFF(pNext);
590	if (!pRam)
591	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
592	off = GCPhys - pRam->GCPhys;
593	} while (off >= pRam->cb);
594	}
595
596	/*
597	* Map the page.
598	* Make a special case for the zero page as it is kind of special.
599	*/
600	PPGMPAGE pPage = &pRam->aPages[off >> PAGE_SHIFT];
601	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
602	if (!PGM_PAGE_IS_ZERO(pPage))
603	{
604	void *pv;
605	PPGMPAGEMAP pMap;
606	int rc = pgmPhysPageMap(PGM2VM(pPGM), pPage, GCPhys, &pMap, &pv);
607	if (RT_FAILURE(rc))
608	return rc;
609	pTlbe->pMap = pMap;
610	pTlbe->pv = pv;
611	}
612	else
613	{
614	Assert(PGM_PAGE_GET_HCPHYS(pPage) == pPGM->HCPhysZeroPg);
615	pTlbe->pMap = NULL;
616	pTlbe->pv = pPGM->CTXALLSUFF(pvZeroPg);
617	}
618	pTlbe->pPage = pPage;
619	return VINF_SUCCESS;
620	}
621	#endif /* !IN_RC && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
622
623
624	/**
625	* Requests the mapping of a guest page into the current context.
626	*
627	* This API should only be used for very short term, as it will consume
628	* scarse resources (R0 and GC) in the mapping cache. When you're done
629	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
630	*
631	* This API will assume your intention is to write to the page, and will
632	* therefore replace shared and zero pages. If you do not intend to modify
633	* the page, use the PGMPhysGCPhys2CCPtrReadOnly() API.
634	*
635	* @returns VBox status code.
636	* @retval VINF_SUCCESS on success.
637	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
638	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
639	*
640	* @param pVM The VM handle.
641	* @param GCPhys The guest physical address of the page that should be mapped.
642	* @param ppv Where to store the address corresponding to GCPhys.
643	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
644	*
645	* @remark Avoid calling this API from within critical sections (other than
646	* the PGM one) because of the deadlock risk.
647	* @thread Any thread.
648	*/
649	VMMDECL(int) PGMPhysGCPhys2CCPtr(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
650	{
651	#ifdef VBOX_WITH_NEW_PHYS_CODE
652	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
653	/** @todo this needs to be fixed, it really ain't right. */
654	/* Until a physical TLB is implemented for GC or/and R0-darwin, let PGMDynMapGCPageEx handle it. */
655	return PGMDynMapGCPageOff(pVM, GCPhys, ppv);
656
657	#else
658	int rc = pgmLock(pVM);
659	AssertRCReturn(rc);
660
661	/*
662	* Query the Physical TLB entry for the page (may fail).
663	*/
664	PGMPHYSTLBE pTlbe;
665	int rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
666	if (RT_SUCCESS(rc))
667	{
668	/*
669	* If the page is shared, the zero page, or being write monitored
670	* it must be converted to an page that's writable if possible.
671	*/
672	PPGMPAGE pPage = pTlbe->pPage;
673	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
674	{
675	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
676	/** @todo stuff is missing here! */
677	}
678	if (RT_SUCCESS(rc))
679	{
680	/*
681	* Now, just perform the locking and calculate the return address.
682	*/
683	PPGMPAGEMAP pMap = pTlbe->pMap;
684	pMap->cRefs++;
685	if (RT_LIKELY(pPage->cLocks != PGM_PAGE_MAX_LOCKS))
686	if (RT_UNLIKELY(++pPage->cLocks == PGM_PAGE_MAX_LOCKS))
687	{
688	AssertMsgFailed(("%RGp is entering permanent locked state!\n", GCPhys));
689	pMap->cRefs++; /* Extra ref to prevent it from going away. */
690	}
691
692	ppv = (void )((uintptr_t)pTlbe->pv \| (GCPhys & PAGE_OFFSET_MASK));
693	pLock->pvPage = pPage;
694	pLock->pvMap = pMap;
695	}
696	}
697
698	pgmUnlock(pVM);
699	return rc;
700
701	#endif /* IN_RING3 \|\| IN_RING0 */
702
703	#else
704	/*
705	* Temporary fallback code.
706	*/
707	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
708	/** @todo @bugref{3202}: check up this path. */
709	return PGMDynMapGCPageOff(pVM, GCPhys, ppv);
710	# else
711	return PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1, (PRTR3PTR)ppv);
712	# endif
713	#endif
714	}
715
716
717	/**
718	* Requests the mapping of a guest page into the current context.
719	*
720	* This API should only be used for very short term, as it will consume
721	* scarse resources (R0 and GC) in the mapping cache. When you're done
722	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
723	*
724	* @returns VBox status code.
725	* @retval VINF_SUCCESS on success.
726	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
727	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
728	*
729	* @param pVM The VM handle.
730	* @param GCPhys The guest physical address of the page that should be mapped.
731	* @param ppv Where to store the address corresponding to GCPhys.
732	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
733	*
734	* @remark Avoid calling this API from within critical sections (other than
735	* the PGM one) because of the deadlock risk.
736	* @thread Any thread.
737	*/
738	VMMDECL(int) PGMPhysGCPhys2CCPtrReadOnly(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
739	{
740	/** @todo implement this */
741	return PGMPhysGCPhys2CCPtr(pVM, GCPhys, (void **)ppv, pLock);
742	}
743
744
745	/**
746	* Requests the mapping of a guest page given by virtual address into the current context.
747	*
748	* This API should only be used for very short term, as it will consume
749	* scarse resources (R0 and GC) in the mapping cache. When you're done
750	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
751	*
752	* This API will assume your intention is to write to the page, and will
753	* therefore replace shared and zero pages. If you do not intend to modify
754	* the page, use the PGMPhysGCPtr2CCPtrReadOnly() API.
755	*
756	* @returns VBox status code.
757	* @retval VINF_SUCCESS on success.
758	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
759	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
760	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
761	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
762	*
763	* @param pVM The VM handle.
764	* @param GCPhys The guest physical address of the page that should be mapped.
765	* @param ppv Where to store the address corresponding to GCPhys.
766	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
767	*
768	* @remark Avoid calling this API from within critical sections (other than
769	* the PGM one) because of the deadlock risk.
770	* @thread EMT
771	*/
772	VMMDECL(int) PGMPhysGCPtr2CCPtr(PVM pVM, RTGCPTR GCPtr, void **ppv, PPGMPAGEMAPLOCK pLock)
773	{
774	RTGCPHYS GCPhys;
775	int rc = PGMPhysGCPtr2GCPhys(pVM, GCPtr, &GCPhys);
776	if (RT_SUCCESS(rc))
777	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, ppv, pLock);
778	return rc;
779	}
780
781
782	/**
783	* Requests the mapping of a guest page given by virtual address into the current context.
784	*
785	* This API should only be used for very short term, as it will consume
786	* scarse resources (R0 and GC) in the mapping cache. When you're done
787	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
788	*
789	* @returns VBox status code.
790	* @retval VINF_SUCCESS on success.
791	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
792	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
793	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
794	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
795	*
796	* @param pVM The VM handle.
797	* @param GCPhys The guest physical address of the page that should be mapped.
798	* @param ppv Where to store the address corresponding to GCPhys.
799	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
800	*
801	* @remark Avoid calling this API from within critical sections (other than
802	* the PGM one) because of the deadlock risk.
803	* @thread EMT
804	*/
805	VMMDECL(int) PGMPhysGCPtr2CCPtrReadOnly(PVM pVM, RTGCPTR GCPtr, void const **ppv, PPGMPAGEMAPLOCK pLock)
806	{
807	RTGCPHYS GCPhys;
808	int rc = PGMPhysGCPtr2GCPhys(pVM, GCPtr, &GCPhys);
809	if (RT_SUCCESS(rc))
810	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, ppv, pLock);
811	return rc;
812	}
813
814
815	/**
816	* Release the mapping of a guest page.
817	*
818	* This is the counter part of PGMPhysGCPhys2CCPtr, PGMPhysGCPhys2CCPtrReadOnly
819	* PGMPhysGCPtr2CCPtr and PGMPhysGCPtr2CCPtrReadOnly.
820	*
821	* @param pVM The VM handle.
822	* @param pLock The lock structure initialized by the mapping function.
823	*/
824	VMMDECL(void) PGMPhysReleasePageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock)
825	{
826	#ifdef VBOX_WITH_NEW_PHYS_CODE
827	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
828	/* currently nothing to do here. */
829	/* --- postponed
830	#elif defined(IN_RING0)
831	*/
832
833	#else /* IN_RING3 */
834	pgmLock(pVM);
835
836	PPGMPAGE pPage = (PPGMPAGE)pLock->pvPage;
837	Assert(pPage->cLocks >= 1);
838	if (pPage->cLocks != PGM_PAGE_MAX_LOCKS)
839	pPage->cLocks--;
840
841	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pLock->pvChunk;
842	Assert(pChunk->cRefs >= 1);
843	pChunk->cRefs--;
844	pChunk->iAge = 0;
845
846	pgmUnlock(pVM);
847	#endif /* IN_RING3 */
848	#else
849	NOREF(pVM);
850	NOREF(pLock);
851	#endif
852	}
853
854
855	/**
856	* Converts a GC physical address to a HC ring-3 pointer.
857	*
858	* @returns VINF_SUCCESS on success.
859	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
860	* page but has no physical backing.
861	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
862	* GC physical address.
863	* @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
864	* a dynamic ram chunk boundary
865	*
866	* @param pVM The VM handle.
867	* @param GCPhys The GC physical address to convert.
868	* @param cbRange Physical range
869	* @param pR3Ptr Where to store the R3 pointer on success.
870	*/
871	VMMDECL(int) PGMPhysGCPhys2R3Ptr(PVM pVM, RTGCPHYS GCPhys, RTUINT cbRange, PRTR3PTR pR3Ptr)
872	{
873	#ifdef VBOX_WITH_NEW_PHYS_CODE
874	VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
875	#endif
876
877	if ((GCPhys & PGM_DYNAMIC_CHUNK_BASE_MASK) != ((GCPhys+cbRange-1) & PGM_DYNAMIC_CHUNK_BASE_MASK))
878	{
879	AssertMsgFailed(("%RGp - %RGp crosses a chunk boundary!!\n", GCPhys, GCPhys+cbRange));
880	LogRel(("PGMPhysGCPhys2HCPtr %RGp - %RGp crosses a chunk boundary!!\n", GCPhys, GCPhys+cbRange));
881	return VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY;
882	}
883
884	PPGMRAMRANGE pRam;
885	PPGMPAGE pPage;
886	int rc = pgmPhysGetPageAndRangeEx(&pVM->pgm.s, GCPhys, &pPage, &pRam);
887	if (RT_FAILURE(rc))
888	return rc;
889
890	#ifndef PGM_IGNORE_RAM_FLAGS_RESERVED
891	if (RT_UNLIKELY(PGM_PAGE_IS_RESERVED(pPage)))
892	return VERR_PGM_PHYS_PAGE_RESERVED;
893	#endif
894
895	RTGCPHYS off = GCPhys - pRam->GCPhys;
896	if (RT_UNLIKELY(off + cbRange > pRam->cb))
897	{
898	AssertMsgFailed(("%RGp - %RGp crosses a chunk boundary!!\n", GCPhys, GCPhys + cbRange));
899	return VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY;
900	}
901
902	if (pRam->fFlags & MM_RAM_FLAGS_DYNAMIC_ALLOC)
903	{
904	unsigned iChunk = (off >> PGM_DYNAMIC_CHUNK_SHIFT);
905	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) /* ASSUMES this is a rare occurence */
906	PRTR3UINTPTR paChunkR3Ptrs = (PRTR3UINTPTR)MMHyperR3ToCC(pVM, pRam->paChunkR3Ptrs);
907	*pR3Ptr = (RTR3PTR)(paChunkR3Ptrs[iChunk] + (off & PGM_DYNAMIC_CHUNK_OFFSET_MASK));
908	#else
909	*pR3Ptr = (RTR3PTR)(pRam->paChunkR3Ptrs[iChunk] + (off & PGM_DYNAMIC_CHUNK_OFFSET_MASK));
910	#endif
911	}
912	else if (RT_LIKELY(pRam->pvR3))
913	*pR3Ptr = (RTR3PTR)((RTR3UINTPTR)pRam->pvR3 + off);
914	else
915	return VERR_PGM_PHYS_PAGE_RESERVED;
916	return VINF_SUCCESS;
917	}
918
919
920	/**
921	* PGMPhysGCPhys2R3Ptr convenience for use with assertions.
922	*
923	* @returns The R3Ptr, NIL_RTR3PTR on failure.
924	* @param pVM The VM handle.
925	* @param GCPhys The GC Physical addresss.
926	* @param cbRange Physical range.
927	*/
928	VMMDECL(RTR3PTR) PGMPhysGCPhys2R3PtrAssert(PVM pVM, RTGCPHYS GCPhys, RTUINT cbRange)
929	{
930	RTR3PTR R3Ptr;
931	int rc = PGMPhysGCPhys2R3Ptr(pVM, GCPhys, cbRange, &R3Ptr);
932	if (RT_SUCCESS(rc))
933	return R3Ptr;
934	return NIL_RTR3PTR;
935	}
936
937
938	/**
939	* Converts a guest pointer to a GC physical address.
940	*
941	* This uses the current CR3/CR0/CR4 of the guest.
942	*
943	* @returns VBox status code.
944	* @param pVM The VM Handle
945	* @param GCPtr The guest pointer to convert.
946	* @param pGCPhys Where to store the GC physical address.
947	*/
948	VMMDECL(int) PGMPhysGCPtr2GCPhys(PVM pVM, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
949	{
950	int rc = PGM_GST_PFN(GetPage,pVM)(pVM, (RTGCUINTPTR)GCPtr, NULL, pGCPhys);
951	if (pGCPhys && RT_SUCCESS(rc))
952	*pGCPhys \|= (RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK;
953	return rc;
954	}
955
956
957	/**
958	* Converts a guest pointer to a HC physical address.
959	*
960	* This uses the current CR3/CR0/CR4 of the guest.
961	*
962	* @returns VBox status code.
963	* @param pVM The VM Handle
964	* @param GCPtr The guest pointer to convert.
965	* @param pHCPhys Where to store the HC physical address.
966	*/
967	VMMDECL(int) PGMPhysGCPtr2HCPhys(PVM pVM, RTGCPTR GCPtr, PRTHCPHYS pHCPhys)
968	{
969	RTGCPHYS GCPhys;
970	int rc = PGM_GST_PFN(GetPage,pVM)(pVM, (RTGCUINTPTR)GCPtr, NULL, &GCPhys);
971	if (RT_SUCCESS(rc))
972	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys \| ((RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK), pHCPhys);
973	return rc;
974	}
975
976
977	/**
978	* Converts a guest pointer to a R3 pointer.
979	*
980	* This uses the current CR3/CR0/CR4 of the guest.
981	*
982	* @returns VBox status code.
983	* @param pVM The VM Handle
984	* @param GCPtr The guest pointer to convert.
985	* @param pR3Ptr Where to store the R3 virtual address.
986	*/
987	VMMDECL(int) PGMPhysGCPtr2R3Ptr(PVM pVM, RTGCPTR GCPtr, PRTR3PTR pR3Ptr)
988	{
989	#ifdef VBOX_WITH_NEW_PHYS_CODE
990	VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
991	#endif
992
993	RTGCPHYS GCPhys;
994	int rc = PGM_GST_PFN(GetPage,pVM)(pVM, (RTGCUINTPTR)GCPtr, NULL, &GCPhys);
995	if (RT_SUCCESS(rc))
996	rc = PGMPhysGCPhys2R3Ptr(pVM, GCPhys \| ((RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK), 1 /* we always stay within one page */, pR3Ptr);
997	return rc;
998	}
999
1000
1001	/**
1002	* Converts a guest virtual address to a HC ring-3 pointer by specfied CR3 and
1003	* flags.
1004	*
1005	* @returns VBox status code.
1006	* @param pVM The VM Handle
1007	* @param GCPtr The guest pointer to convert.
1008	* @param cr3 The guest CR3.
1009	* @param fFlags Flags used for interpreting the PD correctly: X86_CR4_PSE and X86_CR4_PAE
1010	* @param pR3Ptr Where to store the R3 pointer.
1011	*
1012	* @remark This function is used by the REM at a time where PGM could
1013	* potentially not be in sync. It could also be used by a
1014	* future DBGF API to cpu state independent conversions.
1015	*/
1016	VMMDECL(int) PGMPhysGCPtr2R3PtrByGstCR3(PVM pVM, RTGCPTR GCPtr, uint64_t cr3, unsigned fFlags, PRTR3PTR pR3Ptr)
1017	{
1018	#ifdef VBOX_WITH_NEW_PHYS_CODE
1019	VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
1020	#endif
1021	/*
1022	* PAE or 32-bit?
1023	*/
1024	Assert(!CPUMIsGuestInLongMode(pVM));
1025
1026	int rc;
1027	if (!(fFlags & X86_CR4_PAE))
1028	{
1029	PX86PD pPD;
1030	rc = PGM_GCPHYS_2_PTR(pVM, cr3 & X86_CR3_PAGE_MASK, &pPD);
1031	if (RT_SUCCESS(rc))
1032	{
1033	X86PDE Pde = pPD->a[(RTGCUINTPTR)GCPtr >> X86_PD_SHIFT];
1034	if (Pde.n.u1Present)
1035	{
1036	if ((fFlags & X86_CR4_PSE) && Pde.b.u1Size)
1037	{ /* (big page) */
1038	rc = PGMPhysGCPhys2R3Ptr(pVM, pgmGstGet4MBPhysPage(&pVM->pgm.s, Pde) \| ((RTGCUINTPTR)GCPtr & X86_PAGE_4M_OFFSET_MASK),
1039	1 /* we always stay within one page */, pR3Ptr);
1040	}
1041	else
1042	{ /* (normal page) */
1043	PX86PT pPT;
1044	rc = PGM_GCPHYS_2_PTR(pVM, Pde.u & X86_PDE_PG_MASK, &pPT);
1045	if (RT_SUCCESS(rc))
1046	{
1047	X86PTE Pte = pPT->a[((RTGCUINTPTR)GCPtr >> X86_PT_SHIFT) & X86_PT_MASK];
1048	if (Pte.n.u1Present)
1049	return PGMPhysGCPhys2R3Ptr(pVM, (Pte.u & X86_PTE_PG_MASK) \| ((RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK),
1050	1 /* we always stay within one page */, pR3Ptr);
1051	rc = VERR_PAGE_NOT_PRESENT;
1052	}
1053	}
1054	}
1055	else
1056	rc = VERR_PAGE_TABLE_NOT_PRESENT;
1057	}
1058	}
1059	else
1060	{
1061	/** @todo long mode! */
1062	Assert(PGMGetGuestMode(pVM) < PGMMODE_AMD64);
1063
1064	PX86PDPT pPdpt;
1065	rc = PGM_GCPHYS_2_PTR(pVM, cr3 & X86_CR3_PAE_PAGE_MASK, &pPdpt);
1066	if (RT_SUCCESS(rc))
1067	{
1068	X86PDPE Pdpe = pPdpt->a[((RTGCUINTPTR)GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE];
1069	if (Pdpe.n.u1Present)
1070	{
1071	PX86PDPAE pPD;
1072	rc = PGM_GCPHYS_2_PTR(pVM, Pdpe.u & X86_PDPE_PG_MASK, &pPD);
1073	if (RT_SUCCESS(rc))
1074	{
1075	X86PDEPAE Pde = pPD->a[((RTGCUINTPTR)GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK];
1076	if (Pde.n.u1Present)
1077	{
1078	if ((fFlags & X86_CR4_PSE) && Pde.b.u1Size)
1079	{ /* (big page) */
1080	rc = PGMPhysGCPhys2R3Ptr(pVM, (Pde.u & X86_PDE2M_PAE_PG_MASK) \| ((RTGCUINTPTR)GCPtr & X86_PAGE_2M_OFFSET_MASK),
1081	1 /* we always stay within one page */, pR3Ptr);
1082	}
1083	else
1084	{ /* (normal page) */
1085	PX86PTPAE pPT;
1086	rc = PGM_GCPHYS_2_PTR(pVM, (Pde.u & X86_PDE_PAE_PG_MASK), &pPT);
1087	if (RT_SUCCESS(rc))
1088	{
1089	X86PTEPAE Pte = pPT->a[((RTGCUINTPTR)GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK];
1090	if (Pte.n.u1Present)
1091	return PGMPhysGCPhys2R3Ptr(pVM, (Pte.u & X86_PTE_PAE_PG_MASK) \| ((RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK),
1092	1 /* we always stay within one page */, pR3Ptr);
1093	rc = VERR_PAGE_NOT_PRESENT;
1094	}
1095	}
1096	}
1097	else
1098	rc = VERR_PAGE_TABLE_NOT_PRESENT;
1099	}
1100	}
1101	else
1102	rc = VERR_PAGE_TABLE_NOT_PRESENT;
1103	}
1104	}
1105	return rc;
1106	}
1107
1108
1109	#undef LOG_GROUP
1110	#define LOG_GROUP LOG_GROUP_PGM_PHYS_ACCESS
1111
1112
1113	#ifdef IN_RING3
1114	/**
1115	* Cache PGMPhys memory access
1116	*
1117	* @param pVM VM Handle.
1118	* @param pCache Cache structure pointer
1119	* @param GCPhys GC physical address
1120	* @param pbHC HC pointer corresponding to physical page
1121	*
1122	* @thread EMT.
1123	*/
1124	static void pgmPhysCacheAdd(PVM pVM, PGMPHYSCACHE pCache, RTGCPHYS GCPhys, uint8_t pbR3)
1125	{
1126	uint32_t iCacheIndex;
1127
1128	Assert(VM_IS_EMT(pVM));
1129
1130	GCPhys = PHYS_PAGE_ADDRESS(GCPhys);
1131	pbR3 = (uint8_t *)PAGE_ADDRESS(pbR3);
1132
1133	iCacheIndex = ((GCPhys >> PAGE_SHIFT) & PGM_MAX_PHYSCACHE_ENTRIES_MASK);
1134
1135	ASMBitSet(&pCache->aEntries, iCacheIndex);
1136
1137	pCache->Entry[iCacheIndex].GCPhys = GCPhys;
1138	pCache->Entry[iCacheIndex].pbR3 = pbR3;
1139	}
1140	#endif /* IN_RING3 */
1141
1142	/**
1143	* Read physical memory.
1144	*
1145	* This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
1146	* want to ignore those.
1147	*
1148	* @param pVM VM Handle.
1149	* @param GCPhys Physical address start reading from.
1150	* @param pvBuf Where to put the read bits.
1151	* @param cbRead How many bytes to read.
1152	*/
1153	VMMDECL(void) PGMPhysRead(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
1154	{
1155	#ifdef IN_RING3
1156	bool fGrabbedLock = false;
1157	#endif
1158
1159	AssertMsg(cbRead > 0, ("don't even think about reading zero bytes!\n"));
1160	if (cbRead == 0)
1161	return;
1162
1163	LogFlow(("PGMPhysRead: %RGp %d\n", GCPhys, cbRead));
1164
1165	#ifdef IN_RING3
1166	if (!VM_IS_EMT(pVM))
1167	{
1168	pgmLock(pVM);
1169	fGrabbedLock = true;
1170	}
1171	#endif
1172
1173	/*
1174	* Copy loop on ram ranges.
1175	*/
1176	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1177	for (;;)
1178	{
1179	/* Find range. */
1180	while (pRam && GCPhys > pRam->GCPhysLast)
1181	pRam = pRam->CTX_SUFF(pNext);
1182	/* Inside range or not? */
1183	if (pRam && GCPhys >= pRam->GCPhys)
1184	{
1185	/*
1186	* Must work our way thru this page by page.
1187	*/
1188	RTGCPHYS off = GCPhys - pRam->GCPhys;
1189	while (off < pRam->cb)
1190	{
1191	unsigned iPage = off >> PAGE_SHIFT;
1192	PPGMPAGE pPage = &pRam->aPages[iPage];
1193	size_t cb;
1194
1195	/* Physical chunk in dynamically allocated range not present? */
1196	if (RT_UNLIKELY(!PGM_PAGE_GET_HCPHYS(pPage)))
1197	{
1198	/* Treat it as reserved; return zeros */
1199	cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
1200	if (cb >= cbRead)
1201	{
1202	memset(pvBuf, 0, cbRead);
1203	goto end;
1204	}
1205	memset(pvBuf, 0, cb);
1206	}
1207	/* temp hacks, will be reorganized. */
1208	/*
1209	* Physical handler.
1210	*/
1211	else if ( RT_UNLIKELY(PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) >= PGM_PAGE_HNDL_PHYS_STATE_ALL)
1212	&& !(pPage->HCPhys & MM_RAM_FLAGS_MMIO)) /// @todo PAGE FLAGS
1213	{
1214	int rc = VINF_PGM_HANDLER_DO_DEFAULT;
1215	cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
1216
1217	#ifdef IN_RING3 /** @todo deal with this in GC and R0! */
1218	/* find and call the handler */
1219	PPGMPHYSHANDLER pNode = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.pTreesR3->PhysHandlers, GCPhys);
1220	if (pNode && pNode->pfnHandlerR3)
1221	{
1222	size_t cbRange = pNode->Core.KeyLast - GCPhys + 1;
1223	if (cbRange < cb)
1224	cb = cbRange;
1225	if (cb > cbRead)
1226	cb = cbRead;
1227
1228	void *pvSrc = PGMRAMRANGE_GETHCPTR(pRam, off)
1229
1230	/* Note! Dangerous assumption that HC handlers don't do anything that really requires an EMT lock! */
1231	rc = pNode->pfnHandlerR3(pVM, GCPhys, pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, pNode->pvUserR3);
1232	}
1233	#endif /* IN_RING3 */
1234	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
1235	{
1236	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1237	void *pvSrc = NULL;
1238	PGMDynMapHCPageOff(pVM, PGM_PAGE_GET_HCPHYS(pPage) + (off & PAGE_OFFSET_MASK), &pvSrc);
1239	#else
1240	void *pvSrc = PGMRAMRANGE_GETHCPTR(pRam, off)
1241	#endif
1242
1243	if (cb >= cbRead)
1244	{
1245	memcpy(pvBuf, pvSrc, cbRead);
1246	goto end;
1247	}
1248	memcpy(pvBuf, pvSrc, cb);
1249	}
1250	else if (cb >= cbRead)
1251	goto end;
1252	}
1253	/*
1254	* Virtual handlers.
1255	*/
1256	else if ( RT_UNLIKELY(PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) >= PGM_PAGE_HNDL_VIRT_STATE_ALL)
1257	&& !(pPage->HCPhys & MM_RAM_FLAGS_MMIO)) /// @todo PAGE FLAGS
1258	{
1259	int rc = VINF_PGM_HANDLER_DO_DEFAULT;
1260	cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
1261	#ifdef IN_RING3 /** @todo deal with this in GC and R0! */
1262	/* Search the whole tree for matching physical addresses (rather expensive!) */
1263	PPGMVIRTHANDLER pNode;
1264	unsigned iPage;
1265	int rc2 = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pNode, &iPage);
1266	if (RT_SUCCESS(rc2) && pNode->pfnHandlerR3)
1267	{
1268	size_t cbRange = pNode->Core.KeyLast - GCPhys + 1;
1269	if (cbRange < cb)
1270	cb = cbRange;
1271	if (cb > cbRead)
1272	cb = cbRead;
1273	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pNode->Core.Key & PAGE_BASE_GC_MASK)
1274	+ (iPage << PAGE_SHIFT) + (off & PAGE_OFFSET_MASK);
1275
1276	void *pvSrc = PGMRAMRANGE_GETHCPTR(pRam, off)
1277
1278	/* Note! Dangerous assumption that HC handlers don't do anything that really requires an EMT lock! */
1279	rc = pNode->pfnHandlerR3(pVM, (RTGCPTR)GCPtr, pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, 0);
1280	}
1281	#endif /* IN_RING3 */
1282	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
1283	{
1284	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1285	void *pvSrc = NULL;
1286	PGMDynMapHCPageOff(pVM, PGM_PAGE_GET_HCPHYS(pPage) + (off & PAGE_OFFSET_MASK), &pvSrc);
1287	#else
1288	void *pvSrc = PGMRAMRANGE_GETHCPTR(pRam, off)
1289	#endif
1290	if (cb >= cbRead)
1291	{
1292	memcpy(pvBuf, pvSrc, cbRead);
1293	goto end;
1294	}
1295	memcpy(pvBuf, pvSrc, cb);
1296	}
1297	else if (cb >= cbRead)
1298	goto end;
1299	}
1300	else
1301	{
1302	switch (pPage->HCPhys & (MM_RAM_FLAGS_RESERVED \| MM_RAM_FLAGS_MMIO \| MM_RAM_FLAGS_ROM)) /** @todo PAGE FLAGS */
1303	{
1304	/*
1305	* Normal memory or ROM.
1306	*/
1307	case 0:
1308	case MM_RAM_FLAGS_ROM:
1309	case MM_RAM_FLAGS_ROM \| MM_RAM_FLAGS_RESERVED:
1310	//case MM_RAM_FLAGS_ROM \| MM_RAM_FLAGS_MMIO2: /* = shadow */ - //MMIO2 isn't in the mask.
1311	case MM_RAM_FLAGS_MMIO2: // MMIO2 isn't in the mask.
1312	{
1313	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1314	void *pvSrc = NULL;
1315	PGMDynMapHCPageOff(pVM, PGM_PAGE_GET_HCPHYS(pPage) + (off & PAGE_OFFSET_MASK), &pvSrc);
1316	#else
1317	void *pvSrc = PGMRAMRANGE_GETHCPTR(pRam, off)
1318	#endif
1319	cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
1320	if (cb >= cbRead)
1321	{
1322	#if defined(IN_RING3) && defined(PGM_PHYSMEMACCESS_CACHING)
1323	if (cbRead <= 4 && !fGrabbedLock /* i.e. EMT */)
1324	pgmPhysCacheAdd(pVM, &pVM->pgm.s.pgmphysreadcache, GCPhys, (uint8_t*)pvSrc);
1325	#endif /* IN_RING3 && PGM_PHYSMEMACCESS_CACHING */
1326	memcpy(pvBuf, pvSrc, cbRead);
1327	goto end;
1328	}
1329	memcpy(pvBuf, pvSrc, cb);
1330	break;
1331	}
1332
1333	/*
1334	* All reserved, nothing there.
1335	*/
1336	case MM_RAM_FLAGS_RESERVED:
1337	cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
1338	if (cb >= cbRead)
1339	{
1340	memset(pvBuf, 0, cbRead);
1341	goto end;
1342	}
1343	memset(pvBuf, 0, cb);
1344	break;
1345
1346	/*
1347	* The rest needs to be taken more carefully.
1348	*/
1349	default:
1350	#if 1 /** @todo r=bird: Can you do this properly please. */
1351	/** @todo Try MMIO; quick hack */
1352	if (cbRead <= 8 && IOMMMIORead(pVM, GCPhys, (uint32_t *)pvBuf, cbRead) == VINF_SUCCESS)
1353	goto end;
1354	#endif
1355
1356	/** @todo fix me later. */
1357	AssertReleaseMsgFailed(("Unknown read at %RGp size %u implement the complex physical reading case %RHp\n",
1358	GCPhys, cbRead,
1359	pPage->HCPhys & (MM_RAM_FLAGS_RESERVED \| MM_RAM_FLAGS_MMIO \| MM_RAM_FLAGS_ROM))); /** @todo PAGE FLAGS */
1360	cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
1361	break;
1362	}
1363	}
1364	cbRead -= cb;
1365	off += cb;
1366	pvBuf = (char *)pvBuf + cb;
1367	}
1368
1369	GCPhys = pRam->GCPhysLast + 1;
1370	}
1371	else
1372	{
1373	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
1374
1375	/*
1376	* Unassigned address space.
1377	*/
1378	size_t cb;
1379	if ( !pRam
1380	\|\| (cb = pRam->GCPhys - GCPhys) >= cbRead)
1381	{
1382	memset(pvBuf, 0, cbRead);
1383	goto end;
1384	}
1385
1386	memset(pvBuf, 0, cb);
1387	cbRead -= cb;
1388	pvBuf = (char *)pvBuf + cb;
1389	GCPhys += cb;
1390	}
1391	}
1392	end:
1393	#ifdef IN_RING3
1394	if (fGrabbedLock)
1395	pgmUnlock(pVM);
1396	#endif
1397	return;
1398	}
1399
1400
1401	/**
1402	* Write to physical memory.
1403	*
1404	* This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
1405	* want to ignore those.
1406	*
1407	* @param pVM VM Handle.
1408	* @param GCPhys Physical address to write to.
1409	* @param pvBuf What to write.
1410	* @param cbWrite How many bytes to write.
1411	*/
1412	VMMDECL(void) PGMPhysWrite(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite)
1413	{
1414	#ifdef IN_RING3
1415	bool fGrabbedLock = false;
1416	#endif
1417
1418	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMPhysWrite after pgmR3Save()!\n"));
1419	AssertMsg(cbWrite > 0, ("don't even think about writing zero bytes!\n"));
1420	if (cbWrite == 0)
1421	return;
1422
1423	LogFlow(("PGMPhysWrite: %RGp %d\n", GCPhys, cbWrite));
1424
1425	#ifdef IN_RING3
1426	if (!VM_IS_EMT(pVM))
1427	{
1428	pgmLock(pVM);
1429	fGrabbedLock = true;
1430	}
1431	#endif
1432	/*
1433	* Copy loop on ram ranges.
1434	*/
1435	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1436	for (;;)
1437	{
1438	/* Find range. */
1439	while (pRam && GCPhys > pRam->GCPhysLast)
1440	pRam = pRam->CTX_SUFF(pNext);
1441	/* Inside range or not? */
1442	if (pRam && GCPhys >= pRam->GCPhys)
1443	{
1444	/*
1445	* Must work our way thru this page by page.
1446	*/
1447	RTGCPTR off = GCPhys - pRam->GCPhys;
1448	while (off < pRam->cb)
1449	{
1450	RTGCPTR iPage = off >> PAGE_SHIFT;
1451	PPGMPAGE pPage = &pRam->aPages[iPage];
1452
1453	/* Physical chunk in dynamically allocated range not present? */
1454	if (RT_UNLIKELY(!PGM_PAGE_GET_HCPHYS(pPage)))
1455	{
1456	int rc;
1457	#ifdef IN_RING3
1458	if (fGrabbedLock)
1459	{
1460	pgmUnlock(pVM);
1461	rc = pgmr3PhysGrowRange(pVM, GCPhys);
1462	if (rc == VINF_SUCCESS)
1463	PGMPhysWrite(pVM, GCPhys, pvBuf, cbWrite); /* try again; can't assume pRam is still valid (paranoia) */
1464	return;
1465	}
1466	rc = pgmr3PhysGrowRange(pVM, GCPhys);
1467	#else
1468	rc = CTXALLMID(VMM, CallHost)(pVM, VMMCALLHOST_PGM_RAM_GROW_RANGE, GCPhys);
1469	#endif
1470	if (rc != VINF_SUCCESS)
1471	goto end;
1472	}
1473
1474	size_t cb;
1475	/* temporary hack, will reogranize is later. */
1476	/*
1477	* Virtual handlers
1478	*/
1479	if ( PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage)
1480	&& !(pPage->HCPhys & MM_RAM_FLAGS_MMIO)) /// @todo PAGE FLAGS
1481	{
1482	if (PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage))
1483	{
1484	/*
1485	* Physical write handler + virtual write handler.
1486	* Consider this a quick workaround for the CSAM + shadow caching problem.
1487	*
1488	* We hand it to the shadow caching first since it requires the unchanged
1489	* data. CSAM will have to put up with it already being changed.
1490	*/
1491	int rc = VINF_PGM_HANDLER_DO_DEFAULT;
1492	cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
1493	#ifdef IN_RING3 /** @todo deal with this in GC and R0! */
1494	/* 1. The physical handler */
1495	PPGMPHYSHANDLER pPhysNode = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.pTreesR3->PhysHandlers, GCPhys);
1496	if (pPhysNode && pPhysNode->pfnHandlerR3)
1497	{
1498	size_t cbRange = pPhysNode->Core.KeyLast - GCPhys + 1;
1499	if (cbRange < cb)
1500	cb = cbRange;
1501	if (cb > cbWrite)
1502	cb = cbWrite;
1503
1504	void *pvDst = PGMRAMRANGE_GETHCPTR(pRam, off)
1505
1506	/* Note! Dangerous assumption that R3 handlers don't do anything that really requires an EMT lock! */
1507	rc = pPhysNode->pfnHandlerR3(pVM, GCPhys, pvDst, (void *)pvBuf, cb, PGMACCESSTYPE_WRITE, pPhysNode->pvUserR3);
1508	}
1509
1510	/* 2. The virtual handler (will see incorrect data) */
1511	PPGMVIRTHANDLER pVirtNode;
1512	unsigned iPage;
1513	int rc2 = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pVirtNode, &iPage);
1514	if (RT_SUCCESS(rc2) && pVirtNode->pfnHandlerR3)
1515	{
1516	size_t cbRange = pVirtNode->Core.KeyLast - GCPhys + 1;
1517	if (cbRange < cb)
1518	cb = cbRange;
1519	if (cb > cbWrite)
1520	cb = cbWrite;
1521	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirtNode->Core.Key & PAGE_BASE_GC_MASK)
1522	+ (iPage << PAGE_SHIFT) + (off & PAGE_OFFSET_MASK);
1523
1524	void *pvDst = PGMRAMRANGE_GETHCPTR(pRam, off)
1525
1526	/* Note! Dangerous assumption that R3 handlers don't do anything that really requires an EMT lock! */
1527	rc2 = pVirtNode->pfnHandlerR3(pVM, (RTGCPTR)GCPtr, pvDst, (void *)pvBuf, cb, PGMACCESSTYPE_WRITE, 0);
1528	if ( ( rc2 != VINF_PGM_HANDLER_DO_DEFAULT
1529	&& rc == VINF_PGM_HANDLER_DO_DEFAULT)
1530	\|\| ( RT_FAILURE(rc2)
1531	&& RT_SUCCESS(rc)))
1532	rc = rc2;
1533	}
1534	#endif /* IN_RING3 */
1535	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
1536	{
1537	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1538	void *pvDst = NULL;
1539	PGMDynMapHCPageOff(pVM, PGM_PAGE_GET_HCPHYS(pPage) + (off & PAGE_OFFSET_MASK), &pvDst);
1540	#else
1541	void *pvDst = PGMRAMRANGE_GETHCPTR(pRam, off)
1542	#endif
1543	if (cb >= cbWrite)
1544	{
1545	memcpy(pvDst, pvBuf, cbWrite);
1546	goto end;
1547	}
1548	memcpy(pvDst, pvBuf, cb);
1549	}
1550	else if (cb >= cbWrite)
1551	goto end;
1552	}
1553	else
1554	{
1555	int rc = VINF_PGM_HANDLER_DO_DEFAULT;
1556	cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
1557	#ifdef IN_RING3
1558	/** @todo deal with this in GC and R0! */
1559	/* Search the whole tree for matching physical addresses (rather expensive!) */
1560	PPGMVIRTHANDLER pNode;
1561	unsigned iPage;
1562	int rc2 = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pNode, &iPage);
1563	if (RT_SUCCESS(rc2) && pNode->pfnHandlerR3)
1564	{
1565	size_t cbRange = pNode->Core.KeyLast - GCPhys + 1;
1566	if (cbRange < cb)
1567	cb = cbRange;
1568	if (cb > cbWrite)
1569	cb = cbWrite;
1570	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pNode->Core.Key & PAGE_BASE_GC_MASK)
1571	+ (iPage << PAGE_SHIFT) + (off & PAGE_OFFSET_MASK);
1572
1573	void *pvDst = PGMRAMRANGE_GETHCPTR(pRam, off)
1574
1575	/* Note! Dangerous assumption that R3 handlers don't do anything that really requires an EMT lock! */
1576	rc = pNode->pfnHandlerR3(pVM, (RTGCPTR)GCPtr, pvDst, (void *)pvBuf, cb, PGMACCESSTYPE_WRITE, 0);
1577	}
1578	#endif /* IN_RING3 */
1579	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
1580	{
1581	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1582	void *pvDst = NULL;
1583	PGMDynMapHCPageOff(pVM, PGM_PAGE_GET_HCPHYS(pPage) + (off & PAGE_OFFSET_MASK), &pvDst);
1584	#else
1585	void *pvDst = PGMRAMRANGE_GETHCPTR(pRam, off)
1586	#endif
1587	if (cb >= cbWrite)
1588	{
1589	memcpy(pvDst, pvBuf, cbWrite);
1590	goto end;
1591	}
1592	memcpy(pvDst, pvBuf, cb);
1593	}
1594	else if (cb >= cbWrite)
1595	goto end;
1596	}
1597	}
1598	/*
1599	* Physical handler.
1600	*/
1601	else if ( RT_UNLIKELY(PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE)
1602	&& !(pPage->HCPhys & MM_RAM_FLAGS_MMIO)) /// @todo PAGE FLAGS
1603	{
1604	int rc = VINF_PGM_HANDLER_DO_DEFAULT;
1605	cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
1606	#ifdef IN_RING3 /** @todo deal with this in GC and R0! */
1607	/* find and call the handler */
1608	PPGMPHYSHANDLER pNode = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.pTreesR3->PhysHandlers, GCPhys);
1609	if (pNode && pNode->pfnHandlerR3)
1610	{
1611	size_t cbRange = pNode->Core.KeyLast - GCPhys + 1;
1612	if (cbRange < cb)
1613	cb = cbRange;
1614	if (cb > cbWrite)
1615	cb = cbWrite;
1616
1617	void *pvDst = PGMRAMRANGE_GETHCPTR(pRam, off)
1618
1619	/** @todo Dangerous assumption that HC handlers don't do anything that really requires an EMT lock! */
1620	rc = pNode->pfnHandlerR3(pVM, GCPhys, pvDst, (void *)pvBuf, cb, PGMACCESSTYPE_WRITE, pNode->pvUserR3);
1621	}
1622	#endif /* IN_RING3 */
1623	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
1624	{
1625	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1626	void *pvDst = NULL;
1627	PGMDynMapHCPageOff(pVM, PGM_PAGE_GET_HCPHYS(pPage) + (off & PAGE_OFFSET_MASK), &pvDst);
1628	#else
1629	void *pvDst = PGMRAMRANGE_GETHCPTR(pRam, off)
1630	#endif
1631	if (cb >= cbWrite)
1632	{
1633	memcpy(pvDst, pvBuf, cbWrite);
1634	goto end;
1635	}
1636	memcpy(pvDst, pvBuf, cb);
1637	}
1638	else if (cb >= cbWrite)
1639	goto end;
1640	}
1641	else
1642	{
1643	/** @todo r=bird: missing MM_RAM_FLAGS_ROM here, we shall not allow anyone to overwrite the ROM! */
1644	switch (pPage->HCPhys & (MM_RAM_FLAGS_RESERVED \| MM_RAM_FLAGS_MMIO \| MM_RAM_FLAGS_MMIO2)) /** @todo PAGE FLAGS */
1645	{
1646	/*
1647	* Normal memory, MMIO2 or writable shadow ROM.
1648	*/
1649	case 0:
1650	case MM_RAM_FLAGS_MMIO2:
1651	case MM_RAM_FLAGS_ROM \| MM_RAM_FLAGS_MMIO2: /* shadow rom */
1652	{
1653	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1654	void *pvDst = NULL;
1655	PGMDynMapHCPageOff(pVM, PGM_PAGE_GET_HCPHYS(pPage) + (off & PAGE_OFFSET_MASK), &pvDst);
1656	#else
1657	void *pvDst = PGMRAMRANGE_GETHCPTR(pRam, off)
1658	#endif
1659	cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
1660	if (cb >= cbWrite)
1661	{
1662	#if defined(IN_RING3) && defined(PGM_PHYSMEMACCESS_CACHING)
1663	if (cbWrite <= 4 && !fGrabbedLock /* i.e. EMT */)
1664	pgmPhysCacheAdd(pVM, &pVM->pgm.s.pgmphyswritecache, GCPhys, (uint8_t*)pvDst);
1665	#endif /* IN_RING3 && PGM_PHYSMEMACCESS_CACHING */
1666	memcpy(pvDst, pvBuf, cbWrite);
1667	goto end;
1668	}
1669	memcpy(pvDst, pvBuf, cb);
1670	break;
1671	}
1672
1673	/*
1674	* All reserved, nothing there.
1675	*/
1676	case MM_RAM_FLAGS_RESERVED:
1677	case MM_RAM_FLAGS_RESERVED \| MM_RAM_FLAGS_MMIO2:
1678	cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
1679	if (cb >= cbWrite)
1680	goto end;
1681	break;
1682
1683
1684	/*
1685	* The rest needs to be taken more carefully.
1686	*/
1687	default:
1688	#if 1 /** @todo r=bird: Can you do this properly please. */
1689	/** @todo Try MMIO; quick hack */
1690	if (cbWrite <= 8 && IOMMMIOWrite(pVM, GCPhys, (uint32_t )pvBuf, cbWrite) == VINF_SUCCESS)
1691	goto end;
1692	#endif
1693
1694	/** @todo fix me later. */
1695	AssertReleaseMsgFailed(("Unknown write at %RGp size %u implement the complex physical writing case %RHp\n",
1696	GCPhys, cbWrite,
1697	(pPage->HCPhys & (MM_RAM_FLAGS_RESERVED \| MM_RAM_FLAGS_MMIO \| MM_RAM_FLAGS_MMIO2)))); /** @todo PAGE FLAGS */
1698	/* skip the write */
1699	cb = cbWrite;
1700	break;
1701	}
1702	}
1703
1704	cbWrite -= cb;
1705	off += cb;
1706	pvBuf = (const char *)pvBuf + cb;
1707	}
1708
1709	GCPhys = pRam->GCPhysLast + 1;
1710	}
1711	else
1712	{
1713	/*
1714	* Unassigned address space.
1715	*/
1716	size_t cb;
1717	if ( !pRam
1718	\|\| (cb = pRam->GCPhys - GCPhys) >= cbWrite)
1719	goto end;
1720
1721	cbWrite -= cb;
1722	pvBuf = (const char *)pvBuf + cb;
1723	GCPhys += cb;
1724	}
1725	}
1726	end:
1727	#ifdef IN_RING3
1728	if (fGrabbedLock)
1729	pgmUnlock(pVM);
1730	#endif
1731	return;
1732	}
1733
1734	#ifndef IN_RC /* Ring 0 & 3 only. (Just not needed in GC.) */
1735
1736	/**
1737	* Read from guest physical memory by GC physical address, bypassing
1738	* MMIO and access handlers.
1739	*
1740	* @returns VBox status.
1741	* @param pVM VM handle.
1742	* @param pvDst The destination address.
1743	* @param GCPhysSrc The source address (GC physical address).
1744	* @param cb The number of bytes to read.
1745	*/
1746	VMMDECL(int) PGMPhysSimpleReadGCPhys(PVM pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb)
1747	{
1748	/*
1749	* Treat the first page as a special case.
1750	*/
1751	if (!cb)
1752	return VINF_SUCCESS;
1753
1754	/* map the 1st page */
1755	void const *pvSrc;
1756	PGMPAGEMAPLOCK Lock;
1757	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
1758	if (RT_FAILURE(rc))
1759	return rc;
1760
1761	/* optimize for the case where access is completely within the first page. */
1762	size_t cbPage = PAGE_SIZE - (GCPhysSrc & PAGE_OFFSET_MASK);
1763	if (RT_LIKELY(cb <= cbPage))
1764	{
1765	memcpy(pvDst, pvSrc, cb);
1766	PGMPhysReleasePageMappingLock(pVM, &Lock);
1767	return VINF_SUCCESS;
1768	}
1769
1770	/* copy to the end of the page. */
1771	memcpy(pvDst, pvSrc, cbPage);
1772	PGMPhysReleasePageMappingLock(pVM, &Lock);
1773	GCPhysSrc += cbPage;
1774	pvDst = (uint8_t *)pvDst + cbPage;
1775	cb -= cbPage;
1776
1777	/*
1778	* Page by page.
1779	*/
1780	for (;;)
1781	{
1782	/* map the page */
1783	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
1784	if (RT_FAILURE(rc))
1785	return rc;
1786
1787	/* last page? */
1788	if (cb <= PAGE_SIZE)
1789	{
1790	memcpy(pvDst, pvSrc, cb);
1791	PGMPhysReleasePageMappingLock(pVM, &Lock);
1792	return VINF_SUCCESS;
1793	}
1794
1795	/* copy the entire page and advance */
1796	memcpy(pvDst, pvSrc, PAGE_SIZE);
1797	PGMPhysReleasePageMappingLock(pVM, &Lock);
1798	GCPhysSrc += PAGE_SIZE;
1799	pvDst = (uint8_t *)pvDst + PAGE_SIZE;
1800	cb -= PAGE_SIZE;
1801	}
1802	/* won't ever get here. */
1803	}
1804
1805
1806	/**
1807	* Write to guest physical memory referenced by GC pointer.
1808	* Write memory to GC physical address in guest physical memory.
1809	*
1810	* This will bypass MMIO and access handlers.
1811	*
1812	* @returns VBox status.
1813	* @param pVM VM handle.
1814	* @param GCPhysDst The GC physical address of the destination.
1815	* @param pvSrc The source buffer.
1816	* @param cb The number of bytes to write.
1817	*/
1818	VMMDECL(int) PGMPhysSimpleWriteGCPhys(PVM pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb)
1819	{
1820	LogFlow(("PGMPhysSimpleWriteGCPhys: %RGp %zu\n", GCPhysDst, cb));
1821
1822	/*
1823	* Treat the first page as a special case.
1824	*/
1825	if (!cb)
1826	return VINF_SUCCESS;
1827
1828	/* map the 1st page */
1829	void *pvDst;
1830	PGMPAGEMAPLOCK Lock;
1831	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
1832	if (RT_FAILURE(rc))
1833	return rc;
1834
1835	/* optimize for the case where access is completely within the first page. */
1836	size_t cbPage = PAGE_SIZE - (GCPhysDst & PAGE_OFFSET_MASK);
1837	if (RT_LIKELY(cb <= cbPage))
1838	{
1839	memcpy(pvDst, pvSrc, cb);
1840	PGMPhysReleasePageMappingLock(pVM, &Lock);
1841	return VINF_SUCCESS;
1842	}
1843
1844	/* copy to the end of the page. */
1845	memcpy(pvDst, pvSrc, cbPage);
1846	PGMPhysReleasePageMappingLock(pVM, &Lock);
1847	GCPhysDst += cbPage;
1848	pvSrc = (const uint8_t *)pvSrc + cbPage;
1849	cb -= cbPage;
1850
1851	/*
1852	* Page by page.
1853	*/
1854	for (;;)
1855	{
1856	/* map the page */
1857	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
1858	if (RT_FAILURE(rc))
1859	return rc;
1860
1861	/* last page? */
1862	if (cb <= PAGE_SIZE)
1863	{
1864	memcpy(pvDst, pvSrc, cb);
1865	PGMPhysReleasePageMappingLock(pVM, &Lock);
1866	return VINF_SUCCESS;
1867	}
1868
1869	/* copy the entire page and advance */
1870	memcpy(pvDst, pvSrc, PAGE_SIZE);
1871	PGMPhysReleasePageMappingLock(pVM, &Lock);
1872	GCPhysDst += PAGE_SIZE;
1873	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
1874	cb -= PAGE_SIZE;
1875	}
1876	/* won't ever get here. */
1877	}
1878
1879
1880	/**
1881	* Read from guest physical memory referenced by GC pointer.
1882	*
1883	* This function uses the current CR3/CR0/CR4 of the guest and will
1884	* bypass access handlers and not set any accessed bits.
1885	*
1886	* @returns VBox status.
1887	* @param pVM VM handle.
1888	* @param pvDst The destination address.
1889	* @param GCPtrSrc The source address (GC pointer).
1890	* @param cb The number of bytes to read.
1891	*/
1892	VMMDECL(int) PGMPhysSimpleReadGCPtr(PVM pVM, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
1893	{
1894	/*
1895	* Treat the first page as a special case.
1896	*/
1897	if (!cb)
1898	return VINF_SUCCESS;
1899
1900	/* map the 1st page */
1901	void const *pvSrc;
1902	PGMPAGEMAPLOCK Lock;
1903	int rc = PGMPhysGCPtr2CCPtrReadOnly(pVM, GCPtrSrc, &pvSrc, &Lock);
1904	if (RT_FAILURE(rc))
1905	return rc;
1906
1907	/* optimize for the case where access is completely within the first page. */
1908	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
1909	if (RT_LIKELY(cb <= cbPage))
1910	{
1911	memcpy(pvDst, pvSrc, cb);
1912	PGMPhysReleasePageMappingLock(pVM, &Lock);
1913	return VINF_SUCCESS;
1914	}
1915
1916	/* copy to the end of the page. */
1917	memcpy(pvDst, pvSrc, cbPage);
1918	PGMPhysReleasePageMappingLock(pVM, &Lock);
1919	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + cbPage);
1920	pvDst = (uint8_t *)pvDst + cbPage;
1921	cb -= cbPage;
1922
1923	/*
1924	* Page by page.
1925	*/
1926	for (;;)
1927	{
1928	/* map the page */
1929	rc = PGMPhysGCPtr2CCPtrReadOnly(pVM, GCPtrSrc, &pvSrc, &Lock);
1930	if (RT_FAILURE(rc))
1931	return rc;
1932
1933	/* last page? */
1934	if (cb <= PAGE_SIZE)
1935	{
1936	memcpy(pvDst, pvSrc, cb);
1937	PGMPhysReleasePageMappingLock(pVM, &Lock);
1938	return VINF_SUCCESS;
1939	}
1940
1941	/* copy the entire page and advance */
1942	memcpy(pvDst, pvSrc, PAGE_SIZE);
1943	PGMPhysReleasePageMappingLock(pVM, &Lock);
1944	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + PAGE_SIZE);
1945	pvDst = (uint8_t *)pvDst + PAGE_SIZE;
1946	cb -= PAGE_SIZE;
1947	}
1948	/* won't ever get here. */
1949	}
1950
1951
1952	/**
1953	* Write to guest physical memory referenced by GC pointer.
1954	*
1955	* This function uses the current CR3/CR0/CR4 of the guest and will
1956	* bypass access handlers and not set dirty or accessed bits.
1957	*
1958	* @returns VBox status.
1959	* @param pVM VM handle.
1960	* @param GCPtrDst The destination address (GC pointer).
1961	* @param pvSrc The source address.
1962	* @param cb The number of bytes to write.
1963	*/
1964	VMMDECL(int) PGMPhysSimpleWriteGCPtr(PVM pVM, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
1965	{
1966	/*
1967	* Treat the first page as a special case.
1968	*/
1969	if (!cb)
1970	return VINF_SUCCESS;
1971
1972	/* map the 1st page */
1973	void *pvDst;
1974	PGMPAGEMAPLOCK Lock;
1975	int rc = PGMPhysGCPtr2CCPtr(pVM, GCPtrDst, &pvDst, &Lock);
1976	if (RT_FAILURE(rc))
1977	return rc;
1978
1979	/* optimize for the case where access is completely within the first page. */
1980	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
1981	if (RT_LIKELY(cb <= cbPage))
1982	{
1983	memcpy(pvDst, pvSrc, cb);
1984	PGMPhysReleasePageMappingLock(pVM, &Lock);
1985	return VINF_SUCCESS;
1986	}
1987
1988	/* copy to the end of the page. */
1989	memcpy(pvDst, pvSrc, cbPage);
1990	PGMPhysReleasePageMappingLock(pVM, &Lock);
1991	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
1992	pvSrc = (const uint8_t *)pvSrc + cbPage;
1993	cb -= cbPage;
1994
1995	/*
1996	* Page by page.
1997	*/
1998	for (;;)
1999	{
2000	/* map the page */
2001	rc = PGMPhysGCPtr2CCPtr(pVM, GCPtrDst, &pvDst, &Lock);
2002	if (RT_FAILURE(rc))
2003	return rc;
2004
2005	/* last page? */
2006	if (cb <= PAGE_SIZE)
2007	{
2008	memcpy(pvDst, pvSrc, cb);
2009	PGMPhysReleasePageMappingLock(pVM, &Lock);
2010	return VINF_SUCCESS;
2011	}
2012
2013	/* copy the entire page and advance */
2014	memcpy(pvDst, pvSrc, PAGE_SIZE);
2015	PGMPhysReleasePageMappingLock(pVM, &Lock);
2016	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
2017	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
2018	cb -= PAGE_SIZE;
2019	}
2020	/* won't ever get here. */
2021	}
2022
2023
2024	/**
2025	* Write to guest physical memory referenced by GC pointer and update the PTE.
2026	*
2027	* This function uses the current CR3/CR0/CR4 of the guest and will
2028	* bypass access handlers but will set any dirty and accessed bits in the PTE.
2029	*
2030	* If you don't want to set the dirty bit, use PGMPhysSimpleWriteGCPtr().
2031	*
2032	* @returns VBox status.
2033	* @param pVM VM handle.
2034	* @param GCPtrDst The destination address (GC pointer).
2035	* @param pvSrc The source address.
2036	* @param cb The number of bytes to write.
2037	*/
2038	VMMDECL(int) PGMPhysSimpleDirtyWriteGCPtr(PVM pVM, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
2039	{
2040	/*
2041	* Treat the first page as a special case.
2042	* Btw. this is the same code as in PGMPhyssimpleWriteGCPtr excep for the PGMGstModifyPage.
2043	*/
2044	if (!cb)
2045	return VINF_SUCCESS;
2046
2047	/* map the 1st page */
2048	void *pvDst;
2049	PGMPAGEMAPLOCK Lock;
2050	int rc = PGMPhysGCPtr2CCPtr(pVM, GCPtrDst, &pvDst, &Lock);
2051	if (RT_FAILURE(rc))
2052	return rc;
2053
2054	/* optimize for the case where access is completely within the first page. */
2055	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
2056	if (RT_LIKELY(cb <= cbPage))
2057	{
2058	memcpy(pvDst, pvSrc, cb);
2059	PGMPhysReleasePageMappingLock(pVM, &Lock);
2060	rc = PGMGstModifyPage(pVM, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2061	return VINF_SUCCESS;
2062	}
2063
2064	/* copy to the end of the page. */
2065	memcpy(pvDst, pvSrc, cbPage);
2066	PGMPhysReleasePageMappingLock(pVM, &Lock);
2067	rc = PGMGstModifyPage(pVM, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2068	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
2069	pvSrc = (const uint8_t *)pvSrc + cbPage;
2070	cb -= cbPage;
2071
2072	/*
2073	* Page by page.
2074	*/
2075	for (;;)
2076	{
2077	/* map the page */
2078	rc = PGMPhysGCPtr2CCPtr(pVM, GCPtrDst, &pvDst, &Lock);
2079	if (RT_FAILURE(rc))
2080	return rc;
2081
2082	/* last page? */
2083	if (cb <= PAGE_SIZE)
2084	{
2085	memcpy(pvDst, pvSrc, cb);
2086	PGMPhysReleasePageMappingLock(pVM, &Lock);
2087	rc = PGMGstModifyPage(pVM, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2088	return VINF_SUCCESS;
2089	}
2090
2091	/* copy the entire page and advance */
2092	memcpy(pvDst, pvSrc, PAGE_SIZE);
2093	PGMPhysReleasePageMappingLock(pVM, &Lock);
2094	rc = PGMGstModifyPage(pVM, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2095	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
2096	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
2097	cb -= PAGE_SIZE;
2098	}
2099	/* won't ever get here. */
2100	}
2101
2102
2103	/**
2104	* Read from guest physical memory referenced by GC pointer.
2105	*
2106	* This function uses the current CR3/CR0/CR4 of the guest and will
2107	* respect access handlers and set accessed bits.
2108	*
2109	* @returns VBox status.
2110	* @param pVM VM handle.
2111	* @param pvDst The destination address.
2112	* @param GCPtrSrc The source address (GC pointer).
2113	* @param cb The number of bytes to read.
2114	*/
2115	VMMDECL(int) PGMPhysReadGCPtr(PVM pVM, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
2116	{
2117	RTGCPHYS GCPhys;
2118	int rc;
2119
2120	/*
2121	* Anything to do?
2122	*/
2123	if (!cb)
2124	return VINF_SUCCESS;
2125
2126	LogFlow(("PGMPhysReadGCPtr: %RGv %zu\n", GCPtrSrc, cb));
2127
2128	/*
2129	* Optimize reads within a single page.
2130	*/
2131	if (((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
2132	{
2133	/* Convert virtual to physical address */
2134	rc = PGMPhysGCPtr2GCPhys(pVM, GCPtrSrc, &GCPhys);
2135	AssertRCReturn(rc, rc);
2136
2137	/* mark the guest page as accessed. */
2138	rc = PGMGstModifyPage(pVM, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
2139	AssertRC(rc);
2140
2141	PGMPhysRead(pVM, GCPhys, pvDst, cb);
2142	return VINF_SUCCESS;
2143	}
2144
2145	/*
2146	* Page by page.
2147	*/
2148	for (;;)
2149	{
2150	/* Convert virtual to physical address */
2151	rc = PGMPhysGCPtr2GCPhys(pVM, GCPtrSrc, &GCPhys);
2152	AssertRCReturn(rc, rc);
2153
2154	/* mark the guest page as accessed. */
2155	int rc = PGMGstModifyPage(pVM, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
2156	AssertRC(rc);
2157
2158	/* copy */
2159	size_t cbRead = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
2160	if (cbRead >= cb)
2161	{
2162	PGMPhysRead(pVM, GCPhys, pvDst, cb);
2163	return VINF_SUCCESS;
2164	}
2165	PGMPhysRead(pVM, GCPhys, pvDst, cbRead);
2166
2167	/* next */
2168	cb -= cbRead;
2169	pvDst = (uint8_t *)pvDst + cbRead;
2170	GCPtrSrc += cbRead;
2171	}
2172	}
2173
2174
2175	/**
2176	* Write to guest physical memory referenced by GC pointer.
2177	*
2178	* This function uses the current CR3/CR0/CR4 of the guest and will
2179	* respect access handlers and set dirty and accessed bits.
2180	*
2181	* @returns VBox status.
2182	* @param pVM VM handle.
2183	* @param GCPtrDst The destination address (GC pointer).
2184	* @param pvSrc The source address.
2185	* @param cb The number of bytes to write.
2186	*/
2187	VMMDECL(int) PGMPhysWriteGCPtr(PVM pVM, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
2188	{
2189	RTGCPHYS GCPhys;
2190	int rc;
2191
2192	/*
2193	* Anything to do?
2194	*/
2195	if (!cb)
2196	return VINF_SUCCESS;
2197
2198	LogFlow(("PGMPhysWriteGCPtr: %RGv %zu\n", GCPtrDst, cb));
2199
2200	/*
2201	* Optimize writes within a single page.
2202	*/
2203	if (((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
2204	{
2205	/* Convert virtual to physical address */
2206	rc = PGMPhysGCPtr2GCPhys(pVM, GCPtrDst, &GCPhys);
2207	AssertMsgRCReturn(rc, ("PGMPhysGCPtr2GCPhys failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
2208
2209	/* mark the guest page as accessed and dirty. */
2210	rc = PGMGstModifyPage(pVM, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
2211	AssertRC(rc);
2212
2213	PGMPhysWrite(pVM, GCPhys, pvSrc, cb);
2214	return VINF_SUCCESS;
2215	}
2216
2217	/*
2218	* Page by page.
2219	*/
2220	for (;;)
2221	{
2222	/* Convert virtual to physical address */
2223	rc = PGMPhysGCPtr2GCPhys(pVM, GCPtrDst, &GCPhys);
2224	AssertRCReturn(rc, rc);
2225
2226	/* mark the guest page as accessed and dirty. */
2227	rc = PGMGstModifyPage(pVM, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
2228	AssertRC(rc);
2229
2230	/* copy */
2231	size_t cbWrite = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
2232	if (cbWrite >= cb)
2233	{
2234	PGMPhysWrite(pVM, GCPhys, pvSrc, cb);
2235	return VINF_SUCCESS;
2236	}
2237	PGMPhysWrite(pVM, GCPhys, pvSrc, cbWrite);
2238
2239	/* next */
2240	cb -= cbWrite;
2241	pvSrc = (uint8_t *)pvSrc + cbWrite;
2242	GCPtrDst += cbWrite;
2243	}
2244	}
2245
2246	#endif /* !IN_RC */
2247
2248	/**
2249	* Performs a read of guest virtual memory for instruction emulation.
2250	*
2251	* This will check permissions, raise exceptions and update the access bits.
2252	*
2253	* The current implementation will bypass all access handlers. It may later be
2254	* changed to at least respect MMIO.
2255	*
2256	*
2257	* @returns VBox status code suitable to scheduling.
2258	* @retval VINF_SUCCESS if the read was performed successfully.
2259	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
2260	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
2261	*
2262	* @param pVM The VM handle.
2263	* @param pCtxCore The context core.
2264	* @param pvDst Where to put the bytes we've read.
2265	* @param GCPtrSrc The source address.
2266	* @param cb The number of bytes to read. Not more than a page.
2267	*
2268	* @remark This function will dynamically map physical pages in GC. This may unmap
2269	* mappings done by the caller. Be careful!
2270	*/
2271	VMMDECL(int) PGMPhysInterpretedRead(PVM pVM, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb)
2272	{
2273	Assert(cb <= PAGE_SIZE);
2274
2275	/** @todo r=bird: This isn't perfect!
2276	* -# It's not checking for reserved bits being 1.
2277	* -# It's not correctly dealing with the access bit.
2278	* -# It's not respecting MMIO memory or any other access handlers.
2279	*/
2280	/*
2281	* 1. Translate virtual to physical. This may fault.
2282	* 2. Map the physical address.
2283	* 3. Do the read operation.
2284	* 4. Set access bits if required.
2285	*/
2286	int rc;
2287	unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
2288	if (cb <= cb1)
2289	{
2290	/*
2291	* Not crossing pages.
2292	*/
2293	RTGCPHYS GCPhys;
2294	uint64_t fFlags;
2295	rc = PGM_GST_PFN(GetPage,pVM)(pVM, GCPtrSrc, &fFlags, &GCPhys);
2296	if (RT_SUCCESS(rc))
2297	{
2298	/** @todo we should check reserved bits ... */
2299	void *pvSrc;
2300	rc = PGM_GCPHYS_2_PTR(pVM, GCPhys, &pvSrc);
2301	switch (rc)
2302	{
2303	case VINF_SUCCESS:
2304	Log(("PGMPhysInterpretedRead: pvDst=%p pvSrc=%p cb=%d\n", pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb));
2305	memcpy(pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
2306	break;
2307	case VERR_PGM_PHYS_PAGE_RESERVED:
2308	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
2309	memset(pvDst, 0, cb); /** @todo this is wrong, it should be 0xff */
2310	break;
2311	default:
2312	return rc;
2313	}
2314
2315	/** @todo access bit emulation isn't 100% correct. */
2316	if (!(fFlags & X86_PTE_A))
2317	{
2318	rc = PGM_GST_PFN(ModifyPage,pVM)(pVM, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
2319	AssertRC(rc);
2320	}
2321	return VINF_SUCCESS;
2322	}
2323	}
2324	else
2325	{
2326	/*
2327	* Crosses pages.
2328	*/
2329	size_t cb2 = cb - cb1;
2330	uint64_t fFlags1;
2331	RTGCPHYS GCPhys1;
2332	uint64_t fFlags2;
2333	RTGCPHYS GCPhys2;
2334	rc = PGM_GST_PFN(GetPage,pVM)(pVM, GCPtrSrc, &fFlags1, &GCPhys1);
2335	if (RT_SUCCESS(rc))
2336	rc = PGM_GST_PFN(GetPage,pVM)(pVM, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
2337	if (RT_SUCCESS(rc))
2338	{
2339	/** @todo we should check reserved bits ... */
2340	AssertMsgFailed(("cb=%d cb1=%d cb2=%d GCPtrSrc=%RGv\n", cb, cb1, cb2, GCPtrSrc));
2341	void *pvSrc1;
2342	rc = PGM_GCPHYS_2_PTR(pVM, GCPhys1, &pvSrc1);
2343	switch (rc)
2344	{
2345	case VINF_SUCCESS:
2346	memcpy(pvDst, (uint8_t *)pvSrc1 + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
2347	break;
2348	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
2349	memset(pvDst, 0, cb1); /** @todo this is wrong, it should be 0xff */
2350	break;
2351	default:
2352	return rc;
2353	}
2354
2355	void *pvSrc2;
2356	rc = PGM_GCPHYS_2_PTR(pVM, GCPhys2, &pvSrc2);
2357	switch (rc)
2358	{
2359	case VINF_SUCCESS:
2360	memcpy((uint8_t *)pvDst + cb1, pvSrc2, cb2);
2361	break;
2362	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
2363	memset((uint8_t )pvDst + cb1, 0, cb2); /* @todo this is wrong, it should be 0xff */
2364	break;
2365	default:
2366	return rc;
2367	}
2368
2369	if (!(fFlags1 & X86_PTE_A))
2370	{
2371	rc = PGM_GST_PFN(ModifyPage,pVM)(pVM, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
2372	AssertRC(rc);
2373	}
2374	if (!(fFlags2 & X86_PTE_A))
2375	{
2376	rc = PGM_GST_PFN(ModifyPage,pVM)(pVM, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
2377	AssertRC(rc);
2378	}
2379	return VINF_SUCCESS;
2380	}
2381	}
2382
2383	/*
2384	* Raise a #PF.
2385	*/
2386	uint32_t uErr;
2387
2388	/* Get the current privilege level. */
2389	uint32_t cpl = CPUMGetGuestCPL(pVM, pCtxCore);
2390	switch (rc)
2391	{
2392	case VINF_SUCCESS:
2393	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
2394	break;
2395
2396	case VERR_PAGE_NOT_PRESENT:
2397	case VERR_PAGE_TABLE_NOT_PRESENT:
2398	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
2399	break;
2400
2401	default:
2402	AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
2403	return rc;
2404	}
2405	Log(("PGMPhysInterpretedRead: GCPtrSrc=%RGv cb=%#x -> #PF(%#x)\n", GCPtrSrc, cb, uErr));
2406	return TRPMRaiseXcptErrCR2(pVM, pCtxCore, X86_XCPT_PF, uErr, GCPtrSrc);
2407	}
2408
2409	/// @todo VMMDECL(int) PGMPhysInterpretedWrite(PVM pVM, PCPUMCTXCORE pCtxCore, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
2410
2411

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp@ 15348

Download in other formats: