PGMAllPhys.cpp@ 27549

Last change on this file since 27549 was 27545, checked in by vboxsync, 15 years ago
VMM/PGMPhys: provide default RC/R0 physical memory handler in case the caller doesn't specify one when registering
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 128.0 KB

Line
1	/* $Id: PGMAllPhys.cpp 27545 2010-03-19 15:52:10Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22	/*******************************************************************************
23	* Header Files *
24	*******************************************************************************/
25	#define LOG_GROUP LOG_GROUP_PGM_PHYS
26	#include <VBox/pgm.h>
27	#include <VBox/trpm.h>
28	#include <VBox/vmm.h>
29	#include <VBox/iom.h>
30	#include <VBox/em.h>
31	#include <VBox/rem.h>
32	#include "../PGMInternal.h"
33	#include <VBox/vm.h>
34	#include "../PGMInline.h"
35	#include <VBox/param.h>
36	#include <VBox/err.h>
37	#include <iprt/assert.h>
38	#include <iprt/string.h>
39	#include <iprt/asm.h>
40	#include <VBox/log.h>
41	#ifdef IN_RING3
42	# include <iprt/thread.h>
43	#endif
44
45
46	/*******************************************************************************
47	* Defined Constants And Macros *
48	*******************************************************************************/
49	/** Enable the physical TLB. */
50	#define PGM_WITH_PHYS_TLB
51
52
53
54	#ifndef IN_RING3
55
56	/**
57	* \#PF Handler callback for physical memory accesses without a RC/R0 handler.
58	* This simply pushes everything to the HC handler.
59	*
60	* @returns VBox status code (appropritate for trap handling and GC return).
61	* @param pVM VM Handle.
62	* @param uErrorCode CPU Error code.
63	* @param pRegFrame Trap register frame.
64	* @param pvFault The fault address (cr2).
65	* @param GCPhysFault The GC physical address corresponding to pvFault.
66	* @param pvUser User argument.
67	*/
68	VMMDECL(int) pgmPhysHandlerRedirectToHC(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
69	{
70	return (uErrorCode & X86_TRAP_PF_RW) ? VINF_IOM_HC_MMIO_WRITE : VINF_IOM_HC_MMIO_READ;
71	}
72
73
74	/**
75	* \#PF Handler callback for Guest ROM range write access.
76	* We simply ignore the writes or fall back to the recompiler if we don't support the instruction.
77	*
78	* @returns VBox status code (appropritate for trap handling and GC return).
79	* @param pVM VM Handle.
80	* @param uErrorCode CPU Error code.
81	* @param pRegFrame Trap register frame.
82	* @param pvFault The fault address (cr2).
83	* @param GCPhysFault The GC physical address corresponding to pvFault.
84	* @param pvUser User argument. Pointer to the ROM range structure.
85	*/
86	VMMDECL(int) pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
87	{
88	int rc;
89	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
90	uint32_t iPage = (GCPhysFault - pRom->GCPhys) >> PAGE_SHIFT;
91	PVMCPU pVCpu = VMMGetCpu(pVM);
92
93	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
94	switch (pRom->aPages[iPage].enmProt)
95	{
96	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
97	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
98	{
99	/*
100	* If it's a simple instruction which doesn't change the cpu state
101	* we will simply skip it. Otherwise we'll have to defer it to REM.
102	*/
103	uint32_t cbOp;
104	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
105	rc = EMInterpretDisasOne(pVM, pVCpu, pRegFrame, pDis, &cbOp);
106	if ( RT_SUCCESS(rc)
107	&& pDis->mode == CPUMODE_32BIT /** @todo why does this matter? */
108	&& !(pDis->prefix & (PREFIX_REPNE \| PREFIX_REP \| PREFIX_SEG)))
109	{
110	switch (pDis->opcode)
111	{
112	/** @todo Find other instructions we can safely skip, possibly
113	* adding this kind of detection to DIS or EM. */
114	case OP_MOV:
115	pRegFrame->rip += cbOp;
116	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZGuestROMWriteHandled);
117	return VINF_SUCCESS;
118	}
119	}
120	else if (RT_UNLIKELY(rc == VERR_INTERNAL_ERROR))
121	return rc;
122	break;
123	}
124
125	case PGMROMPROT_READ_RAM_WRITE_RAM:
126	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
127	rc = PGMHandlerPhysicalPageTempOff(pVM, pRom->GCPhys, GCPhysFault & X86_PTE_PG_MASK);
128	AssertRC(rc);
129	break; /** @todo Must edit the shadow PT and restart the instruction, not use the interpreter! */
130
131	case PGMROMPROT_READ_ROM_WRITE_RAM:
132	/* Handle it in ring-3 because it's way easier there. */
133	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
134	break;
135
136	default:
137	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhysFault=%RGp\n",
138	pRom->aPages[iPage].enmProt, iPage, GCPhysFault),
139	VERR_INTERNAL_ERROR);
140	}
141
142	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZGuestROMWriteUnhandled);
143	return VINF_EM_RAW_EMULATE_INSTR;
144	}
145
146	#endif /* IN_RING3 */
147
148	/**
149	* Checks if Address Gate 20 is enabled or not.
150	*
151	* @returns true if enabled.
152	* @returns false if disabled.
153	* @param pVCpu VMCPU handle.
154	*/
155	VMMDECL(bool) PGMPhysIsA20Enabled(PVMCPU pVCpu)
156	{
157	LogFlow(("PGMPhysIsA20Enabled %d\n", pVCpu->pgm.s.fA20Enabled));
158	return pVCpu->pgm.s.fA20Enabled;
159	}
160
161
162	/**
163	* Validates a GC physical address.
164	*
165	* @returns true if valid.
166	* @returns false if invalid.
167	* @param pVM The VM handle.
168	* @param GCPhys The physical address to validate.
169	*/
170	VMMDECL(bool) PGMPhysIsGCPhysValid(PVM pVM, RTGCPHYS GCPhys)
171	{
172	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
173	return pPage != NULL;
174	}
175
176
177	/**
178	* Checks if a GC physical address is a normal page,
179	* i.e. not ROM, MMIO or reserved.
180	*
181	* @returns true if normal.
182	* @returns false if invalid, ROM, MMIO or reserved page.
183	* @param pVM The VM handle.
184	* @param GCPhys The physical address to check.
185	*/
186	VMMDECL(bool) PGMPhysIsGCPhysNormal(PVM pVM, RTGCPHYS GCPhys)
187	{
188	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
189	return pPage
190	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM;
191	}
192
193
194	/**
195	* Converts a GC physical address to a HC physical address.
196	*
197	* @returns VINF_SUCCESS on success.
198	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
199	* page but has no physical backing.
200	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
201	* GC physical address.
202	*
203	* @param pVM The VM handle.
204	* @param GCPhys The GC physical address to convert.
205	* @param pHCPhys Where to store the HC physical address on success.
206	*/
207	VMMDECL(int) PGMPhysGCPhys2HCPhys(PVM pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
208	{
209	pgmLock(pVM);
210	PPGMPAGE pPage;
211	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
212	if (RT_SUCCESS(rc))
213	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK);
214	pgmUnlock(pVM);
215	return rc;
216	}
217
218
219	/**
220	* Invalidates all page mapping TLBs.
221	*
222	* @param pVM The VM handle.
223	*/
224	VMMDECL(void) PGMPhysInvalidatePageMapTLB(PVM pVM)
225	{
226	pgmLock(pVM);
227	STAM_COUNTER_INC(&pVM->pgm.s.StatPageMapTlbFlushes);
228	/* Clear the shared R0/R3 TLB completely. */
229	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
230	{
231	pVM->pgm.s.PhysTlbHC.aEntries[i].GCPhys = NIL_RTGCPHYS;
232	pVM->pgm.s.PhysTlbHC.aEntries[i].pPage = 0;
233	pVM->pgm.s.PhysTlbHC.aEntries[i].pMap = 0;
234	pVM->pgm.s.PhysTlbHC.aEntries[i].pv = 0;
235	}
236	/* @todo clear the RC TLB whenever we add it. */
237	pgmUnlock(pVM);
238	}
239
240	/**
241	* Invalidates a page mapping TLB entry
242	*
243	* @param pVM The VM handle.
244	* @param GCPhys GCPhys entry to flush
245	*/
246	VMMDECL(void) PGMPhysInvalidatePageMapTLBEntry(PVM pVM, RTGCPHYS GCPhys)
247	{
248	Assert(PGMIsLocked(pVM));
249
250	STAM_COUNTER_INC(&pVM->pgm.s.StatPageMapTlbFlushEntry);
251	/* Clear the shared R0/R3 TLB entry. */
252	#ifdef IN_RC
253	unsigned idx = PGM_PAGER3MAPTLB_IDX(GCPhys);
254	pVM->pgm.s.PhysTlbHC.aEntries[idx].GCPhys = NIL_RTGCPHYS;
255	pVM->pgm.s.PhysTlbHC.aEntries[idx].pPage = 0;
256	pVM->pgm.s.PhysTlbHC.aEntries[idx].pMap = 0;
257	pVM->pgm.s.PhysTlbHC.aEntries[idx].pv = 0;
258	#else
259	PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
260	pTlbe->GCPhys = NIL_RTGCPHYS;
261	pTlbe->pPage = 0;
262	pTlbe->pMap = 0;
263	pTlbe->pv = 0;
264	#endif
265	/* @todo clear the RC TLB whenever we add it. */
266	}
267
268	/**
269	* Makes sure that there is at least one handy page ready for use.
270	*
271	* This will also take the appropriate actions when reaching water-marks.
272	*
273	* @returns VBox status code.
274	* @retval VINF_SUCCESS on success.
275	* @retval VERR_EM_NO_MEMORY if we're really out of memory.
276	*
277	* @param pVM The VM handle.
278	*
279	* @remarks Must be called from within the PGM critical section. It may
280	* nip back to ring-3/0 in some cases.
281	*/
282	static int pgmPhysEnsureHandyPage(PVM pVM)
283	{
284	AssertMsg(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", pVM->pgm.s.cHandyPages));
285
286	/*
287	* Do we need to do anything special?
288	*/
289	#ifdef IN_RING3
290	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_R3_ALLOC))
291	#else
292	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_RZ_TO_R3))
293	#endif
294	{
295	/*
296	* Allocate pages only if we're out of them, or in ring-3, almost out.
297	*/
298	#ifdef IN_RING3
299	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_R3_ALLOC)
300	#else
301	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_ALLOC)
302	#endif
303	{
304	Log(("PGM: cHandyPages=%u out of %u -> allocate more; VM_FF_PGM_NO_MEMORY=%RTbool\n",
305	pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages), VM_FF_ISSET(pVM, VM_FF_PGM_NO_MEMORY) ));
306	#ifdef IN_RING3
307	int rc = PGMR3PhysAllocateHandyPages(pVM);
308	#else
309	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES, 0);
310	#endif
311	if (RT_UNLIKELY(rc != VINF_SUCCESS))
312	{
313	if (RT_FAILURE(rc))
314	return rc;
315	AssertMsgReturn(rc == VINF_EM_NO_MEMORY, ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
316	if (!pVM->pgm.s.cHandyPages)
317	{
318	LogRel(("PGM: no more handy pages!\n"));
319	return VERR_EM_NO_MEMORY;
320	}
321	Assert(VM_FF_ISSET(pVM, VM_FF_PGM_NEED_HANDY_PAGES));
322	Assert(VM_FF_ISSET(pVM, VM_FF_PGM_NO_MEMORY));
323	#ifdef IN_RING3
324	REMR3NotifyFF(pVM);
325	#else
326	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); /* paranoia */
327	#endif
328	}
329	AssertMsgReturn( pVM->pgm.s.cHandyPages > 0
330	&& pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages),
331	("%u\n", pVM->pgm.s.cHandyPages),
332	VERR_INTERNAL_ERROR);
333	}
334	else
335	{
336	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_SET_FF)
337	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
338	#ifndef IN_RING3
339	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_TO_R3)
340	{
341	Log(("PGM: VM_FF_TO_R3 - cHandyPages=%u out of %u\n", pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
342	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
343	}
344	#endif
345	}
346	}
347
348	return VINF_SUCCESS;
349	}
350
351
352	/**
353	* Replace a zero or shared page with new page that we can write to.
354	*
355	* @returns The following VBox status codes.
356	* @retval VINF_SUCCESS on success, pPage is modified.
357	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
358	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
359	*
360	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
361	*
362	* @param pVM The VM address.
363	* @param pPage The physical page tracking structure. This will
364	* be modified on success.
365	* @param GCPhys The address of the page.
366	*
367	* @remarks Must be called from within the PGM critical section. It may
368	* nip back to ring-3/0 in some cases.
369	*
370	* @remarks This function shouldn't really fail, however if it does
371	* it probably means we've screwed up the size of handy pages and/or
372	* the low-water mark. Or, that some device I/O is causing a lot of
373	* pages to be allocated while while the host is in a low-memory
374	* condition. This latter should be handled elsewhere and in a more
375	* controlled manner, it's on the @bugref{3170} todo list...
376	*/
377	int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
378	{
379	LogFlow(("pgmPhysAllocPage: %R[pgmpage] %RGp\n", pPage, GCPhys));
380
381	/*
382	* Prereqs.
383	*/
384	Assert(PGMIsLocked(pVM));
385	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
386	Assert(!PGM_PAGE_IS_MMIO(pPage));
387
388	# ifdef PGM_WITH_LARGE_PAGES
389	if ( PGMIsUsingLargePages(pVM)
390	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
391	{
392	int rc = pgmPhysAllocLargePage(pVM, GCPhys);
393	if (rc == VINF_SUCCESS)
394	return rc;
395
396	/* fall back to 4kb pages. */
397	}
398	# endif
399
400	/*
401	* Flush any shadow page table mappings of the page.
402	* When VBOX_WITH_NEW_LAZY_PAGE_ALLOC isn't defined, there shouldn't be any.
403	*/
404	bool fFlushTLBs = false;
405	int rc = pgmPoolTrackFlushGCPhys(pVM, GCPhys, pPage, &fFlushTLBs);
406	AssertMsgReturn(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc), RT_FAILURE(rc) ? rc : VERR_IPE_UNEXPECTED_STATUS);
407
408	/*
409	* Ensure that we've got a page handy, take it and use it.
410	*/
411	int rc2 = pgmPhysEnsureHandyPage(pVM);
412	if (RT_FAILURE(rc2))
413	{
414	if (fFlushTLBs)
415	PGM_INVL_ALL_VCPU_TLBS(pVM);
416	Assert(rc2 == VERR_EM_NO_MEMORY);
417	return rc2;
418	}
419	/* re-assert preconditions since pgmPhysEnsureHandyPage may do a context switch. */
420	Assert(PGMIsLocked(pVM));
421	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
422	Assert(!PGM_PAGE_IS_MMIO(pPage));
423
424	uint32_t iHandyPage = --pVM->pgm.s.cHandyPages;
425	AssertMsg(iHandyPage < RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", iHandyPage));
426	Assert(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys != NIL_RTHCPHYS);
427	Assert(!(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
428	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idPage != NIL_GMM_PAGEID);
429	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
430
431	/*
432	* There are one or two action to be taken the next time we allocate handy pages:
433	* - Tell the GMM (global memory manager) what the page is being used for.
434	* (Speeds up replacement operations - sharing and defragmenting.)
435	* - If the current backing is shared, it must be freed.
436	*/
437	const RTHCPHYS HCPhys = pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys;
438	pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
439
440	if (PGM_PAGE_IS_SHARED(pPage))
441	{
442	pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage = PGM_PAGE_GET_PAGEID(pPage);
443	Assert(PGM_PAGE_GET_PAGEID(pPage) != NIL_GMM_PAGEID);
444	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
445
446	Log2(("PGM: Replaced shared page %#x at %RGp with %#x / %RHp\n", PGM_PAGE_GET_PAGEID(pPage),
447	GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
448	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PageReplaceShared));
449	pVM->pgm.s.cSharedPages--;
450	AssertMsgFailed(("TODO: copy shared page content")); /** @todo err.. what about copying the page content? */
451	}
452	else
453	{
454	Log2(("PGM: Replaced zero page %RGp with %#x / %RHp\n", GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
455	STAM_COUNTER_INC(&pVM->pgm.s.StatRZPageReplaceZero);
456	pVM->pgm.s.cZeroPages--;
457	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
458	}
459
460	/*
461	* Do the PGMPAGE modifications.
462	*/
463	pVM->pgm.s.cPrivatePages++;
464	PGM_PAGE_SET_HCPHYS(pPage, HCPhys);
465	PGM_PAGE_SET_PAGEID(pPage, pVM->pgm.s.aHandyPages[iHandyPage].idPage);
466	PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
467	PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_PT);
468	PGMPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
469
470	if ( fFlushTLBs
471	&& rc != VINF_PGM_GCPHYS_ALIASED)
472	PGM_INVL_ALL_VCPU_TLBS(pVM);
473	return rc;
474	}
475
476	#ifdef PGM_WITH_LARGE_PAGES
477	/**
478	* Replace a 2 MB range of zero pages with new pages that we can write to.
479	*
480	* @returns The following VBox status codes.
481	* @retval VINF_SUCCESS on success, pPage is modified.
482	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
483	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
484	*
485	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
486	*
487	* @param pVM The VM address.
488	* @param GCPhys The address of the page.
489	*
490	* @remarks Must be called from within the PGM critical section. It may
491	* nip back to ring-3/0 in some cases.
492	*/
493	int pgmPhysAllocLargePage(PVM pVM, RTGCPHYS GCPhys)
494	{
495	RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
496	LogFlow(("pgmPhysAllocLargePage: %RGp base %RGp\n", GCPhys, GCPhysBase));
497
498	/*
499	* Prereqs.
500	*/
501	Assert(PGMIsLocked(pVM));
502	Assert(PGMIsUsingLargePages(pVM));
503
504	PPGMPAGE pPage;
505	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysBase, &pPage);
506	if ( RT_SUCCESS(rc)
507	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
508	{
509	unsigned uPDEType = PGM_PAGE_GET_PDE_TYPE(pPage);
510
511	/* Don't call this function for already allocated pages. */
512	Assert(uPDEType != PGM_PAGE_PDE_TYPE_PDE);
513
514	if ( uPDEType == PGM_PAGE_PDE_TYPE_DONTCARE
515	&& PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ZERO)
516	{
517	unsigned iPage;
518
519	GCPhys = GCPhysBase;
520
521	/* Lazy approach: check all pages in the 2 MB range.
522	* The whole range must be ram and unallocated
523	*/
524	for (iPage = 0; iPage < _2M/PAGE_SIZE; iPage++)
525	{
526	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
527	if ( RT_FAILURE(rc)
528	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM /* Anything other than ram implies monitoring. */
529	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ZERO) /* allocated, monitored or shared means we can't use a large page here */
530	{
531	LogFlow(("Found page %RGp with wrong attributes (type=%d; state=%d); cancel check. rc=%d\n", GCPhys, PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_STATE(pPage), rc));
532	break;
533	}
534	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_DONTCARE);
535	GCPhys += PAGE_SIZE;
536	}
537	/* Fetch the start page of the 2 MB range again. */
538	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysBase, &pPage);
539	AssertRC(rc); /* can't fail */
540
541	if (iPage != _2M/PAGE_SIZE)
542	{
543	/* Failed. Mark as requiring a PT so we don't check the whole thing again in the future. */
544	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRefused);
545	PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_PT);
546	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
547	}
548	else
549	{
550	# ifdef IN_RING3
551	rc = PGMR3PhysAllocateLargeHandyPage(pVM, GCPhysBase);
552	# else
553	rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE, GCPhysBase);
554	# endif
555	if (RT_SUCCESS(rc))
556	{
557	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
558	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageAlloc);
559	return VINF_SUCCESS;
560	}
561	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
562
563	/* If we fail once, it most likely means the host's memory is too fragmented; don't bother trying again. */
564	PGMSetLargePageUsage(pVM, false);
565	return rc;
566	}
567	}
568	}
569	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
570	}
571
572	/**
573	* Recheck the entire 2 MB range to see if we can use it again as a large page.
574	*
575	* @returns The following VBox status codes.
576	* @retval VINF_SUCCESS on success, the large page can be used again
577	* @retval VERR_PGM_INVALID_LARGE_PAGE_RANGE if it can't be reused
578	*
579	* @param pVM The VM address.
580	* @param GCPhys The address of the page.
581	* @param pLargePage Page structure of the base page
582	*/
583	int pgmPhysIsValidLargePage(PVM pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage)
584	{
585	unsigned i;
586
587	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRecheck);
588
589	GCPhys &= X86_PDE2M_PAE_PG_MASK;
590
591	/* Check the base page. */
592	Assert(PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED);
593	if ( PGM_PAGE_GET_STATE(pLargePage) != PGM_PAGE_STATE_ALLOCATED
594	\|\| PGM_PAGE_GET_TYPE(pLargePage) != PGMPAGETYPE_RAM
595	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
596	{
597	LogFlow(("pgmPhysIsValidLargePage: checks failed for base page %x %x %x\n", PGM_PAGE_GET_STATE(pLargePage), PGM_PAGE_GET_TYPE(pLargePage), PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage)));
598	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
599	}
600
601	STAM_PROFILE_START(&pVM->pgm.s.CTX_MID_Z(Stat,IsValidLargePage), a);
602	/* Check all remaining pages in the 2 MB range. */
603	GCPhys += PAGE_SIZE;
604	for (i = 1; i < _2M/PAGE_SIZE; i++)
605	{
606	PPGMPAGE pPage;
607	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
608	AssertRCBreak(rc);
609
610	if ( PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
611	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
612	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
613	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
614	{
615	LogFlow(("pgmPhysIsValidLargePage: checks failed for page %d; %x %x %x\n", i, PGM_PAGE_GET_STATE(pPage), PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)));
616	break;
617	}
618
619	GCPhys += PAGE_SIZE;
620	}
621	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,IsValidLargePage), a);
622
623	if (i == _2M/PAGE_SIZE)
624	{
625	PGM_PAGE_SET_PDE_TYPE(pLargePage, PGM_PAGE_PDE_TYPE_PDE);
626	Log(("pgmPhysIsValidLargePage: page %RGp can be reused!\n", GCPhys - _2M));
627	return VINF_SUCCESS;
628	}
629
630	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
631	}
632
633	#endif /* PGM_WITH_LARGE_PAGES */
634
635	/**
636	* Deal with a write monitored page.
637	*
638	* @returns VBox strict status code.
639	*
640	* @param pVM The VM address.
641	* @param pPage The physical page tracking structure.
642	*
643	* @remarks Called from within the PGM critical section.
644	*/
645	void pgmPhysPageMakeWriteMonitoredWritable(PVM pVM, PPGMPAGE pPage)
646	{
647	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED);
648	PGM_PAGE_SET_WRITTEN_TO(pPage);
649	PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
650	Assert(pVM->pgm.s.cMonitoredPages > 0);
651	pVM->pgm.s.cMonitoredPages--;
652	pVM->pgm.s.cWrittenToPages++;
653	}
654
655
656	/**
657	* Deal with pages that are not writable, i.e. not in the ALLOCATED state.
658	*
659	* @returns VBox strict status code.
660	* @retval VINF_SUCCESS on success.
661	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
662	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
663	*
664	* @param pVM The VM address.
665	* @param pPage The physical page tracking structure.
666	* @param GCPhys The address of the page.
667	*
668	* @remarks Called from within the PGM critical section.
669	*/
670	int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
671	{
672	Assert(PGMIsLockOwner(pVM));
673	switch (PGM_PAGE_GET_STATE(pPage))
674	{
675	case PGM_PAGE_STATE_WRITE_MONITORED:
676	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
677	/* fall thru */
678	default: /* to shut up GCC */
679	case PGM_PAGE_STATE_ALLOCATED:
680	return VINF_SUCCESS;
681
682	/*
683	* Zero pages can be dummy pages for MMIO or reserved memory,
684	* so we need to check the flags before joining cause with
685	* shared page replacement.
686	*/
687	case PGM_PAGE_STATE_ZERO:
688	if (PGM_PAGE_IS_MMIO(pPage))
689	return VERR_PGM_PHYS_PAGE_RESERVED;
690	/* fall thru */
691	case PGM_PAGE_STATE_SHARED:
692	return pgmPhysAllocPage(pVM, pPage, GCPhys);
693
694	/* Not allowed to write to ballooned pages. */
695	case PGM_PAGE_STATE_BALLOONED:
696	return VERR_PGM_PHYS_PAGE_BALLOONED;
697	}
698	}
699
700
701	/**
702	* Internal usage: Map the page specified by its GMM ID.
703	*
704	* This is similar to pgmPhysPageMap
705	*
706	* @returns VBox status code.
707	*
708	* @param pVM The VM handle.
709	* @param idPage The Page ID.
710	* @param HCPhys The physical address (for RC).
711	* @param ppv Where to store the mapping address.
712	*
713	* @remarks Called from within the PGM critical section. The mapping is only
714	* valid while your inside this section.
715	*/
716	int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv)
717	{
718	/*
719	* Validation.
720	*/
721	Assert(PGMIsLocked(pVM));
722	AssertReturn(HCPhys && !(HCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
723	const uint32_t idChunk = idPage >> GMM_CHUNKID_SHIFT;
724	AssertReturn(idChunk != NIL_GMM_CHUNKID, VERR_INVALID_PARAMETER);
725
726	#ifdef IN_RC
727	/*
728	* Map it by HCPhys.
729	*/
730	return PGMDynMapHCPage(pVM, HCPhys, ppv);
731
732	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
733	/*
734	* Map it by HCPhys.
735	*/
736	return pgmR0DynMapHCPageInlined(&pVM->pgm.s, HCPhys, ppv);
737
738	#else
739	/*
740	* Find/make Chunk TLB entry for the mapping chunk.
741	*/
742	PPGMCHUNKR3MAP pMap;
743	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
744	if (pTlbe->idChunk == idChunk)
745	{
746	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,ChunkR3MapTlbHits));
747	pMap = pTlbe->pChunk;
748	}
749	else
750	{
751	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
752
753	/*
754	* Find the chunk, map it if necessary.
755	*/
756	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
757	if (!pMap)
758	{
759	# ifdef IN_RING0
760	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
761	AssertRCReturn(rc, rc);
762	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
763	Assert(pMap);
764	# else
765	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
766	if (RT_FAILURE(rc))
767	return rc;
768	# endif
769	}
770
771	/*
772	* Enter it into the Chunk TLB.
773	*/
774	pTlbe->idChunk = idChunk;
775	pTlbe->pChunk = pMap;
776	pMap->iAge = 0;
777	}
778
779	ppv = (uint8_t )pMap->pv + ((idPage &GMM_PAGEID_IDX_MASK) << PAGE_SHIFT);
780	return VINF_SUCCESS;
781	#endif
782	}
783
784
785	/**
786	* Maps a page into the current virtual address space so it can be accessed.
787	*
788	* @returns VBox status code.
789	* @retval VINF_SUCCESS on success.
790	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
791	*
792	* @param pVM The VM address.
793	* @param pPage The physical page tracking structure.
794	* @param GCPhys The address of the page.
795	* @param ppMap Where to store the address of the mapping tracking structure.
796	* @param ppv Where to store the mapping address of the page. The page
797	* offset is masked off!
798	*
799	* @remarks Called from within the PGM critical section.
800	*/
801	static int pgmPhysPageMapCommon(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv)
802	{
803	Assert(PGMIsLocked(pVM));
804
805	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
806	/*
807	* Just some sketchy GC/R0-darwin code.
808	*/
809	*ppMap = NULL;
810	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
811	Assert(HCPhys != pVM->pgm.s.HCPhysZeroPg);
812	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
813	pgmR0DynMapHCPageInlined(&pVM->pgm.s, HCPhys, ppv);
814	# else
815	PGMDynMapHCPage(pVM, HCPhys, ppv);
816	# endif
817	return VINF_SUCCESS;
818
819	#else /* IN_RING3 \|\| IN_RING0 */
820
821
822	/*
823	* Special case: ZERO and MMIO2 pages.
824	*/
825	const uint32_t idChunk = PGM_PAGE_GET_CHUNKID(pPage);
826	if (idChunk == NIL_GMM_CHUNKID)
827	{
828	AssertMsgReturn(PGM_PAGE_GET_PAGEID(pPage) == NIL_GMM_PAGEID, ("pPage=%R[pgmpage]\n", pPage), VERR_INTERNAL_ERROR_2);
829	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2)
830	{
831	/* Lookup the MMIO2 range and use pvR3 to calc the address. */
832	PPGMRAMRANGE pRam = pgmPhysGetRange(&pVM->pgm.s, GCPhys);
833	AssertMsgReturn(pRam \|\| !pRam->pvR3, ("pRam=%p pPage=%R[pgmpage]\n", pRam, pPage), VERR_INTERNAL_ERROR_2);
834	ppv = (void )((uintptr_t)pRam->pvR3 + (uintptr_t)((GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK) - pRam->GCPhys));
835	}
836	else if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO)
837	{
838	/** @todo deal with aliased MMIO2 pages somehow...
839	* One solution would be to seed MMIO2 pages to GMM and get unique Page IDs for
840	* them, that would also avoid this mess. It would actually be kind of
841	* elegant... */
842	AssertLogRelMsgFailedReturn(("%RGp\n", GCPhys), VERR_INTERNAL_ERROR_3);
843	}
844	else
845	{
846	/** @todo handle MMIO2 */
847	AssertMsgReturn(PGM_PAGE_IS_ZERO(pPage), ("pPage=%R[pgmpage]\n", pPage), VERR_INTERNAL_ERROR_2);
848	AssertMsgReturn(PGM_PAGE_GET_HCPHYS(pPage) == pVM->pgm.s.HCPhysZeroPg,
849	("pPage=%R[pgmpage]\n", pPage),
850	VERR_INTERNAL_ERROR_2);
851	*ppv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
852	}
853	*ppMap = NULL;
854	return VINF_SUCCESS;
855	}
856
857	/*
858	* Find/make Chunk TLB entry for the mapping chunk.
859	*/
860	PPGMCHUNKR3MAP pMap;
861	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
862	if (pTlbe->idChunk == idChunk)
863	{
864	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,ChunkR3MapTlbHits));
865	pMap = pTlbe->pChunk;
866	}
867	else
868	{
869	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
870
871	/*
872	* Find the chunk, map it if necessary.
873	*/
874	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
875	if (!pMap)
876	{
877	#ifdef IN_RING0
878	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
879	AssertRCReturn(rc, rc);
880	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
881	Assert(pMap);
882	#else
883	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
884	if (RT_FAILURE(rc))
885	return rc;
886	#endif
887	}
888
889	/*
890	* Enter it into the Chunk TLB.
891	*/
892	pTlbe->idChunk = idChunk;
893	pTlbe->pChunk = pMap;
894	pMap->iAge = 0;
895	}
896
897	ppv = (uint8_t )pMap->pv + (PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) << PAGE_SHIFT);
898	*ppMap = pMap;
899	return VINF_SUCCESS;
900	#endif /* IN_RING3 */
901	}
902
903
904	/**
905	* Combination of pgmPhysPageMakeWritable and pgmPhysPageMapWritable.
906	*
907	* This is typically used is paths where we cannot use the TLB methods (like ROM
908	* pages) or where there is no point in using them since we won't get many hits.
909	*
910	* @returns VBox strict status code.
911	* @retval VINF_SUCCESS on success.
912	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
913	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
914	*
915	* @param pVM The VM address.
916	* @param pPage The physical page tracking structure.
917	* @param GCPhys The address of the page.
918	* @param ppv Where to store the mapping address of the page. The page
919	* offset is masked off!
920	*
921	* @remarks Called from within the PGM critical section. The mapping is only
922	* valid while your inside this section.
923	*/
924	int pgmPhysPageMakeWritableAndMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
925	{
926	int rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
927	if (RT_SUCCESS(rc))
928	{
929	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* returned */, ("%Rrc\n", rc));
930	PPGMPAGEMAP pMapIgnore;
931	int rc2 = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
932	if (RT_FAILURE(rc2)) /* preserve rc */
933	rc = rc2;
934	}
935	return rc;
936	}
937
938
939	/**
940	* Maps a page into the current virtual address space so it can be accessed for
941	* both writing and reading.
942	*
943	* This is typically used is paths where we cannot use the TLB methods (like ROM
944	* pages) or where there is no point in using them since we won't get many hits.
945	*
946	* @returns VBox status code.
947	* @retval VINF_SUCCESS on success.
948	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
949	*
950	* @param pVM The VM address.
951	* @param pPage The physical page tracking structure. Must be in the
952	* allocated state.
953	* @param GCPhys The address of the page.
954	* @param ppv Where to store the mapping address of the page. The page
955	* offset is masked off!
956	*
957	* @remarks Called from within the PGM critical section. The mapping is only
958	* valid while your inside this section.
959	*/
960	int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
961	{
962	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
963	PPGMPAGEMAP pMapIgnore;
964	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
965	}
966
967
968	/**
969	* Maps a page into the current virtual address space so it can be accessed for
970	* reading.
971	*
972	* This is typically used is paths where we cannot use the TLB methods (like ROM
973	* pages) or where there is no point in using them since we won't get many hits.
974	*
975	* @returns VBox status code.
976	* @retval VINF_SUCCESS on success.
977	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
978	*
979	* @param pVM The VM address.
980	* @param pPage The physical page tracking structure.
981	* @param GCPhys The address of the page.
982	* @param ppv Where to store the mapping address of the page. The page
983	* offset is masked off!
984	*
985	* @remarks Called from within the PGM critical section. The mapping is only
986	* valid while your inside this section.
987	*/
988	int pgmPhysPageMapReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv)
989	{
990	PPGMPAGEMAP pMapIgnore;
991	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, (void **)ppv);
992	}
993
994
995	#if !defined(IN_RC) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
996	/**
997	* Load a guest page into the ring-3 physical TLB.
998	*
999	* @returns VBox status code.
1000	* @retval VINF_SUCCESS on success
1001	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1002	* @param pPGM The PGM instance pointer.
1003	* @param GCPhys The guest physical address in question.
1004	*/
1005	int pgmPhysPageLoadIntoTlb(PPGM pPGM, RTGCPHYS GCPhys)
1006	{
1007	Assert(PGMIsLocked(PGM2VM(pPGM)));
1008	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbMisses));
1009
1010	/*
1011	* Find the ram range.
1012	* 99.8% of requests are expected to be in the first range.
1013	*/
1014	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
1015	RTGCPHYS off = GCPhys - pRam->GCPhys;
1016	if (RT_UNLIKELY(off >= pRam->cb))
1017	{
1018	do
1019	{
1020	pRam = pRam->CTX_SUFF(pNext);
1021	if (!pRam)
1022	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
1023	off = GCPhys - pRam->GCPhys;
1024	} while (off >= pRam->cb);
1025	}
1026
1027	/*
1028	* Map the page.
1029	* Make a special case for the zero page as it is kind of special.
1030	*/
1031	PPGMPAGE pPage = &pRam->aPages[off >> PAGE_SHIFT];
1032	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
1033	if ( !PGM_PAGE_IS_ZERO(pPage)
1034	&& !PGM_PAGE_IS_BALLOONED(pPage))
1035	{
1036	void *pv;
1037	PPGMPAGEMAP pMap;
1038	int rc = pgmPhysPageMapCommon(PGM2VM(pPGM), pPage, GCPhys, &pMap, &pv);
1039	if (RT_FAILURE(rc))
1040	return rc;
1041	pTlbe->pMap = pMap;
1042	pTlbe->pv = pv;
1043	Assert(!((uintptr_t)pTlbe->pv & PAGE_OFFSET_MASK));
1044	}
1045	else
1046	{
1047	Assert(PGM_PAGE_GET_HCPHYS(pPage) == pPGM->HCPhysZeroPg);
1048	pTlbe->pMap = NULL;
1049	pTlbe->pv = pPGM->CTXALLSUFF(pvZeroPg);
1050	}
1051	#ifdef PGM_WITH_PHYS_TLB
1052	pTlbe->GCPhys = GCPhys & X86_PTE_PAE_PG_MASK;
1053	#else
1054	pTlbe->GCPhys = NIL_RTGCPHYS;
1055	#endif
1056	pTlbe->pPage = pPage;
1057	return VINF_SUCCESS;
1058	}
1059
1060
1061	/**
1062	* Load a guest page into the ring-3 physical TLB.
1063	*
1064	* @returns VBox status code.
1065	* @retval VINF_SUCCESS on success
1066	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1067	*
1068	* @param pPGM The PGM instance pointer.
1069	* @param pPage Pointer to the PGMPAGE structure corresponding to
1070	* GCPhys.
1071	* @param GCPhys The guest physical address in question.
1072	*/
1073	int pgmPhysPageLoadIntoTlbWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys)
1074	{
1075	Assert(PGMIsLocked(PGM2VM(pPGM)));
1076	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbMisses));
1077
1078	/*
1079	* Map the page.
1080	* Make a special case for the zero page as it is kind of special.
1081	*/
1082	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
1083	if ( !PGM_PAGE_IS_ZERO(pPage)
1084	&& !PGM_PAGE_IS_BALLOONED(pPage))
1085	{
1086	void *pv;
1087	PPGMPAGEMAP pMap;
1088	int rc = pgmPhysPageMapCommon(PGM2VM(pPGM), pPage, GCPhys, &pMap, &pv);
1089	if (RT_FAILURE(rc))
1090	return rc;
1091	pTlbe->pMap = pMap;
1092	pTlbe->pv = pv;
1093	Assert(!((uintptr_t)pTlbe->pv & PAGE_OFFSET_MASK));
1094	}
1095	else
1096	{
1097	Assert(PGM_PAGE_GET_HCPHYS(pPage) == pPGM->HCPhysZeroPg);
1098	pTlbe->pMap = NULL;
1099	pTlbe->pv = pPGM->CTXALLSUFF(pvZeroPg);
1100	}
1101	#ifdef PGM_WITH_PHYS_TLB
1102	pTlbe->GCPhys = GCPhys & X86_PTE_PAE_PG_MASK;
1103	#else
1104	pTlbe->GCPhys = NIL_RTGCPHYS;
1105	#endif
1106	pTlbe->pPage = pPage;
1107	return VINF_SUCCESS;
1108	}
1109	#endif /* !IN_RC && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
1110
1111
1112	/**
1113	* Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
1114	* own the PGM lock and therefore not need to lock the mapped page.
1115	*
1116	* @returns VBox status code.
1117	* @retval VINF_SUCCESS on success.
1118	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1119	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1120	*
1121	* @param pVM The VM handle.
1122	* @param GCPhys The guest physical address of the page that should be mapped.
1123	* @param pPage Pointer to the PGMPAGE structure for the page.
1124	* @param ppv Where to store the address corresponding to GCPhys.
1125	*
1126	* @internal
1127	*/
1128	int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1129	{
1130	int rc;
1131	AssertReturn(pPage, VERR_INTERNAL_ERROR);
1132	Assert(PGMIsLocked(pVM));
1133
1134	/*
1135	* Make sure the page is writable.
1136	*/
1137	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1138	{
1139	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1140	if (RT_FAILURE(rc))
1141	return rc;
1142	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1143	}
1144	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1145
1146	/*
1147	* Get the mapping address.
1148	*/
1149	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1150	*ppv = pgmDynMapHCPageOff(&pVM->pgm.s, PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK));
1151	#else
1152	PPGMPAGEMAPTLBE pTlbe;
1153	rc = pgmPhysPageQueryTlbeWithPage(&pVM->pgm.s, pPage, GCPhys, &pTlbe);
1154	if (RT_FAILURE(rc))
1155	return rc;
1156	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1157	#endif
1158	return VINF_SUCCESS;
1159	}
1160
1161
1162	/**
1163	* Internal version of PGMPhysGCPhys2CCPtrReadOnly that expects the caller to
1164	* own the PGM lock and therefore not need to lock the mapped page.
1165	*
1166	* @returns VBox status code.
1167	* @retval VINF_SUCCESS on success.
1168	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1169	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1170	*
1171	* @param pVM The VM handle.
1172	* @param GCPhys The guest physical address of the page that should be mapped.
1173	* @param pPage Pointer to the PGMPAGE structure for the page.
1174	* @param ppv Where to store the address corresponding to GCPhys.
1175	*
1176	* @internal
1177	*/
1178	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv)
1179	{
1180	AssertReturn(pPage, VERR_INTERNAL_ERROR);
1181	Assert(PGMIsLocked(pVM));
1182	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1183
1184	/*
1185	* Get the mapping address.
1186	*/
1187	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1188	ppv = pgmDynMapHCPageOff(&pVM->pgm.s, PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK)); /* @todo add a read only flag? */
1189	#else
1190	PPGMPAGEMAPTLBE pTlbe;
1191	int rc = pgmPhysPageQueryTlbeWithPage(&pVM->pgm.s, pPage, GCPhys, &pTlbe);
1192	if (RT_FAILURE(rc))
1193	return rc;
1194	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1195	#endif
1196	return VINF_SUCCESS;
1197	}
1198
1199
1200	/**
1201	* Requests the mapping of a guest page into the current context.
1202	*
1203	* This API should only be used for very short term, as it will consume
1204	* scarse resources (R0 and GC) in the mapping cache. When you're done
1205	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1206	*
1207	* This API will assume your intention is to write to the page, and will
1208	* therefore replace shared and zero pages. If you do not intend to modify
1209	* the page, use the PGMPhysGCPhys2CCPtrReadOnly() API.
1210	*
1211	* @returns VBox status code.
1212	* @retval VINF_SUCCESS on success.
1213	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1214	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1215	*
1216	* @param pVM The VM handle.
1217	* @param GCPhys The guest physical address of the page that should be mapped.
1218	* @param ppv Where to store the address corresponding to GCPhys.
1219	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1220	*
1221	* @remarks The caller is responsible for dealing with access handlers.
1222	* @todo Add an informational return code for pages with access handlers?
1223	*
1224	* @remark Avoid calling this API from within critical sections (other than the
1225	* PGM one) because of the deadlock risk. External threads may need to
1226	* delegate jobs to the EMTs.
1227	* @thread Any thread.
1228	*/
1229	VMMDECL(int) PGMPhysGCPhys2CCPtr(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
1230	{
1231	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1232
1233	/*
1234	* Find the page and make sure it's writable.
1235	*/
1236	PPGMPAGE pPage;
1237	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1238	if (RT_SUCCESS(rc))
1239	{
1240	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1241	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1242	if (RT_SUCCESS(rc))
1243	{
1244	ppv = pgmDynMapHCPageOff(&pVM->pgm.s, PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK)); /* @todo add a read only flag? */
1245	# if 0
1246	pLock->pvMap = 0;
1247	pLock->pvPage = pPage;
1248	# else
1249	pLock->u32Dummy = UINT32_MAX;
1250	# endif
1251	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1252	rc = VINF_SUCCESS;
1253	}
1254	}
1255
1256	#else /* IN_RING3 \|\| IN_RING0 */
1257	int rc = pgmLock(pVM);
1258	AssertRCReturn(rc, rc);
1259
1260	/*
1261	* Query the Physical TLB entry for the page (may fail).
1262	*/
1263	PPGMPAGEMAPTLBE pTlbe;
1264	rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
1265	if (RT_SUCCESS(rc))
1266	{
1267	/*
1268	* If the page is shared, the zero page, or being write monitored
1269	* it must be converted to an page that's writable if possible.
1270	*/
1271	PPGMPAGE pPage = pTlbe->pPage;
1272	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1273	{
1274	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1275	if (RT_SUCCESS(rc))
1276	{
1277	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1278	rc = pgmPhysPageQueryTlbeWithPage(&pVM->pgm.s, pPage, GCPhys, &pTlbe);
1279	}
1280	}
1281	if (RT_SUCCESS(rc))
1282	{
1283	/*
1284	* Now, just perform the locking and calculate the return address.
1285	*/
1286	PPGMPAGEMAP pMap = pTlbe->pMap;
1287	if (pMap)
1288	pMap->cRefs++;
1289
1290	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
1291	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
1292	{
1293	if (cLocks == 0)
1294	pVM->pgm.s.cWriteLockedPages++;
1295	PGM_PAGE_INC_WRITE_LOCKS(pPage);
1296	}
1297	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
1298	{
1299	PGM_PAGE_INC_WRITE_LOCKS(pPage);
1300	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
1301	if (pMap)
1302	pMap->cRefs++; /* Extra ref to prevent it from going away. */
1303	}
1304
1305	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1306	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
1307	pLock->pvMap = pMap;
1308	}
1309	}
1310
1311	pgmUnlock(pVM);
1312	#endif /* IN_RING3 \|\| IN_RING0 */
1313	return rc;
1314	}
1315
1316
1317	/**
1318	* Requests the mapping of a guest page into the current context.
1319	*
1320	* This API should only be used for very short term, as it will consume
1321	* scarse resources (R0 and GC) in the mapping cache. When you're done
1322	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1323	*
1324	* @returns VBox status code.
1325	* @retval VINF_SUCCESS on success.
1326	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1327	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1328	*
1329	* @param pVM The VM handle.
1330	* @param GCPhys The guest physical address of the page that should be mapped.
1331	* @param ppv Where to store the address corresponding to GCPhys.
1332	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1333	*
1334	* @remarks The caller is responsible for dealing with access handlers.
1335	* @todo Add an informational return code for pages with access handlers?
1336	*
1337	* @remark Avoid calling this API from within critical sections (other than
1338	* the PGM one) because of the deadlock risk.
1339	* @thread Any thread.
1340	*/
1341	VMMDECL(int) PGMPhysGCPhys2CCPtrReadOnly(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
1342	{
1343	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1344
1345	/*
1346	* Find the page and make sure it's readable.
1347	*/
1348	PPGMPAGE pPage;
1349	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1350	if (RT_SUCCESS(rc))
1351	{
1352	if (RT_UNLIKELY(PGM_PAGE_IS_MMIO(pPage)))
1353	rc = VERR_PGM_PHYS_PAGE_RESERVED;
1354	else
1355	{
1356	ppv = pgmDynMapHCPageOff(&pVM->pgm.s, PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK)); /* @todo add a read only flag? */
1357	# if 0
1358	pLock->pvMap = 0;
1359	pLock->pvPage = pPage;
1360	# else
1361	pLock->u32Dummy = UINT32_MAX;
1362	# endif
1363	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1364	rc = VINF_SUCCESS;
1365	}
1366	}
1367
1368	#else /* IN_RING3 \|\| IN_RING0 */
1369	int rc = pgmLock(pVM);
1370	AssertRCReturn(rc, rc);
1371
1372	/*
1373	* Query the Physical TLB entry for the page (may fail).
1374	*/
1375	PPGMPAGEMAPTLBE pTlbe;
1376	rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
1377	if (RT_SUCCESS(rc))
1378	{
1379	/* MMIO pages doesn't have any readable backing. */
1380	PPGMPAGE pPage = pTlbe->pPage;
1381	if (RT_UNLIKELY(PGM_PAGE_IS_MMIO(pPage)))
1382	rc = VERR_PGM_PHYS_PAGE_RESERVED;
1383	else
1384	{
1385	/*
1386	* Now, just perform the locking and calculate the return address.
1387	*/
1388	PPGMPAGEMAP pMap = pTlbe->pMap;
1389	if (pMap)
1390	pMap->cRefs++;
1391
1392	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
1393	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
1394	{
1395	if (cLocks == 0)
1396	pVM->pgm.s.cReadLockedPages++;
1397	PGM_PAGE_INC_READ_LOCKS(pPage);
1398	}
1399	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
1400	{
1401	PGM_PAGE_INC_READ_LOCKS(pPage);
1402	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
1403	if (pMap)
1404	pMap->cRefs++; /* Extra ref to prevent it from going away. */
1405	}
1406
1407	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1408	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
1409	pLock->pvMap = pMap;
1410	}
1411	}
1412
1413	pgmUnlock(pVM);
1414	#endif /* IN_RING3 \|\| IN_RING0 */
1415	return rc;
1416	}
1417
1418
1419	/**
1420	* Requests the mapping of a guest page given by virtual address into the current context.
1421	*
1422	* This API should only be used for very short term, as it will consume
1423	* scarse resources (R0 and GC) in the mapping cache. When you're done
1424	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1425	*
1426	* This API will assume your intention is to write to the page, and will
1427	* therefore replace shared and zero pages. If you do not intend to modify
1428	* the page, use the PGMPhysGCPtr2CCPtrReadOnly() API.
1429	*
1430	* @returns VBox status code.
1431	* @retval VINF_SUCCESS on success.
1432	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
1433	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
1434	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1435	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1436	*
1437	* @param pVCpu VMCPU handle.
1438	* @param GCPhys The guest physical address of the page that should be mapped.
1439	* @param ppv Where to store the address corresponding to GCPhys.
1440	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1441	*
1442	* @remark Avoid calling this API from within critical sections (other than
1443	* the PGM one) because of the deadlock risk.
1444	* @thread EMT
1445	*/
1446	VMMDECL(int) PGMPhysGCPtr2CCPtr(PVMCPU pVCpu, RTGCPTR GCPtr, void **ppv, PPGMPAGEMAPLOCK pLock)
1447	{
1448	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
1449	RTGCPHYS GCPhys;
1450	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
1451	if (RT_SUCCESS(rc))
1452	rc = PGMPhysGCPhys2CCPtr(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
1453	return rc;
1454	}
1455
1456
1457	/**
1458	* Requests the mapping of a guest page given by virtual address into the current context.
1459	*
1460	* This API should only be used for very short term, as it will consume
1461	* scarse resources (R0 and GC) in the mapping cache. When you're done
1462	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1463	*
1464	* @returns VBox status code.
1465	* @retval VINF_SUCCESS on success.
1466	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
1467	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
1468	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1469	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1470	*
1471	* @param pVCpu VMCPU handle.
1472	* @param GCPhys The guest physical address of the page that should be mapped.
1473	* @param ppv Where to store the address corresponding to GCPhys.
1474	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1475	*
1476	* @remark Avoid calling this API from within critical sections (other than
1477	* the PGM one) because of the deadlock risk.
1478	* @thread EMT
1479	*/
1480	VMMDECL(int) PGMPhysGCPtr2CCPtrReadOnly(PVMCPU pVCpu, RTGCPTR GCPtr, void const **ppv, PPGMPAGEMAPLOCK pLock)
1481	{
1482	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
1483	RTGCPHYS GCPhys;
1484	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
1485	if (RT_SUCCESS(rc))
1486	rc = PGMPhysGCPhys2CCPtrReadOnly(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
1487	return rc;
1488	}
1489
1490
1491	/**
1492	* Release the mapping of a guest page.
1493	*
1494	* This is the counter part of PGMPhysGCPhys2CCPtr, PGMPhysGCPhys2CCPtrReadOnly
1495	* PGMPhysGCPtr2CCPtr and PGMPhysGCPtr2CCPtrReadOnly.
1496	*
1497	* @param pVM The VM handle.
1498	* @param pLock The lock structure initialized by the mapping function.
1499	*/
1500	VMMDECL(void) PGMPhysReleasePageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock)
1501	{
1502	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1503	/* currently nothing to do here. */
1504	Assert(pLock->u32Dummy == UINT32_MAX);
1505	pLock->u32Dummy = 0;
1506
1507	#else /* IN_RING3 */
1508	PPGMPAGEMAP pMap = (PPGMPAGEMAP)pLock->pvMap;
1509	PPGMPAGE pPage = (PPGMPAGE)(pLock->uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
1510	bool fWriteLock = (pLock->uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
1511
1512	pLock->uPageAndType = 0;
1513	pLock->pvMap = NULL;
1514
1515	pgmLock(pVM);
1516	if (fWriteLock)
1517	{
1518	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
1519	Assert(cLocks > 0);
1520	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
1521	{
1522	if (cLocks == 1)
1523	{
1524	Assert(pVM->pgm.s.cWriteLockedPages > 0);
1525	pVM->pgm.s.cWriteLockedPages--;
1526	}
1527	PGM_PAGE_DEC_WRITE_LOCKS(pPage);
1528	}
1529
1530	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
1531	{
1532	PGM_PAGE_SET_WRITTEN_TO(pPage);
1533	PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
1534	Assert(pVM->pgm.s.cMonitoredPages > 0);
1535	pVM->pgm.s.cMonitoredPages--;
1536	pVM->pgm.s.cWrittenToPages++;
1537	}
1538	}
1539	else
1540	{
1541	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
1542	Assert(cLocks > 0);
1543	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
1544	{
1545	if (cLocks == 1)
1546	{
1547	Assert(pVM->pgm.s.cReadLockedPages > 0);
1548	pVM->pgm.s.cReadLockedPages--;
1549	}
1550	PGM_PAGE_DEC_READ_LOCKS(pPage);
1551	}
1552	}
1553
1554	if (pMap)
1555	{
1556	Assert(pMap->cRefs >= 1);
1557	pMap->cRefs--;
1558	pMap->iAge = 0;
1559	}
1560	pgmUnlock(pVM);
1561	#endif /* IN_RING3 */
1562	}
1563
1564
1565	/**
1566	* Converts a GC physical address to a HC ring-3 pointer.
1567	*
1568	* @returns VINF_SUCCESS on success.
1569	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
1570	* page but has no physical backing.
1571	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
1572	* GC physical address.
1573	* @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
1574	* a dynamic ram chunk boundary
1575	*
1576	* @param pVM The VM handle.
1577	* @param GCPhys The GC physical address to convert.
1578	* @param cbRange Physical range
1579	* @param pR3Ptr Where to store the R3 pointer on success.
1580	*
1581	* @deprecated Avoid when possible!
1582	*/
1583	VMMDECL(int) PGMPhysGCPhys2R3Ptr(PVM pVM, RTGCPHYS GCPhys, RTUINT cbRange, PRTR3PTR pR3Ptr)
1584	{
1585	/** @todo this is kind of hacky and needs some more work. */
1586	#ifndef DEBUG_sandervl
1587	VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
1588	#endif
1589
1590	Log(("PGMPhysGCPhys2R3Ptr(,%RGp,%#x,): dont use this API!\n", GCPhys, cbRange)); /** @todo eliminate this API! */
1591	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1592	AssertFailedReturn(VERR_NOT_IMPLEMENTED);
1593	#else
1594	pgmLock(pVM);
1595
1596	PPGMRAMRANGE pRam;
1597	PPGMPAGE pPage;
1598	int rc = pgmPhysGetPageAndRangeEx(&pVM->pgm.s, GCPhys, &pPage, &pRam);
1599	if (RT_SUCCESS(rc))
1600	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)pR3Ptr);
1601
1602	pgmUnlock(pVM);
1603	Assert(rc <= VINF_SUCCESS);
1604	return rc;
1605	#endif
1606	}
1607
1608
1609	#ifdef VBOX_STRICT
1610	/**
1611	* PGMPhysGCPhys2R3Ptr convenience for use with assertions.
1612	*
1613	* @returns The R3Ptr, NIL_RTR3PTR on failure.
1614	* @param pVM The VM handle.
1615	* @param GCPhys The GC Physical addresss.
1616	* @param cbRange Physical range.
1617	*
1618	* @deprecated Avoid when possible.
1619	*/
1620	VMMDECL(RTR3PTR) PGMPhysGCPhys2R3PtrAssert(PVM pVM, RTGCPHYS GCPhys, RTUINT cbRange)
1621	{
1622	RTR3PTR R3Ptr;
1623	int rc = PGMPhysGCPhys2R3Ptr(pVM, GCPhys, cbRange, &R3Ptr);
1624	if (RT_SUCCESS(rc))
1625	return R3Ptr;
1626	return NIL_RTR3PTR;
1627	}
1628	#endif /* VBOX_STRICT */
1629
1630
1631	/**
1632	* Converts a guest pointer to a GC physical address.
1633	*
1634	* This uses the current CR3/CR0/CR4 of the guest.
1635	*
1636	* @returns VBox status code.
1637	* @param pVCpu The VMCPU Handle
1638	* @param GCPtr The guest pointer to convert.
1639	* @param pGCPhys Where to store the GC physical address.
1640	*/
1641	VMMDECL(int) PGMPhysGCPtr2GCPhys(PVMCPU pVCpu, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
1642	{
1643	int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, pGCPhys);
1644	if (pGCPhys && RT_SUCCESS(rc))
1645	*pGCPhys \|= (RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK;
1646	return rc;
1647	}
1648
1649
1650	/**
1651	* Converts a guest pointer to a HC physical address.
1652	*
1653	* This uses the current CR3/CR0/CR4 of the guest.
1654	*
1655	* @returns VBox status code.
1656	* @param pVCpu The VMCPU Handle
1657	* @param GCPtr The guest pointer to convert.
1658	* @param pHCPhys Where to store the HC physical address.
1659	*/
1660	VMMDECL(int) PGMPhysGCPtr2HCPhys(PVMCPU pVCpu, RTGCPTR GCPtr, PRTHCPHYS pHCPhys)
1661	{
1662	PVM pVM = pVCpu->CTX_SUFF(pVM);
1663	RTGCPHYS GCPhys;
1664	int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, &GCPhys);
1665	if (RT_SUCCESS(rc))
1666	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys \| ((RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK), pHCPhys);
1667	return rc;
1668	}
1669
1670
1671	/**
1672	* Converts a guest pointer to a R3 pointer.
1673	*
1674	* This uses the current CR3/CR0/CR4 of the guest.
1675	*
1676	* @returns VBox status code.
1677	* @param pVCpu The VMCPU Handle
1678	* @param GCPtr The guest pointer to convert.
1679	* @param pR3Ptr Where to store the R3 virtual address.
1680	*
1681	* @deprecated Don't use this.
1682	*/
1683	VMMDECL(int) PGMPhysGCPtr2R3Ptr(PVMCPU pVCpu, RTGCPTR GCPtr, PRTR3PTR pR3Ptr)
1684	{
1685	PVM pVM = pVCpu->CTX_SUFF(pVM);
1686	VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
1687	RTGCPHYS GCPhys;
1688	int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, &GCPhys);
1689	if (RT_SUCCESS(rc))
1690	rc = PGMPhysGCPhys2R3Ptr(pVM, GCPhys \| ((RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK), 1 /* we always stay within one page */, pR3Ptr);
1691	return rc;
1692	}
1693
1694
1695
1696	#undef LOG_GROUP
1697	#define LOG_GROUP LOG_GROUP_PGM_PHYS_ACCESS
1698
1699
1700	#ifdef IN_RING3
1701	/**
1702	* Cache PGMPhys memory access
1703	*
1704	* @param pVM VM Handle.
1705	* @param pCache Cache structure pointer
1706	* @param GCPhys GC physical address
1707	* @param pbHC HC pointer corresponding to physical page
1708	*
1709	* @thread EMT.
1710	*/
1711	static void pgmPhysCacheAdd(PVM pVM, PGMPHYSCACHE pCache, RTGCPHYS GCPhys, uint8_t pbR3)
1712	{
1713	uint32_t iCacheIndex;
1714
1715	Assert(VM_IS_EMT(pVM));
1716
1717	GCPhys = PHYS_PAGE_ADDRESS(GCPhys);
1718	pbR3 = (uint8_t *)PAGE_ADDRESS(pbR3);
1719
1720	iCacheIndex = ((GCPhys >> PAGE_SHIFT) & PGM_MAX_PHYSCACHE_ENTRIES_MASK);
1721
1722	ASMBitSet(&pCache->aEntries, iCacheIndex);
1723
1724	pCache->Entry[iCacheIndex].GCPhys = GCPhys;
1725	pCache->Entry[iCacheIndex].pbR3 = pbR3;
1726	}
1727	#endif /* IN_RING3 */
1728
1729
1730	/**
1731	* Deals with reading from a page with one or more ALL access handlers.
1732	*
1733	* @returns VBox status code. Can be ignored in ring-3.
1734	* @retval VINF_SUCCESS.
1735	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
1736	*
1737	* @param pVM The VM handle.
1738	* @param pPage The page descriptor.
1739	* @param GCPhys The physical address to start reading at.
1740	* @param pvBuf Where to put the bits we read.
1741	* @param cb How much to read - less or equal to a page.
1742	*/
1743	static int pgmPhysReadHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void *pvBuf, size_t cb)
1744	{
1745	/*
1746	* The most frequent access here is MMIO and shadowed ROM.
1747	* The current code ASSUMES all these access handlers covers full pages!
1748	*/
1749
1750	/*
1751	* Whatever we do we need the source page, map it first.
1752	*/
1753	const void *pvSrc = NULL;
1754	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvSrc);
1755	if (RT_FAILURE(rc))
1756	{
1757	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
1758	GCPhys, pPage, rc));
1759	memset(pvBuf, 0xff, cb);
1760	return VINF_SUCCESS;
1761	}
1762	rc = VINF_PGM_HANDLER_DO_DEFAULT;
1763
1764	/*
1765	* Deal with any physical handlers.
1766	*/
1767	PPGMPHYSHANDLER pPhys = NULL;
1768	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL)
1769	{
1770	#ifdef IN_RING3
1771	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
1772	AssertReleaseMsg(pPhys, ("GCPhys=%RGp cb=%#x\n", GCPhys, cb));
1773	Assert(GCPhys >= pPhys->Core.Key && GCPhys <= pPhys->Core.KeyLast);
1774	Assert((pPhys->Core.Key & PAGE_OFFSET_MASK) == 0);
1775	Assert((pPhys->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
1776	Assert(pPhys->CTX_SUFF(pfnHandler));
1777
1778	PFNPGMR3PHYSHANDLER pfnHandler = pPhys->CTX_SUFF(pfnHandler);
1779	void *pvUser = pPhys->CTX_SUFF(pvUser);
1780
1781	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cb, pPage, R3STRING(pPhys->pszDesc) ));
1782	STAM_PROFILE_START(&pPhys->Stat, h);
1783	Assert(PGMIsLockOwner(pVM));
1784	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
1785	pgmUnlock(pVM);
1786	rc = pfnHandler(pVM, GCPhys, (void *)pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, pvUser);
1787	pgmLock(pVM);
1788	# ifdef VBOX_WITH_STATISTICS
1789	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
1790	if (pPhys)
1791	STAM_PROFILE_STOP(&pPhys->Stat, h);
1792	# else
1793	pPhys = NULL; /* might not be valid anymore. */
1794	# endif
1795	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp\n", rc, GCPhys));
1796	#else
1797	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
1798	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cb=%#x\n", GCPhys, cb));
1799	return VERR_PGM_PHYS_WR_HIT_HANDLER;
1800	#endif
1801	}
1802
1803	/*
1804	* Deal with any virtual handlers.
1805	*/
1806	if (PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) == PGM_PAGE_HNDL_VIRT_STATE_ALL)
1807	{
1808	unsigned iPage;
1809	PPGMVIRTHANDLER pVirt;
1810
1811	int rc2 = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pVirt, &iPage);
1812	AssertReleaseMsg(RT_SUCCESS(rc2), ("GCPhys=%RGp cb=%#x rc2=%Rrc\n", GCPhys, cb, rc2));
1813	Assert((pVirt->Core.Key & PAGE_OFFSET_MASK) == 0);
1814	Assert((pVirt->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
1815	Assert(GCPhys >= pVirt->aPhysToVirt[iPage].Core.Key && GCPhys <= pVirt->aPhysToVirt[iPage].Core.KeyLast);
1816
1817	#ifdef IN_RING3
1818	if (pVirt->pfnHandlerR3)
1819	{
1820	if (!pPhys)
1821	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] virt %s\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc) ));
1822	else
1823	Log(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys/virt %s/%s\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc), R3STRING(pPhys->pszDesc) ));
1824	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
1825	+ (iPage << PAGE_SHIFT)
1826	+ (GCPhys & PAGE_OFFSET_MASK);
1827
1828	STAM_PROFILE_START(&pVirt->Stat, h);
1829	rc2 = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, (void )pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, /pVirt->CTX_SUFF(pvUser)*/ NULL);
1830	STAM_PROFILE_STOP(&pVirt->Stat, h);
1831	if (rc2 == VINF_SUCCESS)
1832	rc = VINF_SUCCESS;
1833	AssertLogRelMsg(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc2, GCPhys, pPage, pVirt->pszDesc));
1834	}
1835	else
1836	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] virt %s [no handler]\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc) ));
1837	#else
1838	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
1839	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cb=%#x\n", GCPhys, cb));
1840	return VERR_PGM_PHYS_WR_HIT_HANDLER;
1841	#endif
1842	}
1843
1844	/*
1845	* Take the default action.
1846	*/
1847	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
1848	memcpy(pvBuf, pvSrc, cb);
1849	return rc;
1850	}
1851
1852
1853	/**
1854	* Read physical memory.
1855	*
1856	* This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
1857	* want to ignore those.
1858	*
1859	* @returns VBox status code. Can be ignored in ring-3.
1860	* @retval VINF_SUCCESS.
1861	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
1862	*
1863	* @param pVM VM Handle.
1864	* @param GCPhys Physical address start reading from.
1865	* @param pvBuf Where to put the read bits.
1866	* @param cbRead How many bytes to read.
1867	*/
1868	VMMDECL(int) PGMPhysRead(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
1869	{
1870	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
1871	LogFlow(("PGMPhysRead: %RGp %d\n", GCPhys, cbRead));
1872
1873	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PhysRead));
1874	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_MID_Z(Stat,PhysReadBytes), cbRead);
1875
1876	pgmLock(pVM);
1877
1878	/*
1879	* Copy loop on ram ranges.
1880	*/
1881	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1882	for (;;)
1883	{
1884	/* Find range. */
1885	while (pRam && GCPhys > pRam->GCPhysLast)
1886	pRam = pRam->CTX_SUFF(pNext);
1887	/* Inside range or not? */
1888	if (pRam && GCPhys >= pRam->GCPhys)
1889	{
1890	/*
1891	* Must work our way thru this page by page.
1892	*/
1893	RTGCPHYS off = GCPhys - pRam->GCPhys;
1894	while (off < pRam->cb)
1895	{
1896	unsigned iPage = off >> PAGE_SHIFT;
1897	PPGMPAGE pPage = &pRam->aPages[iPage];
1898	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
1899	if (cb > cbRead)
1900	cb = cbRead;
1901
1902	/*
1903	* Any ALL access handlers?
1904	*/
1905	if (RT_UNLIKELY(PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)))
1906	{
1907	int rc = pgmPhysReadHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb);
1908	if (RT_FAILURE(rc))
1909	{
1910	pgmUnlock(pVM);
1911	return rc;
1912	}
1913	}
1914	else
1915	{
1916	/*
1917	* Get the pointer to the page.
1918	*/
1919	const void *pvSrc;
1920	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc);
1921	if (RT_SUCCESS(rc))
1922	memcpy(pvBuf, pvSrc, cb);
1923	else
1924	{
1925	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
1926	pRam->GCPhys + off, pPage, rc));
1927	memset(pvBuf, 0xff, cb);
1928	}
1929	}
1930
1931	/* next page */
1932	if (cb >= cbRead)
1933	{
1934	pgmUnlock(pVM);
1935	return VINF_SUCCESS;
1936	}
1937	cbRead -= cb;
1938	off += cb;
1939	pvBuf = (char *)pvBuf + cb;
1940	} /* walk pages in ram range. */
1941
1942	GCPhys = pRam->GCPhysLast + 1;
1943	}
1944	else
1945	{
1946	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
1947
1948	/*
1949	* Unassigned address space.
1950	*/
1951	if (!pRam)
1952	break;
1953	size_t cb = pRam->GCPhys - GCPhys;
1954	if (cb >= cbRead)
1955	{
1956	memset(pvBuf, 0xff, cbRead);
1957	break;
1958	}
1959	memset(pvBuf, 0xff, cb);
1960
1961	cbRead -= cb;
1962	pvBuf = (char *)pvBuf + cb;
1963	GCPhys += cb;
1964	}
1965	} /* Ram range walk */
1966
1967	pgmUnlock(pVM);
1968	return VINF_SUCCESS;
1969	}
1970
1971
1972	/**
1973	* Deals with writing to a page with one or more WRITE or ALL access handlers.
1974	*
1975	* @returns VBox status code. Can be ignored in ring-3.
1976	* @retval VINF_SUCCESS.
1977	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
1978	*
1979	* @param pVM The VM handle.
1980	* @param pPage The page descriptor.
1981	* @param GCPhys The physical address to start writing at.
1982	* @param pvBuf What to write.
1983	* @param cbWrite How much to write - less or equal to a page.
1984	*/
1985	static int pgmPhysWriteHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const *pvBuf, size_t cbWrite)
1986	{
1987	void *pvDst = NULL;
1988	int rc;
1989
1990	/*
1991	* Give priority to physical handlers (like #PF does).
1992	*
1993	* Hope for a lonely physical handler first that covers the whole
1994	* write area. This should be a pretty frequent case with MMIO and
1995	* the heavy usage of full page handlers in the page pool.
1996	*/
1997	if ( !PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage)
1998	\|\| PGM_PAGE_IS_MMIO(pPage) /* screw virtual handlers on MMIO pages */)
1999	{
2000	PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
2001	if (pCur)
2002	{
2003	Assert(GCPhys >= pCur->Core.Key && GCPhys <= pCur->Core.KeyLast);
2004	Assert(pCur->CTX_SUFF(pfnHandler));
2005
2006	size_t cbRange = pCur->Core.KeyLast - GCPhys + 1;
2007	if (cbRange > cbWrite)
2008	cbRange = cbWrite;
2009
2010	#ifndef IN_RING3
2011	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2012	NOREF(cbRange);
2013	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2014	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2015
2016	#else /* IN_RING3 */
2017	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
2018	if (!PGM_PAGE_IS_MMIO(pPage))
2019	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst);
2020	else
2021	rc = VINF_SUCCESS;
2022	if (RT_SUCCESS(rc))
2023	{
2024	PFNPGMR3PHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
2025	void *pvUser = pCur->CTX_SUFF(pvUser);
2026
2027	STAM_PROFILE_START(&pCur->Stat, h);
2028	Assert(PGMIsLockOwner(pVM));
2029	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2030	pgmUnlock(pVM);
2031	rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2032	pgmLock(pVM);
2033	# ifdef VBOX_WITH_STATISTICS
2034	pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
2035	if (pCur)
2036	STAM_PROFILE_STOP(&pCur->Stat, h);
2037	# else
2038	pCur = NULL; /* might not be valid anymore. */
2039	# endif
2040	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
2041	memcpy(pvDst, pvBuf, cbRange);
2042	else
2043	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, (pCur) ? pCur->pszDesc : ""));
2044	}
2045	else
2046	AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2047	GCPhys, pPage, rc), rc);
2048	if (RT_LIKELY(cbRange == cbWrite))
2049	return VINF_SUCCESS;
2050
2051	/* more fun to be had below */
2052	cbWrite -= cbRange;
2053	GCPhys += cbRange;
2054	pvBuf = (uint8_t *)pvBuf + cbRange;
2055	pvDst = (uint8_t *)pvDst + cbRange;
2056	#endif /* IN_RING3 */
2057	}
2058	/* else: the handler is somewhere else in the page, deal with it below. */
2059	Assert(!PGM_PAGE_IS_MMIO(pPage)); /* MMIO handlers are all PAGE_SIZEed! */
2060	}
2061	/*
2062	* A virtual handler without any interfering physical handlers.
2063	* Hopefully it'll conver the whole write.
2064	*/
2065	else if (!PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage))
2066	{
2067	unsigned iPage;
2068	PPGMVIRTHANDLER pCur;
2069	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pCur, &iPage);
2070	if (RT_SUCCESS(rc))
2071	{
2072	size_t cbRange = (PAGE_OFFSET_MASK & pCur->Core.KeyLast) - (PAGE_OFFSET_MASK & GCPhys) + 1;
2073	if (cbRange > cbWrite)
2074	cbRange = cbWrite;
2075
2076	#ifndef IN_RING3
2077	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2078	NOREF(cbRange);
2079	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2080	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2081
2082	#else /* IN_RING3 */
2083
2084	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] virt %s\n", GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
2085	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst);
2086	if (RT_SUCCESS(rc))
2087	{
2088	rc = VINF_PGM_HANDLER_DO_DEFAULT;
2089	if (pCur->pfnHandlerR3)
2090	{
2091	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pCur->Core.Key & PAGE_BASE_GC_MASK)
2092	+ (iPage << PAGE_SHIFT)
2093	+ (GCPhys & PAGE_OFFSET_MASK);
2094
2095	STAM_PROFILE_START(&pCur->Stat, h);
2096	rc = pCur->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void )pvBuf, cbRange, PGMACCESSTYPE_WRITE, /pCur->CTX_SUFF(pvUser)*/ NULL);
2097	STAM_PROFILE_STOP(&pCur->Stat, h);
2098	}
2099	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
2100	memcpy(pvDst, pvBuf, cbRange);
2101	else
2102	AssertLogRelMsg(rc == VINF_SUCCESS, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pCur->pszDesc));
2103	}
2104	else
2105	AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2106	GCPhys, pPage, rc), rc);
2107	if (RT_LIKELY(cbRange == cbWrite))
2108	return VINF_SUCCESS;
2109
2110	/* more fun to be had below */
2111	cbWrite -= cbRange;
2112	GCPhys += cbRange;
2113	pvBuf = (uint8_t *)pvBuf + cbRange;
2114	pvDst = (uint8_t *)pvDst + cbRange;
2115	#endif
2116	}
2117	/* else: the handler is somewhere else in the page, deal with it below. */
2118	}
2119
2120	/*
2121	* Deal with all the odd ends.
2122	*/
2123
2124	/* We need a writable destination page. */
2125	if (!pvDst)
2126	{
2127	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst);
2128	AssertLogRelMsgReturn(RT_SUCCESS(rc),
2129	("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2130	GCPhys, pPage, rc), rc);
2131	}
2132
2133	/* The loop state (big + ugly). */
2134	unsigned iVirtPage = 0;
2135	PPGMVIRTHANDLER pVirt = NULL;
2136	uint32_t offVirt = PAGE_SIZE;
2137	uint32_t offVirtLast = PAGE_SIZE;
2138	bool fMoreVirt = PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage);
2139
2140	PPGMPHYSHANDLER pPhys = NULL;
2141	uint32_t offPhys = PAGE_SIZE;
2142	uint32_t offPhysLast = PAGE_SIZE;
2143	bool fMorePhys = PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage);
2144
2145	/* The loop. */
2146	for (;;)
2147	{
2148	/*
2149	* Find the closest handler at or above GCPhys.
2150	*/
2151	if (fMoreVirt && !pVirt)
2152	{
2153	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pVirt, &iVirtPage);
2154	if (RT_SUCCESS(rc))
2155	{
2156	offVirt = 0;
2157	offVirtLast = (pVirt->aPhysToVirt[iVirtPage].Core.KeyLast & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2158	}
2159	else
2160	{
2161	PPGMPHYS2VIRTHANDLER pVirtPhys;
2162	pVirtPhys = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysToVirtHandlers,
2163	GCPhys, true /* fAbove */);
2164	if ( pVirtPhys
2165	&& (pVirtPhys->Core.Key >> PAGE_SHIFT) == (GCPhys >> PAGE_SHIFT))
2166	{
2167	/* ASSUME that pVirtPhys only covers one page. */
2168	Assert((pVirtPhys->Core.Key >> PAGE_SHIFT) == (pVirtPhys->Core.KeyLast >> PAGE_SHIFT));
2169	Assert(pVirtPhys->Core.Key > GCPhys);
2170
2171	pVirt = (PPGMVIRTHANDLER)((uintptr_t)pVirtPhys + pVirtPhys->offVirtHandler);
2172	iVirtPage = pVirtPhys - &pVirt->aPhysToVirt[0]; Assert(iVirtPage == 0);
2173	offVirt = (pVirtPhys->Core.Key & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2174	offVirtLast = (pVirtPhys->Core.KeyLast & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2175	}
2176	else
2177	{
2178	pVirt = NULL;
2179	fMoreVirt = false;
2180	offVirt = offVirtLast = PAGE_SIZE;
2181	}
2182	}
2183	}
2184
2185	if (fMorePhys && !pPhys)
2186	{
2187	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
2188	if (pPhys)
2189	{
2190	offPhys = 0;
2191	offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
2192	}
2193	else
2194	{
2195	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers,
2196	GCPhys, true /* fAbove */);
2197	if ( pPhys
2198	&& pPhys->Core.Key <= GCPhys + (cbWrite - 1))
2199	{
2200	offPhys = pPhys->Core.Key - GCPhys;
2201	offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
2202	}
2203	else
2204	{
2205	pPhys = NULL;
2206	fMorePhys = false;
2207	offPhys = offPhysLast = PAGE_SIZE;
2208	}
2209	}
2210	}
2211
2212	/*
2213	* Handle access to space without handlers (that's easy).
2214	*/
2215	rc = VINF_PGM_HANDLER_DO_DEFAULT;
2216	uint32_t cbRange = (uint32_t)cbWrite;
2217	if (offPhys && offVirt)
2218	{
2219	if (cbRange > offPhys)
2220	cbRange = offPhys;
2221	if (cbRange > offVirt)
2222	cbRange = offVirt;
2223	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] miss\n", GCPhys, cbRange, pPage));
2224	}
2225	/*
2226	* Physical handler.
2227	*/
2228	else if (!offPhys && offVirt)
2229	{
2230	if (cbRange > offPhysLast + 1)
2231	cbRange = offPhysLast + 1;
2232	if (cbRange > offVirt)
2233	cbRange = offVirt;
2234	#ifdef IN_RING3
2235	PFNPGMR3PHYSHANDLER pfnHandler = pPhys->CTX_SUFF(pfnHandler);
2236	void *pvUser = pPhys->CTX_SUFF(pvUser);
2237
2238	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc) ));
2239	STAM_PROFILE_START(&pPhys->Stat, h);
2240	Assert(PGMIsLockOwner(pVM));
2241	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2242	pgmUnlock(pVM);
2243	rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2244	pgmLock(pVM);
2245	# ifdef VBOX_WITH_STATISTICS
2246	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
2247	if (pPhys)
2248	STAM_PROFILE_STOP(&pPhys->Stat, h);
2249	# else
2250	pPhys = NULL; /* might not be valid anymore. */
2251	# endif
2252	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, (pPhys) ? pPhys->pszDesc : ""));
2253	#else
2254	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2255	NOREF(cbRange);
2256	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2257	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2258	#endif
2259	}
2260	/*
2261	* Virtual handler.
2262	*/
2263	else if (offPhys && !offVirt)
2264	{
2265	if (cbRange > offVirtLast + 1)
2266	cbRange = offVirtLast + 1;
2267	if (cbRange > offPhys)
2268	cbRange = offPhys;
2269	#ifdef IN_RING3
2270	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pVirt->pszDesc) ));
2271	if (pVirt->pfnHandlerR3)
2272	{
2273	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2274	+ (iVirtPage << PAGE_SHIFT)
2275	+ (GCPhys & PAGE_OFFSET_MASK);
2276	STAM_PROFILE_START(&pVirt->Stat, h);
2277	rc = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void )pvBuf, cbRange, PGMACCESSTYPE_WRITE, /pCur->CTX_SUFF(pvUser)*/ NULL);
2278	STAM_PROFILE_STOP(&pVirt->Stat, h);
2279	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pVirt->pszDesc));
2280	}
2281	pVirt = NULL;
2282	#else
2283	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2284	NOREF(cbRange);
2285	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2286	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2287	#endif
2288	}
2289	/*
2290	* Both... give the physical one priority.
2291	*/
2292	else
2293	{
2294	Assert(!offPhys && !offVirt);
2295	if (cbRange > offVirtLast + 1)
2296	cbRange = offVirtLast + 1;
2297	if (cbRange > offPhysLast + 1)
2298	cbRange = offPhysLast + 1;
2299
2300	#ifdef IN_RING3
2301	if (pVirt->pfnHandlerR3)
2302	Log(("pgmPhysWriteHandler: overlapping phys and virt handlers at %RGp %R[pgmpage]; cbRange=%#x\n", GCPhys, pPage, cbRange));
2303	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys/virt %s/%s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc), R3STRING(pVirt->pszDesc) ));
2304
2305	PFNPGMR3PHYSHANDLER pfnHandler = pPhys->CTX_SUFF(pfnHandler);
2306	void *pvUser = pPhys->CTX_SUFF(pvUser);
2307
2308	STAM_PROFILE_START(&pPhys->Stat, h);
2309	Assert(PGMIsLockOwner(pVM));
2310	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2311	pgmUnlock(pVM);
2312	rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2313	pgmLock(pVM);
2314	# ifdef VBOX_WITH_STATISTICS
2315	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
2316	if (pPhys)
2317	STAM_PROFILE_STOP(&pPhys->Stat, h);
2318	# else
2319	pPhys = NULL; /* might not be valid anymore. */
2320	# endif
2321	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, (pPhys) ? pPhys->pszDesc : ""));
2322	if (pVirt->pfnHandlerR3)
2323	{
2324
2325	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2326	+ (iVirtPage << PAGE_SHIFT)
2327	+ (GCPhys & PAGE_OFFSET_MASK);
2328	STAM_PROFILE_START(&pVirt->Stat, h2);
2329	int rc2 = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void )pvBuf, cbRange, PGMACCESSTYPE_WRITE, /pCur->CTX_SUFF(pvUser)*/ NULL);
2330	STAM_PROFILE_STOP(&pVirt->Stat, h2);
2331	if (rc2 == VINF_SUCCESS && rc == VINF_PGM_HANDLER_DO_DEFAULT)
2332	rc = VINF_SUCCESS;
2333	else
2334	AssertLogRelMsg(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pVirt->pszDesc));
2335	}
2336	pPhys = NULL;
2337	pVirt = NULL;
2338	#else
2339	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2340	NOREF(cbRange);
2341	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2342	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2343	#endif
2344	}
2345	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
2346	memcpy(pvDst, pvBuf, cbRange);
2347
2348	/*
2349	* Advance if we've got more stuff to do.
2350	*/
2351	if (cbRange >= cbWrite)
2352	return VINF_SUCCESS;
2353
2354	cbWrite -= cbRange;
2355	GCPhys += cbRange;
2356	pvBuf = (uint8_t *)pvBuf + cbRange;
2357	pvDst = (uint8_t *)pvDst + cbRange;
2358
2359	offPhys -= cbRange;
2360	offPhysLast -= cbRange;
2361	offVirt -= cbRange;
2362	offVirtLast -= cbRange;
2363	}
2364	}
2365
2366
2367	/**
2368	* Write to physical memory.
2369	*
2370	* This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
2371	* want to ignore those.
2372	*
2373	* @returns VBox status code. Can be ignored in ring-3.
2374	* @retval VINF_SUCCESS.
2375	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
2376	*
2377	* @param pVM VM Handle.
2378	* @param GCPhys Physical address to write to.
2379	* @param pvBuf What to write.
2380	* @param cbWrite How many bytes to write.
2381	*/
2382	VMMDECL(int) PGMPhysWrite(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite)
2383	{
2384	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMPhysWrite after pgmR3Save()!\n"));
2385	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
2386	LogFlow(("PGMPhysWrite: %RGp %d\n", GCPhys, cbWrite));
2387
2388	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PhysWrite));
2389	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_MID_Z(Stat,PhysWriteBytes), cbWrite);
2390
2391	pgmLock(pVM);
2392
2393	/*
2394	* Copy loop on ram ranges.
2395	*/
2396	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2397	for (;;)
2398	{
2399	/* Find range. */
2400	while (pRam && GCPhys > pRam->GCPhysLast)
2401	pRam = pRam->CTX_SUFF(pNext);
2402	/* Inside range or not? */
2403	if (pRam && GCPhys >= pRam->GCPhys)
2404	{
2405	/*
2406	* Must work our way thru this page by page.
2407	*/
2408	RTGCPTR off = GCPhys - pRam->GCPhys;
2409	while (off < pRam->cb)
2410	{
2411	RTGCPTR iPage = off >> PAGE_SHIFT;
2412	PPGMPAGE pPage = &pRam->aPages[iPage];
2413	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
2414	if (cb > cbWrite)
2415	cb = cbWrite;
2416
2417	/*
2418	* Any active WRITE or ALL access handlers?
2419	*/
2420	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2421	{
2422	int rc = pgmPhysWriteHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb);
2423	if (RT_FAILURE(rc))
2424	{
2425	pgmUnlock(pVM);
2426	return rc;
2427	}
2428	}
2429	else
2430	{
2431	/*
2432	* Get the pointer to the page.
2433	*/
2434	void *pvDst;
2435	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst);
2436	if (RT_SUCCESS(rc))
2437	memcpy(pvDst, pvBuf, cb);
2438	else
2439	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2440	pRam->GCPhys + off, pPage, rc));
2441	}
2442
2443	/* next page */
2444	if (cb >= cbWrite)
2445	{
2446	pgmUnlock(pVM);
2447	return VINF_SUCCESS;
2448	}
2449
2450	cbWrite -= cb;
2451	off += cb;
2452	pvBuf = (const char *)pvBuf + cb;
2453	} /* walk pages in ram range */
2454
2455	GCPhys = pRam->GCPhysLast + 1;
2456	}
2457	else
2458	{
2459	/*
2460	* Unassigned address space, skip it.
2461	*/
2462	if (!pRam)
2463	break;
2464	size_t cb = pRam->GCPhys - GCPhys;
2465	if (cb >= cbWrite)
2466	break;
2467	cbWrite -= cb;
2468	pvBuf = (const char *)pvBuf + cb;
2469	GCPhys += cb;
2470	}
2471	} /* Ram range walk */
2472
2473	pgmUnlock(pVM);
2474	return VINF_SUCCESS;
2475	}
2476
2477
2478	/**
2479	* Read from guest physical memory by GC physical address, bypassing
2480	* MMIO and access handlers.
2481	*
2482	* @returns VBox status.
2483	* @param pVM VM handle.
2484	* @param pvDst The destination address.
2485	* @param GCPhysSrc The source address (GC physical address).
2486	* @param cb The number of bytes to read.
2487	*/
2488	VMMDECL(int) PGMPhysSimpleReadGCPhys(PVM pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb)
2489	{
2490	/*
2491	* Treat the first page as a special case.
2492	*/
2493	if (!cb)
2494	return VINF_SUCCESS;
2495
2496	/* map the 1st page */
2497	void const *pvSrc;
2498	PGMPAGEMAPLOCK Lock;
2499	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
2500	if (RT_FAILURE(rc))
2501	return rc;
2502
2503	/* optimize for the case where access is completely within the first page. */
2504	size_t cbPage = PAGE_SIZE - (GCPhysSrc & PAGE_OFFSET_MASK);
2505	if (RT_LIKELY(cb <= cbPage))
2506	{
2507	memcpy(pvDst, pvSrc, cb);
2508	PGMPhysReleasePageMappingLock(pVM, &Lock);
2509	return VINF_SUCCESS;
2510	}
2511
2512	/* copy to the end of the page. */
2513	memcpy(pvDst, pvSrc, cbPage);
2514	PGMPhysReleasePageMappingLock(pVM, &Lock);
2515	GCPhysSrc += cbPage;
2516	pvDst = (uint8_t *)pvDst + cbPage;
2517	cb -= cbPage;
2518
2519	/*
2520	* Page by page.
2521	*/
2522	for (;;)
2523	{
2524	/* map the page */
2525	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
2526	if (RT_FAILURE(rc))
2527	return rc;
2528
2529	/* last page? */
2530	if (cb <= PAGE_SIZE)
2531	{
2532	memcpy(pvDst, pvSrc, cb);
2533	PGMPhysReleasePageMappingLock(pVM, &Lock);
2534	return VINF_SUCCESS;
2535	}
2536
2537	/* copy the entire page and advance */
2538	memcpy(pvDst, pvSrc, PAGE_SIZE);
2539	PGMPhysReleasePageMappingLock(pVM, &Lock);
2540	GCPhysSrc += PAGE_SIZE;
2541	pvDst = (uint8_t *)pvDst + PAGE_SIZE;
2542	cb -= PAGE_SIZE;
2543	}
2544	/* won't ever get here. */
2545	}
2546
2547
2548	/**
2549	* Write to guest physical memory referenced by GC pointer.
2550	* Write memory to GC physical address in guest physical memory.
2551	*
2552	* This will bypass MMIO and access handlers.
2553	*
2554	* @returns VBox status.
2555	* @param pVM VM handle.
2556	* @param GCPhysDst The GC physical address of the destination.
2557	* @param pvSrc The source buffer.
2558	* @param cb The number of bytes to write.
2559	*/
2560	VMMDECL(int) PGMPhysSimpleWriteGCPhys(PVM pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb)
2561	{
2562	LogFlow(("PGMPhysSimpleWriteGCPhys: %RGp %zu\n", GCPhysDst, cb));
2563
2564	/*
2565	* Treat the first page as a special case.
2566	*/
2567	if (!cb)
2568	return VINF_SUCCESS;
2569
2570	/* map the 1st page */
2571	void *pvDst;
2572	PGMPAGEMAPLOCK Lock;
2573	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
2574	if (RT_FAILURE(rc))
2575	return rc;
2576
2577	/* optimize for the case where access is completely within the first page. */
2578	size_t cbPage = PAGE_SIZE - (GCPhysDst & PAGE_OFFSET_MASK);
2579	if (RT_LIKELY(cb <= cbPage))
2580	{
2581	memcpy(pvDst, pvSrc, cb);
2582	PGMPhysReleasePageMappingLock(pVM, &Lock);
2583	return VINF_SUCCESS;
2584	}
2585
2586	/* copy to the end of the page. */
2587	memcpy(pvDst, pvSrc, cbPage);
2588	PGMPhysReleasePageMappingLock(pVM, &Lock);
2589	GCPhysDst += cbPage;
2590	pvSrc = (const uint8_t *)pvSrc + cbPage;
2591	cb -= cbPage;
2592
2593	/*
2594	* Page by page.
2595	*/
2596	for (;;)
2597	{
2598	/* map the page */
2599	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
2600	if (RT_FAILURE(rc))
2601	return rc;
2602
2603	/* last page? */
2604	if (cb <= PAGE_SIZE)
2605	{
2606	memcpy(pvDst, pvSrc, cb);
2607	PGMPhysReleasePageMappingLock(pVM, &Lock);
2608	return VINF_SUCCESS;
2609	}
2610
2611	/* copy the entire page and advance */
2612	memcpy(pvDst, pvSrc, PAGE_SIZE);
2613	PGMPhysReleasePageMappingLock(pVM, &Lock);
2614	GCPhysDst += PAGE_SIZE;
2615	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
2616	cb -= PAGE_SIZE;
2617	}
2618	/* won't ever get here. */
2619	}
2620
2621
2622	/**
2623	* Read from guest physical memory referenced by GC pointer.
2624	*
2625	* This function uses the current CR3/CR0/CR4 of the guest and will
2626	* bypass access handlers and not set any accessed bits.
2627	*
2628	* @returns VBox status.
2629	* @param pVCpu The VMCPU handle.
2630	* @param pvDst The destination address.
2631	* @param GCPtrSrc The source address (GC pointer).
2632	* @param cb The number of bytes to read.
2633	*/
2634	VMMDECL(int) PGMPhysSimpleReadGCPtr(PVMCPU pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
2635	{
2636	PVM pVM = pVCpu->CTX_SUFF(pVM);
2637
2638	/*
2639	* Treat the first page as a special case.
2640	*/
2641	if (!cb)
2642	return VINF_SUCCESS;
2643
2644	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PhysSimpleRead));
2645	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_MID_Z(Stat,PhysSimpleReadBytes), cb);
2646
2647	/* Take the PGM lock here, because many called functions take the lock for a very short period. That's counter-productive
2648	* when many VCPUs are fighting for the lock.
2649	*/
2650	pgmLock(pVM);
2651
2652	/* map the 1st page */
2653	void const *pvSrc;
2654	PGMPAGEMAPLOCK Lock;
2655	int rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
2656	if (RT_FAILURE(rc))
2657	{
2658	pgmUnlock(pVM);
2659	return rc;
2660	}
2661
2662	/* optimize for the case where access is completely within the first page. */
2663	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
2664	if (RT_LIKELY(cb <= cbPage))
2665	{
2666	memcpy(pvDst, pvSrc, cb);
2667	PGMPhysReleasePageMappingLock(pVM, &Lock);
2668	pgmUnlock(pVM);
2669	return VINF_SUCCESS;
2670	}
2671
2672	/* copy to the end of the page. */
2673	memcpy(pvDst, pvSrc, cbPage);
2674	PGMPhysReleasePageMappingLock(pVM, &Lock);
2675	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + cbPage);
2676	pvDst = (uint8_t *)pvDst + cbPage;
2677	cb -= cbPage;
2678
2679	/*
2680	* Page by page.
2681	*/
2682	for (;;)
2683	{
2684	/* map the page */
2685	rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
2686	if (RT_FAILURE(rc))
2687	{
2688	pgmUnlock(pVM);
2689	return rc;
2690	}
2691
2692	/* last page? */
2693	if (cb <= PAGE_SIZE)
2694	{
2695	memcpy(pvDst, pvSrc, cb);
2696	PGMPhysReleasePageMappingLock(pVM, &Lock);
2697	pgmUnlock(pVM);
2698	return VINF_SUCCESS;
2699	}
2700
2701	/* copy the entire page and advance */
2702	memcpy(pvDst, pvSrc, PAGE_SIZE);
2703	PGMPhysReleasePageMappingLock(pVM, &Lock);
2704	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + PAGE_SIZE);
2705	pvDst = (uint8_t *)pvDst + PAGE_SIZE;
2706	cb -= PAGE_SIZE;
2707	}
2708	/* won't ever get here. */
2709	}
2710
2711
2712	/**
2713	* Write to guest physical memory referenced by GC pointer.
2714	*
2715	* This function uses the current CR3/CR0/CR4 of the guest and will
2716	* bypass access handlers and not set dirty or accessed bits.
2717	*
2718	* @returns VBox status.
2719	* @param pVCpu The VMCPU handle.
2720	* @param GCPtrDst The destination address (GC pointer).
2721	* @param pvSrc The source address.
2722	* @param cb The number of bytes to write.
2723	*/
2724	VMMDECL(int) PGMPhysSimpleWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
2725	{
2726	PVM pVM = pVCpu->CTX_SUFF(pVM);
2727
2728	/*
2729	* Treat the first page as a special case.
2730	*/
2731	if (!cb)
2732	return VINF_SUCCESS;
2733
2734	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PhysSimpleWrite));
2735	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_MID_Z(Stat,PhysSimpleWriteBytes), cb);
2736
2737	/* map the 1st page */
2738	void *pvDst;
2739	PGMPAGEMAPLOCK Lock;
2740	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
2741	if (RT_FAILURE(rc))
2742	return rc;
2743
2744	/* optimize for the case where access is completely within the first page. */
2745	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
2746	if (RT_LIKELY(cb <= cbPage))
2747	{
2748	memcpy(pvDst, pvSrc, cb);
2749	PGMPhysReleasePageMappingLock(pVM, &Lock);
2750	return VINF_SUCCESS;
2751	}
2752
2753	/* copy to the end of the page. */
2754	memcpy(pvDst, pvSrc, cbPage);
2755	PGMPhysReleasePageMappingLock(pVM, &Lock);
2756	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
2757	pvSrc = (const uint8_t *)pvSrc + cbPage;
2758	cb -= cbPage;
2759
2760	/*
2761	* Page by page.
2762	*/
2763	for (;;)
2764	{
2765	/* map the page */
2766	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
2767	if (RT_FAILURE(rc))
2768	return rc;
2769
2770	/* last page? */
2771	if (cb <= PAGE_SIZE)
2772	{
2773	memcpy(pvDst, pvSrc, cb);
2774	PGMPhysReleasePageMappingLock(pVM, &Lock);
2775	return VINF_SUCCESS;
2776	}
2777
2778	/* copy the entire page and advance */
2779	memcpy(pvDst, pvSrc, PAGE_SIZE);
2780	PGMPhysReleasePageMappingLock(pVM, &Lock);
2781	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
2782	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
2783	cb -= PAGE_SIZE;
2784	}
2785	/* won't ever get here. */
2786	}
2787
2788
2789	/**
2790	* Write to guest physical memory referenced by GC pointer and update the PTE.
2791	*
2792	* This function uses the current CR3/CR0/CR4 of the guest and will
2793	* bypass access handlers but will set any dirty and accessed bits in the PTE.
2794	*
2795	* If you don't want to set the dirty bit, use PGMPhysSimpleWriteGCPtr().
2796	*
2797	* @returns VBox status.
2798	* @param pVCpu The VMCPU handle.
2799	* @param GCPtrDst The destination address (GC pointer).
2800	* @param pvSrc The source address.
2801	* @param cb The number of bytes to write.
2802	*/
2803	VMMDECL(int) PGMPhysSimpleDirtyWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
2804	{
2805	PVM pVM = pVCpu->CTX_SUFF(pVM);
2806
2807	/*
2808	* Treat the first page as a special case.
2809	* Btw. this is the same code as in PGMPhyssimpleWriteGCPtr excep for the PGMGstModifyPage.
2810	*/
2811	if (!cb)
2812	return VINF_SUCCESS;
2813
2814	/* map the 1st page */
2815	void *pvDst;
2816	PGMPAGEMAPLOCK Lock;
2817	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
2818	if (RT_FAILURE(rc))
2819	return rc;
2820
2821	/* optimize for the case where access is completely within the first page. */
2822	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
2823	if (RT_LIKELY(cb <= cbPage))
2824	{
2825	memcpy(pvDst, pvSrc, cb);
2826	PGMPhysReleasePageMappingLock(pVM, &Lock);
2827	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2828	return VINF_SUCCESS;
2829	}
2830
2831	/* copy to the end of the page. */
2832	memcpy(pvDst, pvSrc, cbPage);
2833	PGMPhysReleasePageMappingLock(pVM, &Lock);
2834	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2835	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
2836	pvSrc = (const uint8_t *)pvSrc + cbPage;
2837	cb -= cbPage;
2838
2839	/*
2840	* Page by page.
2841	*/
2842	for (;;)
2843	{
2844	/* map the page */
2845	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
2846	if (RT_FAILURE(rc))
2847	return rc;
2848
2849	/* last page? */
2850	if (cb <= PAGE_SIZE)
2851	{
2852	memcpy(pvDst, pvSrc, cb);
2853	PGMPhysReleasePageMappingLock(pVM, &Lock);
2854	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2855	return VINF_SUCCESS;
2856	}
2857
2858	/* copy the entire page and advance */
2859	memcpy(pvDst, pvSrc, PAGE_SIZE);
2860	PGMPhysReleasePageMappingLock(pVM, &Lock);
2861	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2862	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
2863	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
2864	cb -= PAGE_SIZE;
2865	}
2866	/* won't ever get here. */
2867	}
2868
2869
2870	/**
2871	* Read from guest physical memory referenced by GC pointer.
2872	*
2873	* This function uses the current CR3/CR0/CR4 of the guest and will
2874	* respect access handlers and set accessed bits.
2875	*
2876	* @returns VBox status.
2877	* @param pVCpu The VMCPU handle.
2878	* @param pvDst The destination address.
2879	* @param GCPtrSrc The source address (GC pointer).
2880	* @param cb The number of bytes to read.
2881	* @thread The vCPU EMT.
2882	*/
2883	VMMDECL(int) PGMPhysReadGCPtr(PVMCPU pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
2884	{
2885	RTGCPHYS GCPhys;
2886	uint64_t fFlags;
2887	int rc;
2888	PVM pVM = pVCpu->CTX_SUFF(pVM);
2889
2890	/*
2891	* Anything to do?
2892	*/
2893	if (!cb)
2894	return VINF_SUCCESS;
2895
2896	LogFlow(("PGMPhysReadGCPtr: %RGv %zu\n", GCPtrSrc, cb));
2897
2898	/*
2899	* Optimize reads within a single page.
2900	*/
2901	if (((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
2902	{
2903	/* Convert virtual to physical address + flags */
2904	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
2905	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
2906	GCPhys \|= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
2907
2908	/* mark the guest page as accessed. */
2909	if (!(fFlags & X86_PTE_A))
2910	{
2911	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
2912	AssertRC(rc);
2913	}
2914
2915	return PGMPhysRead(pVM, GCPhys, pvDst, cb);
2916	}
2917
2918	/*
2919	* Page by page.
2920	*/
2921	for (;;)
2922	{
2923	/* Convert virtual to physical address + flags */
2924	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
2925	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
2926	GCPhys \|= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
2927
2928	/* mark the guest page as accessed. */
2929	if (!(fFlags & X86_PTE_A))
2930	{
2931	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
2932	AssertRC(rc);
2933	}
2934
2935	/* copy */
2936	size_t cbRead = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
2937	rc = PGMPhysRead(pVM, GCPhys, pvDst, cbRead);
2938	if (cbRead >= cb \|\| RT_FAILURE(rc))
2939	return rc;
2940
2941	/* next */
2942	cb -= cbRead;
2943	pvDst = (uint8_t *)pvDst + cbRead;
2944	GCPtrSrc += cbRead;
2945	}
2946	}
2947
2948
2949	/**
2950	* Write to guest physical memory referenced by GC pointer.
2951	*
2952	* This function uses the current CR3/CR0/CR4 of the guest and will
2953	* respect access handlers and set dirty and accessed bits.
2954	*
2955	* @returns VBox status.
2956	* @retval VINF_SUCCESS.
2957	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
2958	*
2959	* @param pVCpu The VMCPU handle.
2960	* @param GCPtrDst The destination address (GC pointer).
2961	* @param pvSrc The source address.
2962	* @param cb The number of bytes to write.
2963	*/
2964	VMMDECL(int) PGMPhysWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
2965	{
2966	RTGCPHYS GCPhys;
2967	uint64_t fFlags;
2968	int rc;
2969	PVM pVM = pVCpu->CTX_SUFF(pVM);
2970
2971	/*
2972	* Anything to do?
2973	*/
2974	if (!cb)
2975	return VINF_SUCCESS;
2976
2977	LogFlow(("PGMPhysWriteGCPtr: %RGv %zu\n", GCPtrDst, cb));
2978
2979	/*
2980	* Optimize writes within a single page.
2981	*/
2982	if (((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
2983	{
2984	/* Convert virtual to physical address + flags */
2985	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
2986	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
2987	GCPhys \|= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
2988
2989	/* Mention when we ignore X86_PTE_RW... */
2990	if (!(fFlags & X86_PTE_RW))
2991	Log(("PGMPhysGCPtr2GCPhys: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
2992
2993	/* Mark the guest page as accessed and dirty if necessary. */
2994	if ((fFlags & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
2995	{
2996	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
2997	AssertRC(rc);
2998	}
2999
3000	return PGMPhysWrite(pVM, GCPhys, pvSrc, cb);
3001	}
3002
3003	/*
3004	* Page by page.
3005	*/
3006	for (;;)
3007	{
3008	/* Convert virtual to physical address + flags */
3009	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
3010	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
3011	GCPhys \|= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
3012
3013	/* Mention when we ignore X86_PTE_RW... */
3014	if (!(fFlags & X86_PTE_RW))
3015	Log(("PGMPhysGCPtr2GCPhys: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
3016
3017	/* Mark the guest page as accessed and dirty if necessary. */
3018	if ((fFlags & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
3019	{
3020	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
3021	AssertRC(rc);
3022	}
3023
3024	/* copy */
3025	size_t cbWrite = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
3026	rc = PGMPhysWrite(pVM, GCPhys, pvSrc, cbWrite);
3027	if (cbWrite >= cb \|\| RT_FAILURE(rc))
3028	return rc;
3029
3030	/* next */
3031	cb -= cbWrite;
3032	pvSrc = (uint8_t *)pvSrc + cbWrite;
3033	GCPtrDst += cbWrite;
3034	}
3035	}
3036
3037
3038	/**
3039	* Performs a read of guest virtual memory for instruction emulation.
3040	*
3041	* This will check permissions, raise exceptions and update the access bits.
3042	*
3043	* The current implementation will bypass all access handlers. It may later be
3044	* changed to at least respect MMIO.
3045	*
3046	*
3047	* @returns VBox status code suitable to scheduling.
3048	* @retval VINF_SUCCESS if the read was performed successfully.
3049	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3050	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3051	*
3052	* @param pVCpu The VMCPU handle.
3053	* @param pCtxCore The context core.
3054	* @param pvDst Where to put the bytes we've read.
3055	* @param GCPtrSrc The source address.
3056	* @param cb The number of bytes to read. Not more than a page.
3057	*
3058	* @remark This function will dynamically map physical pages in GC. This may unmap
3059	* mappings done by the caller. Be careful!
3060	*/
3061	VMMDECL(int) PGMPhysInterpretedRead(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb)
3062	{
3063	PVM pVM = pVCpu->CTX_SUFF(pVM);
3064	Assert(cb <= PAGE_SIZE);
3065
3066	/** @todo r=bird: This isn't perfect!
3067	* -# It's not checking for reserved bits being 1.
3068	* -# It's not correctly dealing with the access bit.
3069	* -# It's not respecting MMIO memory or any other access handlers.
3070	*/
3071	/*
3072	* 1. Translate virtual to physical. This may fault.
3073	* 2. Map the physical address.
3074	* 3. Do the read operation.
3075	* 4. Set access bits if required.
3076	*/
3077	int rc;
3078	unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
3079	if (cb <= cb1)
3080	{
3081	/*
3082	* Not crossing pages.
3083	*/
3084	RTGCPHYS GCPhys;
3085	uint64_t fFlags;
3086	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
3087	if (RT_SUCCESS(rc))
3088	{
3089	/** @todo we should check reserved bits ... */
3090	void *pvSrc;
3091	rc = PGM_GCPHYS_2_PTR(pVM, GCPhys, &pvSrc);
3092	switch (rc)
3093	{
3094	case VINF_SUCCESS:
3095	Log(("PGMPhysInterpretedRead: pvDst=%p pvSrc=%p cb=%d\n", pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb));
3096	memcpy(pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
3097	break;
3098	case VERR_PGM_PHYS_PAGE_RESERVED:
3099	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3100	memset(pvDst, 0, cb); /** @todo this is wrong, it should be 0xff */
3101	break;
3102	default:
3103	return rc;
3104	}
3105
3106	/** @todo access bit emulation isn't 100% correct. */
3107	if (!(fFlags & X86_PTE_A))
3108	{
3109	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3110	AssertRC(rc);
3111	}
3112	return VINF_SUCCESS;
3113	}
3114	}
3115	else
3116	{
3117	/*
3118	* Crosses pages.
3119	*/
3120	size_t cb2 = cb - cb1;
3121	uint64_t fFlags1;
3122	RTGCPHYS GCPhys1;
3123	uint64_t fFlags2;
3124	RTGCPHYS GCPhys2;
3125	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
3126	if (RT_SUCCESS(rc))
3127	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
3128	if (RT_SUCCESS(rc))
3129	{
3130	/** @todo we should check reserved bits ... */
3131	AssertMsgFailed(("cb=%d cb1=%d cb2=%d GCPtrSrc=%RGv\n", cb, cb1, cb2, GCPtrSrc));
3132	void *pvSrc1;
3133	rc = PGM_GCPHYS_2_PTR(pVM, GCPhys1, &pvSrc1);
3134	switch (rc)
3135	{
3136	case VINF_SUCCESS:
3137	memcpy(pvDst, (uint8_t *)pvSrc1 + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
3138	break;
3139	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3140	memset(pvDst, 0, cb1); /** @todo this is wrong, it should be 0xff */
3141	break;
3142	default:
3143	return rc;
3144	}
3145
3146	void *pvSrc2;
3147	rc = PGM_GCPHYS_2_PTR(pVM, GCPhys2, &pvSrc2);
3148	switch (rc)
3149	{
3150	case VINF_SUCCESS:
3151	memcpy((uint8_t *)pvDst + cb1, pvSrc2, cb2);
3152	break;
3153	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3154	memset((uint8_t )pvDst + cb1, 0, cb2); /* @todo this is wrong, it should be 0xff */
3155	break;
3156	default:
3157	return rc;
3158	}
3159
3160	if (!(fFlags1 & X86_PTE_A))
3161	{
3162	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3163	AssertRC(rc);
3164	}
3165	if (!(fFlags2 & X86_PTE_A))
3166	{
3167	rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3168	AssertRC(rc);
3169	}
3170	return VINF_SUCCESS;
3171	}
3172	}
3173
3174	/*
3175	* Raise a #PF.
3176	*/
3177	uint32_t uErr;
3178
3179	/* Get the current privilege level. */
3180	uint32_t cpl = CPUMGetGuestCPL(pVCpu, pCtxCore);
3181	switch (rc)
3182	{
3183	case VINF_SUCCESS:
3184	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3185	break;
3186
3187	case VERR_PAGE_NOT_PRESENT:
3188	case VERR_PAGE_TABLE_NOT_PRESENT:
3189	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3190	break;
3191
3192	default:
3193	AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
3194	return rc;
3195	}
3196	Log(("PGMPhysInterpretedRead: GCPtrSrc=%RGv cb=%#x -> #PF(%#x)\n", GCPtrSrc, cb, uErr));
3197	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrSrc);
3198	}
3199
3200
3201	/**
3202	* Performs a read of guest virtual memory for instruction emulation.
3203	*
3204	* This will check permissions, raise exceptions and update the access bits.
3205	*
3206	* The current implementation will bypass all access handlers. It may later be
3207	* changed to at least respect MMIO.
3208	*
3209	*
3210	* @returns VBox status code suitable to scheduling.
3211	* @retval VINF_SUCCESS if the read was performed successfully.
3212	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3213	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3214	*
3215	* @param pVCpu The VMCPU handle.
3216	* @param pCtxCore The context core.
3217	* @param pvDst Where to put the bytes we've read.
3218	* @param GCPtrSrc The source address.
3219	* @param cb The number of bytes to read. Not more than a page.
3220	* @param fRaiseTrap If set the trap will be raised on as per spec, if clear
3221	* an appropriate error status will be returned (no
3222	* informational at all).
3223	*
3224	*
3225	* @remarks Takes the PGM lock.
3226	* @remarks A page fault on the 2nd page of the access will be raised without
3227	* writing the bits on the first page since we're ASSUMING that the
3228	* caller is emulating an instruction access.
3229	* @remarks This function will dynamically map physical pages in GC. This may
3230	* unmap mappings done by the caller. Be careful!
3231	*/
3232	VMMDECL(int) PGMPhysInterpretedReadNoHandlers(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb, bool fRaiseTrap)
3233	{
3234	PVM pVM = pVCpu->CTX_SUFF(pVM);
3235	Assert(cb <= PAGE_SIZE);
3236
3237	/*
3238	* 1. Translate virtual to physical. This may fault.
3239	* 2. Map the physical address.
3240	* 3. Do the read operation.
3241	* 4. Set access bits if required.
3242	*/
3243	int rc;
3244	unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
3245	if (cb <= cb1)
3246	{
3247	/*
3248	* Not crossing pages.
3249	*/
3250	RTGCPHYS GCPhys;
3251	uint64_t fFlags;
3252	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
3253	if (RT_SUCCESS(rc))
3254	{
3255	if (1) /** @todo we should check reserved bits ... */
3256	{
3257	const void *pvSrc;
3258	PGMPAGEMAPLOCK Lock;
3259	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvSrc, &Lock);
3260	switch (rc)
3261	{
3262	case VINF_SUCCESS:
3263	Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d\n",
3264	pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb));
3265	memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
3266	break;
3267	case VERR_PGM_PHYS_PAGE_RESERVED:
3268	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3269	memset(pvDst, 0xff, cb);
3270	break;
3271	default:
3272	AssertMsgFailed(("%Rrc\n", rc));
3273	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3274	return rc;
3275	}
3276	PGMPhysReleasePageMappingLock(pVM, &Lock);
3277
3278	if (!(fFlags & X86_PTE_A))
3279	{
3280	/** @todo access bit emulation isn't 100% correct. */
3281	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3282	AssertRC(rc);
3283	}
3284	return VINF_SUCCESS;
3285	}
3286	}
3287	}
3288	else
3289	{
3290	/*
3291	* Crosses pages.
3292	*/
3293	size_t cb2 = cb - cb1;
3294	uint64_t fFlags1;
3295	RTGCPHYS GCPhys1;
3296	uint64_t fFlags2;
3297	RTGCPHYS GCPhys2;
3298	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
3299	if (RT_SUCCESS(rc))
3300	{
3301	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
3302	if (RT_SUCCESS(rc))
3303	{
3304	if (1) /** @todo we should check reserved bits ... */
3305	{
3306	const void *pvSrc;
3307	PGMPAGEMAPLOCK Lock;
3308	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys1, &pvSrc, &Lock);
3309	switch (rc)
3310	{
3311	case VINF_SUCCESS:
3312	Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d [2]\n",
3313	pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb1));
3314	memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
3315	PGMPhysReleasePageMappingLock(pVM, &Lock);
3316	break;
3317	case VERR_PGM_PHYS_PAGE_RESERVED:
3318	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3319	memset(pvDst, 0xff, cb1);
3320	break;
3321	default:
3322	AssertMsgFailed(("%Rrc\n", rc));
3323	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3324	return rc;
3325	}
3326
3327	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys2, &pvSrc, &Lock);
3328	switch (rc)
3329	{
3330	case VINF_SUCCESS:
3331	memcpy((uint8_t *)pvDst + cb1, pvSrc, cb2);
3332	PGMPhysReleasePageMappingLock(pVM, &Lock);
3333	break;
3334	case VERR_PGM_PHYS_PAGE_RESERVED:
3335	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3336	memset((uint8_t *)pvDst + cb1, 0xff, cb2);
3337	break;
3338	default:
3339	AssertMsgFailed(("%Rrc\n", rc));
3340	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3341	return rc;
3342	}
3343
3344	if (!(fFlags1 & X86_PTE_A))
3345	{
3346	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3347	AssertRC(rc);
3348	}
3349	if (!(fFlags2 & X86_PTE_A))
3350	{
3351	rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3352	AssertRC(rc);
3353	}
3354	return VINF_SUCCESS;
3355	}
3356	/* sort out which page */
3357	}
3358	else
3359	GCPtrSrc += cb1; /* fault on 2nd page */
3360	}
3361	}
3362
3363	/*
3364	* Raise a #PF if we're allowed to do that.
3365	*/
3366	/* Calc the error bits. */
3367	uint32_t cpl = CPUMGetGuestCPL(pVCpu, pCtxCore);
3368	uint32_t uErr;
3369	switch (rc)
3370	{
3371	case VINF_SUCCESS:
3372	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3373	rc = VERR_ACCESS_DENIED;
3374	break;
3375
3376	case VERR_PAGE_NOT_PRESENT:
3377	case VERR_PAGE_TABLE_NOT_PRESENT:
3378	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3379	break;
3380
3381	default:
3382	AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
3383	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3384	return rc;
3385	}
3386	if (fRaiseTrap)
3387	{
3388	Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrSrc, cb, uErr));
3389	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrSrc);
3390	}
3391	Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrSrc, cb, uErr));
3392	return rc;
3393	}
3394
3395
3396	/**
3397	* Performs a write to guest virtual memory for instruction emulation.
3398	*
3399	* This will check permissions, raise exceptions and update the dirty and access
3400	* bits.
3401	*
3402	* @returns VBox status code suitable to scheduling.
3403	* @retval VINF_SUCCESS if the read was performed successfully.
3404	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3405	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3406	*
3407	* @param pVCpu The VMCPU handle.
3408	* @param pCtxCore The context core.
3409	* @param GCPtrDst The destination address.
3410	* @param pvSrc What to write.
3411	* @param cb The number of bytes to write. Not more than a page.
3412	* @param fRaiseTrap If set the trap will be raised on as per spec, if clear
3413	* an appropriate error status will be returned (no
3414	* informational at all).
3415	*
3416	* @remarks Takes the PGM lock.
3417	* @remarks A page fault on the 2nd page of the access will be raised without
3418	* writing the bits on the first page since we're ASSUMING that the
3419	* caller is emulating an instruction access.
3420	* @remarks This function will dynamically map physical pages in GC. This may
3421	* unmap mappings done by the caller. Be careful!
3422	*/
3423	VMMDECL(int) PGMPhysInterpretedWriteNoHandlers(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb, bool fRaiseTrap)
3424	{
3425	Assert(cb <= PAGE_SIZE);
3426	PVM pVM = pVCpu->CTX_SUFF(pVM);
3427
3428	/*
3429	* 1. Translate virtual to physical. This may fault.
3430	* 2. Map the physical address.
3431	* 3. Do the write operation.
3432	* 4. Set access bits if required.
3433	*/
3434	int rc;
3435	unsigned cb1 = PAGE_SIZE - (GCPtrDst & PAGE_OFFSET_MASK);
3436	if (cb <= cb1)
3437	{
3438	/*
3439	* Not crossing pages.
3440	*/
3441	RTGCPHYS GCPhys;
3442	uint64_t fFlags;
3443	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst, &fFlags, &GCPhys);
3444	if (RT_SUCCESS(rc))
3445	{
3446	if ( (fFlags & X86_PTE_RW) /** @todo Also check reserved bits. */
3447	\|\| ( !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
3448	&& CPUMGetGuestCPL(pVCpu, pCtxCore) <= 2) ) /** @todo it's 2, right? Check cpl check below as well. */
3449	{
3450	void *pvDst;
3451	PGMPAGEMAPLOCK Lock;
3452	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, &pvDst, &Lock);
3453	switch (rc)
3454	{
3455	case VINF_SUCCESS:
3456	Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
3457	(uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc, cb));
3458	memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb);
3459	PGMPhysReleasePageMappingLock(pVM, &Lock);
3460	break;
3461	case VERR_PGM_PHYS_PAGE_RESERVED:
3462	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3463	/* bit bucket */
3464	break;
3465	default:
3466	AssertMsgFailed(("%Rrc\n", rc));
3467	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3468	return rc;
3469	}
3470
3471	if (!(fFlags & (X86_PTE_A \| X86_PTE_D)))
3472	{
3473	/** @todo dirty & access bit emulation isn't 100% correct. */
3474	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
3475	AssertRC(rc);
3476	}
3477	return VINF_SUCCESS;
3478	}
3479	rc = VERR_ACCESS_DENIED;
3480	}
3481	}
3482	else
3483	{
3484	/*
3485	* Crosses pages.
3486	*/
3487	size_t cb2 = cb - cb1;
3488	uint64_t fFlags1;
3489	RTGCPHYS GCPhys1;
3490	uint64_t fFlags2;
3491	RTGCPHYS GCPhys2;
3492	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst, &fFlags1, &GCPhys1);
3493	if (RT_SUCCESS(rc))
3494	{
3495	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst + cb1, &fFlags2, &GCPhys2);
3496	if (RT_SUCCESS(rc))
3497	{
3498	if ( ( (fFlags1 & X86_PTE_RW) /** @todo Also check reserved bits. */
3499	&& (fFlags2 & X86_PTE_RW))
3500	\|\| ( !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
3501	&& CPUMGetGuestCPL(pVCpu, pCtxCore) <= 2) )
3502	{
3503	void *pvDst;
3504	PGMPAGEMAPLOCK Lock;
3505	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys1, &pvDst, &Lock);
3506	switch (rc)
3507	{
3508	case VINF_SUCCESS:
3509	Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
3510	(uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc, cb1));
3511	memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb1);
3512	PGMPhysReleasePageMappingLock(pVM, &Lock);
3513	break;
3514	case VERR_PGM_PHYS_PAGE_RESERVED:
3515	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3516	/* bit bucket */
3517	break;
3518	default:
3519	AssertMsgFailed(("%Rrc\n", rc));
3520	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3521	return rc;
3522	}
3523
3524	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys2, &pvDst, &Lock);
3525	switch (rc)
3526	{
3527	case VINF_SUCCESS:
3528	memcpy(pvDst, (const uint8_t *)pvSrc + cb1, cb2);
3529	PGMPhysReleasePageMappingLock(pVM, &Lock);
3530	break;
3531	case VERR_PGM_PHYS_PAGE_RESERVED:
3532	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3533	/* bit bucket */
3534	break;
3535	default:
3536	AssertMsgFailed(("%Rrc\n", rc));
3537	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3538	return rc;
3539	}
3540
3541	if (!(fFlags1 & (X86_PTE_A \| X86_PTE_RW)))
3542	{
3543	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, (X86_PTE_A \| X86_PTE_RW), ~(uint64_t)(X86_PTE_A \| X86_PTE_RW));
3544	AssertRC(rc);
3545	}
3546	if (!(fFlags2 & (X86_PTE_A \| X86_PTE_RW)))
3547	{
3548	rc = PGMGstModifyPage(pVCpu, GCPtrDst + cb1, 1, (X86_PTE_A \| X86_PTE_RW), ~(uint64_t)(X86_PTE_A \| X86_PTE_RW));
3549	AssertRC(rc);
3550	}
3551	return VINF_SUCCESS;
3552	}
3553	if ((fFlags1 & (X86_PTE_RW)) == X86_PTE_RW)
3554	GCPtrDst += cb1; /* fault on the 2nd page. */
3555	rc = VERR_ACCESS_DENIED;
3556	}
3557	else
3558	GCPtrDst += cb1; /* fault on the 2nd page. */
3559	}
3560	}
3561
3562	/*
3563	* Raise a #PF if we're allowed to do that.
3564	*/
3565	/* Calc the error bits. */
3566	uint32_t uErr;
3567	uint32_t cpl = CPUMGetGuestCPL(pVCpu, pCtxCore);
3568	switch (rc)
3569	{
3570	case VINF_SUCCESS:
3571	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3572	rc = VERR_ACCESS_DENIED;
3573	break;
3574
3575	case VERR_ACCESS_DENIED:
3576	uErr = (cpl >= 2) ? X86_TRAP_PF_RW \| X86_TRAP_PF_US : X86_TRAP_PF_RW;
3577	break;
3578
3579	case VERR_PAGE_NOT_PRESENT:
3580	case VERR_PAGE_TABLE_NOT_PRESENT:
3581	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3582	break;
3583
3584	default:
3585	AssertMsgFailed(("rc=%Rrc GCPtrDst=%RGv cb=%#x\n", rc, GCPtrDst, cb));
3586	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3587	return rc;
3588	}
3589	if (fRaiseTrap)
3590	{
3591	Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrDst, cb, uErr));
3592	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrDst);
3593	}
3594	Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrDst, cb, uErr));
3595	return rc;
3596	}
3597

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp@ 27549

Download in other formats: