PGMAllPhys.cpp@ 28112

Last change on this file since 28112 was 27899, checked in by vboxsync, 15 years ago
PGMAllPhys.cpp: Restored pgmPhysHandlerRedirectToHC which got accidentally removed in r59107.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 128.2 KB

Line
1	/* $Id: PGMAllPhys.cpp 27899 2010-03-31 14:05:23Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22	/*******************************************************************************
23	* Header Files *
24	*******************************************************************************/
25	#define LOG_GROUP LOG_GROUP_PGM_PHYS
26	#include <VBox/pgm.h>
27	#include <VBox/trpm.h>
28	#include <VBox/vmm.h>
29	#include <VBox/iom.h>
30	#include <VBox/em.h>
31	#include <VBox/rem.h>
32	#include "../PGMInternal.h"
33	#include <VBox/vm.h>
34	#include "../PGMInline.h"
35	#include <VBox/param.h>
36	#include <VBox/err.h>
37	#include <iprt/assert.h>
38	#include <iprt/string.h>
39	#include <iprt/asm.h>
40	#include <VBox/log.h>
41	#ifdef IN_RING3
42	# include <iprt/thread.h>
43	#endif
44
45
46	/*******************************************************************************
47	* Defined Constants And Macros *
48	*******************************************************************************/
49	/** Enable the physical TLB. */
50	#define PGM_WITH_PHYS_TLB
51
52
53
54	#ifndef IN_RING3
55
56	/**
57	* \#PF Handler callback for physical memory accesses without a RC/R0 handler.
58	* This simply pushes everything to the HC handler.
59	*
60	* @returns VBox status code (appropritate for trap handling and GC return).
61	* @param pVM VM Handle.
62	* @param uErrorCode CPU Error code.
63	* @param pRegFrame Trap register frame.
64	* @param pvFault The fault address (cr2).
65	* @param GCPhysFault The GC physical address corresponding to pvFault.
66	* @param pvUser User argument.
67	*/
68	VMMDECL(int) pgmPhysHandlerRedirectToHC(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
69	{
70	return (uErrorCode & X86_TRAP_PF_RW) ? VINF_IOM_HC_MMIO_WRITE : VINF_IOM_HC_MMIO_READ;
71	}
72
73
74	/**
75	* \#PF Handler callback for Guest ROM range write access.
76	* We simply ignore the writes or fall back to the recompiler if we don't support the instruction.
77	*
78	* @returns VBox status code (appropritate for trap handling and GC return).
79	* @param pVM VM Handle.
80	* @param uErrorCode CPU Error code.
81	* @param pRegFrame Trap register frame.
82	* @param pvFault The fault address (cr2).
83	* @param GCPhysFault The GC physical address corresponding to pvFault.
84	* @param pvUser User argument. Pointer to the ROM range structure.
85	*/
86	VMMDECL(int) pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
87	{
88	int rc;
89	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
90	uint32_t iPage = (GCPhysFault - pRom->GCPhys) >> PAGE_SHIFT;
91	PVMCPU pVCpu = VMMGetCpu(pVM);
92
93	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
94	switch (pRom->aPages[iPage].enmProt)
95	{
96	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
97	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
98	{
99	/*
100	* If it's a simple instruction which doesn't change the cpu state
101	* we will simply skip it. Otherwise we'll have to defer it to REM.
102	*/
103	uint32_t cbOp;
104	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
105	rc = EMInterpretDisasOne(pVM, pVCpu, pRegFrame, pDis, &cbOp);
106	if ( RT_SUCCESS(rc)
107	&& pDis->mode == CPUMODE_32BIT /** @todo why does this matter? */
108	&& !(pDis->prefix & (PREFIX_REPNE \| PREFIX_REP \| PREFIX_SEG)))
109	{
110	switch (pDis->opcode)
111	{
112	/** @todo Find other instructions we can safely skip, possibly
113	* adding this kind of detection to DIS or EM. */
114	case OP_MOV:
115	pRegFrame->rip += cbOp;
116	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZGuestROMWriteHandled);
117	return VINF_SUCCESS;
118	}
119	}
120	else if (RT_UNLIKELY(rc == VERR_INTERNAL_ERROR))
121	return rc;
122	break;
123	}
124
125	case PGMROMPROT_READ_RAM_WRITE_RAM:
126	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
127	rc = PGMHandlerPhysicalPageTempOff(pVM, pRom->GCPhys, GCPhysFault & X86_PTE_PG_MASK);
128	AssertRC(rc);
129	break; /** @todo Must edit the shadow PT and restart the instruction, not use the interpreter! */
130
131	case PGMROMPROT_READ_ROM_WRITE_RAM:
132	/* Handle it in ring-3 because it's way easier there. */
133	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
134	break;
135
136	default:
137	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhysFault=%RGp\n",
138	pRom->aPages[iPage].enmProt, iPage, GCPhysFault),
139	VERR_INTERNAL_ERROR);
140	}
141
142	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZGuestROMWriteUnhandled);
143	return VINF_EM_RAW_EMULATE_INSTR;
144	}
145
146	#endif /* IN_RING3 */
147
148	/**
149	* Checks if Address Gate 20 is enabled or not.
150	*
151	* @returns true if enabled.
152	* @returns false if disabled.
153	* @param pVCpu VMCPU handle.
154	*/
155	VMMDECL(bool) PGMPhysIsA20Enabled(PVMCPU pVCpu)
156	{
157	LogFlow(("PGMPhysIsA20Enabled %d\n", pVCpu->pgm.s.fA20Enabled));
158	return pVCpu->pgm.s.fA20Enabled;
159	}
160
161
162	/**
163	* Validates a GC physical address.
164	*
165	* @returns true if valid.
166	* @returns false if invalid.
167	* @param pVM The VM handle.
168	* @param GCPhys The physical address to validate.
169	*/
170	VMMDECL(bool) PGMPhysIsGCPhysValid(PVM pVM, RTGCPHYS GCPhys)
171	{
172	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
173	return pPage != NULL;
174	}
175
176
177	/**
178	* Checks if a GC physical address is a normal page,
179	* i.e. not ROM, MMIO or reserved.
180	*
181	* @returns true if normal.
182	* @returns false if invalid, ROM, MMIO or reserved page.
183	* @param pVM The VM handle.
184	* @param GCPhys The physical address to check.
185	*/
186	VMMDECL(bool) PGMPhysIsGCPhysNormal(PVM pVM, RTGCPHYS GCPhys)
187	{
188	PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
189	return pPage
190	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM;
191	}
192
193
194	/**
195	* Converts a GC physical address to a HC physical address.
196	*
197	* @returns VINF_SUCCESS on success.
198	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
199	* page but has no physical backing.
200	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
201	* GC physical address.
202	*
203	* @param pVM The VM handle.
204	* @param GCPhys The GC physical address to convert.
205	* @param pHCPhys Where to store the HC physical address on success.
206	*/
207	VMMDECL(int) PGMPhysGCPhys2HCPhys(PVM pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
208	{
209	pgmLock(pVM);
210	PPGMPAGE pPage;
211	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
212	if (RT_SUCCESS(rc))
213	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK);
214	pgmUnlock(pVM);
215	return rc;
216	}
217
218
219	/**
220	* Invalidates all page mapping TLBs.
221	*
222	* @param pVM The VM handle.
223	*/
224	VMMDECL(void) PGMPhysInvalidatePageMapTLB(PVM pVM)
225	{
226	pgmLock(pVM);
227	STAM_COUNTER_INC(&pVM->pgm.s.StatPageMapTlbFlushes);
228	/* Clear the shared R0/R3 TLB completely. */
229	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
230	{
231	pVM->pgm.s.PhysTlbHC.aEntries[i].GCPhys = NIL_RTGCPHYS;
232	pVM->pgm.s.PhysTlbHC.aEntries[i].pPage = 0;
233	pVM->pgm.s.PhysTlbHC.aEntries[i].pMap = 0;
234	pVM->pgm.s.PhysTlbHC.aEntries[i].pv = 0;
235	}
236	/* @todo clear the RC TLB whenever we add it. */
237	pgmUnlock(pVM);
238	}
239
240	/**
241	* Invalidates a page mapping TLB entry
242	*
243	* @param pVM The VM handle.
244	* @param GCPhys GCPhys entry to flush
245	*/
246	VMMDECL(void) PGMPhysInvalidatePageMapTLBEntry(PVM pVM, RTGCPHYS GCPhys)
247	{
248	Assert(PGMIsLocked(pVM));
249
250	STAM_COUNTER_INC(&pVM->pgm.s.StatPageMapTlbFlushEntry);
251	/* Clear the shared R0/R3 TLB entry. */
252	#ifdef IN_RC
253	unsigned idx = PGM_PAGER3MAPTLB_IDX(GCPhys);
254	pVM->pgm.s.PhysTlbHC.aEntries[idx].GCPhys = NIL_RTGCPHYS;
255	pVM->pgm.s.PhysTlbHC.aEntries[idx].pPage = 0;
256	pVM->pgm.s.PhysTlbHC.aEntries[idx].pMap = 0;
257	pVM->pgm.s.PhysTlbHC.aEntries[idx].pv = 0;
258	#else
259	PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
260	pTlbe->GCPhys = NIL_RTGCPHYS;
261	pTlbe->pPage = 0;
262	pTlbe->pMap = 0;
263	pTlbe->pv = 0;
264	#endif
265	/* @todo clear the RC TLB whenever we add it. */
266	}
267
268	/**
269	* Makes sure that there is at least one handy page ready for use.
270	*
271	* This will also take the appropriate actions when reaching water-marks.
272	*
273	* @returns VBox status code.
274	* @retval VINF_SUCCESS on success.
275	* @retval VERR_EM_NO_MEMORY if we're really out of memory.
276	*
277	* @param pVM The VM handle.
278	*
279	* @remarks Must be called from within the PGM critical section. It may
280	* nip back to ring-3/0 in some cases.
281	*/
282	static int pgmPhysEnsureHandyPage(PVM pVM)
283	{
284	AssertMsg(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", pVM->pgm.s.cHandyPages));
285
286	/*
287	* Do we need to do anything special?
288	*/
289	#ifdef IN_RING3
290	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_R3_ALLOC))
291	#else
292	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_RZ_TO_R3))
293	#endif
294	{
295	/*
296	* Allocate pages only if we're out of them, or in ring-3, almost out.
297	*/
298	#ifdef IN_RING3
299	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_R3_ALLOC)
300	#else
301	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_ALLOC)
302	#endif
303	{
304	Log(("PGM: cHandyPages=%u out of %u -> allocate more; VM_FF_PGM_NO_MEMORY=%RTbool\n",
305	pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages), VM_FF_ISSET(pVM, VM_FF_PGM_NO_MEMORY) ));
306	#ifdef IN_RING3
307	int rc = PGMR3PhysAllocateHandyPages(pVM);
308	#else
309	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES, 0);
310	#endif
311	if (RT_UNLIKELY(rc != VINF_SUCCESS))
312	{
313	if (RT_FAILURE(rc))
314	return rc;
315	AssertMsgReturn(rc == VINF_EM_NO_MEMORY, ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
316	if (!pVM->pgm.s.cHandyPages)
317	{
318	LogRel(("PGM: no more handy pages!\n"));
319	return VERR_EM_NO_MEMORY;
320	}
321	Assert(VM_FF_ISSET(pVM, VM_FF_PGM_NEED_HANDY_PAGES));
322	Assert(VM_FF_ISSET(pVM, VM_FF_PGM_NO_MEMORY));
323	#ifdef IN_RING3
324	REMR3NotifyFF(pVM);
325	#else
326	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); /* paranoia */
327	#endif
328	}
329	AssertMsgReturn( pVM->pgm.s.cHandyPages > 0
330	&& pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages),
331	("%u\n", pVM->pgm.s.cHandyPages),
332	VERR_INTERNAL_ERROR);
333	}
334	else
335	{
336	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_SET_FF)
337	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
338	#ifndef IN_RING3
339	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_TO_R3)
340	{
341	Log(("PGM: VM_FF_TO_R3 - cHandyPages=%u out of %u\n", pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
342	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
343	}
344	#endif
345	}
346	}
347
348	return VINF_SUCCESS;
349	}
350
351
352	/**
353	* Replace a zero or shared page with new page that we can write to.
354	*
355	* @returns The following VBox status codes.
356	* @retval VINF_SUCCESS on success, pPage is modified.
357	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
358	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
359	*
360	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
361	*
362	* @param pVM The VM address.
363	* @param pPage The physical page tracking structure. This will
364	* be modified on success.
365	* @param GCPhys The address of the page.
366	*
367	* @remarks Must be called from within the PGM critical section. It may
368	* nip back to ring-3/0 in some cases.
369	*
370	* @remarks This function shouldn't really fail, however if it does
371	* it probably means we've screwed up the size of handy pages and/or
372	* the low-water mark. Or, that some device I/O is causing a lot of
373	* pages to be allocated while while the host is in a low-memory
374	* condition. This latter should be handled elsewhere and in a more
375	* controlled manner, it's on the @bugref{3170} todo list...
376	*/
377	int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
378	{
379	LogFlow(("pgmPhysAllocPage: %R[pgmpage] %RGp\n", pPage, GCPhys));
380
381	/*
382	* Prereqs.
383	*/
384	Assert(PGMIsLocked(pVM));
385	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
386	Assert(!PGM_PAGE_IS_MMIO(pPage));
387
388	# ifdef PGM_WITH_LARGE_PAGES
389	if ( PGMIsUsingLargePages(pVM)
390	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
391	{
392	int rc = pgmPhysAllocLargePage(pVM, GCPhys);
393	if (rc == VINF_SUCCESS)
394	return rc;
395
396	/* fall back to 4kb pages. */
397	}
398	# endif
399
400	/*
401	* Flush any shadow page table mappings of the page.
402	* When VBOX_WITH_NEW_LAZY_PAGE_ALLOC isn't defined, there shouldn't be any.
403	*/
404	bool fFlushTLBs = false;
405	int rc = pgmPoolTrackFlushGCPhys(pVM, GCPhys, pPage, &fFlushTLBs);
406	AssertMsgReturn(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc), RT_FAILURE(rc) ? rc : VERR_IPE_UNEXPECTED_STATUS);
407
408	/*
409	* Ensure that we've got a page handy, take it and use it.
410	*/
411	int rc2 = pgmPhysEnsureHandyPage(pVM);
412	if (RT_FAILURE(rc2))
413	{
414	if (fFlushTLBs)
415	PGM_INVL_ALL_VCPU_TLBS(pVM);
416	Assert(rc2 == VERR_EM_NO_MEMORY);
417	return rc2;
418	}
419	/* re-assert preconditions since pgmPhysEnsureHandyPage may do a context switch. */
420	Assert(PGMIsLocked(pVM));
421	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
422	Assert(!PGM_PAGE_IS_MMIO(pPage));
423
424	uint32_t iHandyPage = --pVM->pgm.s.cHandyPages;
425	AssertMsg(iHandyPage < RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", iHandyPage));
426	Assert(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys != NIL_RTHCPHYS);
427	Assert(!(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
428	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idPage != NIL_GMM_PAGEID);
429	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
430
431	/*
432	* There are one or two action to be taken the next time we allocate handy pages:
433	* - Tell the GMM (global memory manager) what the page is being used for.
434	* (Speeds up replacement operations - sharing and defragmenting.)
435	* - If the current backing is shared, it must be freed.
436	*/
437	const RTHCPHYS HCPhys = pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys;
438	pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
439
440	if (PGM_PAGE_IS_SHARED(pPage))
441	{
442	pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage = PGM_PAGE_GET_PAGEID(pPage);
443	Assert(PGM_PAGE_GET_PAGEID(pPage) != NIL_GMM_PAGEID);
444	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
445
446	Log2(("PGM: Replaced shared page %#x at %RGp with %#x / %RHp\n", PGM_PAGE_GET_PAGEID(pPage),
447	GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
448	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PageReplaceShared));
449	pVM->pgm.s.cSharedPages--;
450	AssertMsgFailed(("TODO: copy shared page content")); /** @todo err.. what about copying the page content? */
451	}
452	else
453	{
454	Log2(("PGM: Replaced zero page %RGp with %#x / %RHp\n", GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
455	STAM_COUNTER_INC(&pVM->pgm.s.StatRZPageReplaceZero);
456	pVM->pgm.s.cZeroPages--;
457	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
458	}
459
460	/*
461	* Do the PGMPAGE modifications.
462	*/
463	pVM->pgm.s.cPrivatePages++;
464	PGM_PAGE_SET_HCPHYS(pPage, HCPhys);
465	PGM_PAGE_SET_PAGEID(pPage, pVM->pgm.s.aHandyPages[iHandyPage].idPage);
466	PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
467	PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_PT);
468	PGMPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
469
470	if ( fFlushTLBs
471	&& rc != VINF_PGM_GCPHYS_ALIASED)
472	PGM_INVL_ALL_VCPU_TLBS(pVM);
473	return rc;
474	}
475
476	#ifdef PGM_WITH_LARGE_PAGES
477	/**
478	* Replace a 2 MB range of zero pages with new pages that we can write to.
479	*
480	* @returns The following VBox status codes.
481	* @retval VINF_SUCCESS on success, pPage is modified.
482	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
483	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
484	*
485	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
486	*
487	* @param pVM The VM address.
488	* @param GCPhys The address of the page.
489	*
490	* @remarks Must be called from within the PGM critical section. It may
491	* nip back to ring-3/0 in some cases.
492	*/
493	int pgmPhysAllocLargePage(PVM pVM, RTGCPHYS GCPhys)
494	{
495	RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
496	LogFlow(("pgmPhysAllocLargePage: %RGp base %RGp\n", GCPhys, GCPhysBase));
497
498	/*
499	* Prereqs.
500	*/
501	Assert(PGMIsLocked(pVM));
502	Assert(PGMIsUsingLargePages(pVM));
503
504	PPGMPAGE pPage;
505	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysBase, &pPage);
506	if ( RT_SUCCESS(rc)
507	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
508	{
509	unsigned uPDEType = PGM_PAGE_GET_PDE_TYPE(pPage);
510
511	/* Don't call this function for already allocated pages. */
512	Assert(uPDEType != PGM_PAGE_PDE_TYPE_PDE);
513
514	if ( uPDEType == PGM_PAGE_PDE_TYPE_DONTCARE
515	&& PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ZERO)
516	{
517	unsigned iPage;
518
519	GCPhys = GCPhysBase;
520
521	/* Lazy approach: check all pages in the 2 MB range.
522	* The whole range must be ram and unallocated
523	*/
524	for (iPage = 0; iPage < _2M/PAGE_SIZE; iPage++)
525	{
526	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
527	if ( RT_FAILURE(rc)
528	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM /* Anything other than ram implies monitoring. */
529	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ZERO) /* allocated, monitored or shared means we can't use a large page here */
530	{
531	LogFlow(("Found page %RGp with wrong attributes (type=%d; state=%d); cancel check. rc=%d\n", GCPhys, PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_STATE(pPage), rc));
532	break;
533	}
534	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_DONTCARE);
535	GCPhys += PAGE_SIZE;
536	}
537	/* Fetch the start page of the 2 MB range again. */
538	rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhysBase, &pPage);
539	AssertRC(rc); /* can't fail */
540
541	if (iPage != _2M/PAGE_SIZE)
542	{
543	/* Failed. Mark as requiring a PT so we don't check the whole thing again in the future. */
544	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRefused);
545	PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_PT);
546	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
547	}
548	else
549	{
550	# ifdef IN_RING3
551	rc = PGMR3PhysAllocateLargeHandyPage(pVM, GCPhysBase);
552	# else
553	rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE, GCPhysBase);
554	# endif
555	if (RT_SUCCESS(rc))
556	{
557	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
558	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageAlloc);
559	return VINF_SUCCESS;
560	}
561	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
562
563	/* If we fail once, it most likely means the host's memory is too fragmented; don't bother trying again. */
564	PGMSetLargePageUsage(pVM, false);
565	return rc;
566	}
567	}
568	}
569	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
570	}
571
572	/**
573	* Recheck the entire 2 MB range to see if we can use it again as a large page.
574	*
575	* @returns The following VBox status codes.
576	* @retval VINF_SUCCESS on success, the large page can be used again
577	* @retval VERR_PGM_INVALID_LARGE_PAGE_RANGE if it can't be reused
578	*
579	* @param pVM The VM address.
580	* @param GCPhys The address of the page.
581	* @param pLargePage Page structure of the base page
582	*/
583	int pgmPhysIsValidLargePage(PVM pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage)
584	{
585	unsigned i;
586
587	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRecheck);
588
589	GCPhys &= X86_PDE2M_PAE_PG_MASK;
590
591	/* Check the base page. */
592	Assert(PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED);
593	if ( PGM_PAGE_GET_STATE(pLargePage) != PGM_PAGE_STATE_ALLOCATED
594	\|\| PGM_PAGE_GET_TYPE(pLargePage) != PGMPAGETYPE_RAM
595	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
596	{
597	LogFlow(("pgmPhysIsValidLargePage: checks failed for base page %x %x %x\n", PGM_PAGE_GET_STATE(pLargePage), PGM_PAGE_GET_TYPE(pLargePage), PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage)));
598	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
599	}
600
601	STAM_PROFILE_START(&pVM->pgm.s.CTX_MID_Z(Stat,IsValidLargePage), a);
602	/* Check all remaining pages in the 2 MB range. */
603	GCPhys += PAGE_SIZE;
604	for (i = 1; i < _2M/PAGE_SIZE; i++)
605	{
606	PPGMPAGE pPage;
607	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
608	AssertRCBreak(rc);
609
610	if ( PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
611	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
612	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
613	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
614	{
615	LogFlow(("pgmPhysIsValidLargePage: checks failed for page %d; %x %x %x\n", i, PGM_PAGE_GET_STATE(pPage), PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)));
616	break;
617	}
618
619	GCPhys += PAGE_SIZE;
620	}
621	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,IsValidLargePage), a);
622
623	if (i == _2M/PAGE_SIZE)
624	{
625	PGM_PAGE_SET_PDE_TYPE(pLargePage, PGM_PAGE_PDE_TYPE_PDE);
626	Log(("pgmPhysIsValidLargePage: page %RGp can be reused!\n", GCPhys - _2M));
627	return VINF_SUCCESS;
628	}
629
630	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
631	}
632
633	#endif /* PGM_WITH_LARGE_PAGES */
634
635	/**
636	* Deal with a write monitored page.
637	*
638	* @returns VBox strict status code.
639	*
640	* @param pVM The VM address.
641	* @param pPage The physical page tracking structure.
642	*
643	* @remarks Called from within the PGM critical section.
644	*/
645	void pgmPhysPageMakeWriteMonitoredWritable(PVM pVM, PPGMPAGE pPage)
646	{
647	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED);
648	PGM_PAGE_SET_WRITTEN_TO(pPage);
649	PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
650	Assert(pVM->pgm.s.cMonitoredPages > 0);
651	pVM->pgm.s.cMonitoredPages--;
652	pVM->pgm.s.cWrittenToPages++;
653	}
654
655
656	/**
657	* Deal with pages that are not writable, i.e. not in the ALLOCATED state.
658	*
659	* @returns VBox strict status code.
660	* @retval VINF_SUCCESS on success.
661	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
662	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
663	*
664	* @param pVM The VM address.
665	* @param pPage The physical page tracking structure.
666	* @param GCPhys The address of the page.
667	*
668	* @remarks Called from within the PGM critical section.
669	*/
670	int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
671	{
672	Assert(PGMIsLockOwner(pVM));
673	switch (PGM_PAGE_GET_STATE(pPage))
674	{
675	case PGM_PAGE_STATE_WRITE_MONITORED:
676	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
677	/* fall thru */
678	default: /* to shut up GCC */
679	case PGM_PAGE_STATE_ALLOCATED:
680	return VINF_SUCCESS;
681
682	/*
683	* Zero pages can be dummy pages for MMIO or reserved memory,
684	* so we need to check the flags before joining cause with
685	* shared page replacement.
686	*/
687	case PGM_PAGE_STATE_ZERO:
688	if (PGM_PAGE_IS_MMIO(pPage))
689	return VERR_PGM_PHYS_PAGE_RESERVED;
690	/* fall thru */
691	case PGM_PAGE_STATE_SHARED:
692	return pgmPhysAllocPage(pVM, pPage, GCPhys);
693
694	/* Not allowed to write to ballooned pages. */
695	case PGM_PAGE_STATE_BALLOONED:
696	return VERR_PGM_PHYS_PAGE_BALLOONED;
697	}
698	}
699
700
701	/**
702	* Internal usage: Map the page specified by its GMM ID.
703	*
704	* This is similar to pgmPhysPageMap
705	*
706	* @returns VBox status code.
707	*
708	* @param pVM The VM handle.
709	* @param idPage The Page ID.
710	* @param HCPhys The physical address (for RC).
711	* @param ppv Where to store the mapping address.
712	*
713	* @remarks Called from within the PGM critical section. The mapping is only
714	* valid while your inside this section.
715	*/
716	int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv)
717	{
718	/*
719	* Validation.
720	*/
721	Assert(PGMIsLocked(pVM));
722	AssertReturn(HCPhys && !(HCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
723	const uint32_t idChunk = idPage >> GMM_CHUNKID_SHIFT;
724	AssertReturn(idChunk != NIL_GMM_CHUNKID, VERR_INVALID_PARAMETER);
725
726	#ifdef IN_RC
727	/*
728	* Map it by HCPhys.
729	*/
730	return PGMDynMapHCPage(pVM, HCPhys, ppv);
731
732	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
733	/*
734	* Map it by HCPhys.
735	*/
736	return pgmR0DynMapHCPageInlined(&pVM->pgm.s, HCPhys, ppv);
737
738	#else
739	/*
740	* Find/make Chunk TLB entry for the mapping chunk.
741	*/
742	PPGMCHUNKR3MAP pMap;
743	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
744	if (pTlbe->idChunk == idChunk)
745	{
746	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,ChunkR3MapTlbHits));
747	pMap = pTlbe->pChunk;
748	}
749	else
750	{
751	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
752
753	/*
754	* Find the chunk, map it if necessary.
755	*/
756	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
757	if (!pMap)
758	{
759	# ifdef IN_RING0
760	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
761	AssertRCReturn(rc, rc);
762	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
763	Assert(pMap);
764	# else
765	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
766	if (RT_FAILURE(rc))
767	return rc;
768	# endif
769	}
770
771	/*
772	* Enter it into the Chunk TLB.
773	*/
774	pTlbe->idChunk = idChunk;
775	pTlbe->pChunk = pMap;
776	pMap->iAge = 0;
777	}
778
779	ppv = (uint8_t )pMap->pv + ((idPage &GMM_PAGEID_IDX_MASK) << PAGE_SHIFT);
780	return VINF_SUCCESS;
781	#endif
782	}
783
784
785	/**
786	* Maps a page into the current virtual address space so it can be accessed.
787	*
788	* @returns VBox status code.
789	* @retval VINF_SUCCESS on success.
790	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
791	*
792	* @param pVM The VM address.
793	* @param pPage The physical page tracking structure.
794	* @param GCPhys The address of the page.
795	* @param ppMap Where to store the address of the mapping tracking structure.
796	* @param ppv Where to store the mapping address of the page. The page
797	* offset is masked off!
798	*
799	* @remarks Called from within the PGM critical section.
800	*/
801	static int pgmPhysPageMapCommon(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv)
802	{
803	Assert(PGMIsLocked(pVM));
804
805	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
806	/*
807	* Just some sketchy GC/R0-darwin code.
808	*/
809	*ppMap = NULL;
810	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
811	Assert(HCPhys != pVM->pgm.s.HCPhysZeroPg);
812	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
813	pgmR0DynMapHCPageInlined(&pVM->pgm.s, HCPhys, ppv);
814	# else
815	PGMDynMapHCPage(pVM, HCPhys, ppv);
816	# endif
817	return VINF_SUCCESS;
818
819	#else /* IN_RING3 \|\| IN_RING0 */
820
821
822	/*
823	* Special case: ZERO and MMIO2 pages.
824	*/
825	const uint32_t idChunk = PGM_PAGE_GET_CHUNKID(pPage);
826	if (idChunk == NIL_GMM_CHUNKID)
827	{
828	AssertMsgReturn(PGM_PAGE_GET_PAGEID(pPage) == NIL_GMM_PAGEID, ("pPage=%R[pgmpage]\n", pPage), VERR_INTERNAL_ERROR_2);
829	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2)
830	{
831	/* Lookup the MMIO2 range and use pvR3 to calc the address. */
832	PPGMRAMRANGE pRam = pgmPhysGetRange(&pVM->pgm.s, GCPhys);
833	AssertMsgReturn(pRam \|\| !pRam->pvR3, ("pRam=%p pPage=%R[pgmpage]\n", pRam, pPage), VERR_INTERNAL_ERROR_2);
834	ppv = (void )((uintptr_t)pRam->pvR3 + (uintptr_t)((GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK) - pRam->GCPhys));
835	}
836	else if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO)
837	{
838	/** @todo deal with aliased MMIO2 pages somehow...
839	* One solution would be to seed MMIO2 pages to GMM and get unique Page IDs for
840	* them, that would also avoid this mess. It would actually be kind of
841	* elegant... */
842	AssertLogRelMsgFailedReturn(("%RGp\n", GCPhys), VERR_INTERNAL_ERROR_3);
843	}
844	else
845	{
846	/** @todo handle MMIO2 */
847	AssertMsgReturn(PGM_PAGE_IS_ZERO(pPage), ("pPage=%R[pgmpage]\n", pPage), VERR_INTERNAL_ERROR_2);
848	AssertMsgReturn(PGM_PAGE_GET_HCPHYS(pPage) == pVM->pgm.s.HCPhysZeroPg,
849	("pPage=%R[pgmpage]\n", pPage),
850	VERR_INTERNAL_ERROR_2);
851	*ppv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
852	}
853	*ppMap = NULL;
854	return VINF_SUCCESS;
855	}
856
857	/*
858	* Find/make Chunk TLB entry for the mapping chunk.
859	*/
860	PPGMCHUNKR3MAP pMap;
861	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
862	if (pTlbe->idChunk == idChunk)
863	{
864	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,ChunkR3MapTlbHits));
865	pMap = pTlbe->pChunk;
866	}
867	else
868	{
869	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
870
871	/*
872	* Find the chunk, map it if necessary.
873	*/
874	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
875	if (!pMap)
876	{
877	#ifdef IN_RING0
878	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
879	AssertRCReturn(rc, rc);
880	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
881	Assert(pMap);
882	#else
883	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
884	if (RT_FAILURE(rc))
885	return rc;
886	#endif
887	}
888
889	/*
890	* Enter it into the Chunk TLB.
891	*/
892	pTlbe->idChunk = idChunk;
893	pTlbe->pChunk = pMap;
894	pMap->iAge = 0;
895	}
896
897	ppv = (uint8_t )pMap->pv + (PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) << PAGE_SHIFT);
898	*ppMap = pMap;
899	return VINF_SUCCESS;
900	#endif /* IN_RING3 */
901	}
902
903
904	/**
905	* Combination of pgmPhysPageMakeWritable and pgmPhysPageMapWritable.
906	*
907	* This is typically used is paths where we cannot use the TLB methods (like ROM
908	* pages) or where there is no point in using them since we won't get many hits.
909	*
910	* @returns VBox strict status code.
911	* @retval VINF_SUCCESS on success.
912	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
913	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
914	*
915	* @param pVM The VM address.
916	* @param pPage The physical page tracking structure.
917	* @param GCPhys The address of the page.
918	* @param ppv Where to store the mapping address of the page. The page
919	* offset is masked off!
920	*
921	* @remarks Called from within the PGM critical section. The mapping is only
922	* valid while your inside this section.
923	*/
924	int pgmPhysPageMakeWritableAndMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
925	{
926	int rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
927	if (RT_SUCCESS(rc))
928	{
929	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* returned */, ("%Rrc\n", rc));
930	PPGMPAGEMAP pMapIgnore;
931	int rc2 = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
932	if (RT_FAILURE(rc2)) /* preserve rc */
933	rc = rc2;
934	}
935	return rc;
936	}
937
938
939	/**
940	* Maps a page into the current virtual address space so it can be accessed for
941	* both writing and reading.
942	*
943	* This is typically used is paths where we cannot use the TLB methods (like ROM
944	* pages) or where there is no point in using them since we won't get many hits.
945	*
946	* @returns VBox status code.
947	* @retval VINF_SUCCESS on success.
948	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
949	*
950	* @param pVM The VM address.
951	* @param pPage The physical page tracking structure. Must be in the
952	* allocated state.
953	* @param GCPhys The address of the page.
954	* @param ppv Where to store the mapping address of the page. The page
955	* offset is masked off!
956	*
957	* @remarks Called from within the PGM critical section. The mapping is only
958	* valid while your inside this section.
959	*/
960	int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
961	{
962	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
963	PPGMPAGEMAP pMapIgnore;
964	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
965	}
966
967
968	/**
969	* Maps a page into the current virtual address space so it can be accessed for
970	* reading.
971	*
972	* This is typically used is paths where we cannot use the TLB methods (like ROM
973	* pages) or where there is no point in using them since we won't get many hits.
974	*
975	* @returns VBox status code.
976	* @retval VINF_SUCCESS on success.
977	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
978	*
979	* @param pVM The VM address.
980	* @param pPage The physical page tracking structure.
981	* @param GCPhys The address of the page.
982	* @param ppv Where to store the mapping address of the page. The page
983	* offset is masked off!
984	*
985	* @remarks Called from within the PGM critical section. The mapping is only
986	* valid while your inside this section.
987	*/
988	int pgmPhysPageMapReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv)
989	{
990	PPGMPAGEMAP pMapIgnore;
991	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, (void **)ppv);
992	}
993
994
995	#if !defined(IN_RC) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
996	/**
997	* Load a guest page into the ring-3 physical TLB.
998	*
999	* @returns VBox status code.
1000	* @retval VINF_SUCCESS on success
1001	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1002	* @param pPGM The PGM instance pointer.
1003	* @param GCPhys The guest physical address in question.
1004	*/
1005	int pgmPhysPageLoadIntoTlb(PPGM pPGM, RTGCPHYS GCPhys)
1006	{
1007	Assert(PGMIsLocked(PGM2VM(pPGM)));
1008	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbMisses));
1009
1010	/*
1011	* Find the ram range.
1012	* 99.8% of requests are expected to be in the first range.
1013	*/
1014	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
1015	RTGCPHYS off = GCPhys - pRam->GCPhys;
1016	if (RT_UNLIKELY(off >= pRam->cb))
1017	{
1018	do
1019	{
1020	pRam = pRam->CTX_SUFF(pNext);
1021	if (!pRam)
1022	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
1023	off = GCPhys - pRam->GCPhys;
1024	} while (off >= pRam->cb);
1025	}
1026
1027	/*
1028	* Map the page.
1029	* Make a special case for the zero page as it is kind of special.
1030	*/
1031	PPGMPAGE pPage = &pRam->aPages[off >> PAGE_SHIFT];
1032	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
1033	if ( !PGM_PAGE_IS_ZERO(pPage)
1034	&& !PGM_PAGE_IS_BALLOONED(pPage))
1035	{
1036	void *pv;
1037	PPGMPAGEMAP pMap;
1038	int rc = pgmPhysPageMapCommon(PGM2VM(pPGM), pPage, GCPhys, &pMap, &pv);
1039	if (RT_FAILURE(rc))
1040	return rc;
1041	pTlbe->pMap = pMap;
1042	pTlbe->pv = pv;
1043	Assert(!((uintptr_t)pTlbe->pv & PAGE_OFFSET_MASK));
1044	}
1045	else
1046	{
1047	Assert(PGM_PAGE_GET_HCPHYS(pPage) == pPGM->HCPhysZeroPg);
1048	pTlbe->pMap = NULL;
1049	pTlbe->pv = pPGM->CTXALLSUFF(pvZeroPg);
1050	}
1051	#ifdef PGM_WITH_PHYS_TLB
1052	pTlbe->GCPhys = GCPhys & X86_PTE_PAE_PG_MASK;
1053	#else
1054	pTlbe->GCPhys = NIL_RTGCPHYS;
1055	#endif
1056	pTlbe->pPage = pPage;
1057	return VINF_SUCCESS;
1058	}
1059
1060
1061	/**
1062	* Load a guest page into the ring-3 physical TLB.
1063	*
1064	* @returns VBox status code.
1065	* @retval VINF_SUCCESS on success
1066	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1067	*
1068	* @param pPGM The PGM instance pointer.
1069	* @param pPage Pointer to the PGMPAGE structure corresponding to
1070	* GCPhys.
1071	* @param GCPhys The guest physical address in question.
1072	*/
1073	int pgmPhysPageLoadIntoTlbWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys)
1074	{
1075	Assert(PGMIsLocked(PGM2VM(pPGM)));
1076	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbMisses));
1077
1078	/*
1079	* Map the page.
1080	* Make a special case for the zero page as it is kind of special.
1081	*/
1082	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
1083	if ( !PGM_PAGE_IS_ZERO(pPage)
1084	&& !PGM_PAGE_IS_BALLOONED(pPage))
1085	{
1086	void *pv;
1087	PPGMPAGEMAP pMap;
1088	int rc = pgmPhysPageMapCommon(PGM2VM(pPGM), pPage, GCPhys, &pMap, &pv);
1089	if (RT_FAILURE(rc))
1090	return rc;
1091	pTlbe->pMap = pMap;
1092	pTlbe->pv = pv;
1093	Assert(!((uintptr_t)pTlbe->pv & PAGE_OFFSET_MASK));
1094	}
1095	else
1096	{
1097	Assert(PGM_PAGE_GET_HCPHYS(pPage) == pPGM->HCPhysZeroPg);
1098	pTlbe->pMap = NULL;
1099	pTlbe->pv = pPGM->CTXALLSUFF(pvZeroPg);
1100	}
1101	#ifdef PGM_WITH_PHYS_TLB
1102	pTlbe->GCPhys = GCPhys & X86_PTE_PAE_PG_MASK;
1103	#else
1104	pTlbe->GCPhys = NIL_RTGCPHYS;
1105	#endif
1106	pTlbe->pPage = pPage;
1107	return VINF_SUCCESS;
1108	}
1109	#endif /* !IN_RC && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
1110
1111
1112	/**
1113	* Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
1114	* own the PGM lock and therefore not need to lock the mapped page.
1115	*
1116	* @returns VBox status code.
1117	* @retval VINF_SUCCESS on success.
1118	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1119	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1120	*
1121	* @param pVM The VM handle.
1122	* @param GCPhys The guest physical address of the page that should be mapped.
1123	* @param pPage Pointer to the PGMPAGE structure for the page.
1124	* @param ppv Where to store the address corresponding to GCPhys.
1125	*
1126	* @internal
1127	*/
1128	int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1129	{
1130	int rc;
1131	AssertReturn(pPage, VERR_INTERNAL_ERROR);
1132	Assert(PGMIsLocked(pVM));
1133
1134	/*
1135	* Make sure the page is writable.
1136	*/
1137	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1138	{
1139	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1140	if (RT_FAILURE(rc))
1141	return rc;
1142	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1143	}
1144	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1145
1146	/*
1147	* Get the mapping address.
1148	*/
1149	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1150	*ppv = pgmDynMapHCPageOff(&pVM->pgm.s, PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK));
1151	#else
1152	PPGMPAGEMAPTLBE pTlbe;
1153	rc = pgmPhysPageQueryTlbeWithPage(&pVM->pgm.s, pPage, GCPhys, &pTlbe);
1154	if (RT_FAILURE(rc))
1155	return rc;
1156	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1157	#endif
1158	return VINF_SUCCESS;
1159	}
1160
1161
1162	/**
1163	* Internal version of PGMPhysGCPhys2CCPtrReadOnly that expects the caller to
1164	* own the PGM lock and therefore not need to lock the mapped page.
1165	*
1166	* @returns VBox status code.
1167	* @retval VINF_SUCCESS on success.
1168	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1169	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1170	*
1171	* @param pVM The VM handle.
1172	* @param GCPhys The guest physical address of the page that should be mapped.
1173	* @param pPage Pointer to the PGMPAGE structure for the page.
1174	* @param ppv Where to store the address corresponding to GCPhys.
1175	*
1176	* @internal
1177	*/
1178	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv)
1179	{
1180	AssertReturn(pPage, VERR_INTERNAL_ERROR);
1181	Assert(PGMIsLocked(pVM));
1182	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1183
1184	/*
1185	* Get the mapping address.
1186	*/
1187	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1188	ppv = pgmDynMapHCPageOff(&pVM->pgm.s, PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK)); /* @todo add a read only flag? */
1189	#else
1190	PPGMPAGEMAPTLBE pTlbe;
1191	int rc = pgmPhysPageQueryTlbeWithPage(&pVM->pgm.s, pPage, GCPhys, &pTlbe);
1192	if (RT_FAILURE(rc))
1193	return rc;
1194	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1195	#endif
1196	return VINF_SUCCESS;
1197	}
1198
1199
1200	/**
1201	* Requests the mapping of a guest page into the current context.
1202	*
1203	* This API should only be used for very short term, as it will consume
1204	* scarse resources (R0 and GC) in the mapping cache. When you're done
1205	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1206	*
1207	* This API will assume your intention is to write to the page, and will
1208	* therefore replace shared and zero pages. If you do not intend to modify
1209	* the page, use the PGMPhysGCPhys2CCPtrReadOnly() API.
1210	*
1211	* @returns VBox status code.
1212	* @retval VINF_SUCCESS on success.
1213	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1214	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1215	*
1216	* @param pVM The VM handle.
1217	* @param GCPhys The guest physical address of the page that should be mapped.
1218	* @param ppv Where to store the address corresponding to GCPhys.
1219	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1220	*
1221	* @remarks The caller is responsible for dealing with access handlers.
1222	* @todo Add an informational return code for pages with access handlers?
1223	*
1224	* @remark Avoid calling this API from within critical sections (other than the
1225	* PGM one) because of the deadlock risk. External threads may need to
1226	* delegate jobs to the EMTs.
1227	* @thread Any thread.
1228	*/
1229	VMMDECL(int) PGMPhysGCPhys2CCPtr(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
1230	{
1231	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1232
1233	/*
1234	* Find the page and make sure it's writable.
1235	*/
1236	PPGMPAGE pPage;
1237	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1238	if (RT_SUCCESS(rc))
1239	{
1240	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1241	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1242	if (RT_SUCCESS(rc))
1243	{
1244	ppv = pgmDynMapHCPageOff(&pVM->pgm.s, PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK)); /* @todo add a read only flag? */
1245	# if 0
1246	pLock->pvMap = 0;
1247	pLock->pvPage = pPage;
1248	# else
1249	pLock->u32Dummy = UINT32_MAX;
1250	# endif
1251	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1252	rc = VINF_SUCCESS;
1253	}
1254	}
1255
1256	#else /* IN_RING3 \|\| IN_RING0 */
1257	int rc = pgmLock(pVM);
1258	AssertRCReturn(rc, rc);
1259
1260	/*
1261	* Query the Physical TLB entry for the page (may fail).
1262	*/
1263	PPGMPAGEMAPTLBE pTlbe;
1264	rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
1265	if (RT_SUCCESS(rc))
1266	{
1267	/*
1268	* If the page is shared, the zero page, or being write monitored
1269	* it must be converted to a page that's writable if possible.
1270	*/
1271	PPGMPAGE pPage = pTlbe->pPage;
1272	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1273	{
1274	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1275	if (RT_SUCCESS(rc))
1276	{
1277	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1278	rc = pgmPhysPageQueryTlbeWithPage(&pVM->pgm.s, pPage, GCPhys, &pTlbe);
1279	}
1280	}
1281	if (RT_SUCCESS(rc))
1282	{
1283	/*
1284	* Now, just perform the locking and calculate the return address.
1285	*/
1286	PPGMPAGEMAP pMap = pTlbe->pMap;
1287	if (pMap)
1288	pMap->cRefs++;
1289
1290	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
1291	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
1292	{
1293	if (cLocks == 0)
1294	pVM->pgm.s.cWriteLockedPages++;
1295	PGM_PAGE_INC_WRITE_LOCKS(pPage);
1296	}
1297	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
1298	{
1299	PGM_PAGE_INC_WRITE_LOCKS(pPage);
1300	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
1301	if (pMap)
1302	pMap->cRefs++; /* Extra ref to prevent it from going away. */
1303	}
1304
1305	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1306	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
1307	pLock->pvMap = pMap;
1308	}
1309	}
1310
1311	pgmUnlock(pVM);
1312	#endif /* IN_RING3 \|\| IN_RING0 */
1313	return rc;
1314	}
1315
1316
1317	/**
1318	* Requests the mapping of a guest page into the current context.
1319	*
1320	* This API should only be used for very short term, as it will consume
1321	* scarse resources (R0 and GC) in the mapping cache. When you're done
1322	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1323	*
1324	* @returns VBox status code.
1325	* @retval VINF_SUCCESS on success.
1326	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1327	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1328	*
1329	* @param pVM The VM handle.
1330	* @param GCPhys The guest physical address of the page that should be mapped.
1331	* @param ppv Where to store the address corresponding to GCPhys.
1332	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1333	*
1334	* @remarks The caller is responsible for dealing with access handlers.
1335	* @todo Add an informational return code for pages with access handlers?
1336	*
1337	* @remark Avoid calling this API from within critical sections (other than
1338	* the PGM one) because of the deadlock risk.
1339	* @thread Any thread.
1340	*/
1341	VMMDECL(int) PGMPhysGCPhys2CCPtrReadOnly(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
1342	{
1343	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1344
1345	/*
1346	* Find the page and make sure it's readable.
1347	*/
1348	PPGMPAGE pPage;
1349	int rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
1350	if (RT_SUCCESS(rc))
1351	{
1352	if (RT_UNLIKELY(PGM_PAGE_IS_MMIO(pPage)))
1353	rc = VERR_PGM_PHYS_PAGE_RESERVED;
1354	else
1355	{
1356	ppv = pgmDynMapHCPageOff(&pVM->pgm.s, PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK)); /* @todo add a read only flag? */
1357	# if 0
1358	pLock->pvMap = 0;
1359	pLock->pvPage = pPage;
1360	# else
1361	pLock->u32Dummy = UINT32_MAX;
1362	# endif
1363	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1364	rc = VINF_SUCCESS;
1365	}
1366	}
1367
1368	#else /* IN_RING3 \|\| IN_RING0 */
1369	int rc = pgmLock(pVM);
1370	AssertRCReturn(rc, rc);
1371
1372	/*
1373	* Query the Physical TLB entry for the page (may fail).
1374	*/
1375	PPGMPAGEMAPTLBE pTlbe;
1376	rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
1377	if (RT_SUCCESS(rc))
1378	{
1379	/* MMIO pages doesn't have any readable backing. */
1380	PPGMPAGE pPage = pTlbe->pPage;
1381	if (RT_UNLIKELY(PGM_PAGE_IS_MMIO(pPage)))
1382	rc = VERR_PGM_PHYS_PAGE_RESERVED;
1383	else
1384	{
1385	/*
1386	* Now, just perform the locking and calculate the return address.
1387	*/
1388	PPGMPAGEMAP pMap = pTlbe->pMap;
1389	if (pMap)
1390	pMap->cRefs++;
1391
1392	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
1393	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
1394	{
1395	if (cLocks == 0)
1396	pVM->pgm.s.cReadLockedPages++;
1397	PGM_PAGE_INC_READ_LOCKS(pPage);
1398	}
1399	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
1400	{
1401	PGM_PAGE_INC_READ_LOCKS(pPage);
1402	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
1403	if (pMap)
1404	pMap->cRefs++; /* Extra ref to prevent it from going away. */
1405	}
1406
1407	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1408	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
1409	pLock->pvMap = pMap;
1410	}
1411	}
1412
1413	pgmUnlock(pVM);
1414	#endif /* IN_RING3 \|\| IN_RING0 */
1415	return rc;
1416	}
1417
1418
1419	/**
1420	* Requests the mapping of a guest page given by virtual address into the current context.
1421	*
1422	* This API should only be used for very short term, as it will consume
1423	* scarse resources (R0 and GC) in the mapping cache. When you're done
1424	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1425	*
1426	* This API will assume your intention is to write to the page, and will
1427	* therefore replace shared and zero pages. If you do not intend to modify
1428	* the page, use the PGMPhysGCPtr2CCPtrReadOnly() API.
1429	*
1430	* @returns VBox status code.
1431	* @retval VINF_SUCCESS on success.
1432	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
1433	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
1434	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1435	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1436	*
1437	* @param pVCpu VMCPU handle.
1438	* @param GCPhys The guest physical address of the page that should be mapped.
1439	* @param ppv Where to store the address corresponding to GCPhys.
1440	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1441	*
1442	* @remark Avoid calling this API from within critical sections (other than
1443	* the PGM one) because of the deadlock risk.
1444	* @thread EMT
1445	*/
1446	VMMDECL(int) PGMPhysGCPtr2CCPtr(PVMCPU pVCpu, RTGCPTR GCPtr, void **ppv, PPGMPAGEMAPLOCK pLock)
1447	{
1448	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
1449	RTGCPHYS GCPhys;
1450	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
1451	if (RT_SUCCESS(rc))
1452	rc = PGMPhysGCPhys2CCPtr(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
1453	return rc;
1454	}
1455
1456
1457	/**
1458	* Requests the mapping of a guest page given by virtual address into the current context.
1459	*
1460	* This API should only be used for very short term, as it will consume
1461	* scarse resources (R0 and GC) in the mapping cache. When you're done
1462	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1463	*
1464	* @returns VBox status code.
1465	* @retval VINF_SUCCESS on success.
1466	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
1467	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
1468	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1469	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1470	*
1471	* @param pVCpu VMCPU handle.
1472	* @param GCPhys The guest physical address of the page that should be mapped.
1473	* @param ppv Where to store the address corresponding to GCPhys.
1474	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1475	*
1476	* @remark Avoid calling this API from within critical sections (other than
1477	* the PGM one) because of the deadlock risk.
1478	* @thread EMT
1479	*/
1480	VMMDECL(int) PGMPhysGCPtr2CCPtrReadOnly(PVMCPU pVCpu, RTGCPTR GCPtr, void const **ppv, PPGMPAGEMAPLOCK pLock)
1481	{
1482	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
1483	RTGCPHYS GCPhys;
1484	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
1485	if (RT_SUCCESS(rc))
1486	rc = PGMPhysGCPhys2CCPtrReadOnly(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
1487	return rc;
1488	}
1489
1490
1491	/**
1492	* Release the mapping of a guest page.
1493	*
1494	* This is the counter part of PGMPhysGCPhys2CCPtr, PGMPhysGCPhys2CCPtrReadOnly
1495	* PGMPhysGCPtr2CCPtr and PGMPhysGCPtr2CCPtrReadOnly.
1496	*
1497	* @param pVM The VM handle.
1498	* @param pLock The lock structure initialized by the mapping function.
1499	*/
1500	VMMDECL(void) PGMPhysReleasePageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock)
1501	{
1502	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1503	/* currently nothing to do here. */
1504	Assert(pLock->u32Dummy == UINT32_MAX);
1505	pLock->u32Dummy = 0;
1506
1507	#else /* IN_RING3 */
1508	PPGMPAGEMAP pMap = (PPGMPAGEMAP)pLock->pvMap;
1509	PPGMPAGE pPage = (PPGMPAGE)(pLock->uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
1510	bool fWriteLock = (pLock->uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
1511
1512	pLock->uPageAndType = 0;
1513	pLock->pvMap = NULL;
1514
1515	pgmLock(pVM);
1516	if (fWriteLock)
1517	{
1518	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
1519	Assert(cLocks > 0);
1520	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
1521	{
1522	if (cLocks == 1)
1523	{
1524	Assert(pVM->pgm.s.cWriteLockedPages > 0);
1525	pVM->pgm.s.cWriteLockedPages--;
1526	}
1527	PGM_PAGE_DEC_WRITE_LOCKS(pPage);
1528	}
1529
1530	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
1531	{
1532	PGM_PAGE_SET_WRITTEN_TO(pPage);
1533	PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
1534	Assert(pVM->pgm.s.cMonitoredPages > 0);
1535	pVM->pgm.s.cMonitoredPages--;
1536	pVM->pgm.s.cWrittenToPages++;
1537	}
1538	}
1539	else
1540	{
1541	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
1542	Assert(cLocks > 0);
1543	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
1544	{
1545	if (cLocks == 1)
1546	{
1547	Assert(pVM->pgm.s.cReadLockedPages > 0);
1548	pVM->pgm.s.cReadLockedPages--;
1549	}
1550	PGM_PAGE_DEC_READ_LOCKS(pPage);
1551	}
1552	}
1553
1554	if (pMap)
1555	{
1556	Assert(pMap->cRefs >= 1);
1557	pMap->cRefs--;
1558	pMap->iAge = 0;
1559	}
1560	pgmUnlock(pVM);
1561	#endif /* IN_RING3 */
1562	}
1563
1564
1565	/**
1566	* Converts a GC physical address to a HC ring-3 pointer.
1567	*
1568	* @returns VINF_SUCCESS on success.
1569	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
1570	* page but has no physical backing.
1571	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
1572	* GC physical address.
1573	* @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
1574	* a dynamic ram chunk boundary
1575	*
1576	* @param pVM The VM handle.
1577	* @param GCPhys The GC physical address to convert.
1578	* @param cbRange Physical range
1579	* @param pR3Ptr Where to store the R3 pointer on success.
1580	*
1581	* @deprecated Avoid when possible!
1582	*/
1583	VMMDECL(int) PGMPhysGCPhys2R3Ptr(PVM pVM, RTGCPHYS GCPhys, RTUINT cbRange, PRTR3PTR pR3Ptr)
1584	{
1585	/** @todo this is kind of hacky and needs some more work. */
1586	#ifndef DEBUG_sandervl
1587	VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
1588	#endif
1589
1590	Log(("PGMPhysGCPhys2R3Ptr(,%RGp,%#x,): dont use this API!\n", GCPhys, cbRange)); /** @todo eliminate this API! */
1591	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1592	AssertFailedReturn(VERR_NOT_IMPLEMENTED);
1593	#else
1594	pgmLock(pVM);
1595
1596	PPGMRAMRANGE pRam;
1597	PPGMPAGE pPage;
1598	int rc = pgmPhysGetPageAndRangeEx(&pVM->pgm.s, GCPhys, &pPage, &pRam);
1599	if (RT_SUCCESS(rc))
1600	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, (void **)pR3Ptr);
1601
1602	pgmUnlock(pVM);
1603	Assert(rc <= VINF_SUCCESS);
1604	return rc;
1605	#endif
1606	}
1607
1608
1609	#ifdef VBOX_STRICT
1610	/**
1611	* PGMPhysGCPhys2R3Ptr convenience for use with assertions.
1612	*
1613	* @returns The R3Ptr, NIL_RTR3PTR on failure.
1614	* @param pVM The VM handle.
1615	* @param GCPhys The GC Physical addresss.
1616	* @param cbRange Physical range.
1617	*
1618	* @deprecated Avoid when possible.
1619	*/
1620	VMMDECL(RTR3PTR) PGMPhysGCPhys2R3PtrAssert(PVM pVM, RTGCPHYS GCPhys, RTUINT cbRange)
1621	{
1622	RTR3PTR R3Ptr;
1623	int rc = PGMPhysGCPhys2R3Ptr(pVM, GCPhys, cbRange, &R3Ptr);
1624	if (RT_SUCCESS(rc))
1625	return R3Ptr;
1626	return NIL_RTR3PTR;
1627	}
1628	#endif /* VBOX_STRICT */
1629
1630
1631	/**
1632	* Converts a guest pointer to a GC physical address.
1633	*
1634	* This uses the current CR3/CR0/CR4 of the guest.
1635	*
1636	* @returns VBox status code.
1637	* @param pVCpu The VMCPU Handle
1638	* @param GCPtr The guest pointer to convert.
1639	* @param pGCPhys Where to store the GC physical address.
1640	*/
1641	VMMDECL(int) PGMPhysGCPtr2GCPhys(PVMCPU pVCpu, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
1642	{
1643	int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, pGCPhys);
1644	if (pGCPhys && RT_SUCCESS(rc))
1645	*pGCPhys \|= (RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK;
1646	return rc;
1647	}
1648
1649
1650	/**
1651	* Converts a guest pointer to a HC physical address.
1652	*
1653	* This uses the current CR3/CR0/CR4 of the guest.
1654	*
1655	* @returns VBox status code.
1656	* @param pVCpu The VMCPU Handle
1657	* @param GCPtr The guest pointer to convert.
1658	* @param pHCPhys Where to store the HC physical address.
1659	*/
1660	VMMDECL(int) PGMPhysGCPtr2HCPhys(PVMCPU pVCpu, RTGCPTR GCPtr, PRTHCPHYS pHCPhys)
1661	{
1662	PVM pVM = pVCpu->CTX_SUFF(pVM);
1663	RTGCPHYS GCPhys;
1664	int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, &GCPhys);
1665	if (RT_SUCCESS(rc))
1666	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys \| ((RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK), pHCPhys);
1667	return rc;
1668	}
1669
1670
1671	/**
1672	* Converts a guest pointer to a R3 pointer.
1673	*
1674	* This uses the current CR3/CR0/CR4 of the guest.
1675	*
1676	* @returns VBox status code.
1677	* @param pVCpu The VMCPU Handle
1678	* @param GCPtr The guest pointer to convert.
1679	* @param pR3Ptr Where to store the R3 virtual address.
1680	*
1681	* @deprecated Don't use this.
1682	*/
1683	VMMDECL(int) PGMPhysGCPtr2R3Ptr(PVMCPU pVCpu, RTGCPTR GCPtr, PRTR3PTR pR3Ptr)
1684	{
1685	PVM pVM = pVCpu->CTX_SUFF(pVM);
1686	VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
1687	RTGCPHYS GCPhys;
1688	int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, &GCPhys);
1689	if (RT_SUCCESS(rc))
1690	rc = PGMPhysGCPhys2R3Ptr(pVM, GCPhys \| ((RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK), 1 /* we always stay within one page */, pR3Ptr);
1691	return rc;
1692	}
1693
1694
1695
1696	#undef LOG_GROUP
1697	#define LOG_GROUP LOG_GROUP_PGM_PHYS_ACCESS
1698
1699
1700	#ifdef IN_RING3
1701	/**
1702	* Cache PGMPhys memory access
1703	*
1704	* @param pVM VM Handle.
1705	* @param pCache Cache structure pointer
1706	* @param GCPhys GC physical address
1707	* @param pbHC HC pointer corresponding to physical page
1708	*
1709	* @thread EMT.
1710	*/
1711	static void pgmPhysCacheAdd(PVM pVM, PGMPHYSCACHE pCache, RTGCPHYS GCPhys, uint8_t pbR3)
1712	{
1713	uint32_t iCacheIndex;
1714
1715	Assert(VM_IS_EMT(pVM));
1716
1717	GCPhys = PHYS_PAGE_ADDRESS(GCPhys);
1718	pbR3 = (uint8_t *)PAGE_ADDRESS(pbR3);
1719
1720	iCacheIndex = ((GCPhys >> PAGE_SHIFT) & PGM_MAX_PHYSCACHE_ENTRIES_MASK);
1721
1722	ASMBitSet(&pCache->aEntries, iCacheIndex);
1723
1724	pCache->Entry[iCacheIndex].GCPhys = GCPhys;
1725	pCache->Entry[iCacheIndex].pbR3 = pbR3;
1726	}
1727	#endif /* IN_RING3 */
1728
1729
1730	/**
1731	* Deals with reading from a page with one or more ALL access handlers.
1732	*
1733	* @returns VBox status code. Can be ignored in ring-3.
1734	* @retval VINF_SUCCESS.
1735	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
1736	*
1737	* @param pVM The VM handle.
1738	* @param pPage The page descriptor.
1739	* @param GCPhys The physical address to start reading at.
1740	* @param pvBuf Where to put the bits we read.
1741	* @param cb How much to read - less or equal to a page.
1742	*/
1743	static int pgmPhysReadHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void *pvBuf, size_t cb)
1744	{
1745	/*
1746	* The most frequent access here is MMIO and shadowed ROM.
1747	* The current code ASSUMES all these access handlers covers full pages!
1748	*/
1749
1750	/*
1751	* Whatever we do we need the source page, map it first.
1752	*/
1753	const void *pvSrc = NULL;
1754	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvSrc);
1755	if (RT_FAILURE(rc))
1756	{
1757	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
1758	GCPhys, pPage, rc));
1759	memset(pvBuf, 0xff, cb);
1760	return VINF_SUCCESS;
1761	}
1762	rc = VINF_PGM_HANDLER_DO_DEFAULT;
1763
1764	/*
1765	* Deal with any physical handlers.
1766	*/
1767	PPGMPHYSHANDLER pPhys = NULL;
1768	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL)
1769	{
1770	#ifdef IN_RING3
1771	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
1772	AssertReleaseMsg(pPhys, ("GCPhys=%RGp cb=%#x\n", GCPhys, cb));
1773	Assert(GCPhys >= pPhys->Core.Key && GCPhys <= pPhys->Core.KeyLast);
1774	Assert((pPhys->Core.Key & PAGE_OFFSET_MASK) == 0);
1775	Assert((pPhys->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
1776	Assert(pPhys->CTX_SUFF(pfnHandler));
1777
1778	PFNPGMR3PHYSHANDLER pfnHandler = pPhys->CTX_SUFF(pfnHandler);
1779	void *pvUser = pPhys->CTX_SUFF(pvUser);
1780
1781	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cb, pPage, R3STRING(pPhys->pszDesc) ));
1782	STAM_PROFILE_START(&pPhys->Stat, h);
1783	Assert(PGMIsLockOwner(pVM));
1784	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
1785	pgmUnlock(pVM);
1786	rc = pfnHandler(pVM, GCPhys, (void *)pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, pvUser);
1787	pgmLock(pVM);
1788	# ifdef VBOX_WITH_STATISTICS
1789	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
1790	if (pPhys)
1791	STAM_PROFILE_STOP(&pPhys->Stat, h);
1792	# else
1793	pPhys = NULL; /* might not be valid anymore. */
1794	# endif
1795	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp\n", rc, GCPhys));
1796	#else
1797	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
1798	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cb=%#x\n", GCPhys, cb));
1799	return VERR_PGM_PHYS_WR_HIT_HANDLER;
1800	#endif
1801	}
1802
1803	/*
1804	* Deal with any virtual handlers.
1805	*/
1806	if (PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) == PGM_PAGE_HNDL_VIRT_STATE_ALL)
1807	{
1808	unsigned iPage;
1809	PPGMVIRTHANDLER pVirt;
1810
1811	int rc2 = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pVirt, &iPage);
1812	AssertReleaseMsg(RT_SUCCESS(rc2), ("GCPhys=%RGp cb=%#x rc2=%Rrc\n", GCPhys, cb, rc2));
1813	Assert((pVirt->Core.Key & PAGE_OFFSET_MASK) == 0);
1814	Assert((pVirt->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
1815	Assert(GCPhys >= pVirt->aPhysToVirt[iPage].Core.Key && GCPhys <= pVirt->aPhysToVirt[iPage].Core.KeyLast);
1816
1817	#ifdef IN_RING3
1818	if (pVirt->pfnHandlerR3)
1819	{
1820	if (!pPhys)
1821	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] virt %s\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc) ));
1822	else
1823	Log(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys/virt %s/%s\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc), R3STRING(pPhys->pszDesc) ));
1824	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
1825	+ (iPage << PAGE_SHIFT)
1826	+ (GCPhys & PAGE_OFFSET_MASK);
1827
1828	STAM_PROFILE_START(&pVirt->Stat, h);
1829	rc2 = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, (void )pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, /pVirt->CTX_SUFF(pvUser)*/ NULL);
1830	STAM_PROFILE_STOP(&pVirt->Stat, h);
1831	if (rc2 == VINF_SUCCESS)
1832	rc = VINF_SUCCESS;
1833	AssertLogRelMsg(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc2, GCPhys, pPage, pVirt->pszDesc));
1834	}
1835	else
1836	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] virt %s [no handler]\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc) ));
1837	#else
1838	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
1839	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cb=%#x\n", GCPhys, cb));
1840	return VERR_PGM_PHYS_WR_HIT_HANDLER;
1841	#endif
1842	}
1843
1844	/*
1845	* Take the default action.
1846	*/
1847	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
1848	memcpy(pvBuf, pvSrc, cb);
1849	return rc;
1850	}
1851
1852
1853	/**
1854	* Read physical memory.
1855	*
1856	* This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
1857	* want to ignore those.
1858	*
1859	* @returns VBox status code. Can be ignored in ring-3.
1860	* @retval VINF_SUCCESS.
1861	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
1862	*
1863	* @param pVM VM Handle.
1864	* @param GCPhys Physical address start reading from.
1865	* @param pvBuf Where to put the read bits.
1866	* @param cbRead How many bytes to read.
1867	*/
1868	VMMDECL(int) PGMPhysRead(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
1869	{
1870	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
1871	LogFlow(("PGMPhysRead: %RGp %d\n", GCPhys, cbRead));
1872
1873	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PhysRead));
1874	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_MID_Z(Stat,PhysReadBytes), cbRead);
1875
1876	pgmLock(pVM);
1877
1878	/*
1879	* Copy loop on ram ranges.
1880	*/
1881	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
1882	for (;;)
1883	{
1884	/* Find range. */
1885	while (pRam && GCPhys > pRam->GCPhysLast)
1886	pRam = pRam->CTX_SUFF(pNext);
1887	/* Inside range or not? */
1888	if (pRam && GCPhys >= pRam->GCPhys)
1889	{
1890	/*
1891	* Must work our way thru this page by page.
1892	*/
1893	RTGCPHYS off = GCPhys - pRam->GCPhys;
1894	while (off < pRam->cb)
1895	{
1896	unsigned iPage = off >> PAGE_SHIFT;
1897	PPGMPAGE pPage = &pRam->aPages[iPage];
1898	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
1899	if (cb > cbRead)
1900	cb = cbRead;
1901
1902	/*
1903	* Any ALL access handlers?
1904	*/
1905	if (RT_UNLIKELY(PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)))
1906	{
1907	int rc = pgmPhysReadHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb);
1908	if (RT_FAILURE(rc))
1909	{
1910	pgmUnlock(pVM);
1911	return rc;
1912	}
1913	}
1914	else
1915	{
1916	/*
1917	* Get the pointer to the page.
1918	*/
1919	const void *pvSrc;
1920	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc);
1921	if (RT_SUCCESS(rc))
1922	memcpy(pvBuf, pvSrc, cb);
1923	else
1924	{
1925	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
1926	pRam->GCPhys + off, pPage, rc));
1927	memset(pvBuf, 0xff, cb);
1928	}
1929	}
1930
1931	/* next page */
1932	if (cb >= cbRead)
1933	{
1934	pgmUnlock(pVM);
1935	return VINF_SUCCESS;
1936	}
1937	cbRead -= cb;
1938	off += cb;
1939	pvBuf = (char *)pvBuf + cb;
1940	} /* walk pages in ram range. */
1941
1942	GCPhys = pRam->GCPhysLast + 1;
1943	}
1944	else
1945	{
1946	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
1947
1948	/*
1949	* Unassigned address space.
1950	*/
1951	if (!pRam)
1952	break;
1953	size_t cb = pRam->GCPhys - GCPhys;
1954	if (cb >= cbRead)
1955	{
1956	memset(pvBuf, 0xff, cbRead);
1957	break;
1958	}
1959	memset(pvBuf, 0xff, cb);
1960
1961	cbRead -= cb;
1962	pvBuf = (char *)pvBuf + cb;
1963	GCPhys += cb;
1964	}
1965	} /* Ram range walk */
1966
1967	pgmUnlock(pVM);
1968	return VINF_SUCCESS;
1969	}
1970
1971
1972	/**
1973	* Deals with writing to a page with one or more WRITE or ALL access handlers.
1974	*
1975	* @returns VBox status code. Can be ignored in ring-3.
1976	* @retval VINF_SUCCESS.
1977	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
1978	*
1979	* @param pVM The VM handle.
1980	* @param pPage The page descriptor.
1981	* @param GCPhys The physical address to start writing at.
1982	* @param pvBuf What to write.
1983	* @param cbWrite How much to write - less or equal to a page.
1984	*/
1985	static int pgmPhysWriteHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const *pvBuf, size_t cbWrite)
1986	{
1987	void *pvDst = NULL;
1988	int rc;
1989
1990	/*
1991	* Give priority to physical handlers (like #PF does).
1992	*
1993	* Hope for a lonely physical handler first that covers the whole
1994	* write area. This should be a pretty frequent case with MMIO and
1995	* the heavy usage of full page handlers in the page pool.
1996	*/
1997	if ( !PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage)
1998	\|\| PGM_PAGE_IS_MMIO(pPage) /* screw virtual handlers on MMIO pages */)
1999	{
2000	PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
2001	if (pCur)
2002	{
2003	Assert(GCPhys >= pCur->Core.Key && GCPhys <= pCur->Core.KeyLast);
2004	Assert(pCur->CTX_SUFF(pfnHandler));
2005
2006	size_t cbRange = pCur->Core.KeyLast - GCPhys + 1;
2007	if (cbRange > cbWrite)
2008	cbRange = cbWrite;
2009
2010	#ifndef IN_RING3
2011	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2012	NOREF(cbRange);
2013	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2014	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2015
2016	#else /* IN_RING3 */
2017	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
2018	if (!PGM_PAGE_IS_MMIO(pPage))
2019	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst);
2020	else
2021	rc = VINF_SUCCESS;
2022	if (RT_SUCCESS(rc))
2023	{
2024	PFNPGMR3PHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
2025	void *pvUser = pCur->CTX_SUFF(pvUser);
2026
2027	STAM_PROFILE_START(&pCur->Stat, h);
2028	Assert(PGMIsLockOwner(pVM));
2029	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2030	pgmUnlock(pVM);
2031	rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2032	pgmLock(pVM);
2033	# ifdef VBOX_WITH_STATISTICS
2034	pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
2035	if (pCur)
2036	STAM_PROFILE_STOP(&pCur->Stat, h);
2037	# else
2038	pCur = NULL; /* might not be valid anymore. */
2039	# endif
2040	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
2041	memcpy(pvDst, pvBuf, cbRange);
2042	else
2043	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, (pCur) ? pCur->pszDesc : ""));
2044	}
2045	else
2046	AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2047	GCPhys, pPage, rc), rc);
2048	if (RT_LIKELY(cbRange == cbWrite))
2049	return VINF_SUCCESS;
2050
2051	/* more fun to be had below */
2052	cbWrite -= cbRange;
2053	GCPhys += cbRange;
2054	pvBuf = (uint8_t *)pvBuf + cbRange;
2055	pvDst = (uint8_t *)pvDst + cbRange;
2056	#endif /* IN_RING3 */
2057	}
2058	/* else: the handler is somewhere else in the page, deal with it below. */
2059	Assert(!PGM_PAGE_IS_MMIO(pPage)); /* MMIO handlers are all PAGE_SIZEed! */
2060	}
2061	/*
2062	* A virtual handler without any interfering physical handlers.
2063	* Hopefully it'll conver the whole write.
2064	*/
2065	else if (!PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage))
2066	{
2067	unsigned iPage;
2068	PPGMVIRTHANDLER pCur;
2069	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pCur, &iPage);
2070	if (RT_SUCCESS(rc))
2071	{
2072	size_t cbRange = (PAGE_OFFSET_MASK & pCur->Core.KeyLast) - (PAGE_OFFSET_MASK & GCPhys) + 1;
2073	if (cbRange > cbWrite)
2074	cbRange = cbWrite;
2075
2076	#ifndef IN_RING3
2077	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2078	NOREF(cbRange);
2079	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2080	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2081
2082	#else /* IN_RING3 */
2083
2084	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] virt %s\n", GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
2085	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst);
2086	if (RT_SUCCESS(rc))
2087	{
2088	rc = VINF_PGM_HANDLER_DO_DEFAULT;
2089	if (pCur->pfnHandlerR3)
2090	{
2091	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pCur->Core.Key & PAGE_BASE_GC_MASK)
2092	+ (iPage << PAGE_SHIFT)
2093	+ (GCPhys & PAGE_OFFSET_MASK);
2094
2095	STAM_PROFILE_START(&pCur->Stat, h);
2096	rc = pCur->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void )pvBuf, cbRange, PGMACCESSTYPE_WRITE, /pCur->CTX_SUFF(pvUser)*/ NULL);
2097	STAM_PROFILE_STOP(&pCur->Stat, h);
2098	}
2099	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
2100	memcpy(pvDst, pvBuf, cbRange);
2101	else
2102	AssertLogRelMsg(rc == VINF_SUCCESS, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pCur->pszDesc));
2103	}
2104	else
2105	AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2106	GCPhys, pPage, rc), rc);
2107	if (RT_LIKELY(cbRange == cbWrite))
2108	return VINF_SUCCESS;
2109
2110	/* more fun to be had below */
2111	cbWrite -= cbRange;
2112	GCPhys += cbRange;
2113	pvBuf = (uint8_t *)pvBuf + cbRange;
2114	pvDst = (uint8_t *)pvDst + cbRange;
2115	#endif
2116	}
2117	/* else: the handler is somewhere else in the page, deal with it below. */
2118	}
2119
2120	/*
2121	* Deal with all the odd ends.
2122	*/
2123
2124	/* We need a writable destination page. */
2125	if (!pvDst)
2126	{
2127	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst);
2128	AssertLogRelMsgReturn(RT_SUCCESS(rc),
2129	("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2130	GCPhys, pPage, rc), rc);
2131	}
2132
2133	/* The loop state (big + ugly). */
2134	unsigned iVirtPage = 0;
2135	PPGMVIRTHANDLER pVirt = NULL;
2136	uint32_t offVirt = PAGE_SIZE;
2137	uint32_t offVirtLast = PAGE_SIZE;
2138	bool fMoreVirt = PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage);
2139
2140	PPGMPHYSHANDLER pPhys = NULL;
2141	uint32_t offPhys = PAGE_SIZE;
2142	uint32_t offPhysLast = PAGE_SIZE;
2143	bool fMorePhys = PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage);
2144
2145	/* The loop. */
2146	for (;;)
2147	{
2148	/*
2149	* Find the closest handler at or above GCPhys.
2150	*/
2151	if (fMoreVirt && !pVirt)
2152	{
2153	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pVirt, &iVirtPage);
2154	if (RT_SUCCESS(rc))
2155	{
2156	offVirt = 0;
2157	offVirtLast = (pVirt->aPhysToVirt[iVirtPage].Core.KeyLast & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2158	}
2159	else
2160	{
2161	PPGMPHYS2VIRTHANDLER pVirtPhys;
2162	pVirtPhys = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysToVirtHandlers,
2163	GCPhys, true /* fAbove */);
2164	if ( pVirtPhys
2165	&& (pVirtPhys->Core.Key >> PAGE_SHIFT) == (GCPhys >> PAGE_SHIFT))
2166	{
2167	/* ASSUME that pVirtPhys only covers one page. */
2168	Assert((pVirtPhys->Core.Key >> PAGE_SHIFT) == (pVirtPhys->Core.KeyLast >> PAGE_SHIFT));
2169	Assert(pVirtPhys->Core.Key > GCPhys);
2170
2171	pVirt = (PPGMVIRTHANDLER)((uintptr_t)pVirtPhys + pVirtPhys->offVirtHandler);
2172	iVirtPage = pVirtPhys - &pVirt->aPhysToVirt[0]; Assert(iVirtPage == 0);
2173	offVirt = (pVirtPhys->Core.Key & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2174	offVirtLast = (pVirtPhys->Core.KeyLast & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2175	}
2176	else
2177	{
2178	pVirt = NULL;
2179	fMoreVirt = false;
2180	offVirt = offVirtLast = PAGE_SIZE;
2181	}
2182	}
2183	}
2184
2185	if (fMorePhys && !pPhys)
2186	{
2187	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
2188	if (pPhys)
2189	{
2190	offPhys = 0;
2191	offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
2192	}
2193	else
2194	{
2195	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers,
2196	GCPhys, true /* fAbove */);
2197	if ( pPhys
2198	&& pPhys->Core.Key <= GCPhys + (cbWrite - 1))
2199	{
2200	offPhys = pPhys->Core.Key - GCPhys;
2201	offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
2202	}
2203	else
2204	{
2205	pPhys = NULL;
2206	fMorePhys = false;
2207	offPhys = offPhysLast = PAGE_SIZE;
2208	}
2209	}
2210	}
2211
2212	/*
2213	* Handle access to space without handlers (that's easy).
2214	*/
2215	rc = VINF_PGM_HANDLER_DO_DEFAULT;
2216	uint32_t cbRange = (uint32_t)cbWrite;
2217	if (offPhys && offVirt)
2218	{
2219	if (cbRange > offPhys)
2220	cbRange = offPhys;
2221	if (cbRange > offVirt)
2222	cbRange = offVirt;
2223	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] miss\n", GCPhys, cbRange, pPage));
2224	}
2225	/*
2226	* Physical handler.
2227	*/
2228	else if (!offPhys && offVirt)
2229	{
2230	if (cbRange > offPhysLast + 1)
2231	cbRange = offPhysLast + 1;
2232	if (cbRange > offVirt)
2233	cbRange = offVirt;
2234	#ifdef IN_RING3
2235	PFNPGMR3PHYSHANDLER pfnHandler = pPhys->CTX_SUFF(pfnHandler);
2236	void *pvUser = pPhys->CTX_SUFF(pvUser);
2237
2238	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc) ));
2239	STAM_PROFILE_START(&pPhys->Stat, h);
2240	Assert(PGMIsLockOwner(pVM));
2241	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2242	pgmUnlock(pVM);
2243	rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2244	pgmLock(pVM);
2245	# ifdef VBOX_WITH_STATISTICS
2246	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
2247	if (pPhys)
2248	STAM_PROFILE_STOP(&pPhys->Stat, h);
2249	# else
2250	pPhys = NULL; /* might not be valid anymore. */
2251	# endif
2252	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, (pPhys) ? pPhys->pszDesc : ""));
2253	#else
2254	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2255	NOREF(cbRange);
2256	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2257	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2258	#endif
2259	}
2260	/*
2261	* Virtual handler.
2262	*/
2263	else if (offPhys && !offVirt)
2264	{
2265	if (cbRange > offVirtLast + 1)
2266	cbRange = offVirtLast + 1;
2267	if (cbRange > offPhys)
2268	cbRange = offPhys;
2269	#ifdef IN_RING3
2270	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pVirt->pszDesc) ));
2271	if (pVirt->pfnHandlerR3)
2272	{
2273	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2274	+ (iVirtPage << PAGE_SHIFT)
2275	+ (GCPhys & PAGE_OFFSET_MASK);
2276	STAM_PROFILE_START(&pVirt->Stat, h);
2277	rc = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void )pvBuf, cbRange, PGMACCESSTYPE_WRITE, /pCur->CTX_SUFF(pvUser)*/ NULL);
2278	STAM_PROFILE_STOP(&pVirt->Stat, h);
2279	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pVirt->pszDesc));
2280	}
2281	pVirt = NULL;
2282	#else
2283	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2284	NOREF(cbRange);
2285	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2286	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2287	#endif
2288	}
2289	/*
2290	* Both... give the physical one priority.
2291	*/
2292	else
2293	{
2294	Assert(!offPhys && !offVirt);
2295	if (cbRange > offVirtLast + 1)
2296	cbRange = offVirtLast + 1;
2297	if (cbRange > offPhysLast + 1)
2298	cbRange = offPhysLast + 1;
2299
2300	#ifdef IN_RING3
2301	if (pVirt->pfnHandlerR3)
2302	Log(("pgmPhysWriteHandler: overlapping phys and virt handlers at %RGp %R[pgmpage]; cbRange=%#x\n", GCPhys, pPage, cbRange));
2303	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys/virt %s/%s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc), R3STRING(pVirt->pszDesc) ));
2304
2305	PFNPGMR3PHYSHANDLER pfnHandler = pPhys->CTX_SUFF(pfnHandler);
2306	void *pvUser = pPhys->CTX_SUFF(pvUser);
2307
2308	STAM_PROFILE_START(&pPhys->Stat, h);
2309	Assert(PGMIsLockOwner(pVM));
2310	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2311	pgmUnlock(pVM);
2312	rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2313	pgmLock(pVM);
2314	# ifdef VBOX_WITH_STATISTICS
2315	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
2316	if (pPhys)
2317	STAM_PROFILE_STOP(&pPhys->Stat, h);
2318	# else
2319	pPhys = NULL; /* might not be valid anymore. */
2320	# endif
2321	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, (pPhys) ? pPhys->pszDesc : ""));
2322	if (pVirt->pfnHandlerR3)
2323	{
2324
2325	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2326	+ (iVirtPage << PAGE_SHIFT)
2327	+ (GCPhys & PAGE_OFFSET_MASK);
2328	STAM_PROFILE_START(&pVirt->Stat, h2);
2329	int rc2 = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void )pvBuf, cbRange, PGMACCESSTYPE_WRITE, /pCur->CTX_SUFF(pvUser)*/ NULL);
2330	STAM_PROFILE_STOP(&pVirt->Stat, h2);
2331	if (rc2 == VINF_SUCCESS && rc == VINF_PGM_HANDLER_DO_DEFAULT)
2332	rc = VINF_SUCCESS;
2333	else
2334	AssertLogRelMsg(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pVirt->pszDesc));
2335	}
2336	pPhys = NULL;
2337	pVirt = NULL;
2338	#else
2339	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2340	NOREF(cbRange);
2341	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2342	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2343	#endif
2344	}
2345	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
2346	memcpy(pvDst, pvBuf, cbRange);
2347
2348	/*
2349	* Advance if we've got more stuff to do.
2350	*/
2351	if (cbRange >= cbWrite)
2352	return VINF_SUCCESS;
2353
2354	cbWrite -= cbRange;
2355	GCPhys += cbRange;
2356	pvBuf = (uint8_t *)pvBuf + cbRange;
2357	pvDst = (uint8_t *)pvDst + cbRange;
2358
2359	offPhys -= cbRange;
2360	offPhysLast -= cbRange;
2361	offVirt -= cbRange;
2362	offVirtLast -= cbRange;
2363	}
2364	}
2365
2366
2367	/**
2368	* Write to physical memory.
2369	*
2370	* This API respects access handlers and MMIO. Use PGMPhysSimpleWriteGCPhys() if you
2371	* want to ignore those.
2372	*
2373	* @returns VBox status code. Can be ignored in ring-3.
2374	* @retval VINF_SUCCESS.
2375	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
2376	*
2377	* @param pVM VM Handle.
2378	* @param GCPhys Physical address to write to.
2379	* @param pvBuf What to write.
2380	* @param cbWrite How many bytes to write.
2381	*/
2382	VMMDECL(int) PGMPhysWrite(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite)
2383	{
2384	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMPhysWrite after pgmR3Save()!\n"));
2385	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
2386	LogFlow(("PGMPhysWrite: %RGp %d\n", GCPhys, cbWrite));
2387
2388	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PhysWrite));
2389	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_MID_Z(Stat,PhysWriteBytes), cbWrite);
2390
2391	pgmLock(pVM);
2392
2393	/*
2394	* Copy loop on ram ranges.
2395	*/
2396	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2397	for (;;)
2398	{
2399	/* Find range. */
2400	while (pRam && GCPhys > pRam->GCPhysLast)
2401	pRam = pRam->CTX_SUFF(pNext);
2402	/* Inside range or not? */
2403	if (pRam && GCPhys >= pRam->GCPhys)
2404	{
2405	/*
2406	* Must work our way thru this page by page.
2407	*/
2408	RTGCPTR off = GCPhys - pRam->GCPhys;
2409	while (off < pRam->cb)
2410	{
2411	RTGCPTR iPage = off >> PAGE_SHIFT;
2412	PPGMPAGE pPage = &pRam->aPages[iPage];
2413	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
2414	if (cb > cbWrite)
2415	cb = cbWrite;
2416
2417	/*
2418	* Any active WRITE or ALL access handlers?
2419	*/
2420	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2421	{
2422	int rc = pgmPhysWriteHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb);
2423	if (RT_FAILURE(rc))
2424	{
2425	pgmUnlock(pVM);
2426	return rc;
2427	}
2428	}
2429	else
2430	{
2431	/*
2432	* Get the pointer to the page.
2433	*/
2434	void *pvDst;
2435	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst);
2436	if (RT_SUCCESS(rc))
2437	{
2438	Assert(!PGM_PAGE_IS_BALLOONED(pPage));
2439	memcpy(pvDst, pvBuf, cb);
2440	}
2441	else
2442	/* Ignore writes to ballooned pages. */
2443	if (!PGM_PAGE_IS_BALLOONED(pPage))
2444	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2445	pRam->GCPhys + off, pPage, rc));
2446	}
2447
2448	/* next page */
2449	if (cb >= cbWrite)
2450	{
2451	pgmUnlock(pVM);
2452	return VINF_SUCCESS;
2453	}
2454
2455	cbWrite -= cb;
2456	off += cb;
2457	pvBuf = (const char *)pvBuf + cb;
2458	} /* walk pages in ram range */
2459
2460	GCPhys = pRam->GCPhysLast + 1;
2461	}
2462	else
2463	{
2464	/*
2465	* Unassigned address space, skip it.
2466	*/
2467	if (!pRam)
2468	break;
2469	size_t cb = pRam->GCPhys - GCPhys;
2470	if (cb >= cbWrite)
2471	break;
2472	cbWrite -= cb;
2473	pvBuf = (const char *)pvBuf + cb;
2474	GCPhys += cb;
2475	}
2476	} /* Ram range walk */
2477
2478	pgmUnlock(pVM);
2479	return VINF_SUCCESS;
2480	}
2481
2482
2483	/**
2484	* Read from guest physical memory by GC physical address, bypassing
2485	* MMIO and access handlers.
2486	*
2487	* @returns VBox status.
2488	* @param pVM VM handle.
2489	* @param pvDst The destination address.
2490	* @param GCPhysSrc The source address (GC physical address).
2491	* @param cb The number of bytes to read.
2492	*/
2493	VMMDECL(int) PGMPhysSimpleReadGCPhys(PVM pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb)
2494	{
2495	/*
2496	* Treat the first page as a special case.
2497	*/
2498	if (!cb)
2499	return VINF_SUCCESS;
2500
2501	/* map the 1st page */
2502	void const *pvSrc;
2503	PGMPAGEMAPLOCK Lock;
2504	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
2505	if (RT_FAILURE(rc))
2506	return rc;
2507
2508	/* optimize for the case where access is completely within the first page. */
2509	size_t cbPage = PAGE_SIZE - (GCPhysSrc & PAGE_OFFSET_MASK);
2510	if (RT_LIKELY(cb <= cbPage))
2511	{
2512	memcpy(pvDst, pvSrc, cb);
2513	PGMPhysReleasePageMappingLock(pVM, &Lock);
2514	return VINF_SUCCESS;
2515	}
2516
2517	/* copy to the end of the page. */
2518	memcpy(pvDst, pvSrc, cbPage);
2519	PGMPhysReleasePageMappingLock(pVM, &Lock);
2520	GCPhysSrc += cbPage;
2521	pvDst = (uint8_t *)pvDst + cbPage;
2522	cb -= cbPage;
2523
2524	/*
2525	* Page by page.
2526	*/
2527	for (;;)
2528	{
2529	/* map the page */
2530	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
2531	if (RT_FAILURE(rc))
2532	return rc;
2533
2534	/* last page? */
2535	if (cb <= PAGE_SIZE)
2536	{
2537	memcpy(pvDst, pvSrc, cb);
2538	PGMPhysReleasePageMappingLock(pVM, &Lock);
2539	return VINF_SUCCESS;
2540	}
2541
2542	/* copy the entire page and advance */
2543	memcpy(pvDst, pvSrc, PAGE_SIZE);
2544	PGMPhysReleasePageMappingLock(pVM, &Lock);
2545	GCPhysSrc += PAGE_SIZE;
2546	pvDst = (uint8_t *)pvDst + PAGE_SIZE;
2547	cb -= PAGE_SIZE;
2548	}
2549	/* won't ever get here. */
2550	}
2551
2552
2553	/**
2554	* Write to guest physical memory referenced by GC pointer.
2555	* Write memory to GC physical address in guest physical memory.
2556	*
2557	* This will bypass MMIO and access handlers.
2558	*
2559	* @returns VBox status.
2560	* @param pVM VM handle.
2561	* @param GCPhysDst The GC physical address of the destination.
2562	* @param pvSrc The source buffer.
2563	* @param cb The number of bytes to write.
2564	*/
2565	VMMDECL(int) PGMPhysSimpleWriteGCPhys(PVM pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb)
2566	{
2567	LogFlow(("PGMPhysSimpleWriteGCPhys: %RGp %zu\n", GCPhysDst, cb));
2568
2569	/*
2570	* Treat the first page as a special case.
2571	*/
2572	if (!cb)
2573	return VINF_SUCCESS;
2574
2575	/* map the 1st page */
2576	void *pvDst;
2577	PGMPAGEMAPLOCK Lock;
2578	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
2579	if (RT_FAILURE(rc))
2580	return rc;
2581
2582	/* optimize for the case where access is completely within the first page. */
2583	size_t cbPage = PAGE_SIZE - (GCPhysDst & PAGE_OFFSET_MASK);
2584	if (RT_LIKELY(cb <= cbPage))
2585	{
2586	memcpy(pvDst, pvSrc, cb);
2587	PGMPhysReleasePageMappingLock(pVM, &Lock);
2588	return VINF_SUCCESS;
2589	}
2590
2591	/* copy to the end of the page. */
2592	memcpy(pvDst, pvSrc, cbPage);
2593	PGMPhysReleasePageMappingLock(pVM, &Lock);
2594	GCPhysDst += cbPage;
2595	pvSrc = (const uint8_t *)pvSrc + cbPage;
2596	cb -= cbPage;
2597
2598	/*
2599	* Page by page.
2600	*/
2601	for (;;)
2602	{
2603	/* map the page */
2604	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
2605	if (RT_FAILURE(rc))
2606	return rc;
2607
2608	/* last page? */
2609	if (cb <= PAGE_SIZE)
2610	{
2611	memcpy(pvDst, pvSrc, cb);
2612	PGMPhysReleasePageMappingLock(pVM, &Lock);
2613	return VINF_SUCCESS;
2614	}
2615
2616	/* copy the entire page and advance */
2617	memcpy(pvDst, pvSrc, PAGE_SIZE);
2618	PGMPhysReleasePageMappingLock(pVM, &Lock);
2619	GCPhysDst += PAGE_SIZE;
2620	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
2621	cb -= PAGE_SIZE;
2622	}
2623	/* won't ever get here. */
2624	}
2625
2626
2627	/**
2628	* Read from guest physical memory referenced by GC pointer.
2629	*
2630	* This function uses the current CR3/CR0/CR4 of the guest and will
2631	* bypass access handlers and not set any accessed bits.
2632	*
2633	* @returns VBox status.
2634	* @param pVCpu The VMCPU handle.
2635	* @param pvDst The destination address.
2636	* @param GCPtrSrc The source address (GC pointer).
2637	* @param cb The number of bytes to read.
2638	*/
2639	VMMDECL(int) PGMPhysSimpleReadGCPtr(PVMCPU pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
2640	{
2641	PVM pVM = pVCpu->CTX_SUFF(pVM);
2642
2643	/*
2644	* Treat the first page as a special case.
2645	*/
2646	if (!cb)
2647	return VINF_SUCCESS;
2648
2649	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PhysSimpleRead));
2650	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_MID_Z(Stat,PhysSimpleReadBytes), cb);
2651
2652	/* Take the PGM lock here, because many called functions take the lock for a very short period. That's counter-productive
2653	* when many VCPUs are fighting for the lock.
2654	*/
2655	pgmLock(pVM);
2656
2657	/* map the 1st page */
2658	void const *pvSrc;
2659	PGMPAGEMAPLOCK Lock;
2660	int rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
2661	if (RT_FAILURE(rc))
2662	{
2663	pgmUnlock(pVM);
2664	return rc;
2665	}
2666
2667	/* optimize for the case where access is completely within the first page. */
2668	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
2669	if (RT_LIKELY(cb <= cbPage))
2670	{
2671	memcpy(pvDst, pvSrc, cb);
2672	PGMPhysReleasePageMappingLock(pVM, &Lock);
2673	pgmUnlock(pVM);
2674	return VINF_SUCCESS;
2675	}
2676
2677	/* copy to the end of the page. */
2678	memcpy(pvDst, pvSrc, cbPage);
2679	PGMPhysReleasePageMappingLock(pVM, &Lock);
2680	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + cbPage);
2681	pvDst = (uint8_t *)pvDst + cbPage;
2682	cb -= cbPage;
2683
2684	/*
2685	* Page by page.
2686	*/
2687	for (;;)
2688	{
2689	/* map the page */
2690	rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
2691	if (RT_FAILURE(rc))
2692	{
2693	pgmUnlock(pVM);
2694	return rc;
2695	}
2696
2697	/* last page? */
2698	if (cb <= PAGE_SIZE)
2699	{
2700	memcpy(pvDst, pvSrc, cb);
2701	PGMPhysReleasePageMappingLock(pVM, &Lock);
2702	pgmUnlock(pVM);
2703	return VINF_SUCCESS;
2704	}
2705
2706	/* copy the entire page and advance */
2707	memcpy(pvDst, pvSrc, PAGE_SIZE);
2708	PGMPhysReleasePageMappingLock(pVM, &Lock);
2709	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + PAGE_SIZE);
2710	pvDst = (uint8_t *)pvDst + PAGE_SIZE;
2711	cb -= PAGE_SIZE;
2712	}
2713	/* won't ever get here. */
2714	}
2715
2716
2717	/**
2718	* Write to guest physical memory referenced by GC pointer.
2719	*
2720	* This function uses the current CR3/CR0/CR4 of the guest and will
2721	* bypass access handlers and not set dirty or accessed bits.
2722	*
2723	* @returns VBox status.
2724	* @param pVCpu The VMCPU handle.
2725	* @param GCPtrDst The destination address (GC pointer).
2726	* @param pvSrc The source address.
2727	* @param cb The number of bytes to write.
2728	*/
2729	VMMDECL(int) PGMPhysSimpleWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
2730	{
2731	PVM pVM = pVCpu->CTX_SUFF(pVM);
2732
2733	/*
2734	* Treat the first page as a special case.
2735	*/
2736	if (!cb)
2737	return VINF_SUCCESS;
2738
2739	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,PhysSimpleWrite));
2740	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_MID_Z(Stat,PhysSimpleWriteBytes), cb);
2741
2742	/* map the 1st page */
2743	void *pvDst;
2744	PGMPAGEMAPLOCK Lock;
2745	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
2746	if (RT_FAILURE(rc))
2747	return rc;
2748
2749	/* optimize for the case where access is completely within the first page. */
2750	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
2751	if (RT_LIKELY(cb <= cbPage))
2752	{
2753	memcpy(pvDst, pvSrc, cb);
2754	PGMPhysReleasePageMappingLock(pVM, &Lock);
2755	return VINF_SUCCESS;
2756	}
2757
2758	/* copy to the end of the page. */
2759	memcpy(pvDst, pvSrc, cbPage);
2760	PGMPhysReleasePageMappingLock(pVM, &Lock);
2761	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
2762	pvSrc = (const uint8_t *)pvSrc + cbPage;
2763	cb -= cbPage;
2764
2765	/*
2766	* Page by page.
2767	*/
2768	for (;;)
2769	{
2770	/* map the page */
2771	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
2772	if (RT_FAILURE(rc))
2773	return rc;
2774
2775	/* last page? */
2776	if (cb <= PAGE_SIZE)
2777	{
2778	memcpy(pvDst, pvSrc, cb);
2779	PGMPhysReleasePageMappingLock(pVM, &Lock);
2780	return VINF_SUCCESS;
2781	}
2782
2783	/* copy the entire page and advance */
2784	memcpy(pvDst, pvSrc, PAGE_SIZE);
2785	PGMPhysReleasePageMappingLock(pVM, &Lock);
2786	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
2787	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
2788	cb -= PAGE_SIZE;
2789	}
2790	/* won't ever get here. */
2791	}
2792
2793
2794	/**
2795	* Write to guest physical memory referenced by GC pointer and update the PTE.
2796	*
2797	* This function uses the current CR3/CR0/CR4 of the guest and will
2798	* bypass access handlers but will set any dirty and accessed bits in the PTE.
2799	*
2800	* If you don't want to set the dirty bit, use PGMPhysSimpleWriteGCPtr().
2801	*
2802	* @returns VBox status.
2803	* @param pVCpu The VMCPU handle.
2804	* @param GCPtrDst The destination address (GC pointer).
2805	* @param pvSrc The source address.
2806	* @param cb The number of bytes to write.
2807	*/
2808	VMMDECL(int) PGMPhysSimpleDirtyWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
2809	{
2810	PVM pVM = pVCpu->CTX_SUFF(pVM);
2811
2812	/*
2813	* Treat the first page as a special case.
2814	* Btw. this is the same code as in PGMPhyssimpleWriteGCPtr excep for the PGMGstModifyPage.
2815	*/
2816	if (!cb)
2817	return VINF_SUCCESS;
2818
2819	/* map the 1st page */
2820	void *pvDst;
2821	PGMPAGEMAPLOCK Lock;
2822	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
2823	if (RT_FAILURE(rc))
2824	return rc;
2825
2826	/* optimize for the case where access is completely within the first page. */
2827	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
2828	if (RT_LIKELY(cb <= cbPage))
2829	{
2830	memcpy(pvDst, pvSrc, cb);
2831	PGMPhysReleasePageMappingLock(pVM, &Lock);
2832	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2833	return VINF_SUCCESS;
2834	}
2835
2836	/* copy to the end of the page. */
2837	memcpy(pvDst, pvSrc, cbPage);
2838	PGMPhysReleasePageMappingLock(pVM, &Lock);
2839	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2840	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
2841	pvSrc = (const uint8_t *)pvSrc + cbPage;
2842	cb -= cbPage;
2843
2844	/*
2845	* Page by page.
2846	*/
2847	for (;;)
2848	{
2849	/* map the page */
2850	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
2851	if (RT_FAILURE(rc))
2852	return rc;
2853
2854	/* last page? */
2855	if (cb <= PAGE_SIZE)
2856	{
2857	memcpy(pvDst, pvSrc, cb);
2858	PGMPhysReleasePageMappingLock(pVM, &Lock);
2859	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2860	return VINF_SUCCESS;
2861	}
2862
2863	/* copy the entire page and advance */
2864	memcpy(pvDst, pvSrc, PAGE_SIZE);
2865	PGMPhysReleasePageMappingLock(pVM, &Lock);
2866	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
2867	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
2868	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
2869	cb -= PAGE_SIZE;
2870	}
2871	/* won't ever get here. */
2872	}
2873
2874
2875	/**
2876	* Read from guest physical memory referenced by GC pointer.
2877	*
2878	* This function uses the current CR3/CR0/CR4 of the guest and will
2879	* respect access handlers and set accessed bits.
2880	*
2881	* @returns VBox status.
2882	* @param pVCpu The VMCPU handle.
2883	* @param pvDst The destination address.
2884	* @param GCPtrSrc The source address (GC pointer).
2885	* @param cb The number of bytes to read.
2886	* @thread The vCPU EMT.
2887	*/
2888	VMMDECL(int) PGMPhysReadGCPtr(PVMCPU pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
2889	{
2890	RTGCPHYS GCPhys;
2891	uint64_t fFlags;
2892	int rc;
2893	PVM pVM = pVCpu->CTX_SUFF(pVM);
2894
2895	/*
2896	* Anything to do?
2897	*/
2898	if (!cb)
2899	return VINF_SUCCESS;
2900
2901	LogFlow(("PGMPhysReadGCPtr: %RGv %zu\n", GCPtrSrc, cb));
2902
2903	/*
2904	* Optimize reads within a single page.
2905	*/
2906	if (((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
2907	{
2908	/* Convert virtual to physical address + flags */
2909	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
2910	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
2911	GCPhys \|= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
2912
2913	/* mark the guest page as accessed. */
2914	if (!(fFlags & X86_PTE_A))
2915	{
2916	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
2917	AssertRC(rc);
2918	}
2919
2920	return PGMPhysRead(pVM, GCPhys, pvDst, cb);
2921	}
2922
2923	/*
2924	* Page by page.
2925	*/
2926	for (;;)
2927	{
2928	/* Convert virtual to physical address + flags */
2929	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
2930	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
2931	GCPhys \|= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
2932
2933	/* mark the guest page as accessed. */
2934	if (!(fFlags & X86_PTE_A))
2935	{
2936	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
2937	AssertRC(rc);
2938	}
2939
2940	/* copy */
2941	size_t cbRead = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
2942	rc = PGMPhysRead(pVM, GCPhys, pvDst, cbRead);
2943	if (cbRead >= cb \|\| RT_FAILURE(rc))
2944	return rc;
2945
2946	/* next */
2947	cb -= cbRead;
2948	pvDst = (uint8_t *)pvDst + cbRead;
2949	GCPtrSrc += cbRead;
2950	}
2951	}
2952
2953
2954	/**
2955	* Write to guest physical memory referenced by GC pointer.
2956	*
2957	* This function uses the current CR3/CR0/CR4 of the guest and will
2958	* respect access handlers and set dirty and accessed bits.
2959	*
2960	* @returns VBox status.
2961	* @retval VINF_SUCCESS.
2962	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
2963	*
2964	* @param pVCpu The VMCPU handle.
2965	* @param GCPtrDst The destination address (GC pointer).
2966	* @param pvSrc The source address.
2967	* @param cb The number of bytes to write.
2968	*/
2969	VMMDECL(int) PGMPhysWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
2970	{
2971	RTGCPHYS GCPhys;
2972	uint64_t fFlags;
2973	int rc;
2974	PVM pVM = pVCpu->CTX_SUFF(pVM);
2975
2976	/*
2977	* Anything to do?
2978	*/
2979	if (!cb)
2980	return VINF_SUCCESS;
2981
2982	LogFlow(("PGMPhysWriteGCPtr: %RGv %zu\n", GCPtrDst, cb));
2983
2984	/*
2985	* Optimize writes within a single page.
2986	*/
2987	if (((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
2988	{
2989	/* Convert virtual to physical address + flags */
2990	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
2991	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
2992	GCPhys \|= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
2993
2994	/* Mention when we ignore X86_PTE_RW... */
2995	if (!(fFlags & X86_PTE_RW))
2996	Log(("PGMPhysGCPtr2GCPhys: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
2997
2998	/* Mark the guest page as accessed and dirty if necessary. */
2999	if ((fFlags & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
3000	{
3001	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
3002	AssertRC(rc);
3003	}
3004
3005	return PGMPhysWrite(pVM, GCPhys, pvSrc, cb);
3006	}
3007
3008	/*
3009	* Page by page.
3010	*/
3011	for (;;)
3012	{
3013	/* Convert virtual to physical address + flags */
3014	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
3015	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
3016	GCPhys \|= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
3017
3018	/* Mention when we ignore X86_PTE_RW... */
3019	if (!(fFlags & X86_PTE_RW))
3020	Log(("PGMPhysGCPtr2GCPhys: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
3021
3022	/* Mark the guest page as accessed and dirty if necessary. */
3023	if ((fFlags & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
3024	{
3025	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
3026	AssertRC(rc);
3027	}
3028
3029	/* copy */
3030	size_t cbWrite = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
3031	rc = PGMPhysWrite(pVM, GCPhys, pvSrc, cbWrite);
3032	if (cbWrite >= cb \|\| RT_FAILURE(rc))
3033	return rc;
3034
3035	/* next */
3036	cb -= cbWrite;
3037	pvSrc = (uint8_t *)pvSrc + cbWrite;
3038	GCPtrDst += cbWrite;
3039	}
3040	}
3041
3042
3043	/**
3044	* Performs a read of guest virtual memory for instruction emulation.
3045	*
3046	* This will check permissions, raise exceptions and update the access bits.
3047	*
3048	* The current implementation will bypass all access handlers. It may later be
3049	* changed to at least respect MMIO.
3050	*
3051	*
3052	* @returns VBox status code suitable to scheduling.
3053	* @retval VINF_SUCCESS if the read was performed successfully.
3054	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3055	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3056	*
3057	* @param pVCpu The VMCPU handle.
3058	* @param pCtxCore The context core.
3059	* @param pvDst Where to put the bytes we've read.
3060	* @param GCPtrSrc The source address.
3061	* @param cb The number of bytes to read. Not more than a page.
3062	*
3063	* @remark This function will dynamically map physical pages in GC. This may unmap
3064	* mappings done by the caller. Be careful!
3065	*/
3066	VMMDECL(int) PGMPhysInterpretedRead(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb)
3067	{
3068	PVM pVM = pVCpu->CTX_SUFF(pVM);
3069	Assert(cb <= PAGE_SIZE);
3070
3071	/** @todo r=bird: This isn't perfect!
3072	* -# It's not checking for reserved bits being 1.
3073	* -# It's not correctly dealing with the access bit.
3074	* -# It's not respecting MMIO memory or any other access handlers.
3075	*/
3076	/*
3077	* 1. Translate virtual to physical. This may fault.
3078	* 2. Map the physical address.
3079	* 3. Do the read operation.
3080	* 4. Set access bits if required.
3081	*/
3082	int rc;
3083	unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
3084	if (cb <= cb1)
3085	{
3086	/*
3087	* Not crossing pages.
3088	*/
3089	RTGCPHYS GCPhys;
3090	uint64_t fFlags;
3091	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
3092	if (RT_SUCCESS(rc))
3093	{
3094	/** @todo we should check reserved bits ... */
3095	void *pvSrc;
3096	rc = PGM_GCPHYS_2_PTR(pVM, GCPhys, &pvSrc);
3097	switch (rc)
3098	{
3099	case VINF_SUCCESS:
3100	Log(("PGMPhysInterpretedRead: pvDst=%p pvSrc=%p cb=%d\n", pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb));
3101	memcpy(pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
3102	break;
3103	case VERR_PGM_PHYS_PAGE_RESERVED:
3104	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3105	memset(pvDst, 0, cb); /** @todo this is wrong, it should be 0xff */
3106	break;
3107	default:
3108	return rc;
3109	}
3110
3111	/** @todo access bit emulation isn't 100% correct. */
3112	if (!(fFlags & X86_PTE_A))
3113	{
3114	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3115	AssertRC(rc);
3116	}
3117	return VINF_SUCCESS;
3118	}
3119	}
3120	else
3121	{
3122	/*
3123	* Crosses pages.
3124	*/
3125	size_t cb2 = cb - cb1;
3126	uint64_t fFlags1;
3127	RTGCPHYS GCPhys1;
3128	uint64_t fFlags2;
3129	RTGCPHYS GCPhys2;
3130	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
3131	if (RT_SUCCESS(rc))
3132	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
3133	if (RT_SUCCESS(rc))
3134	{
3135	/** @todo we should check reserved bits ... */
3136	AssertMsgFailed(("cb=%d cb1=%d cb2=%d GCPtrSrc=%RGv\n", cb, cb1, cb2, GCPtrSrc));
3137	void *pvSrc1;
3138	rc = PGM_GCPHYS_2_PTR(pVM, GCPhys1, &pvSrc1);
3139	switch (rc)
3140	{
3141	case VINF_SUCCESS:
3142	memcpy(pvDst, (uint8_t *)pvSrc1 + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
3143	break;
3144	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3145	memset(pvDst, 0, cb1); /** @todo this is wrong, it should be 0xff */
3146	break;
3147	default:
3148	return rc;
3149	}
3150
3151	void *pvSrc2;
3152	rc = PGM_GCPHYS_2_PTR(pVM, GCPhys2, &pvSrc2);
3153	switch (rc)
3154	{
3155	case VINF_SUCCESS:
3156	memcpy((uint8_t *)pvDst + cb1, pvSrc2, cb2);
3157	break;
3158	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3159	memset((uint8_t )pvDst + cb1, 0, cb2); /* @todo this is wrong, it should be 0xff */
3160	break;
3161	default:
3162	return rc;
3163	}
3164
3165	if (!(fFlags1 & X86_PTE_A))
3166	{
3167	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3168	AssertRC(rc);
3169	}
3170	if (!(fFlags2 & X86_PTE_A))
3171	{
3172	rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3173	AssertRC(rc);
3174	}
3175	return VINF_SUCCESS;
3176	}
3177	}
3178
3179	/*
3180	* Raise a #PF.
3181	*/
3182	uint32_t uErr;
3183
3184	/* Get the current privilege level. */
3185	uint32_t cpl = CPUMGetGuestCPL(pVCpu, pCtxCore);
3186	switch (rc)
3187	{
3188	case VINF_SUCCESS:
3189	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3190	break;
3191
3192	case VERR_PAGE_NOT_PRESENT:
3193	case VERR_PAGE_TABLE_NOT_PRESENT:
3194	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3195	break;
3196
3197	default:
3198	AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
3199	return rc;
3200	}
3201	Log(("PGMPhysInterpretedRead: GCPtrSrc=%RGv cb=%#x -> #PF(%#x)\n", GCPtrSrc, cb, uErr));
3202	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrSrc);
3203	}
3204
3205
3206	/**
3207	* Performs a read of guest virtual memory for instruction emulation.
3208	*
3209	* This will check permissions, raise exceptions and update the access bits.
3210	*
3211	* The current implementation will bypass all access handlers. It may later be
3212	* changed to at least respect MMIO.
3213	*
3214	*
3215	* @returns VBox status code suitable to scheduling.
3216	* @retval VINF_SUCCESS if the read was performed successfully.
3217	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3218	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3219	*
3220	* @param pVCpu The VMCPU handle.
3221	* @param pCtxCore The context core.
3222	* @param pvDst Where to put the bytes we've read.
3223	* @param GCPtrSrc The source address.
3224	* @param cb The number of bytes to read. Not more than a page.
3225	* @param fRaiseTrap If set the trap will be raised on as per spec, if clear
3226	* an appropriate error status will be returned (no
3227	* informational at all).
3228	*
3229	*
3230	* @remarks Takes the PGM lock.
3231	* @remarks A page fault on the 2nd page of the access will be raised without
3232	* writing the bits on the first page since we're ASSUMING that the
3233	* caller is emulating an instruction access.
3234	* @remarks This function will dynamically map physical pages in GC. This may
3235	* unmap mappings done by the caller. Be careful!
3236	*/
3237	VMMDECL(int) PGMPhysInterpretedReadNoHandlers(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb, bool fRaiseTrap)
3238	{
3239	PVM pVM = pVCpu->CTX_SUFF(pVM);
3240	Assert(cb <= PAGE_SIZE);
3241
3242	/*
3243	* 1. Translate virtual to physical. This may fault.
3244	* 2. Map the physical address.
3245	* 3. Do the read operation.
3246	* 4. Set access bits if required.
3247	*/
3248	int rc;
3249	unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
3250	if (cb <= cb1)
3251	{
3252	/*
3253	* Not crossing pages.
3254	*/
3255	RTGCPHYS GCPhys;
3256	uint64_t fFlags;
3257	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
3258	if (RT_SUCCESS(rc))
3259	{
3260	if (1) /** @todo we should check reserved bits ... */
3261	{
3262	const void *pvSrc;
3263	PGMPAGEMAPLOCK Lock;
3264	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvSrc, &Lock);
3265	switch (rc)
3266	{
3267	case VINF_SUCCESS:
3268	Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d\n",
3269	pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb));
3270	memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
3271	break;
3272	case VERR_PGM_PHYS_PAGE_RESERVED:
3273	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3274	memset(pvDst, 0xff, cb);
3275	break;
3276	default:
3277	AssertMsgFailed(("%Rrc\n", rc));
3278	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3279	return rc;
3280	}
3281	PGMPhysReleasePageMappingLock(pVM, &Lock);
3282
3283	if (!(fFlags & X86_PTE_A))
3284	{
3285	/** @todo access bit emulation isn't 100% correct. */
3286	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3287	AssertRC(rc);
3288	}
3289	return VINF_SUCCESS;
3290	}
3291	}
3292	}
3293	else
3294	{
3295	/*
3296	* Crosses pages.
3297	*/
3298	size_t cb2 = cb - cb1;
3299	uint64_t fFlags1;
3300	RTGCPHYS GCPhys1;
3301	uint64_t fFlags2;
3302	RTGCPHYS GCPhys2;
3303	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
3304	if (RT_SUCCESS(rc))
3305	{
3306	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
3307	if (RT_SUCCESS(rc))
3308	{
3309	if (1) /** @todo we should check reserved bits ... */
3310	{
3311	const void *pvSrc;
3312	PGMPAGEMAPLOCK Lock;
3313	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys1, &pvSrc, &Lock);
3314	switch (rc)
3315	{
3316	case VINF_SUCCESS:
3317	Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d [2]\n",
3318	pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb1));
3319	memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
3320	PGMPhysReleasePageMappingLock(pVM, &Lock);
3321	break;
3322	case VERR_PGM_PHYS_PAGE_RESERVED:
3323	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3324	memset(pvDst, 0xff, cb1);
3325	break;
3326	default:
3327	AssertMsgFailed(("%Rrc\n", rc));
3328	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3329	return rc;
3330	}
3331
3332	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys2, &pvSrc, &Lock);
3333	switch (rc)
3334	{
3335	case VINF_SUCCESS:
3336	memcpy((uint8_t *)pvDst + cb1, pvSrc, cb2);
3337	PGMPhysReleasePageMappingLock(pVM, &Lock);
3338	break;
3339	case VERR_PGM_PHYS_PAGE_RESERVED:
3340	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3341	memset((uint8_t *)pvDst + cb1, 0xff, cb2);
3342	break;
3343	default:
3344	AssertMsgFailed(("%Rrc\n", rc));
3345	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3346	return rc;
3347	}
3348
3349	if (!(fFlags1 & X86_PTE_A))
3350	{
3351	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3352	AssertRC(rc);
3353	}
3354	if (!(fFlags2 & X86_PTE_A))
3355	{
3356	rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3357	AssertRC(rc);
3358	}
3359	return VINF_SUCCESS;
3360	}
3361	/* sort out which page */
3362	}
3363	else
3364	GCPtrSrc += cb1; /* fault on 2nd page */
3365	}
3366	}
3367
3368	/*
3369	* Raise a #PF if we're allowed to do that.
3370	*/
3371	/* Calc the error bits. */
3372	uint32_t cpl = CPUMGetGuestCPL(pVCpu, pCtxCore);
3373	uint32_t uErr;
3374	switch (rc)
3375	{
3376	case VINF_SUCCESS:
3377	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3378	rc = VERR_ACCESS_DENIED;
3379	break;
3380
3381	case VERR_PAGE_NOT_PRESENT:
3382	case VERR_PAGE_TABLE_NOT_PRESENT:
3383	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3384	break;
3385
3386	default:
3387	AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
3388	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3389	return rc;
3390	}
3391	if (fRaiseTrap)
3392	{
3393	Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrSrc, cb, uErr));
3394	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrSrc);
3395	}
3396	Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrSrc, cb, uErr));
3397	return rc;
3398	}
3399
3400
3401	/**
3402	* Performs a write to guest virtual memory for instruction emulation.
3403	*
3404	* This will check permissions, raise exceptions and update the dirty and access
3405	* bits.
3406	*
3407	* @returns VBox status code suitable to scheduling.
3408	* @retval VINF_SUCCESS if the read was performed successfully.
3409	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3410	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3411	*
3412	* @param pVCpu The VMCPU handle.
3413	* @param pCtxCore The context core.
3414	* @param GCPtrDst The destination address.
3415	* @param pvSrc What to write.
3416	* @param cb The number of bytes to write. Not more than a page.
3417	* @param fRaiseTrap If set the trap will be raised on as per spec, if clear
3418	* an appropriate error status will be returned (no
3419	* informational at all).
3420	*
3421	* @remarks Takes the PGM lock.
3422	* @remarks A page fault on the 2nd page of the access will be raised without
3423	* writing the bits on the first page since we're ASSUMING that the
3424	* caller is emulating an instruction access.
3425	* @remarks This function will dynamically map physical pages in GC. This may
3426	* unmap mappings done by the caller. Be careful!
3427	*/
3428	VMMDECL(int) PGMPhysInterpretedWriteNoHandlers(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb, bool fRaiseTrap)
3429	{
3430	Assert(cb <= PAGE_SIZE);
3431	PVM pVM = pVCpu->CTX_SUFF(pVM);
3432
3433	/*
3434	* 1. Translate virtual to physical. This may fault.
3435	* 2. Map the physical address.
3436	* 3. Do the write operation.
3437	* 4. Set access bits if required.
3438	*/
3439	int rc;
3440	unsigned cb1 = PAGE_SIZE - (GCPtrDst & PAGE_OFFSET_MASK);
3441	if (cb <= cb1)
3442	{
3443	/*
3444	* Not crossing pages.
3445	*/
3446	RTGCPHYS GCPhys;
3447	uint64_t fFlags;
3448	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst, &fFlags, &GCPhys);
3449	if (RT_SUCCESS(rc))
3450	{
3451	if ( (fFlags & X86_PTE_RW) /** @todo Also check reserved bits. */
3452	\|\| ( !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
3453	&& CPUMGetGuestCPL(pVCpu, pCtxCore) <= 2) ) /** @todo it's 2, right? Check cpl check below as well. */
3454	{
3455	void *pvDst;
3456	PGMPAGEMAPLOCK Lock;
3457	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, &pvDst, &Lock);
3458	switch (rc)
3459	{
3460	case VINF_SUCCESS:
3461	Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
3462	(uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc, cb));
3463	memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb);
3464	PGMPhysReleasePageMappingLock(pVM, &Lock);
3465	break;
3466	case VERR_PGM_PHYS_PAGE_RESERVED:
3467	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3468	/* bit bucket */
3469	break;
3470	default:
3471	AssertMsgFailed(("%Rrc\n", rc));
3472	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3473	return rc;
3474	}
3475
3476	if (!(fFlags & (X86_PTE_A \| X86_PTE_D)))
3477	{
3478	/** @todo dirty & access bit emulation isn't 100% correct. */
3479	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
3480	AssertRC(rc);
3481	}
3482	return VINF_SUCCESS;
3483	}
3484	rc = VERR_ACCESS_DENIED;
3485	}
3486	}
3487	else
3488	{
3489	/*
3490	* Crosses pages.
3491	*/
3492	size_t cb2 = cb - cb1;
3493	uint64_t fFlags1;
3494	RTGCPHYS GCPhys1;
3495	uint64_t fFlags2;
3496	RTGCPHYS GCPhys2;
3497	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst, &fFlags1, &GCPhys1);
3498	if (RT_SUCCESS(rc))
3499	{
3500	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst + cb1, &fFlags2, &GCPhys2);
3501	if (RT_SUCCESS(rc))
3502	{
3503	if ( ( (fFlags1 & X86_PTE_RW) /** @todo Also check reserved bits. */
3504	&& (fFlags2 & X86_PTE_RW))
3505	\|\| ( !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
3506	&& CPUMGetGuestCPL(pVCpu, pCtxCore) <= 2) )
3507	{
3508	void *pvDst;
3509	PGMPAGEMAPLOCK Lock;
3510	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys1, &pvDst, &Lock);
3511	switch (rc)
3512	{
3513	case VINF_SUCCESS:
3514	Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
3515	(uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc, cb1));
3516	memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb1);
3517	PGMPhysReleasePageMappingLock(pVM, &Lock);
3518	break;
3519	case VERR_PGM_PHYS_PAGE_RESERVED:
3520	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3521	/* bit bucket */
3522	break;
3523	default:
3524	AssertMsgFailed(("%Rrc\n", rc));
3525	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3526	return rc;
3527	}
3528
3529	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys2, &pvDst, &Lock);
3530	switch (rc)
3531	{
3532	case VINF_SUCCESS:
3533	memcpy(pvDst, (const uint8_t *)pvSrc + cb1, cb2);
3534	PGMPhysReleasePageMappingLock(pVM, &Lock);
3535	break;
3536	case VERR_PGM_PHYS_PAGE_RESERVED:
3537	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3538	/* bit bucket */
3539	break;
3540	default:
3541	AssertMsgFailed(("%Rrc\n", rc));
3542	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3543	return rc;
3544	}
3545
3546	if (!(fFlags1 & (X86_PTE_A \| X86_PTE_RW)))
3547	{
3548	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, (X86_PTE_A \| X86_PTE_RW), ~(uint64_t)(X86_PTE_A \| X86_PTE_RW));
3549	AssertRC(rc);
3550	}
3551	if (!(fFlags2 & (X86_PTE_A \| X86_PTE_RW)))
3552	{
3553	rc = PGMGstModifyPage(pVCpu, GCPtrDst + cb1, 1, (X86_PTE_A \| X86_PTE_RW), ~(uint64_t)(X86_PTE_A \| X86_PTE_RW));
3554	AssertRC(rc);
3555	}
3556	return VINF_SUCCESS;
3557	}
3558	if ((fFlags1 & (X86_PTE_RW)) == X86_PTE_RW)
3559	GCPtrDst += cb1; /* fault on the 2nd page. */
3560	rc = VERR_ACCESS_DENIED;
3561	}
3562	else
3563	GCPtrDst += cb1; /* fault on the 2nd page. */
3564	}
3565	}
3566
3567	/*
3568	* Raise a #PF if we're allowed to do that.
3569	*/
3570	/* Calc the error bits. */
3571	uint32_t uErr;
3572	uint32_t cpl = CPUMGetGuestCPL(pVCpu, pCtxCore);
3573	switch (rc)
3574	{
3575	case VINF_SUCCESS:
3576	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3577	rc = VERR_ACCESS_DENIED;
3578	break;
3579
3580	case VERR_ACCESS_DENIED:
3581	uErr = (cpl >= 2) ? X86_TRAP_PF_RW \| X86_TRAP_PF_US : X86_TRAP_PF_RW;
3582	break;
3583
3584	case VERR_PAGE_NOT_PRESENT:
3585	case VERR_PAGE_TABLE_NOT_PRESENT:
3586	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3587	break;
3588
3589	default:
3590	AssertMsgFailed(("rc=%Rrc GCPtrDst=%RGv cb=%#x\n", rc, GCPtrDst, cb));
3591	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3592	return rc;
3593	}
3594	if (fRaiseTrap)
3595	{
3596	Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrDst, cb, uErr));
3597	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrDst);
3598	}
3599	Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrDst, cb, uErr));
3600	return rc;
3601	}
3602

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp@ 28112

Download in other formats: