PGMAllPhys.cpp@ 48749

Last change on this file since 48749 was 47792, checked in by vboxsync, 11 years ago
wrong assertion.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 148.3 KB

Line
1	/* $Id: PGMAllPhys.cpp 47792 2013-08-16 09:45:07Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_PGM_PHYS
22	#include <VBox/vmm/pgm.h>
23	#include <VBox/vmm/trpm.h>
24	#include <VBox/vmm/vmm.h>
25	#include <VBox/vmm/iom.h>
26	#include <VBox/vmm/em.h>
27	#ifdef VBOX_WITH_REM
28	# include <VBox/vmm/rem.h>
29	#endif
30	#include "PGMInternal.h"
31	#include <VBox/vmm/vm.h>
32	#include "PGMInline.h"
33	#include <VBox/param.h>
34	#include <VBox/err.h>
35	#include <iprt/assert.h>
36	#include <iprt/string.h>
37	#include <iprt/asm-amd64-x86.h>
38	#include <VBox/log.h>
39	#ifdef IN_RING3
40	# include <iprt/thread.h>
41	#endif
42
43
44	/*******************************************************************************
45	* Defined Constants And Macros *
46	*******************************************************************************/
47	/** Enable the physical TLB. */
48	#define PGM_WITH_PHYS_TLB
49
50
51
52	#ifndef IN_RING3
53
54	/**
55	* \#PF Handler callback for physical memory accesses without a RC/R0 handler.
56	* This simply pushes everything to the HC handler.
57	*
58	* @returns VBox status code (appropriate for trap handling and GC return).
59	* @param pVM Pointer to the VM.
60	* @param uErrorCode CPU Error code.
61	* @param pRegFrame Trap register frame.
62	* @param pvFault The fault address (cr2).
63	* @param GCPhysFault The GC physical address corresponding to pvFault.
64	* @param pvUser User argument.
65	*/
66	VMMDECL(int) pgmPhysHandlerRedirectToHC(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
67	{
68	NOREF(pVM); NOREF(uErrorCode); NOREF(pRegFrame); NOREF(pvFault); NOREF(GCPhysFault); NOREF(pvUser);
69	return (uErrorCode & X86_TRAP_PF_RW) ? VINF_IOM_R3_MMIO_WRITE : VINF_IOM_R3_MMIO_READ;
70	}
71
72
73	/**
74	* \#PF Handler callback for Guest ROM range write access.
75	* We simply ignore the writes or fall back to the recompiler if we don't support the instruction.
76	*
77	* @returns VBox status code (appropriate for trap handling and GC return).
78	* @param pVM Pointer to the VM.
79	* @param uErrorCode CPU Error code.
80	* @param pRegFrame Trap register frame.
81	* @param pvFault The fault address (cr2).
82	* @param GCPhysFault The GC physical address corresponding to pvFault.
83	* @param pvUser User argument. Pointer to the ROM range structure.
84	*/
85	VMMDECL(int) pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
86	{
87	int rc;
88	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
89	uint32_t iPage = (GCPhysFault - pRom->GCPhys) >> PAGE_SHIFT;
90	PVMCPU pVCpu = VMMGetCpu(pVM);
91	NOREF(uErrorCode); NOREF(pvFault);
92
93	Assert(uErrorCode & X86_TRAP_PF_RW); /* This shall not be used for read access! */
94
95	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
96	switch (pRom->aPages[iPage].enmProt)
97	{
98	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
99	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
100	{
101	/*
102	* If it's a simple instruction which doesn't change the cpu state
103	* we will simply skip it. Otherwise we'll have to defer it to REM.
104	*/
105	uint32_t cbOp;
106	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
107	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
108	if ( RT_SUCCESS(rc)
109	&& pDis->uCpuMode == DISCPUMODE_32BIT /** @todo why does this matter? */
110	&& !(pDis->fPrefix & (DISPREFIX_REPNE \| DISPREFIX_REP \| DISPREFIX_SEG)))
111	{
112	switch (pDis->bOpCode)
113	{
114	/** @todo Find other instructions we can safely skip, possibly
115	* adding this kind of detection to DIS or EM. */
116	case OP_MOV:
117	pRegFrame->rip += cbOp;
118	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestROMWriteHandled);
119	return VINF_SUCCESS;
120	}
121	}
122	break;
123	}
124
125	case PGMROMPROT_READ_RAM_WRITE_RAM:
126	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
127	rc = PGMHandlerPhysicalPageTempOff(pVM, pRom->GCPhys, GCPhysFault & X86_PTE_PG_MASK);
128	AssertRC(rc);
129	break; /** @todo Must edit the shadow PT and restart the instruction, not use the interpreter! */
130
131	case PGMROMPROT_READ_ROM_WRITE_RAM:
132	/* Handle it in ring-3 because it's way easier there. */
133	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
134	break;
135
136	default:
137	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhysFault=%RGp\n",
138	pRom->aPages[iPage].enmProt, iPage, GCPhysFault),
139	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
140	}
141
142	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestROMWriteUnhandled);
143	return VINF_EM_RAW_EMULATE_INSTR;
144	}
145
146	#endif /* IN_RING3 */
147
148	/**
149	* Invalidates the RAM range TLBs.
150	*
151	* @param pVM Pointer to the VM.
152	*/
153	void pgmPhysInvalidRamRangeTlbs(PVM pVM)
154	{
155	pgmLock(pVM);
156	for (uint32_t i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
157	{
158	pVM->pgm.s.apRamRangesTlbR3[i] = NIL_RTR3PTR;
159	pVM->pgm.s.apRamRangesTlbR0[i] = NIL_RTR0PTR;
160	pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
161	}
162	pgmUnlock(pVM);
163	}
164
165
166	/**
167	* Tests if a value of type RTGCPHYS is negative if the type had been signed
168	* instead of unsigned.
169	*
170	* @returns @c true if negative, @c false if positive or zero.
171	* @param a_GCPhys The value to test.
172	* @todo Move me to iprt/types.h.
173	*/
174	#define RTGCPHYS_IS_NEGATIVE(a_GCPhys) ((a_GCPhys) & ((RTGCPHYS)1 << (sizeof(RTGCPHYS)*8 - 1)))
175
176
177	/**
178	* Slow worker for pgmPhysGetRange.
179	*
180	* @copydoc pgmPhysGetRange
181	*/
182	PPGMRAMRANGE pgmPhysGetRangeSlow(PVM pVM, RTGCPHYS GCPhys)
183	{
184	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
185
186	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
187	while (pRam)
188	{
189	RTGCPHYS off = GCPhys - pRam->GCPhys;
190	if (off < pRam->cb)
191	{
192	pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
193	return pRam;
194	}
195	if (RTGCPHYS_IS_NEGATIVE(off))
196	pRam = pRam->CTX_SUFF(pLeft);
197	else
198	pRam = pRam->CTX_SUFF(pRight);
199	}
200	return NULL;
201	}
202
203
204	/**
205	* Slow worker for pgmPhysGetRangeAtOrAbove.
206	*
207	* @copydoc pgmPhysGetRangeAtOrAbove
208	*/
209	PPGMRAMRANGE pgmPhysGetRangeAtOrAboveSlow(PVM pVM, RTGCPHYS GCPhys)
210	{
211	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
212
213	PPGMRAMRANGE pLastLeft = NULL;
214	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
215	while (pRam)
216	{
217	RTGCPHYS off = GCPhys - pRam->GCPhys;
218	if (off < pRam->cb)
219	{
220	pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
221	return pRam;
222	}
223	if (RTGCPHYS_IS_NEGATIVE(off))
224	{
225	pLastLeft = pRam;
226	pRam = pRam->CTX_SUFF(pLeft);
227	}
228	else
229	pRam = pRam->CTX_SUFF(pRight);
230	}
231	return pLastLeft;
232	}
233
234
235	/**
236	* Slow worker for pgmPhysGetPage.
237	*
238	* @copydoc pgmPhysGetPage
239	*/
240	PPGMPAGE pgmPhysGetPageSlow(PVM pVM, RTGCPHYS GCPhys)
241	{
242	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
243
244	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
245	while (pRam)
246	{
247	RTGCPHYS off = GCPhys - pRam->GCPhys;
248	if (off < pRam->cb)
249	{
250	pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
251	return &pRam->aPages[off >> PAGE_SHIFT];
252	}
253
254	if (RTGCPHYS_IS_NEGATIVE(off))
255	pRam = pRam->CTX_SUFF(pLeft);
256	else
257	pRam = pRam->CTX_SUFF(pRight);
258	}
259	return NULL;
260	}
261
262
263	/**
264	* Slow worker for pgmPhysGetPageEx.
265	*
266	* @copydoc pgmPhysGetPageEx
267	*/
268	int pgmPhysGetPageExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
269	{
270	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
271
272	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
273	while (pRam)
274	{
275	RTGCPHYS off = GCPhys - pRam->GCPhys;
276	if (off < pRam->cb)
277	{
278	pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
279	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
280	return VINF_SUCCESS;
281	}
282
283	if (RTGCPHYS_IS_NEGATIVE(off))
284	pRam = pRam->CTX_SUFF(pLeft);
285	else
286	pRam = pRam->CTX_SUFF(pRight);
287	}
288
289	*ppPage = NULL;
290	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
291	}
292
293
294	/**
295	* Slow worker for pgmPhysGetPageAndRangeEx.
296	*
297	* @copydoc pgmPhysGetPageAndRangeEx
298	*/
299	int pgmPhysGetPageAndRangeExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
300	{
301	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
302
303	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
304	while (pRam)
305	{
306	RTGCPHYS off = GCPhys - pRam->GCPhys;
307	if (off < pRam->cb)
308	{
309	pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
310	*ppRam = pRam;
311	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
312	return VINF_SUCCESS;
313	}
314
315	if (RTGCPHYS_IS_NEGATIVE(off))
316	pRam = pRam->CTX_SUFF(pLeft);
317	else
318	pRam = pRam->CTX_SUFF(pRight);
319	}
320
321	*ppRam = NULL;
322	*ppPage = NULL;
323	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
324	}
325
326
327	/**
328	* Checks if Address Gate 20 is enabled or not.
329	*
330	* @returns true if enabled.
331	* @returns false if disabled.
332	* @param pVCpu Pointer to the VMCPU.
333	*/
334	VMMDECL(bool) PGMPhysIsA20Enabled(PVMCPU pVCpu)
335	{
336	LogFlow(("PGMPhysIsA20Enabled %d\n", pVCpu->pgm.s.fA20Enabled));
337	return pVCpu->pgm.s.fA20Enabled;
338	}
339
340
341	/**
342	* Validates a GC physical address.
343	*
344	* @returns true if valid.
345	* @returns false if invalid.
346	* @param pVM Pointer to the VM.
347	* @param GCPhys The physical address to validate.
348	*/
349	VMMDECL(bool) PGMPhysIsGCPhysValid(PVM pVM, RTGCPHYS GCPhys)
350	{
351	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
352	return pPage != NULL;
353	}
354
355
356	/**
357	* Checks if a GC physical address is a normal page,
358	* i.e. not ROM, MMIO or reserved.
359	*
360	* @returns true if normal.
361	* @returns false if invalid, ROM, MMIO or reserved page.
362	* @param pVM Pointer to the VM.
363	* @param GCPhys The physical address to check.
364	*/
365	VMMDECL(bool) PGMPhysIsGCPhysNormal(PVM pVM, RTGCPHYS GCPhys)
366	{
367	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
368	return pPage
369	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM;
370	}
371
372
373	/**
374	* Converts a GC physical address to a HC physical address.
375	*
376	* @returns VINF_SUCCESS on success.
377	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
378	* page but has no physical backing.
379	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
380	* GC physical address.
381	*
382	* @param pVM Pointer to the VM.
383	* @param GCPhys The GC physical address to convert.
384	* @param pHCPhys Where to store the HC physical address on success.
385	*/
386	VMMDECL(int) PGMPhysGCPhys2HCPhys(PVM pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
387	{
388	pgmLock(pVM);
389	PPGMPAGE pPage;
390	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
391	if (RT_SUCCESS(rc))
392	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK);
393	pgmUnlock(pVM);
394	return rc;
395	}
396
397
398	/**
399	* Invalidates all page mapping TLBs.
400	*
401	* @param pVM Pointer to the VM.
402	*/
403	void pgmPhysInvalidatePageMapTLB(PVM pVM)
404	{
405	pgmLock(pVM);
406	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatPageMapTlbFlushes);
407
408	/* Clear the shared R0/R3 TLB completely. */
409	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
410	{
411	pVM->pgm.s.PhysTlbHC.aEntries[i].GCPhys = NIL_RTGCPHYS;
412	pVM->pgm.s.PhysTlbHC.aEntries[i].pPage = 0;
413	pVM->pgm.s.PhysTlbHC.aEntries[i].pMap = 0;
414	pVM->pgm.s.PhysTlbHC.aEntries[i].pv = 0;
415	}
416
417	/** @todo clear the RC TLB whenever we add it. */
418
419	pgmUnlock(pVM);
420	}
421
422
423	/**
424	* Invalidates a page mapping TLB entry
425	*
426	* @param pVM Pointer to the VM.
427	* @param GCPhys GCPhys entry to flush
428	*/
429	void pgmPhysInvalidatePageMapTLBEntry(PVM pVM, RTGCPHYS GCPhys)
430	{
431	PGM_LOCK_ASSERT_OWNER(pVM);
432
433	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatPageMapTlbFlushEntry);
434
435	#ifdef IN_RC
436	unsigned idx = PGM_PAGER3MAPTLB_IDX(GCPhys);
437	pVM->pgm.s.PhysTlbHC.aEntries[idx].GCPhys = NIL_RTGCPHYS;
438	pVM->pgm.s.PhysTlbHC.aEntries[idx].pPage = 0;
439	pVM->pgm.s.PhysTlbHC.aEntries[idx].pMap = 0;
440	pVM->pgm.s.PhysTlbHC.aEntries[idx].pv = 0;
441	#else
442	/* Clear the shared R0/R3 TLB entry. */
443	PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
444	pTlbe->GCPhys = NIL_RTGCPHYS;
445	pTlbe->pPage = 0;
446	pTlbe->pMap = 0;
447	pTlbe->pv = 0;
448	#endif
449
450	/** @todo clear the RC TLB whenever we add it. */
451	}
452
453	/**
454	* Makes sure that there is at least one handy page ready for use.
455	*
456	* This will also take the appropriate actions when reaching water-marks.
457	*
458	* @returns VBox status code.
459	* @retval VINF_SUCCESS on success.
460	* @retval VERR_EM_NO_MEMORY if we're really out of memory.
461	*
462	* @param pVM Pointer to the VM.
463	*
464	* @remarks Must be called from within the PGM critical section. It may
465	* nip back to ring-3/0 in some cases.
466	*/
467	static int pgmPhysEnsureHandyPage(PVM pVM)
468	{
469	AssertMsg(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", pVM->pgm.s.cHandyPages));
470
471	/*
472	* Do we need to do anything special?
473	*/
474	#ifdef IN_RING3
475	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_R3_ALLOC))
476	#else
477	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_RZ_TO_R3))
478	#endif
479	{
480	/*
481	* Allocate pages only if we're out of them, or in ring-3, almost out.
482	*/
483	#ifdef IN_RING3
484	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_R3_ALLOC)
485	#else
486	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_ALLOC)
487	#endif
488	{
489	Log(("PGM: cHandyPages=%u out of %u -> allocate more; VM_FF_PGM_NO_MEMORY=%RTbool\n",
490	pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages), VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY) ));
491	#ifdef IN_RING3
492	int rc = PGMR3PhysAllocateHandyPages(pVM);
493	#else
494	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES, 0);
495	#endif
496	if (RT_UNLIKELY(rc != VINF_SUCCESS))
497	{
498	if (RT_FAILURE(rc))
499	return rc;
500	AssertMsgReturn(rc == VINF_EM_NO_MEMORY, ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
501	if (!pVM->pgm.s.cHandyPages)
502	{
503	LogRel(("PGM: no more handy pages!\n"));
504	return VERR_EM_NO_MEMORY;
505	}
506	Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES));
507	Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY));
508	#ifdef IN_RING3
509	# ifdef VBOX_WITH_REM
510	REMR3NotifyFF(pVM);
511	# endif
512	#else
513	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); /* paranoia */
514	#endif
515	}
516	AssertMsgReturn( pVM->pgm.s.cHandyPages > 0
517	&& pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages),
518	("%u\n", pVM->pgm.s.cHandyPages),
519	VERR_PGM_HANDY_PAGE_IPE);
520	}
521	else
522	{
523	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_SET_FF)
524	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
525	#ifndef IN_RING3
526	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_TO_R3)
527	{
528	Log(("PGM: VM_FF_TO_R3 - cHandyPages=%u out of %u\n", pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
529	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
530	}
531	#endif
532	}
533	}
534
535	return VINF_SUCCESS;
536	}
537
538
539	/**
540	* Replace a zero or shared page with new page that we can write to.
541	*
542	* @returns The following VBox status codes.
543	* @retval VINF_SUCCESS on success, pPage is modified.
544	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
545	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
546	*
547	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
548	*
549	* @param pVM Pointer to the VM.
550	* @param pPage The physical page tracking structure. This will
551	* be modified on success.
552	* @param GCPhys The address of the page.
553	*
554	* @remarks Must be called from within the PGM critical section. It may
555	* nip back to ring-3/0 in some cases.
556	*
557	* @remarks This function shouldn't really fail, however if it does
558	* it probably means we've screwed up the size of handy pages and/or
559	* the low-water mark. Or, that some device I/O is causing a lot of
560	* pages to be allocated while while the host is in a low-memory
561	* condition. This latter should be handled elsewhere and in a more
562	* controlled manner, it's on the @bugref{3170} todo list...
563	*/
564	int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
565	{
566	LogFlow(("pgmPhysAllocPage: %R[pgmpage] %RGp\n", pPage, GCPhys));
567
568	/*
569	* Prereqs.
570	*/
571	PGM_LOCK_ASSERT_OWNER(pVM);
572	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
573	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
574
575	# ifdef PGM_WITH_LARGE_PAGES
576	/*
577	* Try allocate a large page if applicable.
578	*/
579	if ( PGMIsUsingLargePages(pVM)
580	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
581	{
582	RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
583	PPGMPAGE pBasePage;
584
585	int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pBasePage);
586	AssertRCReturn(rc, rc); /* paranoia; can't happen. */
587	if (PGM_PAGE_GET_PDE_TYPE(pBasePage) == PGM_PAGE_PDE_TYPE_DONTCARE)
588	{
589	rc = pgmPhysAllocLargePage(pVM, GCPhys);
590	if (rc == VINF_SUCCESS)
591	return rc;
592	}
593	/* Mark the base as type page table, so we don't check over and over again. */
594	PGM_PAGE_SET_PDE_TYPE(pVM, pBasePage, PGM_PAGE_PDE_TYPE_PT);
595
596	/* fall back to 4KB pages. */
597	}
598	# endif
599
600	/*
601	* Flush any shadow page table mappings of the page.
602	* When VBOX_WITH_NEW_LAZY_PAGE_ALLOC isn't defined, there shouldn't be any.
603	*/
604	bool fFlushTLBs = false;
605	int rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhys, pPage, true /fFlushTLBs/, &fFlushTLBs);
606	AssertMsgReturn(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc), RT_FAILURE(rc) ? rc : VERR_IPE_UNEXPECTED_STATUS);
607
608	/*
609	* Ensure that we've got a page handy, take it and use it.
610	*/
611	int rc2 = pgmPhysEnsureHandyPage(pVM);
612	if (RT_FAILURE(rc2))
613	{
614	if (fFlushTLBs)
615	PGM_INVL_ALL_VCPU_TLBS(pVM);
616	Assert(rc2 == VERR_EM_NO_MEMORY);
617	return rc2;
618	}
619	/* re-assert preconditions since pgmPhysEnsureHandyPage may do a context switch. */
620	PGM_LOCK_ASSERT_OWNER(pVM);
621	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
622	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
623
624	uint32_t iHandyPage = --pVM->pgm.s.cHandyPages;
625	AssertMsg(iHandyPage < RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", iHandyPage));
626	Assert(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys != NIL_RTHCPHYS);
627	Assert(!(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
628	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idPage != NIL_GMM_PAGEID);
629	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
630
631	/*
632	* There are one or two action to be taken the next time we allocate handy pages:
633	* - Tell the GMM (global memory manager) what the page is being used for.
634	* (Speeds up replacement operations - sharing and defragmenting.)
635	* - If the current backing is shared, it must be freed.
636	*/
637	const RTHCPHYS HCPhys = pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys;
638	pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
639
640	void const *pvSharedPage = NULL;
641	if (PGM_PAGE_IS_SHARED(pPage))
642	{
643	/* Mark this shared page for freeing/dereferencing. */
644	pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage = PGM_PAGE_GET_PAGEID(pPage);
645	Assert(PGM_PAGE_GET_PAGEID(pPage) != NIL_GMM_PAGEID);
646
647	Log(("PGM: Replaced shared page %#x at %RGp with %#x / %RHp\n", PGM_PAGE_GET_PAGEID(pPage),
648	GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
649	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageReplaceShared));
650	pVM->pgm.s.cSharedPages--;
651
652	/* Grab the address of the page so we can make a copy later on. (safe) */
653	rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvSharedPage);
654	AssertRC(rc);
655	}
656	else
657	{
658	Log2(("PGM: Replaced zero page %RGp with %#x / %RHp\n", GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
659	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
660	pVM->pgm.s.cZeroPages--;
661	}
662
663	/*
664	* Do the PGMPAGE modifications.
665	*/
666	pVM->pgm.s.cPrivatePages++;
667	PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
668	PGM_PAGE_SET_PAGEID(pVM, pPage, pVM->pgm.s.aHandyPages[iHandyPage].idPage);
669	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
670	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PT);
671	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
672
673	/* Copy the shared page contents to the replacement page. */
674	if (pvSharedPage)
675	{
676	/* Get the virtual address of the new page. */
677	PGMPAGEMAPLOCK PgMpLck;
678	void *pvNewPage;
679	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvNewPage, &PgMpLck); AssertRC(rc);
680	if (RT_SUCCESS(rc))
681	{
682	memcpy(pvNewPage, pvSharedPage, PAGE_SIZE); /** @todo todo write ASMMemCopyPage */
683	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
684	}
685	}
686
687	if ( fFlushTLBs
688	&& rc != VINF_PGM_GCPHYS_ALIASED)
689	PGM_INVL_ALL_VCPU_TLBS(pVM);
690	return rc;
691	}
692
693	#ifdef PGM_WITH_LARGE_PAGES
694
695	/**
696	* Replace a 2 MB range of zero pages with new pages that we can write to.
697	*
698	* @returns The following VBox status codes.
699	* @retval VINF_SUCCESS on success, pPage is modified.
700	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
701	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
702	*
703	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
704	*
705	* @param pVM Pointer to the VM.
706	* @param GCPhys The address of the page.
707	*
708	* @remarks Must be called from within the PGM critical section. It may
709	* nip back to ring-3/0 in some cases.
710	*/
711	int pgmPhysAllocLargePage(PVM pVM, RTGCPHYS GCPhys)
712	{
713	RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
714	LogFlow(("pgmPhysAllocLargePage: %RGp base %RGp\n", GCPhys, GCPhysBase));
715
716	/*
717	* Prereqs.
718	*/
719	PGM_LOCK_ASSERT_OWNER(pVM);
720	Assert(PGMIsUsingLargePages(pVM));
721
722	PPGMPAGE pFirstPage;
723	int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pFirstPage);
724	if ( RT_SUCCESS(rc)
725	&& PGM_PAGE_GET_TYPE(pFirstPage) == PGMPAGETYPE_RAM)
726	{
727	unsigned uPDEType = PGM_PAGE_GET_PDE_TYPE(pFirstPage);
728
729	/* Don't call this function for already allocated pages. */
730	Assert(uPDEType != PGM_PAGE_PDE_TYPE_PDE);
731
732	if ( uPDEType == PGM_PAGE_PDE_TYPE_DONTCARE
733	&& PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ZERO)
734	{
735	/* Lazy approach: check all pages in the 2 MB range.
736	* The whole range must be ram and unallocated. */
737	GCPhys = GCPhysBase;
738	unsigned iPage;
739	for (iPage = 0; iPage < _2M/PAGE_SIZE; iPage++)
740	{
741	PPGMPAGE pSubPage;
742	rc = pgmPhysGetPageEx(pVM, GCPhys, &pSubPage);
743	if ( RT_FAILURE(rc)
744	\|\| PGM_PAGE_GET_TYPE(pSubPage) != PGMPAGETYPE_RAM /* Anything other than ram implies monitoring. */
745	\|\| PGM_PAGE_GET_STATE(pSubPage) != PGM_PAGE_STATE_ZERO) /* Allocated, monitored or shared means we can't use a large page here */
746	{
747	LogFlow(("Found page %RGp with wrong attributes (type=%d; state=%d); cancel check. rc=%d\n", GCPhys, PGM_PAGE_GET_TYPE(pSubPage), PGM_PAGE_GET_STATE(pSubPage), rc));
748	break;
749	}
750	Assert(PGM_PAGE_GET_PDE_TYPE(pSubPage) == PGM_PAGE_PDE_TYPE_DONTCARE);
751	GCPhys += PAGE_SIZE;
752	}
753	if (iPage != _2M/PAGE_SIZE)
754	{
755	/* Failed. Mark as requiring a PT so we don't check the whole thing again in the future. */
756	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRefused);
757	PGM_PAGE_SET_PDE_TYPE(pVM, pFirstPage, PGM_PAGE_PDE_TYPE_PT);
758	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
759	}
760
761	/*
762	* Do the allocation.
763	*/
764	# ifdef IN_RING3
765	rc = PGMR3PhysAllocateLargeHandyPage(pVM, GCPhysBase);
766	# else
767	rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE, GCPhysBase);
768	# endif
769	if (RT_SUCCESS(rc))
770	{
771	Assert(PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ALLOCATED);
772	pVM->pgm.s.cLargePages++;
773	return VINF_SUCCESS;
774	}
775
776	/* If we fail once, it most likely means the host's memory is too
777	fragmented; don't bother trying again. */
778	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
779	PGMSetLargePageUsage(pVM, false);
780	return rc;
781	}
782	}
783	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
784	}
785
786
787	/**
788	* Recheck the entire 2 MB range to see if we can use it again as a large page.
789	*
790	* @returns The following VBox status codes.
791	* @retval VINF_SUCCESS on success, the large page can be used again
792	* @retval VERR_PGM_INVALID_LARGE_PAGE_RANGE if it can't be reused
793	*
794	* @param pVM Pointer to the VM.
795	* @param GCPhys The address of the page.
796	* @param pLargePage Page structure of the base page
797	*/
798	int pgmPhysRecheckLargePage(PVM pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage)
799	{
800	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRecheck);
801
802	GCPhys &= X86_PDE2M_PAE_PG_MASK;
803
804	/* Check the base page. */
805	Assert(PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED);
806	if ( PGM_PAGE_GET_STATE(pLargePage) != PGM_PAGE_STATE_ALLOCATED
807	\|\| PGM_PAGE_GET_TYPE(pLargePage) != PGMPAGETYPE_RAM
808	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
809	{
810	LogFlow(("pgmPhysRecheckLargePage: checks failed for base page %x %x %x\n", PGM_PAGE_GET_STATE(pLargePage), PGM_PAGE_GET_TYPE(pLargePage), PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage)));
811	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
812	}
813
814	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,IsValidLargePage), a);
815	/* Check all remaining pages in the 2 MB range. */
816	unsigned i;
817	GCPhys += PAGE_SIZE;
818	for (i = 1; i < _2M/PAGE_SIZE; i++)
819	{
820	PPGMPAGE pPage;
821	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
822	AssertRCBreak(rc);
823
824	if ( PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
825	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
826	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
827	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
828	{
829	LogFlow(("pgmPhysRecheckLargePage: checks failed for page %d; %x %x %x\n", i, PGM_PAGE_GET_STATE(pPage), PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)));
830	break;
831	}
832
833	GCPhys += PAGE_SIZE;
834	}
835	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,IsValidLargePage), a);
836
837	if (i == _2M/PAGE_SIZE)
838	{
839	PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE);
840	pVM->pgm.s.cLargePagesDisabled--;
841	Log(("pgmPhysRecheckLargePage: page %RGp can be reused!\n", GCPhys - _2M));
842	return VINF_SUCCESS;
843	}
844
845	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
846	}
847
848	#endif /* PGM_WITH_LARGE_PAGES */
849
850	/**
851	* Deal with a write monitored page.
852	*
853	* @returns VBox strict status code.
854	*
855	* @param pVM Pointer to the VM.
856	* @param pPage The physical page tracking structure.
857	*
858	* @remarks Called from within the PGM critical section.
859	*/
860	void pgmPhysPageMakeWriteMonitoredWritable(PVM pVM, PPGMPAGE pPage)
861	{
862	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED);
863	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
864	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
865	Assert(pVM->pgm.s.cMonitoredPages > 0);
866	pVM->pgm.s.cMonitoredPages--;
867	pVM->pgm.s.cWrittenToPages++;
868	}
869
870
871	/**
872	* Deal with pages that are not writable, i.e. not in the ALLOCATED state.
873	*
874	* @returns VBox strict status code.
875	* @retval VINF_SUCCESS on success.
876	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
877	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
878	*
879	* @param pVM Pointer to the VM.
880	* @param pPage The physical page tracking structure.
881	* @param GCPhys The address of the page.
882	*
883	* @remarks Called from within the PGM critical section.
884	*/
885	int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
886	{
887	PGM_LOCK_ASSERT_OWNER(pVM);
888	switch (PGM_PAGE_GET_STATE(pPage))
889	{
890	case PGM_PAGE_STATE_WRITE_MONITORED:
891	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
892	/* fall thru */
893	default: /* to shut up GCC */
894	case PGM_PAGE_STATE_ALLOCATED:
895	return VINF_SUCCESS;
896
897	/*
898	* Zero pages can be dummy pages for MMIO or reserved memory,
899	* so we need to check the flags before joining cause with
900	* shared page replacement.
901	*/
902	case PGM_PAGE_STATE_ZERO:
903	if (PGM_PAGE_IS_MMIO(pPage))
904	return VERR_PGM_PHYS_PAGE_RESERVED;
905	/* fall thru */
906	case PGM_PAGE_STATE_SHARED:
907	return pgmPhysAllocPage(pVM, pPage, GCPhys);
908
909	/* Not allowed to write to ballooned pages. */
910	case PGM_PAGE_STATE_BALLOONED:
911	return VERR_PGM_PHYS_PAGE_BALLOONED;
912	}
913	}
914
915
916	/**
917	* Internal usage: Map the page specified by its GMM ID.
918	*
919	* This is similar to pgmPhysPageMap
920	*
921	* @returns VBox status code.
922	*
923	* @param pVM Pointer to the VM.
924	* @param idPage The Page ID.
925	* @param HCPhys The physical address (for RC).
926	* @param ppv Where to store the mapping address.
927	*
928	* @remarks Called from within the PGM critical section. The mapping is only
929	* valid while you are inside this section.
930	*/
931	int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv)
932	{
933	/*
934	* Validation.
935	*/
936	PGM_LOCK_ASSERT_OWNER(pVM);
937	AssertReturn(HCPhys && !(HCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
938	const uint32_t idChunk = idPage >> GMM_CHUNKID_SHIFT;
939	AssertReturn(idChunk != NIL_GMM_CHUNKID, VERR_INVALID_PARAMETER);
940
941	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
942	/*
943	* Map it by HCPhys.
944	*/
945	return pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), HCPhys, ppv RTLOG_COMMA_SRC_POS);
946
947	#else
948	/*
949	* Find/make Chunk TLB entry for the mapping chunk.
950	*/
951	PPGMCHUNKR3MAP pMap;
952	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
953	if (pTlbe->idChunk == idChunk)
954	{
955	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbHits));
956	pMap = pTlbe->pChunk;
957	}
958	else
959	{
960	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
961
962	/*
963	* Find the chunk, map it if necessary.
964	*/
965	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
966	if (pMap)
967	pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
968	else
969	{
970	# ifdef IN_RING0
971	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
972	AssertRCReturn(rc, rc);
973	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
974	Assert(pMap);
975	# else
976	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
977	if (RT_FAILURE(rc))
978	return rc;
979	# endif
980	}
981
982	/*
983	* Enter it into the Chunk TLB.
984	*/
985	pTlbe->idChunk = idChunk;
986	pTlbe->pChunk = pMap;
987	}
988
989	ppv = (uint8_t )pMap->pv + ((idPage &GMM_PAGEID_IDX_MASK) << PAGE_SHIFT);
990	return VINF_SUCCESS;
991	#endif
992	}
993
994
995	/**
996	* Maps a page into the current virtual address space so it can be accessed.
997	*
998	* @returns VBox status code.
999	* @retval VINF_SUCCESS on success.
1000	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1001	*
1002	* @param pVM Pointer to the VM.
1003	* @param pPage The physical page tracking structure.
1004	* @param GCPhys The address of the page.
1005	* @param ppMap Where to store the address of the mapping tracking structure.
1006	* @param ppv Where to store the mapping address of the page. The page
1007	* offset is masked off!
1008	*
1009	* @remarks Called from within the PGM critical section.
1010	*/
1011	static int pgmPhysPageMapCommon(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv)
1012	{
1013	PGM_LOCK_ASSERT_OWNER(pVM);
1014
1015	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1016	/*
1017	* Just some sketchy GC/R0-darwin code.
1018	*/
1019	*ppMap = NULL;
1020	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
1021	Assert(HCPhys != pVM->pgm.s.HCPhysZeroPg);
1022	pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), HCPhys, ppv RTLOG_COMMA_SRC_POS);
1023	NOREF(GCPhys);
1024	return VINF_SUCCESS;
1025
1026	#else /* IN_RING3 \|\| IN_RING0 */
1027
1028
1029	/*
1030	* Special cases: MMIO2, ZERO and specially aliased MMIO pages.
1031	*/
1032	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2
1033	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO)
1034	{
1035	/* Decode the page id to a page in a MMIO2 ram range. */
1036	uint8_t idMmio2 = PGM_MMIO2_PAGEID_GET_MMIO2_ID(PGM_PAGE_GET_PAGEID(pPage));
1037	uint32_t iPage = PGM_MMIO2_PAGEID_GET_IDX(PGM_PAGE_GET_PAGEID(pPage));
1038	AssertLogRelReturn((uint8_t)(idMmio2 - 1U)< RT_ELEMENTS(pVM->pgm.s.CTX_SUFF(apMmio2Ranges)),
1039	VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1040	PPGMMMIO2RANGE pMmio2Range = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[idMmio2 - 1];
1041	AssertLogRelReturn(pMmio2Range, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1042	AssertLogRelReturn(pMmio2Range->idMmio2 == idMmio2, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1043	AssertLogRelReturn(iPage < (pMmio2Range->RamRange.cb >> PAGE_SHIFT), VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1044	ppv = (uint8_t )pMmio2Range->RamRange.pvR3 + ((uintptr_t)iPage << PAGE_SHIFT);
1045	*ppMap = NULL;
1046	return VINF_SUCCESS;
1047	}
1048
1049	const uint32_t idChunk = PGM_PAGE_GET_CHUNKID(pPage);
1050	if (idChunk == NIL_GMM_CHUNKID)
1051	{
1052	AssertMsgReturn(PGM_PAGE_GET_PAGEID(pPage) == NIL_GMM_PAGEID, ("pPage=%R[pgmpage]\n", pPage),
1053	VERR_PGM_PHYS_PAGE_MAP_IPE_1);
1054	if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
1055	{
1056	AssertMsgReturn(PGM_PAGE_IS_ZERO(pPage), ("pPage=%R[pgmpage]\n", pPage),
1057	VERR_PGM_PHYS_PAGE_MAP_IPE_3);
1058	AssertMsgReturn(PGM_PAGE_GET_HCPHYS(pPage)== pVM->pgm.s.HCPhysZeroPg, ("pPage=%R[pgmpage]\n", pPage),
1059	VERR_PGM_PHYS_PAGE_MAP_IPE_4);
1060	*ppv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
1061	}
1062	else
1063	{
1064	static uint8_t s_abPlayItSafe[0x10002]; / I don't dare return the zero page at the moment. */
1065	ppv = (uint8_t )((uintptr_t)&s_abPlayItSafe[0x1000] & ~(uintptr_t)0xfff);
1066	}
1067	*ppMap = NULL;
1068	return VINF_SUCCESS;
1069	}
1070
1071	/*
1072	* Find/make Chunk TLB entry for the mapping chunk.
1073	*/
1074	PPGMCHUNKR3MAP pMap;
1075	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
1076	if (pTlbe->idChunk == idChunk)
1077	{
1078	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbHits));
1079	pMap = pTlbe->pChunk;
1080	AssertPtr(pMap->pv);
1081	}
1082	else
1083	{
1084	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
1085
1086	/*
1087	* Find the chunk, map it if necessary.
1088	*/
1089	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
1090	if (pMap)
1091	{
1092	AssertPtr(pMap->pv);
1093	pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
1094	}
1095	else
1096	{
1097	#ifdef IN_RING0
1098	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
1099	AssertRCReturn(rc, rc);
1100	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
1101	Assert(pMap);
1102	#else
1103	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
1104	if (RT_FAILURE(rc))
1105	return rc;
1106	#endif
1107	AssertPtr(pMap->pv);
1108	}
1109
1110	/*
1111	* Enter it into the Chunk TLB.
1112	*/
1113	pTlbe->idChunk = idChunk;
1114	pTlbe->pChunk = pMap;
1115	}
1116
1117	ppv = (uint8_t )pMap->pv + (PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) << PAGE_SHIFT);
1118	*ppMap = pMap;
1119	return VINF_SUCCESS;
1120	#endif /* IN_RING3 */
1121	}
1122
1123
1124	/**
1125	* Combination of pgmPhysPageMakeWritable and pgmPhysPageMapWritable.
1126	*
1127	* This is typically used is paths where we cannot use the TLB methods (like ROM
1128	* pages) or where there is no point in using them since we won't get many hits.
1129	*
1130	* @returns VBox strict status code.
1131	* @retval VINF_SUCCESS on success.
1132	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
1133	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1134	*
1135	* @param pVM Pointer to the VM.
1136	* @param pPage The physical page tracking structure.
1137	* @param GCPhys The address of the page.
1138	* @param ppv Where to store the mapping address of the page. The page
1139	* offset is masked off!
1140	*
1141	* @remarks Called from within the PGM critical section. The mapping is only
1142	* valid while you are inside section.
1143	*/
1144	int pgmPhysPageMakeWritableAndMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1145	{
1146	int rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1147	if (RT_SUCCESS(rc))
1148	{
1149	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* returned */, ("%Rrc\n", rc));
1150	PPGMPAGEMAP pMapIgnore;
1151	int rc2 = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
1152	if (RT_FAILURE(rc2)) /* preserve rc */
1153	rc = rc2;
1154	}
1155	return rc;
1156	}
1157
1158
1159	/**
1160	* Maps a page into the current virtual address space so it can be accessed for
1161	* both writing and reading.
1162	*
1163	* This is typically used is paths where we cannot use the TLB methods (like ROM
1164	* pages) or where there is no point in using them since we won't get many hits.
1165	*
1166	* @returns VBox status code.
1167	* @retval VINF_SUCCESS on success.
1168	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1169	*
1170	* @param pVM Pointer to the VM.
1171	* @param pPage The physical page tracking structure. Must be in the
1172	* allocated state.
1173	* @param GCPhys The address of the page.
1174	* @param ppv Where to store the mapping address of the page. The page
1175	* offset is masked off!
1176	*
1177	* @remarks Called from within the PGM critical section. The mapping is only
1178	* valid while you are inside section.
1179	*/
1180	int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1181	{
1182	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
1183	PPGMPAGEMAP pMapIgnore;
1184	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
1185	}
1186
1187
1188	/**
1189	* Maps a page into the current virtual address space so it can be accessed for
1190	* reading.
1191	*
1192	* This is typically used is paths where we cannot use the TLB methods (like ROM
1193	* pages) or where there is no point in using them since we won't get many hits.
1194	*
1195	* @returns VBox status code.
1196	* @retval VINF_SUCCESS on success.
1197	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1198	*
1199	* @param pVM Pointer to the VM.
1200	* @param pPage The physical page tracking structure.
1201	* @param GCPhys The address of the page.
1202	* @param ppv Where to store the mapping address of the page. The page
1203	* offset is masked off!
1204	*
1205	* @remarks Called from within the PGM critical section. The mapping is only
1206	* valid while you are inside this section.
1207	*/
1208	int pgmPhysPageMapReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv)
1209	{
1210	PPGMPAGEMAP pMapIgnore;
1211	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, (void **)ppv);
1212	}
1213
1214	#if !defined(IN_RC) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1215
1216	/**
1217	* Load a guest page into the ring-3 physical TLB.
1218	*
1219	* @returns VBox status code.
1220	* @retval VINF_SUCCESS on success
1221	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1222	* @param pPGM The PGM instance pointer.
1223	* @param GCPhys The guest physical address in question.
1224	*/
1225	int pgmPhysPageLoadIntoTlb(PVM pVM, RTGCPHYS GCPhys)
1226	{
1227	PGM_LOCK_ASSERT_OWNER(pVM);
1228
1229	/*
1230	* Find the ram range and page and hand it over to the with-page function.
1231	* 99.8% of requests are expected to be in the first range.
1232	*/
1233	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
1234	if (!pPage)
1235	{
1236	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
1237	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
1238	}
1239
1240	return pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys);
1241	}
1242
1243
1244	/**
1245	* Load a guest page into the ring-3 physical TLB.
1246	*
1247	* @returns VBox status code.
1248	* @retval VINF_SUCCESS on success
1249	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1250	*
1251	* @param pVM Pointer to the VM.
1252	* @param pPage Pointer to the PGMPAGE structure corresponding to
1253	* GCPhys.
1254	* @param GCPhys The guest physical address in question.
1255	*/
1256	int pgmPhysPageLoadIntoTlbWithPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
1257	{
1258	PGM_LOCK_ASSERT_OWNER(pVM);
1259	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
1260
1261	/*
1262	* Map the page.
1263	* Make a special case for the zero page as it is kind of special.
1264	*/
1265	PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
1266	if ( !PGM_PAGE_IS_ZERO(pPage)
1267	&& !PGM_PAGE_IS_BALLOONED(pPage))
1268	{
1269	void *pv;
1270	PPGMPAGEMAP pMap;
1271	int rc = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMap, &pv);
1272	if (RT_FAILURE(rc))
1273	return rc;
1274	pTlbe->pMap = pMap;
1275	pTlbe->pv = pv;
1276	Assert(!((uintptr_t)pTlbe->pv & PAGE_OFFSET_MASK));
1277	}
1278	else
1279	{
1280	AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == pVM->pgm.s.HCPhysZeroPg, ("%RGp/%R[pgmpage]\n", GCPhys, pPage));
1281	pTlbe->pMap = NULL;
1282	pTlbe->pv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
1283	}
1284	#ifdef PGM_WITH_PHYS_TLB
1285	if ( PGM_PAGE_GET_TYPE(pPage) < PGMPAGETYPE_ROM_SHADOW
1286	\|\| PGM_PAGE_GET_TYPE(pPage) > PGMPAGETYPE_ROM)
1287	pTlbe->GCPhys = GCPhys & X86_PTE_PAE_PG_MASK;
1288	else
1289	pTlbe->GCPhys = NIL_RTGCPHYS; /* ROM: Problematic because of the two pages. :-/ */
1290	#else
1291	pTlbe->GCPhys = NIL_RTGCPHYS;
1292	#endif
1293	pTlbe->pPage = pPage;
1294	return VINF_SUCCESS;
1295	}
1296
1297	#endif /* !IN_RC && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
1298
1299	/**
1300	* Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
1301	* own the PGM lock and therefore not need to lock the mapped page.
1302	*
1303	* @returns VBox status code.
1304	* @retval VINF_SUCCESS on success.
1305	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1306	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1307	*
1308	* @param pVM Pointer to the VM.
1309	* @param GCPhys The guest physical address of the page that should be mapped.
1310	* @param pPage Pointer to the PGMPAGE structure for the page.
1311	* @param ppv Where to store the address corresponding to GCPhys.
1312	*
1313	* @internal
1314	* @deprecated Use pgmPhysGCPhys2CCPtrInternalEx.
1315	*/
1316	int pgmPhysGCPhys2CCPtrInternalDepr(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1317	{
1318	int rc;
1319	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
1320	PGM_LOCK_ASSERT_OWNER(pVM);
1321	pVM->pgm.s.cDeprecatedPageLocks++;
1322
1323	/*
1324	* Make sure the page is writable.
1325	*/
1326	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1327	{
1328	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1329	if (RT_FAILURE(rc))
1330	return rc;
1331	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1332	}
1333	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1334
1335	/*
1336	* Get the mapping address.
1337	*/
1338	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1339	void *pv;
1340	rc = pgmRZDynMapHCPageInlined(VMMGetCpu(pVM),
1341	PGM_PAGE_GET_HCPHYS(pPage),
1342	&pv
1343	RTLOG_COMMA_SRC_POS);
1344	if (RT_FAILURE(rc))
1345	return rc;
1346	ppv = (void )((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1347	#else
1348	PPGMPAGEMAPTLBE pTlbe;
1349	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1350	if (RT_FAILURE(rc))
1351	return rc;
1352	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1353	#endif
1354	return VINF_SUCCESS;
1355	}
1356
1357	#if !defined(IN_RC) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1358
1359	/**
1360	* Locks a page mapping for writing.
1361	*
1362	* @param pVM Pointer to the VM.
1363	* @param pPage The page.
1364	* @param pTlbe The mapping TLB entry for the page.
1365	* @param pLock The lock structure (output).
1366	*/
1367	DECLINLINE(void) pgmPhysPageMapLockForWriting(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
1368	{
1369	PPGMPAGEMAP pMap = pTlbe->pMap;
1370	if (pMap)
1371	pMap->cRefs++;
1372
1373	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
1374	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
1375	{
1376	if (cLocks == 0)
1377	pVM->pgm.s.cWriteLockedPages++;
1378	PGM_PAGE_INC_WRITE_LOCKS(pPage);
1379	}
1380	else if (cLocks != PGM_PAGE_MAX_LOCKS)
1381	{
1382	PGM_PAGE_INC_WRITE_LOCKS(pPage);
1383	AssertMsgFailed(("%R[pgmpage] is entering permanent write locked state!\n", pPage));
1384	if (pMap)
1385	pMap->cRefs++; /* Extra ref to prevent it from going away. */
1386	}
1387
1388	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
1389	pLock->pvMap = pMap;
1390	}
1391
1392	/**
1393	* Locks a page mapping for reading.
1394	*
1395	* @param pVM Pointer to the VM.
1396	* @param pPage The page.
1397	* @param pTlbe The mapping TLB entry for the page.
1398	* @param pLock The lock structure (output).
1399	*/
1400	DECLINLINE(void) pgmPhysPageMapLockForReading(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
1401	{
1402	PPGMPAGEMAP pMap = pTlbe->pMap;
1403	if (pMap)
1404	pMap->cRefs++;
1405
1406	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
1407	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
1408	{
1409	if (cLocks == 0)
1410	pVM->pgm.s.cReadLockedPages++;
1411	PGM_PAGE_INC_READ_LOCKS(pPage);
1412	}
1413	else if (cLocks != PGM_PAGE_MAX_LOCKS)
1414	{
1415	PGM_PAGE_INC_READ_LOCKS(pPage);
1416	AssertMsgFailed(("%R[pgmpage] is entering permanent read locked state!\n", pPage));
1417	if (pMap)
1418	pMap->cRefs++; /* Extra ref to prevent it from going away. */
1419	}
1420
1421	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
1422	pLock->pvMap = pMap;
1423	}
1424
1425	#endif /* !IN_RC && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
1426
1427
1428	/**
1429	* Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
1430	* own the PGM lock and have access to the page structure.
1431	*
1432	* @returns VBox status code.
1433	* @retval VINF_SUCCESS on success.
1434	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1435	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1436	*
1437	* @param pVM Pointer to the VM.
1438	* @param GCPhys The guest physical address of the page that should be mapped.
1439	* @param pPage Pointer to the PGMPAGE structure for the page.
1440	* @param ppv Where to store the address corresponding to GCPhys.
1441	* @param pLock Where to store the lock information that
1442	* pgmPhysReleaseInternalPageMappingLock needs.
1443	*
1444	* @internal
1445	*/
1446	int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
1447	{
1448	int rc;
1449	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
1450	PGM_LOCK_ASSERT_OWNER(pVM);
1451
1452	/*
1453	* Make sure the page is writable.
1454	*/
1455	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1456	{
1457	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1458	if (RT_FAILURE(rc))
1459	return rc;
1460	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1461	}
1462	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1463
1464	/*
1465	* Do the job.
1466	*/
1467	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1468	void *pv;
1469	PVMCPU pVCpu = VMMGetCpu(pVM);
1470	rc = pgmRZDynMapHCPageInlined(pVCpu,
1471	PGM_PAGE_GET_HCPHYS(pPage),
1472	&pv
1473	RTLOG_COMMA_SRC_POS);
1474	if (RT_FAILURE(rc))
1475	return rc;
1476	ppv = (void )((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1477	pLock->pvPage = pv;
1478	pLock->pVCpu = pVCpu;
1479
1480	#else
1481	PPGMPAGEMAPTLBE pTlbe;
1482	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1483	if (RT_FAILURE(rc))
1484	return rc;
1485	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
1486	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1487	#endif
1488	return VINF_SUCCESS;
1489	}
1490
1491
1492	/**
1493	* Internal version of PGMPhysGCPhys2CCPtrReadOnly that expects the caller to
1494	* own the PGM lock and have access to the page structure.
1495	*
1496	* @returns VBox status code.
1497	* @retval VINF_SUCCESS on success.
1498	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1499	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1500	*
1501	* @param pVM Pointer to the VM.
1502	* @param GCPhys The guest physical address of the page that should be mapped.
1503	* @param pPage Pointer to the PGMPAGE structure for the page.
1504	* @param ppv Where to store the address corresponding to GCPhys.
1505	* @param pLock Where to store the lock information that
1506	* pgmPhysReleaseInternalPageMappingLock needs.
1507	*
1508	* @internal
1509	*/
1510	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock)
1511	{
1512	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
1513	PGM_LOCK_ASSERT_OWNER(pVM);
1514	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1515
1516	/*
1517	* Do the job.
1518	*/
1519	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1520	void *pv;
1521	PVMCPU pVCpu = VMMGetCpu(pVM);
1522	int rc = pgmRZDynMapHCPageInlined(pVCpu,
1523	PGM_PAGE_GET_HCPHYS(pPage),
1524	&pv
1525	RTLOG_COMMA_SRC_POS); /** @todo add a read only flag? */
1526	if (RT_FAILURE(rc))
1527	return rc;
1528	ppv = (void )((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1529	pLock->pvPage = pv;
1530	pLock->pVCpu = pVCpu;
1531
1532	#else
1533	PPGMPAGEMAPTLBE pTlbe;
1534	int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1535	if (RT_FAILURE(rc))
1536	return rc;
1537	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
1538	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1539	#endif
1540	return VINF_SUCCESS;
1541	}
1542
1543
1544	/**
1545	* Requests the mapping of a guest page into the current context.
1546	*
1547	* This API should only be used for very short term, as it will consume scarse
1548	* resources (R0 and GC) in the mapping cache. When you're done with the page,
1549	* call PGMPhysReleasePageMappingLock() ASAP to release it.
1550	*
1551	* This API will assume your intention is to write to the page, and will
1552	* therefore replace shared and zero pages. If you do not intend to modify
1553	* the page, use the PGMPhysGCPhys2CCPtrReadOnly() API.
1554	*
1555	* @returns VBox status code.
1556	* @retval VINF_SUCCESS on success.
1557	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1558	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1559	*
1560	* @param pVM Pointer to the VM.
1561	* @param GCPhys The guest physical address of the page that should be
1562	* mapped.
1563	* @param ppv Where to store the address corresponding to GCPhys.
1564	* @param pLock Where to store the lock information that
1565	* PGMPhysReleasePageMappingLock needs.
1566	*
1567	* @remarks The caller is responsible for dealing with access handlers.
1568	* @todo Add an informational return code for pages with access handlers?
1569	*
1570	* @remark Avoid calling this API from within critical sections (other than
1571	* the PGM one) because of the deadlock risk. External threads may
1572	* need to delegate jobs to the EMTs.
1573	* @remarks Only one page is mapped! Make no assumption about what's after or
1574	* before the returned page!
1575	* @thread Any thread.
1576	*/
1577	VMMDECL(int) PGMPhysGCPhys2CCPtr(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
1578	{
1579	int rc = pgmLock(pVM);
1580	AssertRCReturn(rc, rc);
1581
1582	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1583	/*
1584	* Find the page and make sure it's writable.
1585	*/
1586	PPGMPAGE pPage;
1587	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1588	if (RT_SUCCESS(rc))
1589	{
1590	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1591	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1592	if (RT_SUCCESS(rc))
1593	{
1594	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1595
1596	PVMCPU pVCpu = VMMGetCpu(pVM);
1597	void *pv;
1598	rc = pgmRZDynMapHCPageInlined(pVCpu,
1599	PGM_PAGE_GET_HCPHYS(pPage),
1600	&pv
1601	RTLOG_COMMA_SRC_POS);
1602	if (RT_SUCCESS(rc))
1603	{
1604	AssertRCSuccess(rc);
1605
1606	pv = (void *)((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1607	*ppv = pv;
1608	pLock->pvPage = pv;
1609	pLock->pVCpu = pVCpu;
1610	}
1611	}
1612	}
1613
1614	#else /* IN_RING3 \|\| IN_RING0 */
1615	/*
1616	* Query the Physical TLB entry for the page (may fail).
1617	*/
1618	PPGMPAGEMAPTLBE pTlbe;
1619	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
1620	if (RT_SUCCESS(rc))
1621	{
1622	/*
1623	* If the page is shared, the zero page, or being write monitored
1624	* it must be converted to a page that's writable if possible.
1625	*/
1626	PPGMPAGE pPage = pTlbe->pPage;
1627	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1628	{
1629	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1630	if (RT_SUCCESS(rc))
1631	{
1632	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1633	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1634	}
1635	}
1636	if (RT_SUCCESS(rc))
1637	{
1638	/*
1639	* Now, just perform the locking and calculate the return address.
1640	*/
1641	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
1642	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1643	}
1644	}
1645
1646	#endif /* IN_RING3 \|\| IN_RING0 */
1647	pgmUnlock(pVM);
1648	return rc;
1649	}
1650
1651
1652	/**
1653	* Requests the mapping of a guest page into the current context.
1654	*
1655	* This API should only be used for very short term, as it will consume scarse
1656	* resources (R0 and GC) in the mapping cache. When you're done with the page,
1657	* call PGMPhysReleasePageMappingLock() ASAP to release it.
1658	*
1659	* @returns VBox status code.
1660	* @retval VINF_SUCCESS on success.
1661	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1662	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1663	*
1664	* @param pVM Pointer to the VM.
1665	* @param GCPhys The guest physical address of the page that should be
1666	* mapped.
1667	* @param ppv Where to store the address corresponding to GCPhys.
1668	* @param pLock Where to store the lock information that
1669	* PGMPhysReleasePageMappingLock needs.
1670	*
1671	* @remarks The caller is responsible for dealing with access handlers.
1672	* @todo Add an informational return code for pages with access handlers?
1673	*
1674	* @remarks Avoid calling this API from within critical sections (other than
1675	* the PGM one) because of the deadlock risk.
1676	* @remarks Only one page is mapped! Make no assumption about what's after or
1677	* before the returned page!
1678	* @thread Any thread.
1679	*/
1680	VMMDECL(int) PGMPhysGCPhys2CCPtrReadOnly(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
1681	{
1682	int rc = pgmLock(pVM);
1683	AssertRCReturn(rc, rc);
1684
1685	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1686	/*
1687	* Find the page and make sure it's readable.
1688	*/
1689	PPGMPAGE pPage;
1690	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1691	if (RT_SUCCESS(rc))
1692	{
1693	if (RT_UNLIKELY(PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)))
1694	rc = VERR_PGM_PHYS_PAGE_RESERVED;
1695	else
1696	{
1697	PVMCPU pVCpu = VMMGetCpu(pVM);
1698	void *pv;
1699	rc = pgmRZDynMapHCPageInlined(pVCpu,
1700	PGM_PAGE_GET_HCPHYS(pPage),
1701	&pv
1702	RTLOG_COMMA_SRC_POS); /** @todo add a read only flag? */
1703	if (RT_SUCCESS(rc))
1704	{
1705	AssertRCSuccess(rc);
1706
1707	pv = (void *)((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1708	*ppv = pv;
1709	pLock->pvPage = pv;
1710	pLock->pVCpu = pVCpu;
1711	}
1712	}
1713	}
1714
1715	#else /* IN_RING3 \|\| IN_RING0 */
1716	/*
1717	* Query the Physical TLB entry for the page (may fail).
1718	*/
1719	PPGMPAGEMAPTLBE pTlbe;
1720	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
1721	if (RT_SUCCESS(rc))
1722	{
1723	/* MMIO pages doesn't have any readable backing. */
1724	PPGMPAGE pPage = pTlbe->pPage;
1725	if (RT_UNLIKELY(PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)))
1726	rc = VERR_PGM_PHYS_PAGE_RESERVED;
1727	else
1728	{
1729	/*
1730	* Now, just perform the locking and calculate the return address.
1731	*/
1732	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
1733	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1734	}
1735	}
1736
1737	#endif /* IN_RING3 \|\| IN_RING0 */
1738	pgmUnlock(pVM);
1739	return rc;
1740	}
1741
1742
1743	/**
1744	* Requests the mapping of a guest page given by virtual address into the current context.
1745	*
1746	* This API should only be used for very short term, as it will consume
1747	* scarse resources (R0 and GC) in the mapping cache. When you're done
1748	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1749	*
1750	* This API will assume your intention is to write to the page, and will
1751	* therefore replace shared and zero pages. If you do not intend to modify
1752	* the page, use the PGMPhysGCPtr2CCPtrReadOnly() API.
1753	*
1754	* @returns VBox status code.
1755	* @retval VINF_SUCCESS on success.
1756	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
1757	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
1758	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1759	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1760	*
1761	* @param pVCpu Pointer to the VMCPU.
1762	* @param GCPhys The guest physical address of the page that should be mapped.
1763	* @param ppv Where to store the address corresponding to GCPhys.
1764	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1765	*
1766	* @remark Avoid calling this API from within critical sections (other than
1767	* the PGM one) because of the deadlock risk.
1768	* @thread EMT
1769	*/
1770	VMMDECL(int) PGMPhysGCPtr2CCPtr(PVMCPU pVCpu, RTGCPTR GCPtr, void **ppv, PPGMPAGEMAPLOCK pLock)
1771	{
1772	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
1773	RTGCPHYS GCPhys;
1774	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
1775	if (RT_SUCCESS(rc))
1776	rc = PGMPhysGCPhys2CCPtr(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
1777	return rc;
1778	}
1779
1780
1781	/**
1782	* Requests the mapping of a guest page given by virtual address into the current context.
1783	*
1784	* This API should only be used for very short term, as it will consume
1785	* scarse resources (R0 and GC) in the mapping cache. When you're done
1786	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1787	*
1788	* @returns VBox status code.
1789	* @retval VINF_SUCCESS on success.
1790	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
1791	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
1792	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1793	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1794	*
1795	* @param pVCpu Pointer to the VMCPU.
1796	* @param GCPhys The guest physical address of the page that should be mapped.
1797	* @param ppv Where to store the address corresponding to GCPhys.
1798	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1799	*
1800	* @remark Avoid calling this API from within critical sections (other than
1801	* the PGM one) because of the deadlock risk.
1802	* @thread EMT
1803	*/
1804	VMMDECL(int) PGMPhysGCPtr2CCPtrReadOnly(PVMCPU pVCpu, RTGCPTR GCPtr, void const **ppv, PPGMPAGEMAPLOCK pLock)
1805	{
1806	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
1807	RTGCPHYS GCPhys;
1808	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
1809	if (RT_SUCCESS(rc))
1810	rc = PGMPhysGCPhys2CCPtrReadOnly(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
1811	return rc;
1812	}
1813
1814
1815	/**
1816	* Release the mapping of a guest page.
1817	*
1818	* This is the counter part of PGMPhysGCPhys2CCPtr, PGMPhysGCPhys2CCPtrReadOnly
1819	* PGMPhysGCPtr2CCPtr and PGMPhysGCPtr2CCPtrReadOnly.
1820	*
1821	* @param pVM Pointer to the VM.
1822	* @param pLock The lock structure initialized by the mapping function.
1823	*/
1824	VMMDECL(void) PGMPhysReleasePageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock)
1825	{
1826	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1827	Assert(pLock->pvPage != NULL);
1828	Assert(pLock->pVCpu == VMMGetCpu(pVM));
1829	PGM_DYNMAP_UNUSED_HINT(pLock->pVCpu, pLock->pvPage);
1830	pLock->pVCpu = NULL;
1831	pLock->pvPage = NULL;
1832
1833	#else
1834	PPGMPAGEMAP pMap = (PPGMPAGEMAP)pLock->pvMap;
1835	PPGMPAGE pPage = (PPGMPAGE)(pLock->uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
1836	bool fWriteLock = (pLock->uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
1837
1838	pLock->uPageAndType = 0;
1839	pLock->pvMap = NULL;
1840
1841	pgmLock(pVM);
1842	if (fWriteLock)
1843	{
1844	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
1845	Assert(cLocks > 0);
1846	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
1847	{
1848	if (cLocks == 1)
1849	{
1850	Assert(pVM->pgm.s.cWriteLockedPages > 0);
1851	pVM->pgm.s.cWriteLockedPages--;
1852	}
1853	PGM_PAGE_DEC_WRITE_LOCKS(pPage);
1854	}
1855
1856	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
1857	{
1858	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
1859	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
1860	Assert(pVM->pgm.s.cMonitoredPages > 0);
1861	pVM->pgm.s.cMonitoredPages--;
1862	pVM->pgm.s.cWrittenToPages++;
1863	}
1864	}
1865	else
1866	{
1867	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
1868	Assert(cLocks > 0);
1869	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
1870	{
1871	if (cLocks == 1)
1872	{
1873	Assert(pVM->pgm.s.cReadLockedPages > 0);
1874	pVM->pgm.s.cReadLockedPages--;
1875	}
1876	PGM_PAGE_DEC_READ_LOCKS(pPage);
1877	}
1878	}
1879
1880	if (pMap)
1881	{
1882	Assert(pMap->cRefs >= 1);
1883	pMap->cRefs--;
1884	}
1885	pgmUnlock(pVM);
1886	#endif /* IN_RING3 */
1887	}
1888
1889
1890	/**
1891	* Release the internal mapping of a guest page.
1892	*
1893	* This is the counter part of pgmPhysGCPhys2CCPtrInternalEx and
1894	* pgmPhysGCPhys2CCPtrInternalReadOnly.
1895	*
1896	* @param pVM Pointer to the VM.
1897	* @param pLock The lock structure initialized by the mapping function.
1898	*
1899	* @remarks Caller must hold the PGM lock.
1900	*/
1901	void pgmPhysReleaseInternalPageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock)
1902	{
1903	PGM_LOCK_ASSERT_OWNER(pVM);
1904	PGMPhysReleasePageMappingLock(pVM, pLock); /* lazy for now */
1905	}
1906
1907
1908	/**
1909	* Converts a GC physical address to a HC ring-3 pointer.
1910	*
1911	* @returns VINF_SUCCESS on success.
1912	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
1913	* page but has no physical backing.
1914	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
1915	* GC physical address.
1916	* @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
1917	* a dynamic ram chunk boundary
1918	*
1919	* @param pVM Pointer to the VM.
1920	* @param GCPhys The GC physical address to convert.
1921	* @param pR3Ptr Where to store the R3 pointer on success.
1922	*
1923	* @deprecated Avoid when possible!
1924	*/
1925	int pgmPhysGCPhys2R3Ptr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
1926	{
1927	/** @todo this is kind of hacky and needs some more work. */
1928	#ifndef DEBUG_sandervl
1929	VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
1930	#endif
1931
1932	Log(("pgmPhysGCPhys2R3Ptr(,%RGp,): dont use this API!\n", GCPhys)); /** @todo eliminate this API! */
1933	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1934	NOREF(pVM); NOREF(pR3Ptr);
1935	AssertFailedReturn(VERR_NOT_IMPLEMENTED);
1936	#else
1937	pgmLock(pVM);
1938
1939	PPGMRAMRANGE pRam;
1940	PPGMPAGE pPage;
1941	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
1942	if (RT_SUCCESS(rc))
1943	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
1944
1945	pgmUnlock(pVM);
1946	Assert(rc <= VINF_SUCCESS);
1947	return rc;
1948	#endif
1949	}
1950
1951	#if 0 /defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)/
1952
1953	/**
1954	* Maps and locks a guest CR3 or PD (PAE) page.
1955	*
1956	* @returns VINF_SUCCESS on success.
1957	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
1958	* page but has no physical backing.
1959	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
1960	* GC physical address.
1961	* @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
1962	* a dynamic ram chunk boundary
1963	*
1964	* @param pVM Pointer to the VM.
1965	* @param GCPhys The GC physical address to convert.
1966	* @param pR3Ptr Where to store the R3 pointer on success. This may or
1967	* may not be valid in ring-0 depending on the
1968	* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 build option.
1969	*
1970	* @remarks The caller must own the PGM lock.
1971	*/
1972	int pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
1973	{
1974
1975	PPGMRAMRANGE pRam;
1976	PPGMPAGE pPage;
1977	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
1978	if (RT_SUCCESS(rc))
1979	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
1980	Assert(rc <= VINF_SUCCESS);
1981	return rc;
1982	}
1983
1984
1985	int pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
1986	{
1987
1988	}
1989
1990	#endif
1991
1992	/**
1993	* Converts a guest pointer to a GC physical address.
1994	*
1995	* This uses the current CR3/CR0/CR4 of the guest.
1996	*
1997	* @returns VBox status code.
1998	* @param pVCpu Pointer to the VMCPU.
1999	* @param GCPtr The guest pointer to convert.
2000	* @param pGCPhys Where to store the GC physical address.
2001	*/
2002	VMMDECL(int) PGMPhysGCPtr2GCPhys(PVMCPU pVCpu, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
2003	{
2004	int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, pGCPhys);
2005	if (pGCPhys && RT_SUCCESS(rc))
2006	*pGCPhys \|= (RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK;
2007	return rc;
2008	}
2009
2010
2011	/**
2012	* Converts a guest pointer to a HC physical address.
2013	*
2014	* This uses the current CR3/CR0/CR4 of the guest.
2015	*
2016	* @returns VBox status code.
2017	* @param pVCpu Pointer to the VMCPU.
2018	* @param GCPtr The guest pointer to convert.
2019	* @param pHCPhys Where to store the HC physical address.
2020	*/
2021	VMMDECL(int) PGMPhysGCPtr2HCPhys(PVMCPU pVCpu, RTGCPTR GCPtr, PRTHCPHYS pHCPhys)
2022	{
2023	PVM pVM = pVCpu->CTX_SUFF(pVM);
2024	RTGCPHYS GCPhys;
2025	int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, &GCPhys);
2026	if (RT_SUCCESS(rc))
2027	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys \| ((RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK), pHCPhys);
2028	return rc;
2029	}
2030
2031
2032
2033	#undef LOG_GROUP
2034	#define LOG_GROUP LOG_GROUP_PGM_PHYS_ACCESS
2035
2036
2037	#if defined(IN_RING3) && defined(SOME_UNUSED_FUNCTION)
2038	/**
2039	* Cache PGMPhys memory access
2040	*
2041	* @param pVM Pointer to the VM.
2042	* @param pCache Cache structure pointer
2043	* @param GCPhys GC physical address
2044	* @param pbHC HC pointer corresponding to physical page
2045	*
2046	* @thread EMT.
2047	*/
2048	static void pgmPhysCacheAdd(PVM pVM, PGMPHYSCACHE pCache, RTGCPHYS GCPhys, uint8_t pbR3)
2049	{
2050	uint32_t iCacheIndex;
2051
2052	Assert(VM_IS_EMT(pVM));
2053
2054	GCPhys = PHYS_PAGE_ADDRESS(GCPhys);
2055	pbR3 = (uint8_t *)PAGE_ADDRESS(pbR3);
2056
2057	iCacheIndex = ((GCPhys >> PAGE_SHIFT) & PGM_MAX_PHYSCACHE_ENTRIES_MASK);
2058
2059	ASMBitSet(&pCache->aEntries, iCacheIndex);
2060
2061	pCache->Entry[iCacheIndex].GCPhys = GCPhys;
2062	pCache->Entry[iCacheIndex].pbR3 = pbR3;
2063	}
2064	#endif /* IN_RING3 */
2065
2066
2067	/**
2068	* Deals with reading from a page with one or more ALL access handlers.
2069	*
2070	* @returns VBox status code. Can be ignored in ring-3.
2071	* @retval VINF_SUCCESS.
2072	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
2073	*
2074	* @param pVM Pointer to the VM.
2075	* @param pPage The page descriptor.
2076	* @param GCPhys The physical address to start reading at.
2077	* @param pvBuf Where to put the bits we read.
2078	* @param cb How much to read - less or equal to a page.
2079	*/
2080	static int pgmPhysReadHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void *pvBuf, size_t cb)
2081	{
2082	/*
2083	* The most frequent access here is MMIO and shadowed ROM.
2084	* The current code ASSUMES all these access handlers covers full pages!
2085	*/
2086
2087	/*
2088	* Whatever we do we need the source page, map it first.
2089	*/
2090	PGMPAGEMAPLOCK PgMpLck;
2091	const void *pvSrc = NULL;
2092	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvSrc, &PgMpLck);
2093	if (RT_FAILURE(rc))
2094	{
2095	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
2096	GCPhys, pPage, rc));
2097	memset(pvBuf, 0xff, cb);
2098	return VINF_SUCCESS;
2099	}
2100	rc = VINF_PGM_HANDLER_DO_DEFAULT;
2101
2102	/*
2103	* Deal with any physical handlers.
2104	*/
2105	#ifdef IN_RING3
2106	PPGMPHYSHANDLER pPhys = NULL;
2107	#endif
2108	if ( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL
2109	\|\| PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
2110	{
2111	#ifdef IN_RING3
2112	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2113	AssertReleaseMsg(pPhys, ("GCPhys=%RGp cb=%#x\n", GCPhys, cb));
2114	Assert(GCPhys >= pPhys->Core.Key && GCPhys <= pPhys->Core.KeyLast);
2115	Assert((pPhys->Core.Key & PAGE_OFFSET_MASK) == 0);
2116	Assert((pPhys->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
2117	Assert(pPhys->CTX_SUFF(pfnHandler));
2118
2119	PFNPGMR3PHYSHANDLER pfnHandler = pPhys->CTX_SUFF(pfnHandler);
2120	void *pvUser = pPhys->CTX_SUFF(pvUser);
2121
2122	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cb, pPage, R3STRING(pPhys->pszDesc) ));
2123	STAM_PROFILE_START(&pPhys->Stat, h);
2124	PGM_LOCK_ASSERT_OWNER(pVM);
2125	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2126	pgmUnlock(pVM);
2127	rc = pfnHandler(pVM, GCPhys, (void *)pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, pvUser);
2128	pgmLock(pVM);
2129	# ifdef VBOX_WITH_STATISTICS
2130	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2131	if (pPhys)
2132	STAM_PROFILE_STOP(&pPhys->Stat, h);
2133	# else
2134	pPhys = NULL; /* might not be valid anymore. */
2135	# endif
2136	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp\n", rc, GCPhys));
2137	#else
2138	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2139	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cb=%#x\n", GCPhys, cb));
2140	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2141	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2142	#endif
2143	}
2144
2145	/*
2146	* Deal with any virtual handlers.
2147	*/
2148	if (PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) == PGM_PAGE_HNDL_VIRT_STATE_ALL)
2149	{
2150	unsigned iPage;
2151	PPGMVIRTHANDLER pVirt;
2152
2153	int rc2 = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pVirt, &iPage);
2154	AssertReleaseMsg(RT_SUCCESS(rc2), ("GCPhys=%RGp cb=%#x rc2=%Rrc\n", GCPhys, cb, rc2));
2155	Assert((pVirt->Core.Key & PAGE_OFFSET_MASK) == 0);
2156	Assert((pVirt->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
2157	Assert(GCPhys >= pVirt->aPhysToVirt[iPage].Core.Key && GCPhys <= pVirt->aPhysToVirt[iPage].Core.KeyLast);
2158
2159	#ifdef IN_RING3
2160	if (pVirt->pfnHandlerR3)
2161	{
2162	if (!pPhys)
2163	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] virt %s\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc) ));
2164	else
2165	Log(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys/virt %s/%s\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc), R3STRING(pPhys->pszDesc) ));
2166	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2167	+ (iPage << PAGE_SHIFT)
2168	+ (GCPhys & PAGE_OFFSET_MASK);
2169
2170	STAM_PROFILE_START(&pVirt->Stat, h);
2171	rc2 = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, (void )pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, /pVirt->CTX_SUFF(pvUser)*/ NULL);
2172	STAM_PROFILE_STOP(&pVirt->Stat, h);
2173	if (rc2 == VINF_SUCCESS)
2174	rc = VINF_SUCCESS;
2175	AssertLogRelMsg(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc2, GCPhys, pPage, pVirt->pszDesc));
2176	}
2177	else
2178	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] virt %s [no handler]\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc) ));
2179	#else
2180	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2181	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cb=%#x\n", GCPhys, cb));
2182	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2183	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2184	#endif
2185	}
2186
2187	/*
2188	* Take the default action.
2189	*/
2190	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
2191	memcpy(pvBuf, pvSrc, cb);
2192	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2193	return rc;
2194	}
2195
2196
2197	/**
2198	* Read physical memory.
2199	*
2200	* This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
2201	* want to ignore those.
2202	*
2203	* @returns VBox status code. Can be ignored in ring-3.
2204	* @retval VINF_SUCCESS.
2205	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
2206	*
2207	* @param pVM Pointer to the VM.
2208	* @param GCPhys Physical address start reading from.
2209	* @param pvBuf Where to put the read bits.
2210	* @param cbRead How many bytes to read.
2211	*/
2212	VMMDECL(int) PGMPhysRead(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
2213	{
2214	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
2215	LogFlow(("PGMPhysRead: %RGp %d\n", GCPhys, cbRead));
2216
2217	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysRead));
2218	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysReadBytes), cbRead);
2219
2220	pgmLock(pVM);
2221
2222	/*
2223	* Copy loop on ram ranges.
2224	*/
2225	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
2226	for (;;)
2227	{
2228	/* Inside range or not? */
2229	if (pRam && GCPhys >= pRam->GCPhys)
2230	{
2231	/*
2232	* Must work our way thru this page by page.
2233	*/
2234	RTGCPHYS off = GCPhys - pRam->GCPhys;
2235	while (off < pRam->cb)
2236	{
2237	unsigned iPage = off >> PAGE_SHIFT;
2238	PPGMPAGE pPage = &pRam->aPages[iPage];
2239	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
2240	if (cb > cbRead)
2241	cb = cbRead;
2242
2243	/*
2244	* Any ALL access handlers?
2245	*/
2246	if (RT_UNLIKELY( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
2247	\|\| PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage)))
2248	{
2249	int rc = pgmPhysReadHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb);
2250	if (RT_FAILURE(rc))
2251	{
2252	pgmUnlock(pVM);
2253	return rc;
2254	}
2255	}
2256	else
2257	{
2258	/*
2259	* Get the pointer to the page.
2260	*/
2261	PGMPAGEMAPLOCK PgMpLck;
2262	const void *pvSrc;
2263	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
2264	if (RT_SUCCESS(rc))
2265	{
2266	memcpy(pvBuf, pvSrc, cb);
2267	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2268	}
2269	else
2270	{
2271	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
2272	pRam->GCPhys + off, pPage, rc));
2273	memset(pvBuf, 0xff, cb);
2274	}
2275	}
2276
2277	/* next page */
2278	if (cb >= cbRead)
2279	{
2280	pgmUnlock(pVM);
2281	return VINF_SUCCESS;
2282	}
2283	cbRead -= cb;
2284	off += cb;
2285	pvBuf = (char *)pvBuf + cb;
2286	} /* walk pages in ram range. */
2287
2288	GCPhys = pRam->GCPhysLast + 1;
2289	}
2290	else
2291	{
2292	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
2293
2294	/*
2295	* Unassigned address space.
2296	*/
2297	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
2298	if (cb >= cbRead)
2299	{
2300	memset(pvBuf, 0xff, cbRead);
2301	break;
2302	}
2303	memset(pvBuf, 0xff, cb);
2304
2305	cbRead -= cb;
2306	pvBuf = (char *)pvBuf + cb;
2307	GCPhys += cb;
2308	}
2309
2310	/* Advance range if necessary. */
2311	while (pRam && GCPhys > pRam->GCPhysLast)
2312	pRam = pRam->CTX_SUFF(pNext);
2313	} /* Ram range walk */
2314
2315	pgmUnlock(pVM);
2316	return VINF_SUCCESS;
2317	}
2318
2319
2320	/**
2321	* Deals with writing to a page with one or more WRITE or ALL access handlers.
2322	*
2323	* @returns VBox status code. Can be ignored in ring-3.
2324	* @retval VINF_SUCCESS.
2325	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
2326	*
2327	* @param pVM Pointer to the VM.
2328	* @param pPage The page descriptor.
2329	* @param GCPhys The physical address to start writing at.
2330	* @param pvBuf What to write.
2331	* @param cbWrite How much to write - less or equal to a page.
2332	*/
2333	static int pgmPhysWriteHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const *pvBuf, size_t cbWrite)
2334	{
2335	PGMPAGEMAPLOCK PgMpLck;
2336	void *pvDst = NULL;
2337	int rc;
2338
2339	/*
2340	* Give priority to physical handlers (like #PF does).
2341	*
2342	* Hope for a lonely physical handler first that covers the whole
2343	* write area. This should be a pretty frequent case with MMIO and
2344	* the heavy usage of full page handlers in the page pool.
2345	*/
2346	if ( !PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage)
2347	\|\| PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage) /* screw virtual handlers on MMIO pages */)
2348	{
2349	PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
2350	if (pCur)
2351	{
2352	Assert(GCPhys >= pCur->Core.Key && GCPhys <= pCur->Core.KeyLast);
2353	Assert(pCur->CTX_SUFF(pfnHandler));
2354
2355	size_t cbRange = pCur->Core.KeyLast - GCPhys + 1;
2356	if (cbRange > cbWrite)
2357	cbRange = cbWrite;
2358
2359	#ifndef IN_RING3
2360	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2361	NOREF(cbRange);
2362	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2363	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2364
2365	#else /* IN_RING3 */
2366	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
2367	if (!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
2368	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
2369	else
2370	rc = VINF_SUCCESS;
2371	if (RT_SUCCESS(rc))
2372	{
2373	PFNPGMR3PHYSHANDLER pfnHandler = pCur->CTX_SUFF(pfnHandler);
2374	void *pvUser = pCur->CTX_SUFF(pvUser);
2375
2376	STAM_PROFILE_START(&pCur->Stat, h);
2377	PGM_LOCK_ASSERT_OWNER(pVM);
2378	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2379	pgmUnlock(pVM);
2380	rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2381	pgmLock(pVM);
2382	# ifdef VBOX_WITH_STATISTICS
2383	pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
2384	if (pCur)
2385	STAM_PROFILE_STOP(&pCur->Stat, h);
2386	# else
2387	pCur = NULL; /* might not be valid anymore. */
2388	# endif
2389	if (rc == VINF_PGM_HANDLER_DO_DEFAULT && pvDst)
2390	{
2391	if (pvDst)
2392	memcpy(pvDst, pvBuf, cbRange);
2393	}
2394	else
2395	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, (pCur) ? pCur->pszDesc : ""));
2396	}
2397	else
2398	AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2399	GCPhys, pPage, rc), rc);
2400	if (RT_LIKELY(cbRange == cbWrite))
2401	{
2402	if (pvDst)
2403	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2404	return VINF_SUCCESS;
2405	}
2406
2407	/* more fun to be had below */
2408	cbWrite -= cbRange;
2409	GCPhys += cbRange;
2410	pvBuf = (uint8_t *)pvBuf + cbRange;
2411	pvDst = (uint8_t *)pvDst + cbRange;
2412	#endif /* IN_RING3 */
2413	}
2414	/* else: the handler is somewhere else in the page, deal with it below. */
2415	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)); /* MMIO handlers are all PAGE_SIZEed! */
2416	}
2417	/*
2418	* A virtual handler without any interfering physical handlers.
2419	* Hopefully it'll cover the whole write.
2420	*/
2421	else if (!PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage))
2422	{
2423	unsigned iPage;
2424	PPGMVIRTHANDLER pCur;
2425	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pCur, &iPage);
2426	if (RT_SUCCESS(rc))
2427	{
2428	size_t cbRange = (PAGE_OFFSET_MASK & pCur->Core.KeyLast) - (PAGE_OFFSET_MASK & GCPhys) + 1;
2429	if (cbRange > cbWrite)
2430	cbRange = cbWrite;
2431
2432	#ifndef IN_RING3
2433	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2434	NOREF(cbRange);
2435	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2436	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2437
2438	#else /* IN_RING3 */
2439
2440	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] virt %s\n", GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
2441	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
2442	if (RT_SUCCESS(rc))
2443	{
2444	rc = VINF_PGM_HANDLER_DO_DEFAULT;
2445	if (pCur->pfnHandlerR3)
2446	{
2447	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pCur->Core.Key & PAGE_BASE_GC_MASK)
2448	+ (iPage << PAGE_SHIFT)
2449	+ (GCPhys & PAGE_OFFSET_MASK);
2450
2451	STAM_PROFILE_START(&pCur->Stat, h);
2452	rc = pCur->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void )pvBuf, cbRange, PGMACCESSTYPE_WRITE, /pCur->CTX_SUFF(pvUser)*/ NULL);
2453	STAM_PROFILE_STOP(&pCur->Stat, h);
2454	}
2455	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
2456	memcpy(pvDst, pvBuf, cbRange);
2457	else
2458	AssertLogRelMsg(rc == VINF_SUCCESS, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pCur->pszDesc));
2459	}
2460	else
2461	AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2462	GCPhys, pPage, rc), rc);
2463	if (RT_LIKELY(cbRange == cbWrite))
2464	{
2465	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2466	return VINF_SUCCESS;
2467	}
2468
2469	/* more fun to be had below */
2470	cbWrite -= cbRange;
2471	GCPhys += cbRange;
2472	pvBuf = (uint8_t *)pvBuf + cbRange;
2473	pvDst = (uint8_t *)pvDst + cbRange;
2474	#endif
2475	}
2476	/* else: the handler is somewhere else in the page, deal with it below. */
2477	}
2478
2479	/*
2480	* Deal with all the odd ends.
2481	*/
2482
2483	/* We need a writable destination page. */
2484	if (!pvDst)
2485	{
2486	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
2487	AssertLogRelMsgReturn(RT_SUCCESS(rc),
2488	("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2489	GCPhys, pPage, rc), rc);
2490	}
2491
2492	/* The loop state (big + ugly). */
2493	unsigned iVirtPage = 0;
2494	PPGMVIRTHANDLER pVirt = NULL;
2495	uint32_t offVirt = PAGE_SIZE;
2496	uint32_t offVirtLast = PAGE_SIZE;
2497	bool fMoreVirt = PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage);
2498
2499	PPGMPHYSHANDLER pPhys = NULL;
2500	uint32_t offPhys = PAGE_SIZE;
2501	uint32_t offPhysLast = PAGE_SIZE;
2502	bool fMorePhys = PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage);
2503
2504	/* The loop. */
2505	for (;;)
2506	{
2507	/*
2508	* Find the closest handler at or above GCPhys.
2509	*/
2510	if (fMoreVirt && !pVirt)
2511	{
2512	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pVirt, &iVirtPage);
2513	if (RT_SUCCESS(rc))
2514	{
2515	offVirt = 0;
2516	offVirtLast = (pVirt->aPhysToVirt[iVirtPage].Core.KeyLast & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2517	}
2518	else
2519	{
2520	PPGMPHYS2VIRTHANDLER pVirtPhys;
2521	pVirtPhys = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysToVirtHandlers,
2522	GCPhys, true /* fAbove */);
2523	if ( pVirtPhys
2524	&& (pVirtPhys->Core.Key >> PAGE_SHIFT) == (GCPhys >> PAGE_SHIFT))
2525	{
2526	/* ASSUME that pVirtPhys only covers one page. */
2527	Assert((pVirtPhys->Core.Key >> PAGE_SHIFT) == (pVirtPhys->Core.KeyLast >> PAGE_SHIFT));
2528	Assert(pVirtPhys->Core.Key > GCPhys);
2529
2530	pVirt = (PPGMVIRTHANDLER)((uintptr_t)pVirtPhys + pVirtPhys->offVirtHandler);
2531	iVirtPage = pVirtPhys - &pVirt->aPhysToVirt[0]; Assert(iVirtPage == 0);
2532	offVirt = (pVirtPhys->Core.Key & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2533	offVirtLast = (pVirtPhys->Core.KeyLast & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2534	}
2535	else
2536	{
2537	pVirt = NULL;
2538	fMoreVirt = false;
2539	offVirt = offVirtLast = PAGE_SIZE;
2540	}
2541	}
2542	}
2543
2544	if (fMorePhys && !pPhys)
2545	{
2546	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2547	if (pPhys)
2548	{
2549	offPhys = 0;
2550	offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
2551	}
2552	else
2553	{
2554	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers,
2555	GCPhys, true /* fAbove */);
2556	if ( pPhys
2557	&& pPhys->Core.Key <= GCPhys + (cbWrite - 1))
2558	{
2559	offPhys = pPhys->Core.Key - GCPhys;
2560	offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
2561	}
2562	else
2563	{
2564	pPhys = NULL;
2565	fMorePhys = false;
2566	offPhys = offPhysLast = PAGE_SIZE;
2567	}
2568	}
2569	}
2570
2571	/*
2572	* Handle access to space without handlers (that's easy).
2573	*/
2574	rc = VINF_PGM_HANDLER_DO_DEFAULT;
2575	uint32_t cbRange = (uint32_t)cbWrite;
2576	if (offPhys && offVirt)
2577	{
2578	if (cbRange > offPhys)
2579	cbRange = offPhys;
2580	if (cbRange > offVirt)
2581	cbRange = offVirt;
2582	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] miss\n", GCPhys, cbRange, pPage));
2583	}
2584	/*
2585	* Physical handler.
2586	*/
2587	else if (!offPhys && offVirt)
2588	{
2589	if (cbRange > offPhysLast + 1)
2590	cbRange = offPhysLast + 1;
2591	if (cbRange > offVirt)
2592	cbRange = offVirt;
2593	#ifdef IN_RING3
2594	PFNPGMR3PHYSHANDLER pfnHandler = pPhys->CTX_SUFF(pfnHandler);
2595	void *pvUser = pPhys->CTX_SUFF(pvUser);
2596
2597	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc) ));
2598	STAM_PROFILE_START(&pPhys->Stat, h);
2599	PGM_LOCK_ASSERT_OWNER(pVM);
2600	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2601	pgmUnlock(pVM);
2602	rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2603	pgmLock(pVM);
2604	# ifdef VBOX_WITH_STATISTICS
2605	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2606	if (pPhys)
2607	STAM_PROFILE_STOP(&pPhys->Stat, h);
2608	# else
2609	pPhys = NULL; /* might not be valid anymore. */
2610	# endif
2611	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, (pPhys) ? pPhys->pszDesc : ""));
2612	#else
2613	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2614	NOREF(cbRange);
2615	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2616	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2617	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2618	#endif
2619	}
2620	/*
2621	* Virtual handler.
2622	*/
2623	else if (offPhys && !offVirt)
2624	{
2625	if (cbRange > offVirtLast + 1)
2626	cbRange = offVirtLast + 1;
2627	if (cbRange > offPhys)
2628	cbRange = offPhys;
2629	#ifdef IN_RING3
2630	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pVirt->pszDesc) ));
2631	if (pVirt->pfnHandlerR3)
2632	{
2633	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2634	+ (iVirtPage << PAGE_SHIFT)
2635	+ (GCPhys & PAGE_OFFSET_MASK);
2636	STAM_PROFILE_START(&pVirt->Stat, h);
2637	rc = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void )pvBuf, cbRange, PGMACCESSTYPE_WRITE, /pCur->CTX_SUFF(pvUser)*/ NULL);
2638	STAM_PROFILE_STOP(&pVirt->Stat, h);
2639	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pVirt->pszDesc));
2640	}
2641	pVirt = NULL;
2642	#else
2643	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2644	NOREF(cbRange);
2645	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2646	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2647	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2648	#endif
2649	}
2650	/*
2651	* Both... give the physical one priority.
2652	*/
2653	else
2654	{
2655	Assert(!offPhys && !offVirt);
2656	if (cbRange > offVirtLast + 1)
2657	cbRange = offVirtLast + 1;
2658	if (cbRange > offPhysLast + 1)
2659	cbRange = offPhysLast + 1;
2660
2661	#ifdef IN_RING3
2662	if (pVirt->pfnHandlerR3)
2663	Log(("pgmPhysWriteHandler: overlapping phys and virt handlers at %RGp %R[pgmpage]; cbRange=%#x\n", GCPhys, pPage, cbRange));
2664	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys/virt %s/%s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc), R3STRING(pVirt->pszDesc) ));
2665
2666	PFNPGMR3PHYSHANDLER pfnHandler = pPhys->CTX_SUFF(pfnHandler);
2667	void *pvUser = pPhys->CTX_SUFF(pvUser);
2668
2669	STAM_PROFILE_START(&pPhys->Stat, h);
2670	PGM_LOCK_ASSERT_OWNER(pVM);
2671	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2672	pgmUnlock(pVM);
2673	rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2674	pgmLock(pVM);
2675	# ifdef VBOX_WITH_STATISTICS
2676	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2677	if (pPhys)
2678	STAM_PROFILE_STOP(&pPhys->Stat, h);
2679	# else
2680	pPhys = NULL; /* might not be valid anymore. */
2681	# endif
2682	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, (pPhys) ? pPhys->pszDesc : ""));
2683	if (pVirt->pfnHandlerR3)
2684	{
2685
2686	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2687	+ (iVirtPage << PAGE_SHIFT)
2688	+ (GCPhys & PAGE_OFFSET_MASK);
2689	STAM_PROFILE_START(&pVirt->Stat, h2);
2690	int rc2 = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void )pvBuf, cbRange, PGMACCESSTYPE_WRITE, /pCur->CTX_SUFF(pvUser)*/ NULL);
2691	STAM_PROFILE_STOP(&pVirt->Stat, h2);
2692	if (rc2 == VINF_SUCCESS && rc == VINF_PGM_HANDLER_DO_DEFAULT)
2693	rc = VINF_SUCCESS;
2694	else
2695	AssertLogRelMsg(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pVirt->pszDesc));
2696	}
2697	pPhys = NULL;
2698	pVirt = NULL;
2699	#else
2700	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2701	NOREF(cbRange);
2702	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2703	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2704	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2705	#endif
2706	}
2707	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
2708	memcpy(pvDst, pvBuf, cbRange);
2709
2710	/*
2711	* Advance if we've got more stuff to do.
2712	*/
2713	if (cbRange >= cbWrite)
2714	{
2715	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2716	return VINF_SUCCESS;
2717	}
2718
2719	cbWrite -= cbRange;
2720	GCPhys += cbRange;
2721	pvBuf = (uint8_t *)pvBuf + cbRange;
2722	pvDst = (uint8_t *)pvDst + cbRange;
2723
2724	offPhys -= cbRange;
2725	offPhysLast -= cbRange;
2726	offVirt -= cbRange;
2727	offVirtLast -= cbRange;
2728	}
2729	}
2730
2731
2732	/**
2733	* Write to physical memory.
2734	*
2735	* This API respects access handlers and MMIO. Use PGMPhysSimpleWriteGCPhys() if you
2736	* want to ignore those.
2737	*
2738	* @returns VBox status code. Can be ignored in ring-3.
2739	* @retval VINF_SUCCESS.
2740	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
2741	*
2742	* @param pVM Pointer to the VM.
2743	* @param GCPhys Physical address to write to.
2744	* @param pvBuf What to write.
2745	* @param cbWrite How many bytes to write.
2746	*/
2747	VMMDECL(int) PGMPhysWrite(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite)
2748	{
2749	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMPhysWrite after pgmR3Save()!\n"));
2750	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
2751	LogFlow(("PGMPhysWrite: %RGp %d\n", GCPhys, cbWrite));
2752
2753	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysWrite));
2754	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysWriteBytes), cbWrite);
2755
2756	pgmLock(pVM);
2757
2758	/*
2759	* Copy loop on ram ranges.
2760	*/
2761	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
2762	for (;;)
2763	{
2764	/* Inside range or not? */
2765	if (pRam && GCPhys >= pRam->GCPhys)
2766	{
2767	/*
2768	* Must work our way thru this page by page.
2769	*/
2770	RTGCPTR off = GCPhys - pRam->GCPhys;
2771	while (off < pRam->cb)
2772	{
2773	RTGCPTR iPage = off >> PAGE_SHIFT;
2774	PPGMPAGE pPage = &pRam->aPages[iPage];
2775	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
2776	if (cb > cbWrite)
2777	cb = cbWrite;
2778
2779	/*
2780	* Any active WRITE or ALL access handlers?
2781	*/
2782	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
2783	\|\| PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
2784	{
2785	int rc = pgmPhysWriteHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb);
2786	if (RT_FAILURE(rc))
2787	{
2788	pgmUnlock(pVM);
2789	return rc;
2790	}
2791	}
2792	else
2793	{
2794	/*
2795	* Get the pointer to the page.
2796	*/
2797	PGMPAGEMAPLOCK PgMpLck;
2798	void *pvDst;
2799	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
2800	if (RT_SUCCESS(rc))
2801	{
2802	Assert(!PGM_PAGE_IS_BALLOONED(pPage));
2803	memcpy(pvDst, pvBuf, cb);
2804	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2805	}
2806	/* Ignore writes to ballooned pages. */
2807	else if (!PGM_PAGE_IS_BALLOONED(pPage))
2808	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2809	pRam->GCPhys + off, pPage, rc));
2810	}
2811
2812	/* next page */
2813	if (cb >= cbWrite)
2814	{
2815	pgmUnlock(pVM);
2816	return VINF_SUCCESS;
2817	}
2818
2819	cbWrite -= cb;
2820	off += cb;
2821	pvBuf = (const char *)pvBuf + cb;
2822	} /* walk pages in ram range */
2823
2824	GCPhys = pRam->GCPhysLast + 1;
2825	}
2826	else
2827	{
2828	/*
2829	* Unassigned address space, skip it.
2830	*/
2831	if (!pRam)
2832	break;
2833	size_t cb = pRam->GCPhys - GCPhys;
2834	if (cb >= cbWrite)
2835	break;
2836	cbWrite -= cb;
2837	pvBuf = (const char *)pvBuf + cb;
2838	GCPhys += cb;
2839	}
2840
2841	/* Advance range if necessary. */
2842	while (pRam && GCPhys > pRam->GCPhysLast)
2843	pRam = pRam->CTX_SUFF(pNext);
2844	} /* Ram range walk */
2845
2846	pgmUnlock(pVM);
2847	return VINF_SUCCESS;
2848	}
2849
2850
2851	/**
2852	* Read from guest physical memory by GC physical address, bypassing
2853	* MMIO and access handlers.
2854	*
2855	* @returns VBox status.
2856	* @param pVM Pointer to the VM.
2857	* @param pvDst The destination address.
2858	* @param GCPhysSrc The source address (GC physical address).
2859	* @param cb The number of bytes to read.
2860	*/
2861	VMMDECL(int) PGMPhysSimpleReadGCPhys(PVM pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb)
2862	{
2863	/*
2864	* Treat the first page as a special case.
2865	*/
2866	if (!cb)
2867	return VINF_SUCCESS;
2868
2869	/* map the 1st page */
2870	void const *pvSrc;
2871	PGMPAGEMAPLOCK Lock;
2872	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
2873	if (RT_FAILURE(rc))
2874	return rc;
2875
2876	/* optimize for the case where access is completely within the first page. */
2877	size_t cbPage = PAGE_SIZE - (GCPhysSrc & PAGE_OFFSET_MASK);
2878	if (RT_LIKELY(cb <= cbPage))
2879	{
2880	memcpy(pvDst, pvSrc, cb);
2881	PGMPhysReleasePageMappingLock(pVM, &Lock);
2882	return VINF_SUCCESS;
2883	}
2884
2885	/* copy to the end of the page. */
2886	memcpy(pvDst, pvSrc, cbPage);
2887	PGMPhysReleasePageMappingLock(pVM, &Lock);
2888	GCPhysSrc += cbPage;
2889	pvDst = (uint8_t *)pvDst + cbPage;
2890	cb -= cbPage;
2891
2892	/*
2893	* Page by page.
2894	*/
2895	for (;;)
2896	{
2897	/* map the page */
2898	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
2899	if (RT_FAILURE(rc))
2900	return rc;
2901
2902	/* last page? */
2903	if (cb <= PAGE_SIZE)
2904	{
2905	memcpy(pvDst, pvSrc, cb);
2906	PGMPhysReleasePageMappingLock(pVM, &Lock);
2907	return VINF_SUCCESS;
2908	}
2909
2910	/* copy the entire page and advance */
2911	memcpy(pvDst, pvSrc, PAGE_SIZE);
2912	PGMPhysReleasePageMappingLock(pVM, &Lock);
2913	GCPhysSrc += PAGE_SIZE;
2914	pvDst = (uint8_t *)pvDst + PAGE_SIZE;
2915	cb -= PAGE_SIZE;
2916	}
2917	/* won't ever get here. */
2918	}
2919
2920
2921	/**
2922	* Write to guest physical memory referenced by GC pointer.
2923	* Write memory to GC physical address in guest physical memory.
2924	*
2925	* This will bypass MMIO and access handlers.
2926	*
2927	* @returns VBox status.
2928	* @param pVM Pointer to the VM.
2929	* @param GCPhysDst The GC physical address of the destination.
2930	* @param pvSrc The source buffer.
2931	* @param cb The number of bytes to write.
2932	*/
2933	VMMDECL(int) PGMPhysSimpleWriteGCPhys(PVM pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb)
2934	{
2935	LogFlow(("PGMPhysSimpleWriteGCPhys: %RGp %zu\n", GCPhysDst, cb));
2936
2937	/*
2938	* Treat the first page as a special case.
2939	*/
2940	if (!cb)
2941	return VINF_SUCCESS;
2942
2943	/* map the 1st page */
2944	void *pvDst;
2945	PGMPAGEMAPLOCK Lock;
2946	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
2947	if (RT_FAILURE(rc))
2948	return rc;
2949
2950	/* optimize for the case where access is completely within the first page. */
2951	size_t cbPage = PAGE_SIZE - (GCPhysDst & PAGE_OFFSET_MASK);
2952	if (RT_LIKELY(cb <= cbPage))
2953	{
2954	memcpy(pvDst, pvSrc, cb);
2955	PGMPhysReleasePageMappingLock(pVM, &Lock);
2956	return VINF_SUCCESS;
2957	}
2958
2959	/* copy to the end of the page. */
2960	memcpy(pvDst, pvSrc, cbPage);
2961	PGMPhysReleasePageMappingLock(pVM, &Lock);
2962	GCPhysDst += cbPage;
2963	pvSrc = (const uint8_t *)pvSrc + cbPage;
2964	cb -= cbPage;
2965
2966	/*
2967	* Page by page.
2968	*/
2969	for (;;)
2970	{
2971	/* map the page */
2972	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
2973	if (RT_FAILURE(rc))
2974	return rc;
2975
2976	/* last page? */
2977	if (cb <= PAGE_SIZE)
2978	{
2979	memcpy(pvDst, pvSrc, cb);
2980	PGMPhysReleasePageMappingLock(pVM, &Lock);
2981	return VINF_SUCCESS;
2982	}
2983
2984	/* copy the entire page and advance */
2985	memcpy(pvDst, pvSrc, PAGE_SIZE);
2986	PGMPhysReleasePageMappingLock(pVM, &Lock);
2987	GCPhysDst += PAGE_SIZE;
2988	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
2989	cb -= PAGE_SIZE;
2990	}
2991	/* won't ever get here. */
2992	}
2993
2994
2995	/**
2996	* Read from guest physical memory referenced by GC pointer.
2997	*
2998	* This function uses the current CR3/CR0/CR4 of the guest and will
2999	* bypass access handlers and not set any accessed bits.
3000	*
3001	* @returns VBox status.
3002	* @param pVCpu Handle to the current virtual CPU.
3003	* @param pvDst The destination address.
3004	* @param GCPtrSrc The source address (GC pointer).
3005	* @param cb The number of bytes to read.
3006	*/
3007	VMMDECL(int) PGMPhysSimpleReadGCPtr(PVMCPU pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
3008	{
3009	PVM pVM = pVCpu->CTX_SUFF(pVM);
3010	/** @todo fix the macro / state handling: VMCPU_ASSERT_EMT_OR_GURU(pVCpu); */
3011
3012	/*
3013	* Treat the first page as a special case.
3014	*/
3015	if (!cb)
3016	return VINF_SUCCESS;
3017
3018	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleRead));
3019	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleReadBytes), cb);
3020
3021	/* Take the PGM lock here, because many called functions take the lock for a very short period. That's counter-productive
3022	* when many VCPUs are fighting for the lock.
3023	*/
3024	pgmLock(pVM);
3025
3026	/* map the 1st page */
3027	void const *pvSrc;
3028	PGMPAGEMAPLOCK Lock;
3029	int rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
3030	if (RT_FAILURE(rc))
3031	{
3032	pgmUnlock(pVM);
3033	return rc;
3034	}
3035
3036	/* optimize for the case where access is completely within the first page. */
3037	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
3038	if (RT_LIKELY(cb <= cbPage))
3039	{
3040	memcpy(pvDst, pvSrc, cb);
3041	PGMPhysReleasePageMappingLock(pVM, &Lock);
3042	pgmUnlock(pVM);
3043	return VINF_SUCCESS;
3044	}
3045
3046	/* copy to the end of the page. */
3047	memcpy(pvDst, pvSrc, cbPage);
3048	PGMPhysReleasePageMappingLock(pVM, &Lock);
3049	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + cbPage);
3050	pvDst = (uint8_t *)pvDst + cbPage;
3051	cb -= cbPage;
3052
3053	/*
3054	* Page by page.
3055	*/
3056	for (;;)
3057	{
3058	/* map the page */
3059	rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
3060	if (RT_FAILURE(rc))
3061	{
3062	pgmUnlock(pVM);
3063	return rc;
3064	}
3065
3066	/* last page? */
3067	if (cb <= PAGE_SIZE)
3068	{
3069	memcpy(pvDst, pvSrc, cb);
3070	PGMPhysReleasePageMappingLock(pVM, &Lock);
3071	pgmUnlock(pVM);
3072	return VINF_SUCCESS;
3073	}
3074
3075	/* copy the entire page and advance */
3076	memcpy(pvDst, pvSrc, PAGE_SIZE);
3077	PGMPhysReleasePageMappingLock(pVM, &Lock);
3078	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + PAGE_SIZE);
3079	pvDst = (uint8_t *)pvDst + PAGE_SIZE;
3080	cb -= PAGE_SIZE;
3081	}
3082	/* won't ever get here. */
3083	}
3084
3085
3086	/**
3087	* Write to guest physical memory referenced by GC pointer.
3088	*
3089	* This function uses the current CR3/CR0/CR4 of the guest and will
3090	* bypass access handlers and not set dirty or accessed bits.
3091	*
3092	* @returns VBox status.
3093	* @param pVCpu Handle to the current virtual CPU.
3094	* @param GCPtrDst The destination address (GC pointer).
3095	* @param pvSrc The source address.
3096	* @param cb The number of bytes to write.
3097	*/
3098	VMMDECL(int) PGMPhysSimpleWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
3099	{
3100	PVM pVM = pVCpu->CTX_SUFF(pVM);
3101	VMCPU_ASSERT_EMT(pVCpu);
3102
3103	/*
3104	* Treat the first page as a special case.
3105	*/
3106	if (!cb)
3107	return VINF_SUCCESS;
3108
3109	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleWrite));
3110	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleWriteBytes), cb);
3111
3112	/* map the 1st page */
3113	void *pvDst;
3114	PGMPAGEMAPLOCK Lock;
3115	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3116	if (RT_FAILURE(rc))
3117	return rc;
3118
3119	/* optimize for the case where access is completely within the first page. */
3120	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
3121	if (RT_LIKELY(cb <= cbPage))
3122	{
3123	memcpy(pvDst, pvSrc, cb);
3124	PGMPhysReleasePageMappingLock(pVM, &Lock);
3125	return VINF_SUCCESS;
3126	}
3127
3128	/* copy to the end of the page. */
3129	memcpy(pvDst, pvSrc, cbPage);
3130	PGMPhysReleasePageMappingLock(pVM, &Lock);
3131	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
3132	pvSrc = (const uint8_t *)pvSrc + cbPage;
3133	cb -= cbPage;
3134
3135	/*
3136	* Page by page.
3137	*/
3138	for (;;)
3139	{
3140	/* map the page */
3141	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3142	if (RT_FAILURE(rc))
3143	return rc;
3144
3145	/* last page? */
3146	if (cb <= PAGE_SIZE)
3147	{
3148	memcpy(pvDst, pvSrc, cb);
3149	PGMPhysReleasePageMappingLock(pVM, &Lock);
3150	return VINF_SUCCESS;
3151	}
3152
3153	/* copy the entire page and advance */
3154	memcpy(pvDst, pvSrc, PAGE_SIZE);
3155	PGMPhysReleasePageMappingLock(pVM, &Lock);
3156	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
3157	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
3158	cb -= PAGE_SIZE;
3159	}
3160	/* won't ever get here. */
3161	}
3162
3163
3164	/**
3165	* Write to guest physical memory referenced by GC pointer and update the PTE.
3166	*
3167	* This function uses the current CR3/CR0/CR4 of the guest and will
3168	* bypass access handlers but will set any dirty and accessed bits in the PTE.
3169	*
3170	* If you don't want to set the dirty bit, use PGMPhysSimpleWriteGCPtr().
3171	*
3172	* @returns VBox status.
3173	* @param pVCpu Handle to the current virtual CPU.
3174	* @param GCPtrDst The destination address (GC pointer).
3175	* @param pvSrc The source address.
3176	* @param cb The number of bytes to write.
3177	*/
3178	VMMDECL(int) PGMPhysSimpleDirtyWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
3179	{
3180	PVM pVM = pVCpu->CTX_SUFF(pVM);
3181	VMCPU_ASSERT_EMT(pVCpu);
3182
3183	/*
3184	* Treat the first page as a special case.
3185	* Btw. this is the same code as in PGMPhyssimpleWriteGCPtr excep for the PGMGstModifyPage.
3186	*/
3187	if (!cb)
3188	return VINF_SUCCESS;
3189
3190	/* map the 1st page */
3191	void *pvDst;
3192	PGMPAGEMAPLOCK Lock;
3193	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3194	if (RT_FAILURE(rc))
3195	return rc;
3196
3197	/* optimize for the case where access is completely within the first page. */
3198	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
3199	if (RT_LIKELY(cb <= cbPage))
3200	{
3201	memcpy(pvDst, pvSrc, cb);
3202	PGMPhysReleasePageMappingLock(pVM, &Lock);
3203	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
3204	return VINF_SUCCESS;
3205	}
3206
3207	/* copy to the end of the page. */
3208	memcpy(pvDst, pvSrc, cbPage);
3209	PGMPhysReleasePageMappingLock(pVM, &Lock);
3210	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
3211	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
3212	pvSrc = (const uint8_t *)pvSrc + cbPage;
3213	cb -= cbPage;
3214
3215	/*
3216	* Page by page.
3217	*/
3218	for (;;)
3219	{
3220	/* map the page */
3221	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3222	if (RT_FAILURE(rc))
3223	return rc;
3224
3225	/* last page? */
3226	if (cb <= PAGE_SIZE)
3227	{
3228	memcpy(pvDst, pvSrc, cb);
3229	PGMPhysReleasePageMappingLock(pVM, &Lock);
3230	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
3231	return VINF_SUCCESS;
3232	}
3233
3234	/* copy the entire page and advance */
3235	memcpy(pvDst, pvSrc, PAGE_SIZE);
3236	PGMPhysReleasePageMappingLock(pVM, &Lock);
3237	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
3238	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
3239	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
3240	cb -= PAGE_SIZE;
3241	}
3242	/* won't ever get here. */
3243	}
3244
3245
3246	/**
3247	* Read from guest physical memory referenced by GC pointer.
3248	*
3249	* This function uses the current CR3/CR0/CR4 of the guest and will
3250	* respect access handlers and set accessed bits.
3251	*
3252	* @returns VBox status.
3253	* @param pVCpu Handle to the current virtual CPU.
3254	* @param pvDst The destination address.
3255	* @param GCPtrSrc The source address (GC pointer).
3256	* @param cb The number of bytes to read.
3257	* @thread The vCPU EMT.
3258	*/
3259	VMMDECL(int) PGMPhysReadGCPtr(PVMCPU pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
3260	{
3261	RTGCPHYS GCPhys;
3262	uint64_t fFlags;
3263	int rc;
3264	PVM pVM = pVCpu->CTX_SUFF(pVM);
3265	VMCPU_ASSERT_EMT(pVCpu);
3266
3267	/*
3268	* Anything to do?
3269	*/
3270	if (!cb)
3271	return VINF_SUCCESS;
3272
3273	LogFlow(("PGMPhysReadGCPtr: %RGv %zu\n", GCPtrSrc, cb));
3274
3275	/*
3276	* Optimize reads within a single page.
3277	*/
3278	if (((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
3279	{
3280	/* Convert virtual to physical address + flags */
3281	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
3282	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
3283	GCPhys \|= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
3284
3285	/* mark the guest page as accessed. */
3286	if (!(fFlags & X86_PTE_A))
3287	{
3288	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
3289	AssertRC(rc);
3290	}
3291
3292	return PGMPhysRead(pVM, GCPhys, pvDst, cb);
3293	}
3294
3295	/*
3296	* Page by page.
3297	*/
3298	for (;;)
3299	{
3300	/* Convert virtual to physical address + flags */
3301	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
3302	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
3303	GCPhys \|= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
3304
3305	/* mark the guest page as accessed. */
3306	if (!(fFlags & X86_PTE_A))
3307	{
3308	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
3309	AssertRC(rc);
3310	}
3311
3312	/* copy */
3313	size_t cbRead = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
3314	rc = PGMPhysRead(pVM, GCPhys, pvDst, cbRead);
3315	if (cbRead >= cb \|\| RT_FAILURE(rc))
3316	return rc;
3317
3318	/* next */
3319	cb -= cbRead;
3320	pvDst = (uint8_t *)pvDst + cbRead;
3321	GCPtrSrc += cbRead;
3322	}
3323	}
3324
3325
3326	/**
3327	* Write to guest physical memory referenced by GC pointer.
3328	*
3329	* This function uses the current CR3/CR0/CR4 of the guest and will
3330	* respect access handlers and set dirty and accessed bits.
3331	*
3332	* @returns VBox status.
3333	* @retval VINF_SUCCESS.
3334	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
3335	*
3336	* @param pVCpu Handle to the current virtual CPU.
3337	* @param GCPtrDst The destination address (GC pointer).
3338	* @param pvSrc The source address.
3339	* @param cb The number of bytes to write.
3340	*/
3341	VMMDECL(int) PGMPhysWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
3342	{
3343	RTGCPHYS GCPhys;
3344	uint64_t fFlags;
3345	int rc;
3346	PVM pVM = pVCpu->CTX_SUFF(pVM);
3347	VMCPU_ASSERT_EMT(pVCpu);
3348
3349	/*
3350	* Anything to do?
3351	*/
3352	if (!cb)
3353	return VINF_SUCCESS;
3354
3355	LogFlow(("PGMPhysWriteGCPtr: %RGv %zu\n", GCPtrDst, cb));
3356
3357	/*
3358	* Optimize writes within a single page.
3359	*/
3360	if (((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
3361	{
3362	/* Convert virtual to physical address + flags */
3363	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
3364	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
3365	GCPhys \|= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
3366
3367	/* Mention when we ignore X86_PTE_RW... */
3368	if (!(fFlags & X86_PTE_RW))
3369	Log(("PGMPhysGCPtr2GCPhys: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
3370
3371	/* Mark the guest page as accessed and dirty if necessary. */
3372	if ((fFlags & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
3373	{
3374	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
3375	AssertRC(rc);
3376	}
3377
3378	return PGMPhysWrite(pVM, GCPhys, pvSrc, cb);
3379	}
3380
3381	/*
3382	* Page by page.
3383	*/
3384	for (;;)
3385	{
3386	/* Convert virtual to physical address + flags */
3387	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
3388	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
3389	GCPhys \|= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
3390
3391	/* Mention when we ignore X86_PTE_RW... */
3392	if (!(fFlags & X86_PTE_RW))
3393	Log(("PGMPhysGCPtr2GCPhys: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
3394
3395	/* Mark the guest page as accessed and dirty if necessary. */
3396	if ((fFlags & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
3397	{
3398	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
3399	AssertRC(rc);
3400	}
3401
3402	/* copy */
3403	size_t cbWrite = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
3404	rc = PGMPhysWrite(pVM, GCPhys, pvSrc, cbWrite);
3405	if (cbWrite >= cb \|\| RT_FAILURE(rc))
3406	return rc;
3407
3408	/* next */
3409	cb -= cbWrite;
3410	pvSrc = (uint8_t *)pvSrc + cbWrite;
3411	GCPtrDst += cbWrite;
3412	}
3413	}
3414
3415
3416	/**
3417	* Performs a read of guest virtual memory for instruction emulation.
3418	*
3419	* This will check permissions, raise exceptions and update the access bits.
3420	*
3421	* The current implementation will bypass all access handlers. It may later be
3422	* changed to at least respect MMIO.
3423	*
3424	*
3425	* @returns VBox status code suitable to scheduling.
3426	* @retval VINF_SUCCESS if the read was performed successfully.
3427	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3428	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3429	*
3430	* @param pVCpu Handle to the current virtual CPU.
3431	* @param pCtxCore The context core.
3432	* @param pvDst Where to put the bytes we've read.
3433	* @param GCPtrSrc The source address.
3434	* @param cb The number of bytes to read. Not more than a page.
3435	*
3436	* @remark This function will dynamically map physical pages in GC. This may unmap
3437	* mappings done by the caller. Be careful!
3438	*/
3439	VMMDECL(int) PGMPhysInterpretedRead(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb)
3440	{
3441	PVM pVM = pVCpu->CTX_SUFF(pVM);
3442	Assert(cb <= PAGE_SIZE);
3443	VMCPU_ASSERT_EMT(pVCpu);
3444
3445	/** @todo r=bird: This isn't perfect!
3446	* -# It's not checking for reserved bits being 1.
3447	* -# It's not correctly dealing with the access bit.
3448	* -# It's not respecting MMIO memory or any other access handlers.
3449	*/
3450	/*
3451	* 1. Translate virtual to physical. This may fault.
3452	* 2. Map the physical address.
3453	* 3. Do the read operation.
3454	* 4. Set access bits if required.
3455	*/
3456	int rc;
3457	unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
3458	if (cb <= cb1)
3459	{
3460	/*
3461	* Not crossing pages.
3462	*/
3463	RTGCPHYS GCPhys;
3464	uint64_t fFlags;
3465	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
3466	if (RT_SUCCESS(rc))
3467	{
3468	/** @todo we should check reserved bits ... */
3469	PGMPAGEMAPLOCK PgMpLck;
3470	void const *pvSrc;
3471	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvSrc, &PgMpLck);
3472	switch (rc)
3473	{
3474	case VINF_SUCCESS:
3475	Log(("PGMPhysInterpretedRead: pvDst=%p pvSrc=%p cb=%d\n", pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb));
3476	memcpy(pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
3477	PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
3478	break;
3479	case VERR_PGM_PHYS_PAGE_RESERVED:
3480	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3481	memset(pvDst, 0xff, cb);
3482	break;
3483	default:
3484	Assert(RT_FAILURE_NP(rc));
3485	return rc;
3486	}
3487
3488	/** @todo access bit emulation isn't 100% correct. */
3489	if (!(fFlags & X86_PTE_A))
3490	{
3491	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3492	AssertRC(rc);
3493	}
3494	return VINF_SUCCESS;
3495	}
3496	}
3497	else
3498	{
3499	/*
3500	* Crosses pages.
3501	*/
3502	size_t cb2 = cb - cb1;
3503	uint64_t fFlags1;
3504	RTGCPHYS GCPhys1;
3505	uint64_t fFlags2;
3506	RTGCPHYS GCPhys2;
3507	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
3508	if (RT_SUCCESS(rc))
3509	{
3510	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
3511	if (RT_SUCCESS(rc))
3512	{
3513	/** @todo we should check reserved bits ... */
3514	AssertMsgFailed(("cb=%d cb1=%d cb2=%d GCPtrSrc=%RGv\n", cb, cb1, cb2, GCPtrSrc));
3515	PGMPAGEMAPLOCK PgMpLck;
3516	void const *pvSrc1;
3517	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys1, &pvSrc1, &PgMpLck);
3518	switch (rc)
3519	{
3520	case VINF_SUCCESS:
3521	memcpy(pvDst, (uint8_t *)pvSrc1 + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
3522	PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
3523	break;
3524	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3525	memset(pvDst, 0xff, cb1);
3526	break;
3527	default:
3528	Assert(RT_FAILURE_NP(rc));
3529	return rc;
3530	}
3531
3532	void const *pvSrc2;
3533	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys2, &pvSrc2, &PgMpLck);
3534	switch (rc)
3535	{
3536	case VINF_SUCCESS:
3537	memcpy((uint8_t *)pvDst + cb1, pvSrc2, cb2);
3538	PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
3539	break;
3540	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3541	memset((uint8_t *)pvDst + cb1, 0xff, cb2);
3542	break;
3543	default:
3544	Assert(RT_FAILURE_NP(rc));
3545	return rc;
3546	}
3547
3548	if (!(fFlags1 & X86_PTE_A))
3549	{
3550	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3551	AssertRC(rc);
3552	}
3553	if (!(fFlags2 & X86_PTE_A))
3554	{
3555	rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3556	AssertRC(rc);
3557	}
3558	return VINF_SUCCESS;
3559	}
3560	}
3561	}
3562
3563	/*
3564	* Raise a #PF.
3565	*/
3566	uint32_t uErr;
3567
3568	/* Get the current privilege level. */
3569	uint32_t cpl = CPUMGetGuestCPL(pVCpu);
3570	switch (rc)
3571	{
3572	case VINF_SUCCESS:
3573	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3574	break;
3575
3576	case VERR_PAGE_NOT_PRESENT:
3577	case VERR_PAGE_TABLE_NOT_PRESENT:
3578	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3579	break;
3580
3581	default:
3582	AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
3583	return rc;
3584	}
3585	Log(("PGMPhysInterpretedRead: GCPtrSrc=%RGv cb=%#x -> #PF(%#x)\n", GCPtrSrc, cb, uErr));
3586	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrSrc);
3587	}
3588
3589
3590	/**
3591	* Performs a read of guest virtual memory for instruction emulation.
3592	*
3593	* This will check permissions, raise exceptions and update the access bits.
3594	*
3595	* The current implementation will bypass all access handlers. It may later be
3596	* changed to at least respect MMIO.
3597	*
3598	*
3599	* @returns VBox status code suitable to scheduling.
3600	* @retval VINF_SUCCESS if the read was performed successfully.
3601	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3602	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3603	*
3604	* @param pVCpu Handle to the current virtual CPU.
3605	* @param pCtxCore The context core.
3606	* @param pvDst Where to put the bytes we've read.
3607	* @param GCPtrSrc The source address.
3608	* @param cb The number of bytes to read. Not more than a page.
3609	* @param fRaiseTrap If set the trap will be raised on as per spec, if clear
3610	* an appropriate error status will be returned (no
3611	* informational at all).
3612	*
3613	*
3614	* @remarks Takes the PGM lock.
3615	* @remarks A page fault on the 2nd page of the access will be raised without
3616	* writing the bits on the first page since we're ASSUMING that the
3617	* caller is emulating an instruction access.
3618	* @remarks This function will dynamically map physical pages in GC. This may
3619	* unmap mappings done by the caller. Be careful!
3620	*/
3621	VMMDECL(int) PGMPhysInterpretedReadNoHandlers(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb,
3622	bool fRaiseTrap)
3623	{
3624	PVM pVM = pVCpu->CTX_SUFF(pVM);
3625	Assert(cb <= PAGE_SIZE);
3626	VMCPU_ASSERT_EMT(pVCpu);
3627
3628	/*
3629	* 1. Translate virtual to physical. This may fault.
3630	* 2. Map the physical address.
3631	* 3. Do the read operation.
3632	* 4. Set access bits if required.
3633	*/
3634	int rc;
3635	unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
3636	if (cb <= cb1)
3637	{
3638	/*
3639	* Not crossing pages.
3640	*/
3641	RTGCPHYS GCPhys;
3642	uint64_t fFlags;
3643	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
3644	if (RT_SUCCESS(rc))
3645	{
3646	if (1) /** @todo we should check reserved bits ... */
3647	{
3648	const void *pvSrc;
3649	PGMPAGEMAPLOCK Lock;
3650	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvSrc, &Lock);
3651	switch (rc)
3652	{
3653	case VINF_SUCCESS:
3654	Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d\n",
3655	pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb));
3656	memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
3657	PGMPhysReleasePageMappingLock(pVM, &Lock);
3658	break;
3659	case VERR_PGM_PHYS_PAGE_RESERVED:
3660	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3661	memset(pvDst, 0xff, cb);
3662	break;
3663	default:
3664	AssertMsgFailed(("%Rrc\n", rc));
3665	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3666	return rc;
3667	}
3668
3669	if (!(fFlags & X86_PTE_A))
3670	{
3671	/** @todo access bit emulation isn't 100% correct. */
3672	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3673	AssertRC(rc);
3674	}
3675	return VINF_SUCCESS;
3676	}
3677	}
3678	}
3679	else
3680	{
3681	/*
3682	* Crosses pages.
3683	*/
3684	size_t cb2 = cb - cb1;
3685	uint64_t fFlags1;
3686	RTGCPHYS GCPhys1;
3687	uint64_t fFlags2;
3688	RTGCPHYS GCPhys2;
3689	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
3690	if (RT_SUCCESS(rc))
3691	{
3692	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
3693	if (RT_SUCCESS(rc))
3694	{
3695	if (1) /** @todo we should check reserved bits ... */
3696	{
3697	const void *pvSrc;
3698	PGMPAGEMAPLOCK Lock;
3699	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys1, &pvSrc, &Lock);
3700	switch (rc)
3701	{
3702	case VINF_SUCCESS:
3703	Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d [2]\n",
3704	pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb1));
3705	memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
3706	PGMPhysReleasePageMappingLock(pVM, &Lock);
3707	break;
3708	case VERR_PGM_PHYS_PAGE_RESERVED:
3709	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3710	memset(pvDst, 0xff, cb1);
3711	break;
3712	default:
3713	AssertMsgFailed(("%Rrc\n", rc));
3714	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3715	return rc;
3716	}
3717
3718	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys2, &pvSrc, &Lock);
3719	switch (rc)
3720	{
3721	case VINF_SUCCESS:
3722	memcpy((uint8_t *)pvDst + cb1, pvSrc, cb2);
3723	PGMPhysReleasePageMappingLock(pVM, &Lock);
3724	break;
3725	case VERR_PGM_PHYS_PAGE_RESERVED:
3726	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3727	memset((uint8_t *)pvDst + cb1, 0xff, cb2);
3728	break;
3729	default:
3730	AssertMsgFailed(("%Rrc\n", rc));
3731	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3732	return rc;
3733	}
3734
3735	if (!(fFlags1 & X86_PTE_A))
3736	{
3737	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3738	AssertRC(rc);
3739	}
3740	if (!(fFlags2 & X86_PTE_A))
3741	{
3742	rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3743	AssertRC(rc);
3744	}
3745	return VINF_SUCCESS;
3746	}
3747	/* sort out which page */
3748	}
3749	else
3750	GCPtrSrc += cb1; /* fault on 2nd page */
3751	}
3752	}
3753
3754	/*
3755	* Raise a #PF if we're allowed to do that.
3756	*/
3757	/* Calc the error bits. */
3758	uint32_t cpl = CPUMGetGuestCPL(pVCpu);
3759	uint32_t uErr;
3760	switch (rc)
3761	{
3762	case VINF_SUCCESS:
3763	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3764	rc = VERR_ACCESS_DENIED;
3765	break;
3766
3767	case VERR_PAGE_NOT_PRESENT:
3768	case VERR_PAGE_TABLE_NOT_PRESENT:
3769	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3770	break;
3771
3772	default:
3773	AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
3774	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3775	return rc;
3776	}
3777	if (fRaiseTrap)
3778	{
3779	Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrSrc, cb, uErr));
3780	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrSrc);
3781	}
3782	Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrSrc, cb, uErr));
3783	return rc;
3784	}
3785
3786
3787	/**
3788	* Performs a write to guest virtual memory for instruction emulation.
3789	*
3790	* This will check permissions, raise exceptions and update the dirty and access
3791	* bits.
3792	*
3793	* @returns VBox status code suitable to scheduling.
3794	* @retval VINF_SUCCESS if the read was performed successfully.
3795	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3796	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3797	*
3798	* @param pVCpu Handle to the current virtual CPU.
3799	* @param pCtxCore The context core.
3800	* @param GCPtrDst The destination address.
3801	* @param pvSrc What to write.
3802	* @param cb The number of bytes to write. Not more than a page.
3803	* @param fRaiseTrap If set the trap will be raised on as per spec, if clear
3804	* an appropriate error status will be returned (no
3805	* informational at all).
3806	*
3807	* @remarks Takes the PGM lock.
3808	* @remarks A page fault on the 2nd page of the access will be raised without
3809	* writing the bits on the first page since we're ASSUMING that the
3810	* caller is emulating an instruction access.
3811	* @remarks This function will dynamically map physical pages in GC. This may
3812	* unmap mappings done by the caller. Be careful!
3813	*/
3814	VMMDECL(int) PGMPhysInterpretedWriteNoHandlers(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, RTGCPTR GCPtrDst, const void *pvSrc,
3815	size_t cb, bool fRaiseTrap)
3816	{
3817	Assert(cb <= PAGE_SIZE);
3818	PVM pVM = pVCpu->CTX_SUFF(pVM);
3819	VMCPU_ASSERT_EMT(pVCpu);
3820
3821	/*
3822	* 1. Translate virtual to physical. This may fault.
3823	* 2. Map the physical address.
3824	* 3. Do the write operation.
3825	* 4. Set access bits if required.
3826	*/
3827	/** @todo Since this method is frequently used by EMInterpret or IOM
3828	* upon a write fault to an write access monitored page, we can
3829	* reuse the guest page table walking from the \#PF code. */
3830	int rc;
3831	unsigned cb1 = PAGE_SIZE - (GCPtrDst & PAGE_OFFSET_MASK);
3832	if (cb <= cb1)
3833	{
3834	/*
3835	* Not crossing pages.
3836	*/
3837	RTGCPHYS GCPhys;
3838	uint64_t fFlags;
3839	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst, &fFlags, &GCPhys);
3840	if (RT_SUCCESS(rc))
3841	{
3842	if ( (fFlags & X86_PTE_RW) /** @todo Also check reserved bits. */
3843	\|\| ( !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
3844	&& CPUMGetGuestCPL(pVCpu) <= 2) ) /** @todo it's 2, right? Check cpl check below as well. */
3845	{
3846	void *pvDst;
3847	PGMPAGEMAPLOCK Lock;
3848	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, &pvDst, &Lock);
3849	switch (rc)
3850	{
3851	case VINF_SUCCESS:
3852	Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
3853	(uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc, cb));
3854	memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb);
3855	PGMPhysReleasePageMappingLock(pVM, &Lock);
3856	break;
3857	case VERR_PGM_PHYS_PAGE_RESERVED:
3858	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3859	/* bit bucket */
3860	break;
3861	default:
3862	AssertMsgFailed(("%Rrc\n", rc));
3863	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3864	return rc;
3865	}
3866
3867	if (!(fFlags & (X86_PTE_A \| X86_PTE_D)))
3868	{
3869	/** @todo dirty & access bit emulation isn't 100% correct. */
3870	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
3871	AssertRC(rc);
3872	}
3873	return VINF_SUCCESS;
3874	}
3875	rc = VERR_ACCESS_DENIED;
3876	}
3877	}
3878	else
3879	{
3880	/*
3881	* Crosses pages.
3882	*/
3883	size_t cb2 = cb - cb1;
3884	uint64_t fFlags1;
3885	RTGCPHYS GCPhys1;
3886	uint64_t fFlags2;
3887	RTGCPHYS GCPhys2;
3888	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst, &fFlags1, &GCPhys1);
3889	if (RT_SUCCESS(rc))
3890	{
3891	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst + cb1, &fFlags2, &GCPhys2);
3892	if (RT_SUCCESS(rc))
3893	{
3894	if ( ( (fFlags1 & X86_PTE_RW) /** @todo Also check reserved bits. */
3895	&& (fFlags2 & X86_PTE_RW))
3896	\|\| ( !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
3897	&& CPUMGetGuestCPL(pVCpu) <= 2) )
3898	{
3899	void *pvDst;
3900	PGMPAGEMAPLOCK Lock;
3901	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys1, &pvDst, &Lock);
3902	switch (rc)
3903	{
3904	case VINF_SUCCESS:
3905	Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
3906	(uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc, cb1));
3907	memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb1);
3908	PGMPhysReleasePageMappingLock(pVM, &Lock);
3909	break;
3910	case VERR_PGM_PHYS_PAGE_RESERVED:
3911	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3912	/* bit bucket */
3913	break;
3914	default:
3915	AssertMsgFailed(("%Rrc\n", rc));
3916	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3917	return rc;
3918	}
3919
3920	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys2, &pvDst, &Lock);
3921	switch (rc)
3922	{
3923	case VINF_SUCCESS:
3924	memcpy(pvDst, (const uint8_t *)pvSrc + cb1, cb2);
3925	PGMPhysReleasePageMappingLock(pVM, &Lock);
3926	break;
3927	case VERR_PGM_PHYS_PAGE_RESERVED:
3928	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3929	/* bit bucket */
3930	break;
3931	default:
3932	AssertMsgFailed(("%Rrc\n", rc));
3933	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3934	return rc;
3935	}
3936
3937	if (!(fFlags1 & (X86_PTE_A \| X86_PTE_RW)))
3938	{
3939	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, (X86_PTE_A \| X86_PTE_RW), ~(uint64_t)(X86_PTE_A \| X86_PTE_RW));
3940	AssertRC(rc);
3941	}
3942	if (!(fFlags2 & (X86_PTE_A \| X86_PTE_RW)))
3943	{
3944	rc = PGMGstModifyPage(pVCpu, GCPtrDst + cb1, 1, (X86_PTE_A \| X86_PTE_RW), ~(uint64_t)(X86_PTE_A \| X86_PTE_RW));
3945	AssertRC(rc);
3946	}
3947	return VINF_SUCCESS;
3948	}
3949	if ((fFlags1 & (X86_PTE_RW)) == X86_PTE_RW)
3950	GCPtrDst += cb1; /* fault on the 2nd page. */
3951	rc = VERR_ACCESS_DENIED;
3952	}
3953	else
3954	GCPtrDst += cb1; /* fault on the 2nd page. */
3955	}
3956	}
3957
3958	/*
3959	* Raise a #PF if we're allowed to do that.
3960	*/
3961	/* Calc the error bits. */
3962	uint32_t uErr;
3963	uint32_t cpl = CPUMGetGuestCPL(pVCpu);
3964	switch (rc)
3965	{
3966	case VINF_SUCCESS:
3967	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3968	rc = VERR_ACCESS_DENIED;
3969	break;
3970
3971	case VERR_ACCESS_DENIED:
3972	uErr = (cpl >= 2) ? X86_TRAP_PF_RW \| X86_TRAP_PF_US : X86_TRAP_PF_RW;
3973	break;
3974
3975	case VERR_PAGE_NOT_PRESENT:
3976	case VERR_PAGE_TABLE_NOT_PRESENT:
3977	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3978	break;
3979
3980	default:
3981	AssertMsgFailed(("rc=%Rrc GCPtrDst=%RGv cb=%#x\n", rc, GCPtrDst, cb));
3982	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3983	return rc;
3984	}
3985	if (fRaiseTrap)
3986	{
3987	Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrDst, cb, uErr));
3988	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrDst);
3989	}
3990	Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrDst, cb, uErr));
3991	return rc;
3992	}
3993
3994
3995	/**
3996	* Return the page type of the specified physical address.
3997	*
3998	* @returns The page type.
3999	* @param pVM Pointer to the VM.
4000	* @param GCPhys Guest physical address
4001	*/
4002	VMMDECL(PGMPAGETYPE) PGMPhysGetPageType(PVM pVM, RTGCPHYS GCPhys)
4003	{
4004	pgmLock(pVM);
4005	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
4006	PGMPAGETYPE enmPgType = pPage ? (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage) : PGMPAGETYPE_INVALID;
4007	pgmUnlock(pVM);
4008
4009	return enmPgType;
4010	}
4011
4012
4013
4014
4015	/**
4016	* Converts a GC physical address to a HC ring-3 pointer, with some
4017	* additional checks.
4018	*
4019	* @returns VBox status code (no informational statuses).
4020	* @retval VINF_SUCCESS on success.
4021	* @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4022	* access handler of some kind.
4023	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4024	* accesses or is odd in any way.
4025	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4026	*
4027	* @param pVM Pointer to the VM.
4028	* @param GCPhys The GC physical address to convert. Since this is only
4029	* used for filling the REM TLB, the A20 mask must be
4030	* applied before calling this API.
4031	* @param fWritable Whether write access is required.
4032	* @param ppv Where to store the pointer corresponding to GCPhys on
4033	* success.
4034	* @param pLock
4035	*
4036	* @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr.
4037	*/
4038	VMM_INT_DECL(int) PGMPhysIemGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers,
4039	void **ppv, PPGMPAGEMAPLOCK pLock)
4040	{
4041	pgmLock(pVM);
4042	PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
4043
4044	PPGMRAMRANGE pRam;
4045	PPGMPAGE pPage;
4046	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4047	if (RT_SUCCESS(rc))
4048	{
4049	if (PGM_PAGE_IS_BALLOONED(pPage))
4050	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4051	else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
4052	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4053	else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
4054	\|\| (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
4055	rc = VINF_SUCCESS;
4056	else
4057	{
4058	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4059	{
4060	Assert(!fByPassHandlers \|\| PGM_PAGE_IS_MMIO(pPage));
4061	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4062	}
4063	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
4064	{
4065	Assert(!fByPassHandlers);
4066	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4067	}
4068	}
4069	if (RT_SUCCESS(rc))
4070	{
4071	int rc2;
4072
4073	/* Make sure what we return is writable. */
4074	if (fWritable)
4075	switch (PGM_PAGE_GET_STATE(pPage))
4076	{
4077	case PGM_PAGE_STATE_ALLOCATED:
4078	break;
4079	case PGM_PAGE_STATE_BALLOONED:
4080	AssertFailed();
4081	case PGM_PAGE_STATE_ZERO:
4082	case PGM_PAGE_STATE_SHARED:
4083	case PGM_PAGE_STATE_WRITE_MONITORED:
4084	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
4085	AssertLogRelRCReturn(rc2, rc2);
4086	break;
4087	}
4088
4089	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4090	PVMCPU pVCpu = VMMGetCpu(pVM);
4091	void *pv;
4092	rc = pgmRZDynMapHCPageInlined(pVCpu,
4093	PGM_PAGE_GET_HCPHYS(pPage),
4094	&pv
4095	RTLOG_COMMA_SRC_POS);
4096	if (RT_FAILURE(rc))
4097	return rc;
4098	ppv = (void )((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4099	pLock->pvPage = pv;
4100	pLock->pVCpu = pVCpu;
4101
4102	#else
4103	/* Get a ring-3 mapping of the address. */
4104	PPGMPAGER3MAPTLBE pTlbe;
4105	rc2 = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
4106	AssertLogRelRCReturn(rc2, rc2);
4107
4108	/* Lock it and calculate the address. */
4109	if (fWritable)
4110	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
4111	else
4112	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
4113	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4114	#endif
4115
4116	Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
4117	}
4118	else
4119	Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
4120
4121	/* else: handler catching all access, no pointer returned. */
4122	}
4123	else
4124	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
4125
4126	pgmUnlock(pVM);
4127	return rc;
4128	}
4129
4130
4131	/**
4132	* Checks if the give GCPhys page requires special handling for the given access
4133	* because it's MMIO or otherwise monitored.
4134	*
4135	* @returns VBox status code (no informational statuses).
4136	* @retval VINF_SUCCESS on success.
4137	* @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4138	* access handler of some kind.
4139	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4140	* accesses or is odd in any way.
4141	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4142	*
4143	* @param pVM Pointer to the VM.
4144	* @param GCPhys The GC physical address to convert. Since this is only
4145	* used for filling the REM TLB, the A20 mask must be
4146	* applied before calling this API.
4147	* @param fWritable Whether write access is required.
4148	*
4149	* @remarks This is a watered down version PGMPhysIemGCPhys2Ptr and really just
4150	* a stop gap thing that should be removed once there is a better TLB
4151	* for virtual address accesses.
4152	*/
4153	VMM_INT_DECL(int) PGMPhysIemQueryAccess(PVM pVM, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers)
4154	{
4155	pgmLock(pVM);
4156	PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
4157
4158	PPGMRAMRANGE pRam;
4159	PPGMPAGE pPage;
4160	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4161	if (RT_SUCCESS(rc))
4162	{
4163	if (PGM_PAGE_IS_BALLOONED(pPage))
4164	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4165	else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
4166	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4167	else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
4168	\|\| (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
4169	rc = VINF_SUCCESS;
4170	else
4171	{
4172	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4173	{
4174	Assert(!fByPassHandlers \|\| PGM_PAGE_IS_MMIO(pPage));
4175	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4176	}
4177	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
4178	{
4179	Assert(!fByPassHandlers);
4180	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4181	}
4182	}
4183	}
4184
4185	pgmUnlock(pVM);
4186	return rc;
4187	}
4188

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp@ 48749

Download in other formats: