PGMAllPhys.cpp@ 55747

Last change on this file since 55747 was 55493, checked in by vboxsync, 10 years ago
PGM,++: Separated physical access handler callback function pointers from the access handler registrations to reduce footprint and simplify adding a couple of more callbacks.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 149.2 KB

Line
1	/* $Id: PGMAllPhys.cpp 55493 2015-04-28 16:51:35Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_PGM_PHYS
22	#include <VBox/vmm/pgm.h>
23	#include <VBox/vmm/trpm.h>
24	#include <VBox/vmm/vmm.h>
25	#include <VBox/vmm/iom.h>
26	#include <VBox/vmm/em.h>
27	#ifdef VBOX_WITH_REM
28	# include <VBox/vmm/rem.h>
29	#endif
30	#include "PGMInternal.h"
31	#include <VBox/vmm/vm.h>
32	#include "PGMInline.h"
33	#include <VBox/param.h>
34	#include <VBox/err.h>
35	#include <iprt/assert.h>
36	#include <iprt/string.h>
37	#include <iprt/asm-amd64-x86.h>
38	#include <VBox/log.h>
39	#ifdef IN_RING3
40	# include <iprt/thread.h>
41	#endif
42
43
44	/*******************************************************************************
45	* Defined Constants And Macros *
46	*******************************************************************************/
47	/** Enable the physical TLB. */
48	#define PGM_WITH_PHYS_TLB
49
50
51
52	#ifndef IN_RING3
53
54	/**
55	* \#PF Handler callback for physical memory accesses without a RC/R0 handler.
56	* This simply pushes everything to the HC handler.
57	*
58	* @returns VBox status code (appropriate for trap handling and GC return).
59	* @param pVM Pointer to the VM.
60	* @param uErrorCode CPU Error code.
61	* @param pRegFrame Trap register frame.
62	* @param pvFault The fault address (cr2).
63	* @param GCPhysFault The GC physical address corresponding to pvFault.
64	* @param pvUser User argument.
65	*/
66	VMMDECL(int) pgmPhysHandlerRedirectToHC(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
67	{
68	NOREF(pVM); NOREF(uErrorCode); NOREF(pRegFrame); NOREF(pvFault); NOREF(GCPhysFault); NOREF(pvUser);
69	return (uErrorCode & X86_TRAP_PF_RW) ? VINF_IOM_R3_MMIO_WRITE : VINF_IOM_R3_MMIO_READ;
70	}
71
72
73	/**
74	* \#PF Handler callback for Guest ROM range write access.
75	* We simply ignore the writes or fall back to the recompiler if we don't support the instruction.
76	*
77	* @returns VBox status code (appropriate for trap handling and GC return).
78	* @param pVM Pointer to the VM.
79	* @param uErrorCode CPU Error code.
80	* @param pRegFrame Trap register frame.
81	* @param pvFault The fault address (cr2).
82	* @param GCPhysFault The GC physical address corresponding to pvFault.
83	* @param pvUser User argument. Pointer to the ROM range structure.
84	*/
85	VMMDECL(int) pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
86	{
87	int rc;
88	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
89	uint32_t iPage = (GCPhysFault - pRom->GCPhys) >> PAGE_SHIFT;
90	PVMCPU pVCpu = VMMGetCpu(pVM);
91	NOREF(uErrorCode); NOREF(pvFault);
92
93	Assert(uErrorCode & X86_TRAP_PF_RW); /* This shall not be used for read access! */
94
95	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
96	switch (pRom->aPages[iPage].enmProt)
97	{
98	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
99	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
100	{
101	/*
102	* If it's a simple instruction which doesn't change the cpu state
103	* we will simply skip it. Otherwise we'll have to defer it to REM.
104	*/
105	uint32_t cbOp;
106	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
107	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
108	if ( RT_SUCCESS(rc)
109	&& pDis->uCpuMode == DISCPUMODE_32BIT /** @todo why does this matter? */
110	&& !(pDis->fPrefix & (DISPREFIX_REPNE \| DISPREFIX_REP \| DISPREFIX_SEG)))
111	{
112	switch (pDis->bOpCode)
113	{
114	/** @todo Find other instructions we can safely skip, possibly
115	* adding this kind of detection to DIS or EM. */
116	case OP_MOV:
117	pRegFrame->rip += cbOp;
118	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestROMWriteHandled);
119	return VINF_SUCCESS;
120	}
121	}
122	break;
123	}
124
125	case PGMROMPROT_READ_RAM_WRITE_RAM:
126	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
127	rc = PGMHandlerPhysicalPageTempOff(pVM, pRom->GCPhys, GCPhysFault & X86_PTE_PG_MASK);
128	AssertRC(rc);
129	break; /** @todo Must edit the shadow PT and restart the instruction, not use the interpreter! */
130
131	case PGMROMPROT_READ_ROM_WRITE_RAM:
132	/* Handle it in ring-3 because it's way easier there. */
133	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
134	break;
135
136	default:
137	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhysFault=%RGp\n",
138	pRom->aPages[iPage].enmProt, iPage, GCPhysFault),
139	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
140	}
141
142	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestROMWriteUnhandled);
143	return VINF_EM_RAW_EMULATE_INSTR;
144	}
145
146	#endif /* IN_RING3 */
147
148	/**
149	* Invalidates the RAM range TLBs.
150	*
151	* @param pVM Pointer to the VM.
152	*/
153	void pgmPhysInvalidRamRangeTlbs(PVM pVM)
154	{
155	pgmLock(pVM);
156	for (uint32_t i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
157	{
158	pVM->pgm.s.apRamRangesTlbR3[i] = NIL_RTR3PTR;
159	pVM->pgm.s.apRamRangesTlbR0[i] = NIL_RTR0PTR;
160	pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
161	}
162	pgmUnlock(pVM);
163	}
164
165
166	/**
167	* Tests if a value of type RTGCPHYS is negative if the type had been signed
168	* instead of unsigned.
169	*
170	* @returns @c true if negative, @c false if positive or zero.
171	* @param a_GCPhys The value to test.
172	* @todo Move me to iprt/types.h.
173	*/
174	#define RTGCPHYS_IS_NEGATIVE(a_GCPhys) ((a_GCPhys) & ((RTGCPHYS)1 << (sizeof(RTGCPHYS)*8 - 1)))
175
176
177	/**
178	* Slow worker for pgmPhysGetRange.
179	*
180	* @copydoc pgmPhysGetRange
181	*/
182	PPGMRAMRANGE pgmPhysGetRangeSlow(PVM pVM, RTGCPHYS GCPhys)
183	{
184	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
185
186	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
187	while (pRam)
188	{
189	RTGCPHYS off = GCPhys - pRam->GCPhys;
190	if (off < pRam->cb)
191	{
192	pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
193	return pRam;
194	}
195	if (RTGCPHYS_IS_NEGATIVE(off))
196	pRam = pRam->CTX_SUFF(pLeft);
197	else
198	pRam = pRam->CTX_SUFF(pRight);
199	}
200	return NULL;
201	}
202
203
204	/**
205	* Slow worker for pgmPhysGetRangeAtOrAbove.
206	*
207	* @copydoc pgmPhysGetRangeAtOrAbove
208	*/
209	PPGMRAMRANGE pgmPhysGetRangeAtOrAboveSlow(PVM pVM, RTGCPHYS GCPhys)
210	{
211	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
212
213	PPGMRAMRANGE pLastLeft = NULL;
214	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
215	while (pRam)
216	{
217	RTGCPHYS off = GCPhys - pRam->GCPhys;
218	if (off < pRam->cb)
219	{
220	pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
221	return pRam;
222	}
223	if (RTGCPHYS_IS_NEGATIVE(off))
224	{
225	pLastLeft = pRam;
226	pRam = pRam->CTX_SUFF(pLeft);
227	}
228	else
229	pRam = pRam->CTX_SUFF(pRight);
230	}
231	return pLastLeft;
232	}
233
234
235	/**
236	* Slow worker for pgmPhysGetPage.
237	*
238	* @copydoc pgmPhysGetPage
239	*/
240	PPGMPAGE pgmPhysGetPageSlow(PVM pVM, RTGCPHYS GCPhys)
241	{
242	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
243
244	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
245	while (pRam)
246	{
247	RTGCPHYS off = GCPhys - pRam->GCPhys;
248	if (off < pRam->cb)
249	{
250	pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
251	return &pRam->aPages[off >> PAGE_SHIFT];
252	}
253
254	if (RTGCPHYS_IS_NEGATIVE(off))
255	pRam = pRam->CTX_SUFF(pLeft);
256	else
257	pRam = pRam->CTX_SUFF(pRight);
258	}
259	return NULL;
260	}
261
262
263	/**
264	* Slow worker for pgmPhysGetPageEx.
265	*
266	* @copydoc pgmPhysGetPageEx
267	*/
268	int pgmPhysGetPageExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
269	{
270	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
271
272	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
273	while (pRam)
274	{
275	RTGCPHYS off = GCPhys - pRam->GCPhys;
276	if (off < pRam->cb)
277	{
278	pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
279	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
280	return VINF_SUCCESS;
281	}
282
283	if (RTGCPHYS_IS_NEGATIVE(off))
284	pRam = pRam->CTX_SUFF(pLeft);
285	else
286	pRam = pRam->CTX_SUFF(pRight);
287	}
288
289	*ppPage = NULL;
290	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
291	}
292
293
294	/**
295	* Slow worker for pgmPhysGetPageAndRangeEx.
296	*
297	* @copydoc pgmPhysGetPageAndRangeEx
298	*/
299	int pgmPhysGetPageAndRangeExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
300	{
301	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
302
303	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
304	while (pRam)
305	{
306	RTGCPHYS off = GCPhys - pRam->GCPhys;
307	if (off < pRam->cb)
308	{
309	pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
310	*ppRam = pRam;
311	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
312	return VINF_SUCCESS;
313	}
314
315	if (RTGCPHYS_IS_NEGATIVE(off))
316	pRam = pRam->CTX_SUFF(pLeft);
317	else
318	pRam = pRam->CTX_SUFF(pRight);
319	}
320
321	*ppRam = NULL;
322	*ppPage = NULL;
323	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
324	}
325
326
327	/**
328	* Checks if Address Gate 20 is enabled or not.
329	*
330	* @returns true if enabled.
331	* @returns false if disabled.
332	* @param pVCpu Pointer to the VMCPU.
333	*/
334	VMMDECL(bool) PGMPhysIsA20Enabled(PVMCPU pVCpu)
335	{
336	LogFlow(("PGMPhysIsA20Enabled %d\n", pVCpu->pgm.s.fA20Enabled));
337	return pVCpu->pgm.s.fA20Enabled;
338	}
339
340
341	/**
342	* Validates a GC physical address.
343	*
344	* @returns true if valid.
345	* @returns false if invalid.
346	* @param pVM Pointer to the VM.
347	* @param GCPhys The physical address to validate.
348	*/
349	VMMDECL(bool) PGMPhysIsGCPhysValid(PVM pVM, RTGCPHYS GCPhys)
350	{
351	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
352	return pPage != NULL;
353	}
354
355
356	/**
357	* Checks if a GC physical address is a normal page,
358	* i.e. not ROM, MMIO or reserved.
359	*
360	* @returns true if normal.
361	* @returns false if invalid, ROM, MMIO or reserved page.
362	* @param pVM Pointer to the VM.
363	* @param GCPhys The physical address to check.
364	*/
365	VMMDECL(bool) PGMPhysIsGCPhysNormal(PVM pVM, RTGCPHYS GCPhys)
366	{
367	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
368	return pPage
369	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM;
370	}
371
372
373	/**
374	* Converts a GC physical address to a HC physical address.
375	*
376	* @returns VINF_SUCCESS on success.
377	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
378	* page but has no physical backing.
379	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
380	* GC physical address.
381	*
382	* @param pVM Pointer to the VM.
383	* @param GCPhys The GC physical address to convert.
384	* @param pHCPhys Where to store the HC physical address on success.
385	*/
386	VMMDECL(int) PGMPhysGCPhys2HCPhys(PVM pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
387	{
388	pgmLock(pVM);
389	PPGMPAGE pPage;
390	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
391	if (RT_SUCCESS(rc))
392	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK);
393	pgmUnlock(pVM);
394	return rc;
395	}
396
397
398	/**
399	* Invalidates all page mapping TLBs.
400	*
401	* @param pVM Pointer to the VM.
402	*/
403	void pgmPhysInvalidatePageMapTLB(PVM pVM)
404	{
405	pgmLock(pVM);
406	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatPageMapTlbFlushes);
407
408	/* Clear the shared R0/R3 TLB completely. */
409	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
410	{
411	pVM->pgm.s.PhysTlbHC.aEntries[i].GCPhys = NIL_RTGCPHYS;
412	pVM->pgm.s.PhysTlbHC.aEntries[i].pPage = 0;
413	pVM->pgm.s.PhysTlbHC.aEntries[i].pMap = 0;
414	pVM->pgm.s.PhysTlbHC.aEntries[i].pv = 0;
415	}
416
417	/** @todo clear the RC TLB whenever we add it. */
418
419	pgmUnlock(pVM);
420	}
421
422
423	/**
424	* Invalidates a page mapping TLB entry
425	*
426	* @param pVM Pointer to the VM.
427	* @param GCPhys GCPhys entry to flush
428	*/
429	void pgmPhysInvalidatePageMapTLBEntry(PVM pVM, RTGCPHYS GCPhys)
430	{
431	PGM_LOCK_ASSERT_OWNER(pVM);
432
433	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatPageMapTlbFlushEntry);
434
435	#ifdef IN_RC
436	unsigned idx = PGM_PAGER3MAPTLB_IDX(GCPhys);
437	pVM->pgm.s.PhysTlbHC.aEntries[idx].GCPhys = NIL_RTGCPHYS;
438	pVM->pgm.s.PhysTlbHC.aEntries[idx].pPage = 0;
439	pVM->pgm.s.PhysTlbHC.aEntries[idx].pMap = 0;
440	pVM->pgm.s.PhysTlbHC.aEntries[idx].pv = 0;
441	#else
442	/* Clear the shared R0/R3 TLB entry. */
443	PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
444	pTlbe->GCPhys = NIL_RTGCPHYS;
445	pTlbe->pPage = 0;
446	pTlbe->pMap = 0;
447	pTlbe->pv = 0;
448	#endif
449
450	/** @todo clear the RC TLB whenever we add it. */
451	}
452
453	/**
454	* Makes sure that there is at least one handy page ready for use.
455	*
456	* This will also take the appropriate actions when reaching water-marks.
457	*
458	* @returns VBox status code.
459	* @retval VINF_SUCCESS on success.
460	* @retval VERR_EM_NO_MEMORY if we're really out of memory.
461	*
462	* @param pVM Pointer to the VM.
463	*
464	* @remarks Must be called from within the PGM critical section. It may
465	* nip back to ring-3/0 in some cases.
466	*/
467	static int pgmPhysEnsureHandyPage(PVM pVM)
468	{
469	AssertMsg(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", pVM->pgm.s.cHandyPages));
470
471	/*
472	* Do we need to do anything special?
473	*/
474	#ifdef IN_RING3
475	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_R3_ALLOC))
476	#else
477	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_RZ_TO_R3))
478	#endif
479	{
480	/*
481	* Allocate pages only if we're out of them, or in ring-3, almost out.
482	*/
483	#ifdef IN_RING3
484	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_R3_ALLOC)
485	#else
486	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_ALLOC)
487	#endif
488	{
489	Log(("PGM: cHandyPages=%u out of %u -> allocate more; VM_FF_PGM_NO_MEMORY=%RTbool\n",
490	pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages), VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY) ));
491	#ifdef IN_RING3
492	int rc = PGMR3PhysAllocateHandyPages(pVM);
493	#else
494	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES, 0);
495	#endif
496	if (RT_UNLIKELY(rc != VINF_SUCCESS))
497	{
498	if (RT_FAILURE(rc))
499	return rc;
500	AssertMsgReturn(rc == VINF_EM_NO_MEMORY, ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
501	if (!pVM->pgm.s.cHandyPages)
502	{
503	LogRel(("PGM: no more handy pages!\n"));
504	return VERR_EM_NO_MEMORY;
505	}
506	Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES));
507	Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY));
508	#ifdef IN_RING3
509	# ifdef VBOX_WITH_REM
510	REMR3NotifyFF(pVM);
511	# endif
512	#else
513	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); /* paranoia */
514	#endif
515	}
516	AssertMsgReturn( pVM->pgm.s.cHandyPages > 0
517	&& pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages),
518	("%u\n", pVM->pgm.s.cHandyPages),
519	VERR_PGM_HANDY_PAGE_IPE);
520	}
521	else
522	{
523	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_SET_FF)
524	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
525	#ifndef IN_RING3
526	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_TO_R3)
527	{
528	Log(("PGM: VM_FF_TO_R3 - cHandyPages=%u out of %u\n", pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
529	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
530	}
531	#endif
532	}
533	}
534
535	return VINF_SUCCESS;
536	}
537
538
539	/**
540	* Replace a zero or shared page with new page that we can write to.
541	*
542	* @returns The following VBox status codes.
543	* @retval VINF_SUCCESS on success, pPage is modified.
544	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
545	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
546	*
547	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
548	*
549	* @param pVM Pointer to the VM.
550	* @param pPage The physical page tracking structure. This will
551	* be modified on success.
552	* @param GCPhys The address of the page.
553	*
554	* @remarks Must be called from within the PGM critical section. It may
555	* nip back to ring-3/0 in some cases.
556	*
557	* @remarks This function shouldn't really fail, however if it does
558	* it probably means we've screwed up the size of handy pages and/or
559	* the low-water mark. Or, that some device I/O is causing a lot of
560	* pages to be allocated while while the host is in a low-memory
561	* condition. This latter should be handled elsewhere and in a more
562	* controlled manner, it's on the @bugref{3170} todo list...
563	*/
564	int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
565	{
566	LogFlow(("pgmPhysAllocPage: %R[pgmpage] %RGp\n", pPage, GCPhys));
567
568	/*
569	* Prereqs.
570	*/
571	PGM_LOCK_ASSERT_OWNER(pVM);
572	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
573	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
574
575	# ifdef PGM_WITH_LARGE_PAGES
576	/*
577	* Try allocate a large page if applicable.
578	*/
579	if ( PGMIsUsingLargePages(pVM)
580	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
581	{
582	RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
583	PPGMPAGE pBasePage;
584
585	int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pBasePage);
586	AssertRCReturn(rc, rc); /* paranoia; can't happen. */
587	if (PGM_PAGE_GET_PDE_TYPE(pBasePage) == PGM_PAGE_PDE_TYPE_DONTCARE)
588	{
589	rc = pgmPhysAllocLargePage(pVM, GCPhys);
590	if (rc == VINF_SUCCESS)
591	return rc;
592	}
593	/* Mark the base as type page table, so we don't check over and over again. */
594	PGM_PAGE_SET_PDE_TYPE(pVM, pBasePage, PGM_PAGE_PDE_TYPE_PT);
595
596	/* fall back to 4KB pages. */
597	}
598	# endif
599
600	/*
601	* Flush any shadow page table mappings of the page.
602	* When VBOX_WITH_NEW_LAZY_PAGE_ALLOC isn't defined, there shouldn't be any.
603	*/
604	bool fFlushTLBs = false;
605	int rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhys, pPage, true /fFlushTLBs/, &fFlushTLBs);
606	AssertMsgReturn(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc), RT_FAILURE(rc) ? rc : VERR_IPE_UNEXPECTED_STATUS);
607
608	/*
609	* Ensure that we've got a page handy, take it and use it.
610	*/
611	int rc2 = pgmPhysEnsureHandyPage(pVM);
612	if (RT_FAILURE(rc2))
613	{
614	if (fFlushTLBs)
615	PGM_INVL_ALL_VCPU_TLBS(pVM);
616	Assert(rc2 == VERR_EM_NO_MEMORY);
617	return rc2;
618	}
619	/* re-assert preconditions since pgmPhysEnsureHandyPage may do a context switch. */
620	PGM_LOCK_ASSERT_OWNER(pVM);
621	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
622	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
623
624	uint32_t iHandyPage = --pVM->pgm.s.cHandyPages;
625	AssertMsg(iHandyPage < RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", iHandyPage));
626	Assert(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys != NIL_RTHCPHYS);
627	Assert(!(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
628	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idPage != NIL_GMM_PAGEID);
629	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
630
631	/*
632	* There are one or two action to be taken the next time we allocate handy pages:
633	* - Tell the GMM (global memory manager) what the page is being used for.
634	* (Speeds up replacement operations - sharing and defragmenting.)
635	* - If the current backing is shared, it must be freed.
636	*/
637	const RTHCPHYS HCPhys = pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys;
638	pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
639
640	void const *pvSharedPage = NULL;
641	if (PGM_PAGE_IS_SHARED(pPage))
642	{
643	/* Mark this shared page for freeing/dereferencing. */
644	pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage = PGM_PAGE_GET_PAGEID(pPage);
645	Assert(PGM_PAGE_GET_PAGEID(pPage) != NIL_GMM_PAGEID);
646
647	Log(("PGM: Replaced shared page %#x at %RGp with %#x / %RHp\n", PGM_PAGE_GET_PAGEID(pPage),
648	GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
649	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageReplaceShared));
650	pVM->pgm.s.cSharedPages--;
651
652	/* Grab the address of the page so we can make a copy later on. (safe) */
653	rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvSharedPage);
654	AssertRC(rc);
655	}
656	else
657	{
658	Log2(("PGM: Replaced zero page %RGp with %#x / %RHp\n", GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
659	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
660	pVM->pgm.s.cZeroPages--;
661	}
662
663	/*
664	* Do the PGMPAGE modifications.
665	*/
666	pVM->pgm.s.cPrivatePages++;
667	PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
668	PGM_PAGE_SET_PAGEID(pVM, pPage, pVM->pgm.s.aHandyPages[iHandyPage].idPage);
669	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
670	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PT);
671	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
672
673	/* Copy the shared page contents to the replacement page. */
674	if (pvSharedPage)
675	{
676	/* Get the virtual address of the new page. */
677	PGMPAGEMAPLOCK PgMpLck;
678	void *pvNewPage;
679	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvNewPage, &PgMpLck); AssertRC(rc);
680	if (RT_SUCCESS(rc))
681	{
682	memcpy(pvNewPage, pvSharedPage, PAGE_SIZE); /** @todo todo write ASMMemCopyPage */
683	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
684	}
685	}
686
687	if ( fFlushTLBs
688	&& rc != VINF_PGM_GCPHYS_ALIASED)
689	PGM_INVL_ALL_VCPU_TLBS(pVM);
690	return rc;
691	}
692
693	#ifdef PGM_WITH_LARGE_PAGES
694
695	/**
696	* Replace a 2 MB range of zero pages with new pages that we can write to.
697	*
698	* @returns The following VBox status codes.
699	* @retval VINF_SUCCESS on success, pPage is modified.
700	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
701	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
702	*
703	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
704	*
705	* @param pVM Pointer to the VM.
706	* @param GCPhys The address of the page.
707	*
708	* @remarks Must be called from within the PGM critical section. It may
709	* nip back to ring-3/0 in some cases.
710	*/
711	int pgmPhysAllocLargePage(PVM pVM, RTGCPHYS GCPhys)
712	{
713	RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
714	LogFlow(("pgmPhysAllocLargePage: %RGp base %RGp\n", GCPhys, GCPhysBase));
715
716	/*
717	* Prereqs.
718	*/
719	PGM_LOCK_ASSERT_OWNER(pVM);
720	Assert(PGMIsUsingLargePages(pVM));
721
722	PPGMPAGE pFirstPage;
723	int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pFirstPage);
724	if ( RT_SUCCESS(rc)
725	&& PGM_PAGE_GET_TYPE(pFirstPage) == PGMPAGETYPE_RAM)
726	{
727	unsigned uPDEType = PGM_PAGE_GET_PDE_TYPE(pFirstPage);
728
729	/* Don't call this function for already allocated pages. */
730	Assert(uPDEType != PGM_PAGE_PDE_TYPE_PDE);
731
732	if ( uPDEType == PGM_PAGE_PDE_TYPE_DONTCARE
733	&& PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ZERO)
734	{
735	/* Lazy approach: check all pages in the 2 MB range.
736	* The whole range must be ram and unallocated. */
737	GCPhys = GCPhysBase;
738	unsigned iPage;
739	for (iPage = 0; iPage < _2M/PAGE_SIZE; iPage++)
740	{
741	PPGMPAGE pSubPage;
742	rc = pgmPhysGetPageEx(pVM, GCPhys, &pSubPage);
743	if ( RT_FAILURE(rc)
744	\|\| PGM_PAGE_GET_TYPE(pSubPage) != PGMPAGETYPE_RAM /* Anything other than ram implies monitoring. */
745	\|\| PGM_PAGE_GET_STATE(pSubPage) != PGM_PAGE_STATE_ZERO) /* Allocated, monitored or shared means we can't use a large page here */
746	{
747	LogFlow(("Found page %RGp with wrong attributes (type=%d; state=%d); cancel check. rc=%d\n", GCPhys, PGM_PAGE_GET_TYPE(pSubPage), PGM_PAGE_GET_STATE(pSubPage), rc));
748	break;
749	}
750	Assert(PGM_PAGE_GET_PDE_TYPE(pSubPage) == PGM_PAGE_PDE_TYPE_DONTCARE);
751	GCPhys += PAGE_SIZE;
752	}
753	if (iPage != _2M/PAGE_SIZE)
754	{
755	/* Failed. Mark as requiring a PT so we don't check the whole thing again in the future. */
756	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRefused);
757	PGM_PAGE_SET_PDE_TYPE(pVM, pFirstPage, PGM_PAGE_PDE_TYPE_PT);
758	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
759	}
760
761	/*
762	* Do the allocation.
763	*/
764	# ifdef IN_RING3
765	rc = PGMR3PhysAllocateLargeHandyPage(pVM, GCPhysBase);
766	# else
767	rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE, GCPhysBase);
768	# endif
769	if (RT_SUCCESS(rc))
770	{
771	Assert(PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ALLOCATED);
772	pVM->pgm.s.cLargePages++;
773	return VINF_SUCCESS;
774	}
775
776	/* If we fail once, it most likely means the host's memory is too
777	fragmented; don't bother trying again. */
778	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
779	PGMSetLargePageUsage(pVM, false);
780	return rc;
781	}
782	}
783	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
784	}
785
786
787	/**
788	* Recheck the entire 2 MB range to see if we can use it again as a large page.
789	*
790	* @returns The following VBox status codes.
791	* @retval VINF_SUCCESS on success, the large page can be used again
792	* @retval VERR_PGM_INVALID_LARGE_PAGE_RANGE if it can't be reused
793	*
794	* @param pVM Pointer to the VM.
795	* @param GCPhys The address of the page.
796	* @param pLargePage Page structure of the base page
797	*/
798	int pgmPhysRecheckLargePage(PVM pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage)
799	{
800	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRecheck);
801
802	GCPhys &= X86_PDE2M_PAE_PG_MASK;
803
804	/* Check the base page. */
805	Assert(PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED);
806	if ( PGM_PAGE_GET_STATE(pLargePage) != PGM_PAGE_STATE_ALLOCATED
807	\|\| PGM_PAGE_GET_TYPE(pLargePage) != PGMPAGETYPE_RAM
808	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
809	{
810	LogFlow(("pgmPhysRecheckLargePage: checks failed for base page %x %x %x\n", PGM_PAGE_GET_STATE(pLargePage), PGM_PAGE_GET_TYPE(pLargePage), PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage)));
811	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
812	}
813
814	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,IsValidLargePage), a);
815	/* Check all remaining pages in the 2 MB range. */
816	unsigned i;
817	GCPhys += PAGE_SIZE;
818	for (i = 1; i < _2M/PAGE_SIZE; i++)
819	{
820	PPGMPAGE pPage;
821	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
822	AssertRCBreak(rc);
823
824	if ( PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
825	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
826	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
827	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
828	{
829	LogFlow(("pgmPhysRecheckLargePage: checks failed for page %d; %x %x %x\n", i, PGM_PAGE_GET_STATE(pPage), PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)));
830	break;
831	}
832
833	GCPhys += PAGE_SIZE;
834	}
835	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,IsValidLargePage), a);
836
837	if (i == _2M/PAGE_SIZE)
838	{
839	PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE);
840	pVM->pgm.s.cLargePagesDisabled--;
841	Log(("pgmPhysRecheckLargePage: page %RGp can be reused!\n", GCPhys - _2M));
842	return VINF_SUCCESS;
843	}
844
845	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
846	}
847
848	#endif /* PGM_WITH_LARGE_PAGES */
849
850	/**
851	* Deal with a write monitored page.
852	*
853	* @returns VBox strict status code.
854	*
855	* @param pVM Pointer to the VM.
856	* @param pPage The physical page tracking structure.
857	*
858	* @remarks Called from within the PGM critical section.
859	*/
860	void pgmPhysPageMakeWriteMonitoredWritable(PVM pVM, PPGMPAGE pPage)
861	{
862	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED);
863	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
864	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
865	Assert(pVM->pgm.s.cMonitoredPages > 0);
866	pVM->pgm.s.cMonitoredPages--;
867	pVM->pgm.s.cWrittenToPages++;
868	}
869
870
871	/**
872	* Deal with pages that are not writable, i.e. not in the ALLOCATED state.
873	*
874	* @returns VBox strict status code.
875	* @retval VINF_SUCCESS on success.
876	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
877	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
878	*
879	* @param pVM Pointer to the VM.
880	* @param pPage The physical page tracking structure.
881	* @param GCPhys The address of the page.
882	*
883	* @remarks Called from within the PGM critical section.
884	*/
885	int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
886	{
887	PGM_LOCK_ASSERT_OWNER(pVM);
888	switch (PGM_PAGE_GET_STATE(pPage))
889	{
890	case PGM_PAGE_STATE_WRITE_MONITORED:
891	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
892	/* fall thru */
893	default: /* to shut up GCC */
894	case PGM_PAGE_STATE_ALLOCATED:
895	return VINF_SUCCESS;
896
897	/*
898	* Zero pages can be dummy pages for MMIO or reserved memory,
899	* so we need to check the flags before joining cause with
900	* shared page replacement.
901	*/
902	case PGM_PAGE_STATE_ZERO:
903	if (PGM_PAGE_IS_MMIO(pPage))
904	return VERR_PGM_PHYS_PAGE_RESERVED;
905	/* fall thru */
906	case PGM_PAGE_STATE_SHARED:
907	return pgmPhysAllocPage(pVM, pPage, GCPhys);
908
909	/* Not allowed to write to ballooned pages. */
910	case PGM_PAGE_STATE_BALLOONED:
911	return VERR_PGM_PHYS_PAGE_BALLOONED;
912	}
913	}
914
915
916	/**
917	* Internal usage: Map the page specified by its GMM ID.
918	*
919	* This is similar to pgmPhysPageMap
920	*
921	* @returns VBox status code.
922	*
923	* @param pVM Pointer to the VM.
924	* @param idPage The Page ID.
925	* @param HCPhys The physical address (for RC).
926	* @param ppv Where to store the mapping address.
927	*
928	* @remarks Called from within the PGM critical section. The mapping is only
929	* valid while you are inside this section.
930	*/
931	int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv)
932	{
933	/*
934	* Validation.
935	*/
936	PGM_LOCK_ASSERT_OWNER(pVM);
937	AssertReturn(HCPhys && !(HCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
938	const uint32_t idChunk = idPage >> GMM_CHUNKID_SHIFT;
939	AssertReturn(idChunk != NIL_GMM_CHUNKID, VERR_INVALID_PARAMETER);
940
941	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
942	/*
943	* Map it by HCPhys.
944	*/
945	return pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), HCPhys, ppv RTLOG_COMMA_SRC_POS);
946
947	#else
948	/*
949	* Find/make Chunk TLB entry for the mapping chunk.
950	*/
951	PPGMCHUNKR3MAP pMap;
952	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
953	if (pTlbe->idChunk == idChunk)
954	{
955	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbHits));
956	pMap = pTlbe->pChunk;
957	}
958	else
959	{
960	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
961
962	/*
963	* Find the chunk, map it if necessary.
964	*/
965	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
966	if (pMap)
967	pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
968	else
969	{
970	# ifdef IN_RING0
971	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
972	AssertRCReturn(rc, rc);
973	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
974	Assert(pMap);
975	# else
976	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
977	if (RT_FAILURE(rc))
978	return rc;
979	# endif
980	}
981
982	/*
983	* Enter it into the Chunk TLB.
984	*/
985	pTlbe->idChunk = idChunk;
986	pTlbe->pChunk = pMap;
987	}
988
989	ppv = (uint8_t )pMap->pv + ((idPage &GMM_PAGEID_IDX_MASK) << PAGE_SHIFT);
990	return VINF_SUCCESS;
991	#endif
992	}
993
994
995	/**
996	* Maps a page into the current virtual address space so it can be accessed.
997	*
998	* @returns VBox status code.
999	* @retval VINF_SUCCESS on success.
1000	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1001	*
1002	* @param pVM Pointer to the VM.
1003	* @param pPage The physical page tracking structure.
1004	* @param GCPhys The address of the page.
1005	* @param ppMap Where to store the address of the mapping tracking structure.
1006	* @param ppv Where to store the mapping address of the page. The page
1007	* offset is masked off!
1008	*
1009	* @remarks Called from within the PGM critical section.
1010	*/
1011	static int pgmPhysPageMapCommon(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv)
1012	{
1013	PGM_LOCK_ASSERT_OWNER(pVM);
1014	NOREF(GCPhys);
1015
1016	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1017	/*
1018	* Just some sketchy GC/R0-darwin code.
1019	*/
1020	*ppMap = NULL;
1021	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
1022	Assert(HCPhys != pVM->pgm.s.HCPhysZeroPg);
1023	pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), HCPhys, ppv RTLOG_COMMA_SRC_POS);
1024	return VINF_SUCCESS;
1025
1026	#else /* IN_RING3 \|\| IN_RING0 */
1027
1028
1029	/*
1030	* Special cases: MMIO2, ZERO and specially aliased MMIO pages.
1031	*/
1032	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2
1033	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO)
1034	{
1035	/* Decode the page id to a page in a MMIO2 ram range. */
1036	uint8_t idMmio2 = PGM_MMIO2_PAGEID_GET_MMIO2_ID(PGM_PAGE_GET_PAGEID(pPage));
1037	uint32_t iPage = PGM_MMIO2_PAGEID_GET_IDX(PGM_PAGE_GET_PAGEID(pPage));
1038	AssertLogRelMsgReturn((uint8_t)(idMmio2 - 1U) < RT_ELEMENTS(pVM->pgm.s.CTX_SUFF(apMmio2Ranges)),
1039	("idMmio2=%u size=%u type=%u GCPHys=%#RGp Id=%u State=%u", idMmio2,
1040	RT_ELEMENTS(pVM->pgm.s.CTX_SUFF(apMmio2Ranges)), PGM_PAGE_GET_TYPE(pPage), GCPhys,
1041	pPage->s.idPage, pPage->s.uStateY),
1042	VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1043	PPGMMMIO2RANGE pMmio2Range = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[idMmio2 - 1];
1044	AssertLogRelReturn(pMmio2Range, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1045	AssertLogRelReturn(pMmio2Range->idMmio2 == idMmio2, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1046	AssertLogRelReturn(iPage < (pMmio2Range->RamRange.cb >> PAGE_SHIFT), VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1047	ppv = (uint8_t )pMmio2Range->RamRange.pvR3 + ((uintptr_t)iPage << PAGE_SHIFT);
1048	*ppMap = NULL;
1049	return VINF_SUCCESS;
1050	}
1051
1052	const uint32_t idChunk = PGM_PAGE_GET_CHUNKID(pPage);
1053	if (idChunk == NIL_GMM_CHUNKID)
1054	{
1055	AssertMsgReturn(PGM_PAGE_GET_PAGEID(pPage) == NIL_GMM_PAGEID, ("pPage=%R[pgmpage]\n", pPage),
1056	VERR_PGM_PHYS_PAGE_MAP_IPE_1);
1057	if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
1058	{
1059	AssertMsgReturn(PGM_PAGE_IS_ZERO(pPage), ("pPage=%R[pgmpage]\n", pPage),
1060	VERR_PGM_PHYS_PAGE_MAP_IPE_3);
1061	AssertMsgReturn(PGM_PAGE_GET_HCPHYS(pPage)== pVM->pgm.s.HCPhysZeroPg, ("pPage=%R[pgmpage]\n", pPage),
1062	VERR_PGM_PHYS_PAGE_MAP_IPE_4);
1063	*ppv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
1064	}
1065	else
1066	{
1067	static uint8_t s_abPlayItSafe[0x10002]; / I don't dare return the zero page at the moment. */
1068	ppv = (uint8_t )((uintptr_t)&s_abPlayItSafe[0x1000] & ~(uintptr_t)0xfff);
1069	}
1070	*ppMap = NULL;
1071	return VINF_SUCCESS;
1072	}
1073
1074	/*
1075	* Find/make Chunk TLB entry for the mapping chunk.
1076	*/
1077	PPGMCHUNKR3MAP pMap;
1078	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
1079	if (pTlbe->idChunk == idChunk)
1080	{
1081	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbHits));
1082	pMap = pTlbe->pChunk;
1083	AssertPtr(pMap->pv);
1084	}
1085	else
1086	{
1087	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
1088
1089	/*
1090	* Find the chunk, map it if necessary.
1091	*/
1092	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
1093	if (pMap)
1094	{
1095	AssertPtr(pMap->pv);
1096	pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
1097	}
1098	else
1099	{
1100	#ifdef IN_RING0
1101	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
1102	AssertRCReturn(rc, rc);
1103	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
1104	Assert(pMap);
1105	#else
1106	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
1107	if (RT_FAILURE(rc))
1108	return rc;
1109	#endif
1110	AssertPtr(pMap->pv);
1111	}
1112
1113	/*
1114	* Enter it into the Chunk TLB.
1115	*/
1116	pTlbe->idChunk = idChunk;
1117	pTlbe->pChunk = pMap;
1118	}
1119
1120	ppv = (uint8_t )pMap->pv + (PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) << PAGE_SHIFT);
1121	*ppMap = pMap;
1122	return VINF_SUCCESS;
1123	#endif /* IN_RING3 */
1124	}
1125
1126
1127	/**
1128	* Combination of pgmPhysPageMakeWritable and pgmPhysPageMapWritable.
1129	*
1130	* This is typically used is paths where we cannot use the TLB methods (like ROM
1131	* pages) or where there is no point in using them since we won't get many hits.
1132	*
1133	* @returns VBox strict status code.
1134	* @retval VINF_SUCCESS on success.
1135	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
1136	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1137	*
1138	* @param pVM Pointer to the VM.
1139	* @param pPage The physical page tracking structure.
1140	* @param GCPhys The address of the page.
1141	* @param ppv Where to store the mapping address of the page. The page
1142	* offset is masked off!
1143	*
1144	* @remarks Called from within the PGM critical section. The mapping is only
1145	* valid while you are inside section.
1146	*/
1147	int pgmPhysPageMakeWritableAndMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1148	{
1149	int rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1150	if (RT_SUCCESS(rc))
1151	{
1152	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* returned */, ("%Rrc\n", rc));
1153	PPGMPAGEMAP pMapIgnore;
1154	int rc2 = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
1155	if (RT_FAILURE(rc2)) /* preserve rc */
1156	rc = rc2;
1157	}
1158	return rc;
1159	}
1160
1161
1162	/**
1163	* Maps a page into the current virtual address space so it can be accessed for
1164	* both writing and reading.
1165	*
1166	* This is typically used is paths where we cannot use the TLB methods (like ROM
1167	* pages) or where there is no point in using them since we won't get many hits.
1168	*
1169	* @returns VBox status code.
1170	* @retval VINF_SUCCESS on success.
1171	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1172	*
1173	* @param pVM Pointer to the VM.
1174	* @param pPage The physical page tracking structure. Must be in the
1175	* allocated state.
1176	* @param GCPhys The address of the page.
1177	* @param ppv Where to store the mapping address of the page. The page
1178	* offset is masked off!
1179	*
1180	* @remarks Called from within the PGM critical section. The mapping is only
1181	* valid while you are inside section.
1182	*/
1183	int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1184	{
1185	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
1186	PPGMPAGEMAP pMapIgnore;
1187	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
1188	}
1189
1190
1191	/**
1192	* Maps a page into the current virtual address space so it can be accessed for
1193	* reading.
1194	*
1195	* This is typically used is paths where we cannot use the TLB methods (like ROM
1196	* pages) or where there is no point in using them since we won't get many hits.
1197	*
1198	* @returns VBox status code.
1199	* @retval VINF_SUCCESS on success.
1200	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1201	*
1202	* @param pVM Pointer to the VM.
1203	* @param pPage The physical page tracking structure.
1204	* @param GCPhys The address of the page.
1205	* @param ppv Where to store the mapping address of the page. The page
1206	* offset is masked off!
1207	*
1208	* @remarks Called from within the PGM critical section. The mapping is only
1209	* valid while you are inside this section.
1210	*/
1211	int pgmPhysPageMapReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv)
1212	{
1213	PPGMPAGEMAP pMapIgnore;
1214	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, (void **)ppv);
1215	}
1216
1217	#if !defined(IN_RC) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1218
1219	/**
1220	* Load a guest page into the ring-3 physical TLB.
1221	*
1222	* @returns VBox status code.
1223	* @retval VINF_SUCCESS on success
1224	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1225	* @param pPGM The PGM instance pointer.
1226	* @param GCPhys The guest physical address in question.
1227	*/
1228	int pgmPhysPageLoadIntoTlb(PVM pVM, RTGCPHYS GCPhys)
1229	{
1230	PGM_LOCK_ASSERT_OWNER(pVM);
1231
1232	/*
1233	* Find the ram range and page and hand it over to the with-page function.
1234	* 99.8% of requests are expected to be in the first range.
1235	*/
1236	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
1237	if (!pPage)
1238	{
1239	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
1240	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
1241	}
1242
1243	return pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys);
1244	}
1245
1246
1247	/**
1248	* Load a guest page into the ring-3 physical TLB.
1249	*
1250	* @returns VBox status code.
1251	* @retval VINF_SUCCESS on success
1252	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1253	*
1254	* @param pVM Pointer to the VM.
1255	* @param pPage Pointer to the PGMPAGE structure corresponding to
1256	* GCPhys.
1257	* @param GCPhys The guest physical address in question.
1258	*/
1259	int pgmPhysPageLoadIntoTlbWithPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
1260	{
1261	PGM_LOCK_ASSERT_OWNER(pVM);
1262	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
1263
1264	/*
1265	* Map the page.
1266	* Make a special case for the zero page as it is kind of special.
1267	*/
1268	PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
1269	if ( !PGM_PAGE_IS_ZERO(pPage)
1270	&& !PGM_PAGE_IS_BALLOONED(pPage))
1271	{
1272	void *pv;
1273	PPGMPAGEMAP pMap;
1274	int rc = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMap, &pv);
1275	if (RT_FAILURE(rc))
1276	return rc;
1277	pTlbe->pMap = pMap;
1278	pTlbe->pv = pv;
1279	Assert(!((uintptr_t)pTlbe->pv & PAGE_OFFSET_MASK));
1280	}
1281	else
1282	{
1283	AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == pVM->pgm.s.HCPhysZeroPg, ("%RGp/%R[pgmpage]\n", GCPhys, pPage));
1284	pTlbe->pMap = NULL;
1285	pTlbe->pv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
1286	}
1287	#ifdef PGM_WITH_PHYS_TLB
1288	if ( PGM_PAGE_GET_TYPE(pPage) < PGMPAGETYPE_ROM_SHADOW
1289	\|\| PGM_PAGE_GET_TYPE(pPage) > PGMPAGETYPE_ROM)
1290	pTlbe->GCPhys = GCPhys & X86_PTE_PAE_PG_MASK;
1291	else
1292	pTlbe->GCPhys = NIL_RTGCPHYS; /* ROM: Problematic because of the two pages. :-/ */
1293	#else
1294	pTlbe->GCPhys = NIL_RTGCPHYS;
1295	#endif
1296	pTlbe->pPage = pPage;
1297	return VINF_SUCCESS;
1298	}
1299
1300	#endif /* !IN_RC && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
1301
1302	/**
1303	* Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
1304	* own the PGM lock and therefore not need to lock the mapped page.
1305	*
1306	* @returns VBox status code.
1307	* @retval VINF_SUCCESS on success.
1308	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1309	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1310	*
1311	* @param pVM Pointer to the VM.
1312	* @param GCPhys The guest physical address of the page that should be mapped.
1313	* @param pPage Pointer to the PGMPAGE structure for the page.
1314	* @param ppv Where to store the address corresponding to GCPhys.
1315	*
1316	* @internal
1317	* @deprecated Use pgmPhysGCPhys2CCPtrInternalEx.
1318	*/
1319	int pgmPhysGCPhys2CCPtrInternalDepr(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1320	{
1321	int rc;
1322	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
1323	PGM_LOCK_ASSERT_OWNER(pVM);
1324	pVM->pgm.s.cDeprecatedPageLocks++;
1325
1326	/*
1327	* Make sure the page is writable.
1328	*/
1329	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1330	{
1331	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1332	if (RT_FAILURE(rc))
1333	return rc;
1334	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1335	}
1336	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1337
1338	/*
1339	* Get the mapping address.
1340	*/
1341	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1342	void *pv;
1343	rc = pgmRZDynMapHCPageInlined(VMMGetCpu(pVM),
1344	PGM_PAGE_GET_HCPHYS(pPage),
1345	&pv
1346	RTLOG_COMMA_SRC_POS);
1347	if (RT_FAILURE(rc))
1348	return rc;
1349	ppv = (void )((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1350	#else
1351	PPGMPAGEMAPTLBE pTlbe;
1352	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1353	if (RT_FAILURE(rc))
1354	return rc;
1355	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1356	#endif
1357	return VINF_SUCCESS;
1358	}
1359
1360	#if !defined(IN_RC) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1361
1362	/**
1363	* Locks a page mapping for writing.
1364	*
1365	* @param pVM Pointer to the VM.
1366	* @param pPage The page.
1367	* @param pTlbe The mapping TLB entry for the page.
1368	* @param pLock The lock structure (output).
1369	*/
1370	DECLINLINE(void) pgmPhysPageMapLockForWriting(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
1371	{
1372	PPGMPAGEMAP pMap = pTlbe->pMap;
1373	if (pMap)
1374	pMap->cRefs++;
1375
1376	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
1377	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
1378	{
1379	if (cLocks == 0)
1380	pVM->pgm.s.cWriteLockedPages++;
1381	PGM_PAGE_INC_WRITE_LOCKS(pPage);
1382	}
1383	else if (cLocks != PGM_PAGE_MAX_LOCKS)
1384	{
1385	PGM_PAGE_INC_WRITE_LOCKS(pPage);
1386	AssertMsgFailed(("%R[pgmpage] is entering permanent write locked state!\n", pPage));
1387	if (pMap)
1388	pMap->cRefs++; /* Extra ref to prevent it from going away. */
1389	}
1390
1391	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
1392	pLock->pvMap = pMap;
1393	}
1394
1395	/**
1396	* Locks a page mapping for reading.
1397	*
1398	* @param pVM Pointer to the VM.
1399	* @param pPage The page.
1400	* @param pTlbe The mapping TLB entry for the page.
1401	* @param pLock The lock structure (output).
1402	*/
1403	DECLINLINE(void) pgmPhysPageMapLockForReading(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
1404	{
1405	PPGMPAGEMAP pMap = pTlbe->pMap;
1406	if (pMap)
1407	pMap->cRefs++;
1408
1409	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
1410	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
1411	{
1412	if (cLocks == 0)
1413	pVM->pgm.s.cReadLockedPages++;
1414	PGM_PAGE_INC_READ_LOCKS(pPage);
1415	}
1416	else if (cLocks != PGM_PAGE_MAX_LOCKS)
1417	{
1418	PGM_PAGE_INC_READ_LOCKS(pPage);
1419	AssertMsgFailed(("%R[pgmpage] is entering permanent read locked state!\n", pPage));
1420	if (pMap)
1421	pMap->cRefs++; /* Extra ref to prevent it from going away. */
1422	}
1423
1424	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
1425	pLock->pvMap = pMap;
1426	}
1427
1428	#endif /* !IN_RC && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
1429
1430
1431	/**
1432	* Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
1433	* own the PGM lock and have access to the page structure.
1434	*
1435	* @returns VBox status code.
1436	* @retval VINF_SUCCESS on success.
1437	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1438	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1439	*
1440	* @param pVM Pointer to the VM.
1441	* @param GCPhys The guest physical address of the page that should be mapped.
1442	* @param pPage Pointer to the PGMPAGE structure for the page.
1443	* @param ppv Where to store the address corresponding to GCPhys.
1444	* @param pLock Where to store the lock information that
1445	* pgmPhysReleaseInternalPageMappingLock needs.
1446	*
1447	* @internal
1448	*/
1449	int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
1450	{
1451	int rc;
1452	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
1453	PGM_LOCK_ASSERT_OWNER(pVM);
1454
1455	/*
1456	* Make sure the page is writable.
1457	*/
1458	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1459	{
1460	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1461	if (RT_FAILURE(rc))
1462	return rc;
1463	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1464	}
1465	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1466
1467	/*
1468	* Do the job.
1469	*/
1470	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1471	void *pv;
1472	PVMCPU pVCpu = VMMGetCpu(pVM);
1473	rc = pgmRZDynMapHCPageInlined(pVCpu,
1474	PGM_PAGE_GET_HCPHYS(pPage),
1475	&pv
1476	RTLOG_COMMA_SRC_POS);
1477	if (RT_FAILURE(rc))
1478	return rc;
1479	ppv = (void )((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1480	pLock->pvPage = pv;
1481	pLock->pVCpu = pVCpu;
1482
1483	#else
1484	PPGMPAGEMAPTLBE pTlbe;
1485	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1486	if (RT_FAILURE(rc))
1487	return rc;
1488	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
1489	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1490	#endif
1491	return VINF_SUCCESS;
1492	}
1493
1494
1495	/**
1496	* Internal version of PGMPhysGCPhys2CCPtrReadOnly that expects the caller to
1497	* own the PGM lock and have access to the page structure.
1498	*
1499	* @returns VBox status code.
1500	* @retval VINF_SUCCESS on success.
1501	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1502	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1503	*
1504	* @param pVM Pointer to the VM.
1505	* @param GCPhys The guest physical address of the page that should be mapped.
1506	* @param pPage Pointer to the PGMPAGE structure for the page.
1507	* @param ppv Where to store the address corresponding to GCPhys.
1508	* @param pLock Where to store the lock information that
1509	* pgmPhysReleaseInternalPageMappingLock needs.
1510	*
1511	* @internal
1512	*/
1513	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock)
1514	{
1515	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
1516	PGM_LOCK_ASSERT_OWNER(pVM);
1517	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1518
1519	/*
1520	* Do the job.
1521	*/
1522	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1523	void *pv;
1524	PVMCPU pVCpu = VMMGetCpu(pVM);
1525	int rc = pgmRZDynMapHCPageInlined(pVCpu,
1526	PGM_PAGE_GET_HCPHYS(pPage),
1527	&pv
1528	RTLOG_COMMA_SRC_POS); /** @todo add a read only flag? */
1529	if (RT_FAILURE(rc))
1530	return rc;
1531	ppv = (void )((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1532	pLock->pvPage = pv;
1533	pLock->pVCpu = pVCpu;
1534
1535	#else
1536	PPGMPAGEMAPTLBE pTlbe;
1537	int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1538	if (RT_FAILURE(rc))
1539	return rc;
1540	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
1541	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1542	#endif
1543	return VINF_SUCCESS;
1544	}
1545
1546
1547	/**
1548	* Requests the mapping of a guest page into the current context.
1549	*
1550	* This API should only be used for very short term, as it will consume scarse
1551	* resources (R0 and GC) in the mapping cache. When you're done with the page,
1552	* call PGMPhysReleasePageMappingLock() ASAP to release it.
1553	*
1554	* This API will assume your intention is to write to the page, and will
1555	* therefore replace shared and zero pages. If you do not intend to modify
1556	* the page, use the PGMPhysGCPhys2CCPtrReadOnly() API.
1557	*
1558	* @returns VBox status code.
1559	* @retval VINF_SUCCESS on success.
1560	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1561	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1562	*
1563	* @param pVM Pointer to the VM.
1564	* @param GCPhys The guest physical address of the page that should be
1565	* mapped.
1566	* @param ppv Where to store the address corresponding to GCPhys.
1567	* @param pLock Where to store the lock information that
1568	* PGMPhysReleasePageMappingLock needs.
1569	*
1570	* @remarks The caller is responsible for dealing with access handlers.
1571	* @todo Add an informational return code for pages with access handlers?
1572	*
1573	* @remark Avoid calling this API from within critical sections (other than
1574	* the PGM one) because of the deadlock risk. External threads may
1575	* need to delegate jobs to the EMTs.
1576	* @remarks Only one page is mapped! Make no assumption about what's after or
1577	* before the returned page!
1578	* @thread Any thread.
1579	*/
1580	VMMDECL(int) PGMPhysGCPhys2CCPtr(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
1581	{
1582	int rc = pgmLock(pVM);
1583	AssertRCReturn(rc, rc);
1584
1585	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1586	/*
1587	* Find the page and make sure it's writable.
1588	*/
1589	PPGMPAGE pPage;
1590	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1591	if (RT_SUCCESS(rc))
1592	{
1593	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1594	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1595	if (RT_SUCCESS(rc))
1596	{
1597	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1598
1599	PVMCPU pVCpu = VMMGetCpu(pVM);
1600	void *pv;
1601	rc = pgmRZDynMapHCPageInlined(pVCpu,
1602	PGM_PAGE_GET_HCPHYS(pPage),
1603	&pv
1604	RTLOG_COMMA_SRC_POS);
1605	if (RT_SUCCESS(rc))
1606	{
1607	AssertRCSuccess(rc);
1608
1609	pv = (void *)((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1610	*ppv = pv;
1611	pLock->pvPage = pv;
1612	pLock->pVCpu = pVCpu;
1613	}
1614	}
1615	}
1616
1617	#else /* IN_RING3 \|\| IN_RING0 */
1618	/*
1619	* Query the Physical TLB entry for the page (may fail).
1620	*/
1621	PPGMPAGEMAPTLBE pTlbe;
1622	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
1623	if (RT_SUCCESS(rc))
1624	{
1625	/*
1626	* If the page is shared, the zero page, or being write monitored
1627	* it must be converted to a page that's writable if possible.
1628	*/
1629	PPGMPAGE pPage = pTlbe->pPage;
1630	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1631	{
1632	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1633	if (RT_SUCCESS(rc))
1634	{
1635	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1636	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1637	}
1638	}
1639	if (RT_SUCCESS(rc))
1640	{
1641	/*
1642	* Now, just perform the locking and calculate the return address.
1643	*/
1644	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
1645	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1646	}
1647	}
1648
1649	#endif /* IN_RING3 \|\| IN_RING0 */
1650	pgmUnlock(pVM);
1651	return rc;
1652	}
1653
1654
1655	/**
1656	* Requests the mapping of a guest page into the current context.
1657	*
1658	* This API should only be used for very short term, as it will consume scarse
1659	* resources (R0 and GC) in the mapping cache. When you're done with the page,
1660	* call PGMPhysReleasePageMappingLock() ASAP to release it.
1661	*
1662	* @returns VBox status code.
1663	* @retval VINF_SUCCESS on success.
1664	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1665	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1666	*
1667	* @param pVM Pointer to the VM.
1668	* @param GCPhys The guest physical address of the page that should be
1669	* mapped.
1670	* @param ppv Where to store the address corresponding to GCPhys.
1671	* @param pLock Where to store the lock information that
1672	* PGMPhysReleasePageMappingLock needs.
1673	*
1674	* @remarks The caller is responsible for dealing with access handlers.
1675	* @todo Add an informational return code for pages with access handlers?
1676	*
1677	* @remarks Avoid calling this API from within critical sections (other than
1678	* the PGM one) because of the deadlock risk.
1679	* @remarks Only one page is mapped! Make no assumption about what's after or
1680	* before the returned page!
1681	* @thread Any thread.
1682	*/
1683	VMMDECL(int) PGMPhysGCPhys2CCPtrReadOnly(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
1684	{
1685	int rc = pgmLock(pVM);
1686	AssertRCReturn(rc, rc);
1687
1688	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1689	/*
1690	* Find the page and make sure it's readable.
1691	*/
1692	PPGMPAGE pPage;
1693	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1694	if (RT_SUCCESS(rc))
1695	{
1696	if (RT_UNLIKELY(PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)))
1697	rc = VERR_PGM_PHYS_PAGE_RESERVED;
1698	else
1699	{
1700	PVMCPU pVCpu = VMMGetCpu(pVM);
1701	void *pv;
1702	rc = pgmRZDynMapHCPageInlined(pVCpu,
1703	PGM_PAGE_GET_HCPHYS(pPage),
1704	&pv
1705	RTLOG_COMMA_SRC_POS); /** @todo add a read only flag? */
1706	if (RT_SUCCESS(rc))
1707	{
1708	AssertRCSuccess(rc);
1709
1710	pv = (void *)((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1711	*ppv = pv;
1712	pLock->pvPage = pv;
1713	pLock->pVCpu = pVCpu;
1714	}
1715	}
1716	}
1717
1718	#else /* IN_RING3 \|\| IN_RING0 */
1719	/*
1720	* Query the Physical TLB entry for the page (may fail).
1721	*/
1722	PPGMPAGEMAPTLBE pTlbe;
1723	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
1724	if (RT_SUCCESS(rc))
1725	{
1726	/* MMIO pages doesn't have any readable backing. */
1727	PPGMPAGE pPage = pTlbe->pPage;
1728	if (RT_UNLIKELY(PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)))
1729	rc = VERR_PGM_PHYS_PAGE_RESERVED;
1730	else
1731	{
1732	/*
1733	* Now, just perform the locking and calculate the return address.
1734	*/
1735	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
1736	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1737	}
1738	}
1739
1740	#endif /* IN_RING3 \|\| IN_RING0 */
1741	pgmUnlock(pVM);
1742	return rc;
1743	}
1744
1745
1746	/**
1747	* Requests the mapping of a guest page given by virtual address into the current context.
1748	*
1749	* This API should only be used for very short term, as it will consume
1750	* scarse resources (R0 and GC) in the mapping cache. When you're done
1751	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1752	*
1753	* This API will assume your intention is to write to the page, and will
1754	* therefore replace shared and zero pages. If you do not intend to modify
1755	* the page, use the PGMPhysGCPtr2CCPtrReadOnly() API.
1756	*
1757	* @returns VBox status code.
1758	* @retval VINF_SUCCESS on success.
1759	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
1760	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
1761	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1762	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1763	*
1764	* @param pVCpu Pointer to the VMCPU.
1765	* @param GCPhys The guest physical address of the page that should be mapped.
1766	* @param ppv Where to store the address corresponding to GCPhys.
1767	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1768	*
1769	* @remark Avoid calling this API from within critical sections (other than
1770	* the PGM one) because of the deadlock risk.
1771	* @thread EMT
1772	*/
1773	VMMDECL(int) PGMPhysGCPtr2CCPtr(PVMCPU pVCpu, RTGCPTR GCPtr, void **ppv, PPGMPAGEMAPLOCK pLock)
1774	{
1775	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
1776	RTGCPHYS GCPhys;
1777	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
1778	if (RT_SUCCESS(rc))
1779	rc = PGMPhysGCPhys2CCPtr(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
1780	return rc;
1781	}
1782
1783
1784	/**
1785	* Requests the mapping of a guest page given by virtual address into the current context.
1786	*
1787	* This API should only be used for very short term, as it will consume
1788	* scarse resources (R0 and GC) in the mapping cache. When you're done
1789	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1790	*
1791	* @returns VBox status code.
1792	* @retval VINF_SUCCESS on success.
1793	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
1794	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
1795	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1796	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1797	*
1798	* @param pVCpu Pointer to the VMCPU.
1799	* @param GCPhys The guest physical address of the page that should be mapped.
1800	* @param ppv Where to store the address corresponding to GCPhys.
1801	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
1802	*
1803	* @remark Avoid calling this API from within critical sections (other than
1804	* the PGM one) because of the deadlock risk.
1805	* @thread EMT
1806	*/
1807	VMMDECL(int) PGMPhysGCPtr2CCPtrReadOnly(PVMCPU pVCpu, RTGCPTR GCPtr, void const **ppv, PPGMPAGEMAPLOCK pLock)
1808	{
1809	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
1810	RTGCPHYS GCPhys;
1811	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
1812	if (RT_SUCCESS(rc))
1813	rc = PGMPhysGCPhys2CCPtrReadOnly(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
1814	return rc;
1815	}
1816
1817
1818	/**
1819	* Release the mapping of a guest page.
1820	*
1821	* This is the counter part of PGMPhysGCPhys2CCPtr, PGMPhysGCPhys2CCPtrReadOnly
1822	* PGMPhysGCPtr2CCPtr and PGMPhysGCPtr2CCPtrReadOnly.
1823	*
1824	* @param pVM Pointer to the VM.
1825	* @param pLock The lock structure initialized by the mapping function.
1826	*/
1827	VMMDECL(void) PGMPhysReleasePageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock)
1828	{
1829	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1830	Assert(pLock->pvPage != NULL);
1831	Assert(pLock->pVCpu == VMMGetCpu(pVM));
1832	PGM_DYNMAP_UNUSED_HINT(pLock->pVCpu, pLock->pvPage);
1833	pLock->pVCpu = NULL;
1834	pLock->pvPage = NULL;
1835
1836	#else
1837	PPGMPAGEMAP pMap = (PPGMPAGEMAP)pLock->pvMap;
1838	PPGMPAGE pPage = (PPGMPAGE)(pLock->uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
1839	bool fWriteLock = (pLock->uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
1840
1841	pLock->uPageAndType = 0;
1842	pLock->pvMap = NULL;
1843
1844	pgmLock(pVM);
1845	if (fWriteLock)
1846	{
1847	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
1848	Assert(cLocks > 0);
1849	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
1850	{
1851	if (cLocks == 1)
1852	{
1853	Assert(pVM->pgm.s.cWriteLockedPages > 0);
1854	pVM->pgm.s.cWriteLockedPages--;
1855	}
1856	PGM_PAGE_DEC_WRITE_LOCKS(pPage);
1857	}
1858
1859	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
1860	{
1861	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
1862	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
1863	Assert(pVM->pgm.s.cMonitoredPages > 0);
1864	pVM->pgm.s.cMonitoredPages--;
1865	pVM->pgm.s.cWrittenToPages++;
1866	}
1867	}
1868	else
1869	{
1870	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
1871	Assert(cLocks > 0);
1872	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
1873	{
1874	if (cLocks == 1)
1875	{
1876	Assert(pVM->pgm.s.cReadLockedPages > 0);
1877	pVM->pgm.s.cReadLockedPages--;
1878	}
1879	PGM_PAGE_DEC_READ_LOCKS(pPage);
1880	}
1881	}
1882
1883	if (pMap)
1884	{
1885	Assert(pMap->cRefs >= 1);
1886	pMap->cRefs--;
1887	}
1888	pgmUnlock(pVM);
1889	#endif /* IN_RING3 */
1890	}
1891
1892
1893	/**
1894	* Release the internal mapping of a guest page.
1895	*
1896	* This is the counter part of pgmPhysGCPhys2CCPtrInternalEx and
1897	* pgmPhysGCPhys2CCPtrInternalReadOnly.
1898	*
1899	* @param pVM Pointer to the VM.
1900	* @param pLock The lock structure initialized by the mapping function.
1901	*
1902	* @remarks Caller must hold the PGM lock.
1903	*/
1904	void pgmPhysReleaseInternalPageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock)
1905	{
1906	PGM_LOCK_ASSERT_OWNER(pVM);
1907	PGMPhysReleasePageMappingLock(pVM, pLock); /* lazy for now */
1908	}
1909
1910
1911	/**
1912	* Converts a GC physical address to a HC ring-3 pointer.
1913	*
1914	* @returns VINF_SUCCESS on success.
1915	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
1916	* page but has no physical backing.
1917	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
1918	* GC physical address.
1919	* @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
1920	* a dynamic ram chunk boundary
1921	*
1922	* @param pVM Pointer to the VM.
1923	* @param GCPhys The GC physical address to convert.
1924	* @param pR3Ptr Where to store the R3 pointer on success.
1925	*
1926	* @deprecated Avoid when possible!
1927	*/
1928	int pgmPhysGCPhys2R3Ptr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
1929	{
1930	/** @todo this is kind of hacky and needs some more work. */
1931	#ifndef DEBUG_sandervl
1932	VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
1933	#endif
1934
1935	Log(("pgmPhysGCPhys2R3Ptr(,%RGp,): dont use this API!\n", GCPhys)); /** @todo eliminate this API! */
1936	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1937	NOREF(pVM); NOREF(pR3Ptr);
1938	AssertFailedReturn(VERR_NOT_IMPLEMENTED);
1939	#else
1940	pgmLock(pVM);
1941
1942	PPGMRAMRANGE pRam;
1943	PPGMPAGE pPage;
1944	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
1945	if (RT_SUCCESS(rc))
1946	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
1947
1948	pgmUnlock(pVM);
1949	Assert(rc <= VINF_SUCCESS);
1950	return rc;
1951	#endif
1952	}
1953
1954	#if 0 /defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)/
1955
1956	/**
1957	* Maps and locks a guest CR3 or PD (PAE) page.
1958	*
1959	* @returns VINF_SUCCESS on success.
1960	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
1961	* page but has no physical backing.
1962	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
1963	* GC physical address.
1964	* @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
1965	* a dynamic ram chunk boundary
1966	*
1967	* @param pVM Pointer to the VM.
1968	* @param GCPhys The GC physical address to convert.
1969	* @param pR3Ptr Where to store the R3 pointer on success. This may or
1970	* may not be valid in ring-0 depending on the
1971	* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 build option.
1972	*
1973	* @remarks The caller must own the PGM lock.
1974	*/
1975	int pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
1976	{
1977
1978	PPGMRAMRANGE pRam;
1979	PPGMPAGE pPage;
1980	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
1981	if (RT_SUCCESS(rc))
1982	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
1983	Assert(rc <= VINF_SUCCESS);
1984	return rc;
1985	}
1986
1987
1988	int pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
1989	{
1990
1991	}
1992
1993	#endif
1994
1995	/**
1996	* Converts a guest pointer to a GC physical address.
1997	*
1998	* This uses the current CR3/CR0/CR4 of the guest.
1999	*
2000	* @returns VBox status code.
2001	* @param pVCpu Pointer to the VMCPU.
2002	* @param GCPtr The guest pointer to convert.
2003	* @param pGCPhys Where to store the GC physical address.
2004	*/
2005	VMMDECL(int) PGMPhysGCPtr2GCPhys(PVMCPU pVCpu, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
2006	{
2007	int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, pGCPhys);
2008	if (pGCPhys && RT_SUCCESS(rc))
2009	*pGCPhys \|= (RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK;
2010	return rc;
2011	}
2012
2013
2014	/**
2015	* Converts a guest pointer to a HC physical address.
2016	*
2017	* This uses the current CR3/CR0/CR4 of the guest.
2018	*
2019	* @returns VBox status code.
2020	* @param pVCpu Pointer to the VMCPU.
2021	* @param GCPtr The guest pointer to convert.
2022	* @param pHCPhys Where to store the HC physical address.
2023	*/
2024	VMMDECL(int) PGMPhysGCPtr2HCPhys(PVMCPU pVCpu, RTGCPTR GCPtr, PRTHCPHYS pHCPhys)
2025	{
2026	PVM pVM = pVCpu->CTX_SUFF(pVM);
2027	RTGCPHYS GCPhys;
2028	int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, &GCPhys);
2029	if (RT_SUCCESS(rc))
2030	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys \| ((RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK), pHCPhys);
2031	return rc;
2032	}
2033
2034
2035
2036	#undef LOG_GROUP
2037	#define LOG_GROUP LOG_GROUP_PGM_PHYS_ACCESS
2038
2039
2040	#if defined(IN_RING3) && defined(SOME_UNUSED_FUNCTION)
2041	/**
2042	* Cache PGMPhys memory access
2043	*
2044	* @param pVM Pointer to the VM.
2045	* @param pCache Cache structure pointer
2046	* @param GCPhys GC physical address
2047	* @param pbHC HC pointer corresponding to physical page
2048	*
2049	* @thread EMT.
2050	*/
2051	static void pgmPhysCacheAdd(PVM pVM, PGMPHYSCACHE pCache, RTGCPHYS GCPhys, uint8_t pbR3)
2052	{
2053	uint32_t iCacheIndex;
2054
2055	Assert(VM_IS_EMT(pVM));
2056
2057	GCPhys = PHYS_PAGE_ADDRESS(GCPhys);
2058	pbR3 = (uint8_t *)PAGE_ADDRESS(pbR3);
2059
2060	iCacheIndex = ((GCPhys >> PAGE_SHIFT) & PGM_MAX_PHYSCACHE_ENTRIES_MASK);
2061
2062	ASMBitSet(&pCache->aEntries, iCacheIndex);
2063
2064	pCache->Entry[iCacheIndex].GCPhys = GCPhys;
2065	pCache->Entry[iCacheIndex].pbR3 = pbR3;
2066	}
2067	#endif /* IN_RING3 */
2068
2069
2070	/**
2071	* Deals with reading from a page with one or more ALL access handlers.
2072	*
2073	* @returns VBox status code. Can be ignored in ring-3.
2074	* @retval VINF_SUCCESS.
2075	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
2076	*
2077	* @param pVM Pointer to the VM.
2078	* @param pPage The page descriptor.
2079	* @param GCPhys The physical address to start reading at.
2080	* @param pvBuf Where to put the bits we read.
2081	* @param cb How much to read - less or equal to a page.
2082	*/
2083	static int pgmPhysReadHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void *pvBuf, size_t cb)
2084	{
2085	/*
2086	* The most frequent access here is MMIO and shadowed ROM.
2087	* The current code ASSUMES all these access handlers covers full pages!
2088	*/
2089
2090	/*
2091	* Whatever we do we need the source page, map it first.
2092	*/
2093	PGMPAGEMAPLOCK PgMpLck;
2094	const void *pvSrc = NULL;
2095	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvSrc, &PgMpLck);
2096	if (RT_FAILURE(rc))
2097	{
2098	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
2099	GCPhys, pPage, rc));
2100	memset(pvBuf, 0xff, cb);
2101	return VINF_SUCCESS;
2102	}
2103	rc = VINF_PGM_HANDLER_DO_DEFAULT;
2104
2105	/*
2106	* Deal with any physical handlers.
2107	*/
2108	#ifdef IN_RING3
2109	PPGMPHYSHANDLER pPhys = NULL;
2110	#endif
2111	if ( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL
2112	\|\| PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
2113	{
2114	#ifdef IN_RING3
2115	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2116	AssertReleaseMsg(pPhys, ("GCPhys=%RGp cb=%#x\n", GCPhys, cb));
2117	Assert(GCPhys >= pPhys->Core.Key && GCPhys <= pPhys->Core.KeyLast);
2118	Assert((pPhys->Core.Key & PAGE_OFFSET_MASK) == 0);
2119	Assert((pPhys->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
2120
2121	PFNPGMR3PHYSHANDLER pfnHandler = PGMPHYSHANDLER_GET_TYPE(pVM, pPhys)->CTX_SUFF(pfnHandler); Assert(pfnHandler);
2122	void *pvUser = pPhys->CTX_SUFF(pvUser);
2123
2124	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cb, pPage, R3STRING(pPhys->pszDesc) ));
2125	STAM_PROFILE_START(&pPhys->Stat, h);
2126	PGM_LOCK_ASSERT_OWNER(pVM);
2127	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2128	pgmUnlock(pVM);
2129	rc = pfnHandler(pVM, GCPhys, (void *)pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, pvUser);
2130	pgmLock(pVM);
2131	# ifdef VBOX_WITH_STATISTICS
2132	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2133	if (pPhys)
2134	STAM_PROFILE_STOP(&pPhys->Stat, h);
2135	# else
2136	pPhys = NULL; /* might not be valid anymore. */
2137	# endif
2138	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp\n", rc, GCPhys));
2139	#else
2140	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2141	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cb=%#x\n", GCPhys, cb));
2142	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2143	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2144	#endif
2145	}
2146
2147	/*
2148	* Deal with any virtual handlers.
2149	*/
2150	if (PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) == PGM_PAGE_HNDL_VIRT_STATE_ALL)
2151	{
2152	unsigned iPage;
2153	PPGMVIRTHANDLER pVirt;
2154
2155	int rc2 = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pVirt, &iPage);
2156	AssertReleaseMsg(RT_SUCCESS(rc2), ("GCPhys=%RGp cb=%#x rc2=%Rrc\n", GCPhys, cb, rc2));
2157	Assert((pVirt->Core.Key & PAGE_OFFSET_MASK) == 0);
2158	Assert((pVirt->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
2159	Assert(GCPhys >= pVirt->aPhysToVirt[iPage].Core.Key && GCPhys <= pVirt->aPhysToVirt[iPage].Core.KeyLast);
2160
2161	#ifdef IN_RING3
2162	if (pVirt->pfnHandlerR3)
2163	{
2164	if (!pPhys)
2165	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] virt %s\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc) ));
2166	else
2167	Log(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys/virt %s/%s\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc), R3STRING(pPhys->pszDesc) ));
2168	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2169	+ (iPage << PAGE_SHIFT)
2170	+ (GCPhys & PAGE_OFFSET_MASK);
2171
2172	STAM_PROFILE_START(&pVirt->Stat, h);
2173	rc2 = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, (void )pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, /pVirt->CTX_SUFF(pvUser)*/ NULL);
2174	STAM_PROFILE_STOP(&pVirt->Stat, h);
2175	if (rc2 == VINF_SUCCESS)
2176	rc = VINF_SUCCESS;
2177	AssertLogRelMsg(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc2, GCPhys, pPage, pVirt->pszDesc));
2178	}
2179	else
2180	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] virt %s [no handler]\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc) ));
2181	#else
2182	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2183	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cb=%#x\n", GCPhys, cb));
2184	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2185	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2186	#endif
2187	}
2188
2189	/*
2190	* Take the default action.
2191	*/
2192	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
2193	memcpy(pvBuf, pvSrc, cb);
2194	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2195	return rc;
2196	}
2197
2198
2199	/**
2200	* Read physical memory.
2201	*
2202	* This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
2203	* want to ignore those.
2204	*
2205	* @returns VBox status code. Can be ignored in ring-3.
2206	* @retval VINF_SUCCESS.
2207	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
2208	*
2209	* @param pVM Pointer to the VM.
2210	* @param GCPhys Physical address start reading from.
2211	* @param pvBuf Where to put the read bits.
2212	* @param cbRead How many bytes to read.
2213	*/
2214	VMMDECL(int) PGMPhysRead(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
2215	{
2216	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
2217	LogFlow(("PGMPhysRead: %RGp %d\n", GCPhys, cbRead));
2218
2219	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysRead));
2220	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysReadBytes), cbRead);
2221
2222	pgmLock(pVM);
2223
2224	/*
2225	* Copy loop on ram ranges.
2226	*/
2227	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
2228	for (;;)
2229	{
2230	/* Inside range or not? */
2231	if (pRam && GCPhys >= pRam->GCPhys)
2232	{
2233	/*
2234	* Must work our way thru this page by page.
2235	*/
2236	RTGCPHYS off = GCPhys - pRam->GCPhys;
2237	while (off < pRam->cb)
2238	{
2239	unsigned iPage = off >> PAGE_SHIFT;
2240	PPGMPAGE pPage = &pRam->aPages[iPage];
2241	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
2242	if (cb > cbRead)
2243	cb = cbRead;
2244
2245	/*
2246	* Any ALL access handlers?
2247	*/
2248	if (RT_UNLIKELY( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
2249	\|\| PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage)))
2250	{
2251	int rc = pgmPhysReadHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb);
2252	if (RT_FAILURE(rc))
2253	{
2254	pgmUnlock(pVM);
2255	return rc;
2256	}
2257	}
2258	else
2259	{
2260	/*
2261	* Get the pointer to the page.
2262	*/
2263	PGMPAGEMAPLOCK PgMpLck;
2264	const void *pvSrc;
2265	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
2266	if (RT_SUCCESS(rc))
2267	{
2268	memcpy(pvBuf, pvSrc, cb);
2269	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2270	}
2271	else
2272	{
2273	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
2274	pRam->GCPhys + off, pPage, rc));
2275	memset(pvBuf, 0xff, cb);
2276	}
2277	}
2278
2279	/* next page */
2280	if (cb >= cbRead)
2281	{
2282	pgmUnlock(pVM);
2283	return VINF_SUCCESS;
2284	}
2285	cbRead -= cb;
2286	off += cb;
2287	pvBuf = (char *)pvBuf + cb;
2288	} /* walk pages in ram range. */
2289
2290	GCPhys = pRam->GCPhysLast + 1;
2291	}
2292	else
2293	{
2294	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
2295
2296	/*
2297	* Unassigned address space.
2298	*/
2299	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
2300	if (cb >= cbRead)
2301	{
2302	memset(pvBuf, 0xff, cbRead);
2303	break;
2304	}
2305	memset(pvBuf, 0xff, cb);
2306
2307	cbRead -= cb;
2308	pvBuf = (char *)pvBuf + cb;
2309	GCPhys += cb;
2310	}
2311
2312	/* Advance range if necessary. */
2313	while (pRam && GCPhys > pRam->GCPhysLast)
2314	pRam = pRam->CTX_SUFF(pNext);
2315	} /* Ram range walk */
2316
2317	pgmUnlock(pVM);
2318	return VINF_SUCCESS;
2319	}
2320
2321
2322	/**
2323	* Deals with writing to a page with one or more WRITE or ALL access handlers.
2324	*
2325	* @returns VBox status code. Can be ignored in ring-3.
2326	* @retval VINF_SUCCESS.
2327	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
2328	*
2329	* @param pVM Pointer to the VM.
2330	* @param pPage The page descriptor.
2331	* @param GCPhys The physical address to start writing at.
2332	* @param pvBuf What to write.
2333	* @param cbWrite How much to write - less or equal to a page.
2334	*/
2335	static int pgmPhysWriteHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const *pvBuf, size_t cbWrite)
2336	{
2337	PGMPAGEMAPLOCK PgMpLck;
2338	void *pvDst = NULL;
2339	int rc;
2340
2341	/*
2342	* Give priority to physical handlers (like #PF does).
2343	*
2344	* Hope for a lonely physical handler first that covers the whole
2345	* write area. This should be a pretty frequent case with MMIO and
2346	* the heavy usage of full page handlers in the page pool.
2347	*/
2348	if ( !PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage)
2349	\|\| PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage) /* screw virtual handlers on MMIO pages */)
2350	{
2351	PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
2352	if (pCur)
2353	{
2354	Assert(GCPhys >= pCur->Core.Key && GCPhys <= pCur->Core.KeyLast);
2355	Assert(PGMPHYSHANDLER_GET_TYPE(pVM, pCur)->CTX_SUFF(pfnHandler));
2356
2357	size_t cbRange = pCur->Core.KeyLast - GCPhys + 1;
2358	if (cbRange > cbWrite)
2359	cbRange = cbWrite;
2360
2361	#ifndef IN_RING3
2362	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2363	NOREF(cbRange);
2364	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2365	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2366
2367	#else /* IN_RING3 */
2368	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n",
2369	GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
2370	if (!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
2371	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
2372	else
2373	rc = VINF_SUCCESS;
2374	if (RT_SUCCESS(rc))
2375	{
2376	PFNPGMR3PHYSHANDLER pfnHandler = PGMPHYSHANDLER_GET_TYPE(pVM, pCur)->CTX_SUFF(pfnHandler);
2377	void *pvUser = pCur->CTX_SUFF(pvUser);
2378
2379	STAM_PROFILE_START(&pCur->Stat, h);
2380	PGM_LOCK_ASSERT_OWNER(pVM);
2381	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2382	pgmUnlock(pVM);
2383	rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2384	pgmLock(pVM);
2385	# ifdef VBOX_WITH_STATISTICS
2386	pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
2387	if (pCur)
2388	STAM_PROFILE_STOP(&pCur->Stat, h);
2389	# else
2390	pCur = NULL; /* might not be valid anymore. */
2391	# endif
2392	if (rc == VINF_PGM_HANDLER_DO_DEFAULT && pvDst)
2393	{
2394	if (pvDst)
2395	memcpy(pvDst, pvBuf, cbRange);
2396	}
2397	else
2398	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT,
2399	("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pCur ? pCur->pszDesc : ""));
2400	}
2401	else
2402	AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2403	GCPhys, pPage, rc), rc);
2404	if (RT_LIKELY(cbRange == cbWrite))
2405	{
2406	if (pvDst)
2407	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2408	return VINF_SUCCESS;
2409	}
2410
2411	/* more fun to be had below */
2412	cbWrite -= cbRange;
2413	GCPhys += cbRange;
2414	pvBuf = (uint8_t *)pvBuf + cbRange;
2415	pvDst = (uint8_t *)pvDst + cbRange;
2416	#endif /* IN_RING3 */
2417	}
2418	/* else: the handler is somewhere else in the page, deal with it below. */
2419	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)); /* MMIO handlers are all PAGE_SIZEed! */
2420	}
2421	/*
2422	* A virtual handler without any interfering physical handlers.
2423	* Hopefully it'll cover the whole write.
2424	*/
2425	else if (!PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage))
2426	{
2427	unsigned iPage;
2428	PPGMVIRTHANDLER pCur;
2429	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pCur, &iPage);
2430	if (RT_SUCCESS(rc))
2431	{
2432	size_t cbRange = (PAGE_OFFSET_MASK & pCur->Core.KeyLast) - (PAGE_OFFSET_MASK & GCPhys) + 1;
2433	if (cbRange > cbWrite)
2434	cbRange = cbWrite;
2435
2436	#ifndef IN_RING3
2437	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2438	NOREF(cbRange);
2439	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2440	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2441
2442	#else /* IN_RING3 */
2443
2444	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] virt %s\n", GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
2445	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
2446	if (RT_SUCCESS(rc))
2447	{
2448	rc = VINF_PGM_HANDLER_DO_DEFAULT;
2449	if (pCur->pfnHandlerR3)
2450	{
2451	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pCur->Core.Key & PAGE_BASE_GC_MASK)
2452	+ (iPage << PAGE_SHIFT)
2453	+ (GCPhys & PAGE_OFFSET_MASK);
2454
2455	STAM_PROFILE_START(&pCur->Stat, h);
2456	rc = pCur->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void )pvBuf, cbRange, PGMACCESSTYPE_WRITE, /pCur->CTX_SUFF(pvUser)*/ NULL);
2457	STAM_PROFILE_STOP(&pCur->Stat, h);
2458	}
2459	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
2460	memcpy(pvDst, pvBuf, cbRange);
2461	else
2462	AssertLogRelMsg(rc == VINF_SUCCESS, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pCur->pszDesc));
2463	}
2464	else
2465	AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2466	GCPhys, pPage, rc), rc);
2467	if (RT_LIKELY(cbRange == cbWrite))
2468	{
2469	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2470	return VINF_SUCCESS;
2471	}
2472
2473	/* more fun to be had below */
2474	cbWrite -= cbRange;
2475	GCPhys += cbRange;
2476	pvBuf = (uint8_t *)pvBuf + cbRange;
2477	pvDst = (uint8_t *)pvDst + cbRange;
2478	#endif
2479	}
2480	/* else: the handler is somewhere else in the page, deal with it below. */
2481	}
2482
2483	/*
2484	* Deal with all the odd ends.
2485	*/
2486
2487	/* We need a writable destination page. */
2488	if (!pvDst)
2489	{
2490	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
2491	AssertLogRelMsgReturn(RT_SUCCESS(rc),
2492	("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2493	GCPhys, pPage, rc), rc);
2494	}
2495
2496	/* The loop state (big + ugly). */
2497	unsigned iVirtPage = 0;
2498	PPGMVIRTHANDLER pVirt = NULL;
2499	uint32_t offVirt = PAGE_SIZE;
2500	uint32_t offVirtLast = PAGE_SIZE;
2501	bool fMoreVirt = PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage);
2502
2503	PPGMPHYSHANDLER pPhys = NULL;
2504	uint32_t offPhys = PAGE_SIZE;
2505	uint32_t offPhysLast = PAGE_SIZE;
2506	bool fMorePhys = PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage);
2507
2508	/* The loop. */
2509	for (;;)
2510	{
2511	/*
2512	* Find the closest handler at or above GCPhys.
2513	*/
2514	if (fMoreVirt && !pVirt)
2515	{
2516	rc = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &pVirt, &iVirtPage);
2517	if (RT_SUCCESS(rc))
2518	{
2519	offVirt = 0;
2520	offVirtLast = (pVirt->aPhysToVirt[iVirtPage].Core.KeyLast & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2521	}
2522	else
2523	{
2524	PPGMPHYS2VIRTHANDLER pVirtPhys;
2525	pVirtPhys = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysToVirtHandlers,
2526	GCPhys, true /* fAbove */);
2527	if ( pVirtPhys
2528	&& (pVirtPhys->Core.Key >> PAGE_SHIFT) == (GCPhys >> PAGE_SHIFT))
2529	{
2530	/* ASSUME that pVirtPhys only covers one page. */
2531	Assert((pVirtPhys->Core.Key >> PAGE_SHIFT) == (pVirtPhys->Core.KeyLast >> PAGE_SHIFT));
2532	Assert(pVirtPhys->Core.Key > GCPhys);
2533
2534	pVirt = (PPGMVIRTHANDLER)((uintptr_t)pVirtPhys + pVirtPhys->offVirtHandler);
2535	iVirtPage = pVirtPhys - &pVirt->aPhysToVirt[0]; Assert(iVirtPage == 0);
2536	offVirt = (pVirtPhys->Core.Key & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2537	offVirtLast = (pVirtPhys->Core.KeyLast & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2538	}
2539	else
2540	{
2541	pVirt = NULL;
2542	fMoreVirt = false;
2543	offVirt = offVirtLast = PAGE_SIZE;
2544	}
2545	}
2546	}
2547
2548	if (fMorePhys && !pPhys)
2549	{
2550	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2551	if (pPhys)
2552	{
2553	offPhys = 0;
2554	offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
2555	}
2556	else
2557	{
2558	pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers,
2559	GCPhys, true /* fAbove */);
2560	if ( pPhys
2561	&& pPhys->Core.Key <= GCPhys + (cbWrite - 1))
2562	{
2563	offPhys = pPhys->Core.Key - GCPhys;
2564	offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
2565	}
2566	else
2567	{
2568	pPhys = NULL;
2569	fMorePhys = false;
2570	offPhys = offPhysLast = PAGE_SIZE;
2571	}
2572	}
2573	}
2574
2575	/*
2576	* Handle access to space without handlers (that's easy).
2577	*/
2578	rc = VINF_PGM_HANDLER_DO_DEFAULT;
2579	uint32_t cbRange = (uint32_t)cbWrite;
2580	if (offPhys && offVirt)
2581	{
2582	if (cbRange > offPhys)
2583	cbRange = offPhys;
2584	if (cbRange > offVirt)
2585	cbRange = offVirt;
2586	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] miss\n", GCPhys, cbRange, pPage));
2587	}
2588	/*
2589	* Physical handler.
2590	*/
2591	else if (!offPhys && offVirt)
2592	{
2593	if (cbRange > offPhysLast + 1)
2594	cbRange = offPhysLast + 1;
2595	if (cbRange > offVirt)
2596	cbRange = offVirt;
2597	#ifdef IN_RING3
2598	PFNPGMR3PHYSHANDLER pfnHandler = PGMPHYSHANDLER_GET_TYPE(pVM, pPhys)->CTX_SUFF(pfnHandler);
2599	void *pvUser = pPhys->CTX_SUFF(pvUser);
2600
2601	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc) ));
2602	STAM_PROFILE_START(&pPhys->Stat, h);
2603	PGM_LOCK_ASSERT_OWNER(pVM);
2604	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2605	pgmUnlock(pVM);
2606	rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2607	pgmLock(pVM);
2608	# ifdef VBOX_WITH_STATISTICS
2609	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2610	if (pPhys)
2611	STAM_PROFILE_STOP(&pPhys->Stat, h);
2612	# else
2613	pPhys = NULL; /* might not be valid anymore. */
2614	# endif
2615	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, (pPhys) ? pPhys->pszDesc : ""));
2616	#else
2617	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2618	NOREF(cbRange);
2619	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2620	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2621	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2622	#endif
2623	}
2624	/*
2625	* Virtual handler.
2626	*/
2627	else if (offPhys && !offVirt)
2628	{
2629	if (cbRange > offVirtLast + 1)
2630	cbRange = offVirtLast + 1;
2631	if (cbRange > offPhys)
2632	cbRange = offPhys;
2633	#ifdef IN_RING3
2634	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pVirt->pszDesc) ));
2635	if (pVirt->pfnHandlerR3)
2636	{
2637	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2638	+ (iVirtPage << PAGE_SHIFT)
2639	+ (GCPhys & PAGE_OFFSET_MASK);
2640	STAM_PROFILE_START(&pVirt->Stat, h);
2641	rc = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void )pvBuf, cbRange, PGMACCESSTYPE_WRITE, /pCur->CTX_SUFF(pvUser)*/ NULL);
2642	STAM_PROFILE_STOP(&pVirt->Stat, h);
2643	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pVirt->pszDesc));
2644	}
2645	pVirt = NULL;
2646	#else
2647	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2648	NOREF(cbRange);
2649	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2650	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2651	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2652	#endif
2653	}
2654	/*
2655	* Both... give the physical one priority.
2656	*/
2657	else
2658	{
2659	Assert(!offPhys && !offVirt);
2660	if (cbRange > offVirtLast + 1)
2661	cbRange = offVirtLast + 1;
2662	if (cbRange > offPhysLast + 1)
2663	cbRange = offPhysLast + 1;
2664
2665	#ifdef IN_RING3
2666	if (pVirt->pfnHandlerR3)
2667	Log(("pgmPhysWriteHandler: overlapping phys and virt handlers at %RGp %R[pgmpage]; cbRange=%#x\n", GCPhys, pPage, cbRange));
2668	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys/virt %s/%s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc), R3STRING(pVirt->pszDesc) ));
2669
2670	PFNPGMR3PHYSHANDLER pfnHandler = PGMPHYSHANDLER_GET_TYPE(pVM, pPhys)->CTX_SUFF(pfnHandler);
2671	void *pvUser = pPhys->CTX_SUFF(pvUser);
2672
2673	STAM_PROFILE_START(&pPhys->Stat, h);
2674	PGM_LOCK_ASSERT_OWNER(pVM);
2675	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2676	pgmUnlock(pVM);
2677	rc = pfnHandler(pVM, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, pvUser);
2678	pgmLock(pVM);
2679	# ifdef VBOX_WITH_STATISTICS
2680	pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2681	if (pPhys)
2682	STAM_PROFILE_STOP(&pPhys->Stat, h);
2683	# else
2684	pPhys = NULL; /* might not be valid anymore. */
2685	# endif
2686	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, (pPhys) ? pPhys->pszDesc : ""));
2687	if (pVirt->pfnHandlerR3)
2688	{
2689
2690	RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2691	+ (iVirtPage << PAGE_SHIFT)
2692	+ (GCPhys & PAGE_OFFSET_MASK);
2693	STAM_PROFILE_START(&pVirt->Stat, h2);
2694	int rc2 = pVirt->CTX_SUFF(pfnHandler)(pVM, GCPtr, pvDst, (void )pvBuf, cbRange, PGMACCESSTYPE_WRITE, /pCur->CTX_SUFF(pvUser)*/ NULL);
2695	STAM_PROFILE_STOP(&pVirt->Stat, h2);
2696	if (rc2 == VINF_SUCCESS && rc == VINF_PGM_HANDLER_DO_DEFAULT)
2697	rc = VINF_SUCCESS;
2698	else
2699	AssertLogRelMsg(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_HANDLER_DO_DEFAULT, ("rc=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n", rc, GCPhys, pPage, pVirt->pszDesc));
2700	}
2701	pPhys = NULL;
2702	pVirt = NULL;
2703	#else
2704	/* In R0 and RC the callbacks cannot handle this context, so we'll fail. */
2705	NOREF(cbRange);
2706	//AssertReleaseMsgFailed(("Wrong API! GCPhys=%RGp cbRange=%#x\n", GCPhys, cbRange));
2707	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2708	return VERR_PGM_PHYS_WR_HIT_HANDLER;
2709	#endif
2710	}
2711	if (rc == VINF_PGM_HANDLER_DO_DEFAULT)
2712	memcpy(pvDst, pvBuf, cbRange);
2713
2714	/*
2715	* Advance if we've got more stuff to do.
2716	*/
2717	if (cbRange >= cbWrite)
2718	{
2719	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2720	return VINF_SUCCESS;
2721	}
2722
2723	cbWrite -= cbRange;
2724	GCPhys += cbRange;
2725	pvBuf = (uint8_t *)pvBuf + cbRange;
2726	pvDst = (uint8_t *)pvDst + cbRange;
2727
2728	offPhys -= cbRange;
2729	offPhysLast -= cbRange;
2730	offVirt -= cbRange;
2731	offVirtLast -= cbRange;
2732	}
2733	}
2734
2735
2736	/**
2737	* Write to physical memory.
2738	*
2739	* This API respects access handlers and MMIO. Use PGMPhysSimpleWriteGCPhys() if you
2740	* want to ignore those.
2741	*
2742	* @returns VBox status code. Can be ignored in ring-3.
2743	* @retval VINF_SUCCESS.
2744	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
2745	*
2746	* @param pVM Pointer to the VM.
2747	* @param GCPhys Physical address to write to.
2748	* @param pvBuf What to write.
2749	* @param cbWrite How many bytes to write.
2750	*/
2751	VMMDECL(int) PGMPhysWrite(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite)
2752	{
2753	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMPhysWrite after pgmR3Save()!\n"));
2754	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
2755	LogFlow(("PGMPhysWrite: %RGp %d\n", GCPhys, cbWrite));
2756
2757	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysWrite));
2758	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysWriteBytes), cbWrite);
2759
2760	pgmLock(pVM);
2761
2762	/*
2763	* Copy loop on ram ranges.
2764	*/
2765	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
2766	for (;;)
2767	{
2768	/* Inside range or not? */
2769	if (pRam && GCPhys >= pRam->GCPhys)
2770	{
2771	/*
2772	* Must work our way thru this page by page.
2773	*/
2774	RTGCPTR off = GCPhys - pRam->GCPhys;
2775	while (off < pRam->cb)
2776	{
2777	RTGCPTR iPage = off >> PAGE_SHIFT;
2778	PPGMPAGE pPage = &pRam->aPages[iPage];
2779	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
2780	if (cb > cbWrite)
2781	cb = cbWrite;
2782
2783	/*
2784	* Any active WRITE or ALL access handlers?
2785	*/
2786	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
2787	\|\| PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
2788	{
2789	int rc = pgmPhysWriteHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb);
2790	if (RT_FAILURE(rc))
2791	{
2792	pgmUnlock(pVM);
2793	return rc;
2794	}
2795	}
2796	else
2797	{
2798	/*
2799	* Get the pointer to the page.
2800	*/
2801	PGMPAGEMAPLOCK PgMpLck;
2802	void *pvDst;
2803	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
2804	if (RT_SUCCESS(rc))
2805	{
2806	Assert(!PGM_PAGE_IS_BALLOONED(pPage));
2807	memcpy(pvDst, pvBuf, cb);
2808	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2809	}
2810	/* Ignore writes to ballooned pages. */
2811	else if (!PGM_PAGE_IS_BALLOONED(pPage))
2812	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2813	pRam->GCPhys + off, pPage, rc));
2814	}
2815
2816	/* next page */
2817	if (cb >= cbWrite)
2818	{
2819	pgmUnlock(pVM);
2820	return VINF_SUCCESS;
2821	}
2822
2823	cbWrite -= cb;
2824	off += cb;
2825	pvBuf = (const char *)pvBuf + cb;
2826	} /* walk pages in ram range */
2827
2828	GCPhys = pRam->GCPhysLast + 1;
2829	}
2830	else
2831	{
2832	/*
2833	* Unassigned address space, skip it.
2834	*/
2835	if (!pRam)
2836	break;
2837	size_t cb = pRam->GCPhys - GCPhys;
2838	if (cb >= cbWrite)
2839	break;
2840	cbWrite -= cb;
2841	pvBuf = (const char *)pvBuf + cb;
2842	GCPhys += cb;
2843	}
2844
2845	/* Advance range if necessary. */
2846	while (pRam && GCPhys > pRam->GCPhysLast)
2847	pRam = pRam->CTX_SUFF(pNext);
2848	} /* Ram range walk */
2849
2850	pgmUnlock(pVM);
2851	return VINF_SUCCESS;
2852	}
2853
2854
2855	/**
2856	* Read from guest physical memory by GC physical address, bypassing
2857	* MMIO and access handlers.
2858	*
2859	* @returns VBox status.
2860	* @param pVM Pointer to the VM.
2861	* @param pvDst The destination address.
2862	* @param GCPhysSrc The source address (GC physical address).
2863	* @param cb The number of bytes to read.
2864	*/
2865	VMMDECL(int) PGMPhysSimpleReadGCPhys(PVM pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb)
2866	{
2867	/*
2868	* Treat the first page as a special case.
2869	*/
2870	if (!cb)
2871	return VINF_SUCCESS;
2872
2873	/* map the 1st page */
2874	void const *pvSrc;
2875	PGMPAGEMAPLOCK Lock;
2876	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
2877	if (RT_FAILURE(rc))
2878	return rc;
2879
2880	/* optimize for the case where access is completely within the first page. */
2881	size_t cbPage = PAGE_SIZE - (GCPhysSrc & PAGE_OFFSET_MASK);
2882	if (RT_LIKELY(cb <= cbPage))
2883	{
2884	memcpy(pvDst, pvSrc, cb);
2885	PGMPhysReleasePageMappingLock(pVM, &Lock);
2886	return VINF_SUCCESS;
2887	}
2888
2889	/* copy to the end of the page. */
2890	memcpy(pvDst, pvSrc, cbPage);
2891	PGMPhysReleasePageMappingLock(pVM, &Lock);
2892	GCPhysSrc += cbPage;
2893	pvDst = (uint8_t *)pvDst + cbPage;
2894	cb -= cbPage;
2895
2896	/*
2897	* Page by page.
2898	*/
2899	for (;;)
2900	{
2901	/* map the page */
2902	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
2903	if (RT_FAILURE(rc))
2904	return rc;
2905
2906	/* last page? */
2907	if (cb <= PAGE_SIZE)
2908	{
2909	memcpy(pvDst, pvSrc, cb);
2910	PGMPhysReleasePageMappingLock(pVM, &Lock);
2911	return VINF_SUCCESS;
2912	}
2913
2914	/* copy the entire page and advance */
2915	memcpy(pvDst, pvSrc, PAGE_SIZE);
2916	PGMPhysReleasePageMappingLock(pVM, &Lock);
2917	GCPhysSrc += PAGE_SIZE;
2918	pvDst = (uint8_t *)pvDst + PAGE_SIZE;
2919	cb -= PAGE_SIZE;
2920	}
2921	/* won't ever get here. */
2922	}
2923
2924
2925	/**
2926	* Write to guest physical memory referenced by GC pointer.
2927	* Write memory to GC physical address in guest physical memory.
2928	*
2929	* This will bypass MMIO and access handlers.
2930	*
2931	* @returns VBox status.
2932	* @param pVM Pointer to the VM.
2933	* @param GCPhysDst The GC physical address of the destination.
2934	* @param pvSrc The source buffer.
2935	* @param cb The number of bytes to write.
2936	*/
2937	VMMDECL(int) PGMPhysSimpleWriteGCPhys(PVM pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb)
2938	{
2939	LogFlow(("PGMPhysSimpleWriteGCPhys: %RGp %zu\n", GCPhysDst, cb));
2940
2941	/*
2942	* Treat the first page as a special case.
2943	*/
2944	if (!cb)
2945	return VINF_SUCCESS;
2946
2947	/* map the 1st page */
2948	void *pvDst;
2949	PGMPAGEMAPLOCK Lock;
2950	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
2951	if (RT_FAILURE(rc))
2952	return rc;
2953
2954	/* optimize for the case where access is completely within the first page. */
2955	size_t cbPage = PAGE_SIZE - (GCPhysDst & PAGE_OFFSET_MASK);
2956	if (RT_LIKELY(cb <= cbPage))
2957	{
2958	memcpy(pvDst, pvSrc, cb);
2959	PGMPhysReleasePageMappingLock(pVM, &Lock);
2960	return VINF_SUCCESS;
2961	}
2962
2963	/* copy to the end of the page. */
2964	memcpy(pvDst, pvSrc, cbPage);
2965	PGMPhysReleasePageMappingLock(pVM, &Lock);
2966	GCPhysDst += cbPage;
2967	pvSrc = (const uint8_t *)pvSrc + cbPage;
2968	cb -= cbPage;
2969
2970	/*
2971	* Page by page.
2972	*/
2973	for (;;)
2974	{
2975	/* map the page */
2976	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
2977	if (RT_FAILURE(rc))
2978	return rc;
2979
2980	/* last page? */
2981	if (cb <= PAGE_SIZE)
2982	{
2983	memcpy(pvDst, pvSrc, cb);
2984	PGMPhysReleasePageMappingLock(pVM, &Lock);
2985	return VINF_SUCCESS;
2986	}
2987
2988	/* copy the entire page and advance */
2989	memcpy(pvDst, pvSrc, PAGE_SIZE);
2990	PGMPhysReleasePageMappingLock(pVM, &Lock);
2991	GCPhysDst += PAGE_SIZE;
2992	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
2993	cb -= PAGE_SIZE;
2994	}
2995	/* won't ever get here. */
2996	}
2997
2998
2999	/**
3000	* Read from guest physical memory referenced by GC pointer.
3001	*
3002	* This function uses the current CR3/CR0/CR4 of the guest and will
3003	* bypass access handlers and not set any accessed bits.
3004	*
3005	* @returns VBox status.
3006	* @param pVCpu Handle to the current virtual CPU.
3007	* @param pvDst The destination address.
3008	* @param GCPtrSrc The source address (GC pointer).
3009	* @param cb The number of bytes to read.
3010	*/
3011	VMMDECL(int) PGMPhysSimpleReadGCPtr(PVMCPU pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
3012	{
3013	PVM pVM = pVCpu->CTX_SUFF(pVM);
3014	/** @todo fix the macro / state handling: VMCPU_ASSERT_EMT_OR_GURU(pVCpu); */
3015
3016	/*
3017	* Treat the first page as a special case.
3018	*/
3019	if (!cb)
3020	return VINF_SUCCESS;
3021
3022	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleRead));
3023	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleReadBytes), cb);
3024
3025	/* Take the PGM lock here, because many called functions take the lock for a very short period. That's counter-productive
3026	* when many VCPUs are fighting for the lock.
3027	*/
3028	pgmLock(pVM);
3029
3030	/* map the 1st page */
3031	void const *pvSrc;
3032	PGMPAGEMAPLOCK Lock;
3033	int rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
3034	if (RT_FAILURE(rc))
3035	{
3036	pgmUnlock(pVM);
3037	return rc;
3038	}
3039
3040	/* optimize for the case where access is completely within the first page. */
3041	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
3042	if (RT_LIKELY(cb <= cbPage))
3043	{
3044	memcpy(pvDst, pvSrc, cb);
3045	PGMPhysReleasePageMappingLock(pVM, &Lock);
3046	pgmUnlock(pVM);
3047	return VINF_SUCCESS;
3048	}
3049
3050	/* copy to the end of the page. */
3051	memcpy(pvDst, pvSrc, cbPage);
3052	PGMPhysReleasePageMappingLock(pVM, &Lock);
3053	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + cbPage);
3054	pvDst = (uint8_t *)pvDst + cbPage;
3055	cb -= cbPage;
3056
3057	/*
3058	* Page by page.
3059	*/
3060	for (;;)
3061	{
3062	/* map the page */
3063	rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
3064	if (RT_FAILURE(rc))
3065	{
3066	pgmUnlock(pVM);
3067	return rc;
3068	}
3069
3070	/* last page? */
3071	if (cb <= PAGE_SIZE)
3072	{
3073	memcpy(pvDst, pvSrc, cb);
3074	PGMPhysReleasePageMappingLock(pVM, &Lock);
3075	pgmUnlock(pVM);
3076	return VINF_SUCCESS;
3077	}
3078
3079	/* copy the entire page and advance */
3080	memcpy(pvDst, pvSrc, PAGE_SIZE);
3081	PGMPhysReleasePageMappingLock(pVM, &Lock);
3082	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + PAGE_SIZE);
3083	pvDst = (uint8_t *)pvDst + PAGE_SIZE;
3084	cb -= PAGE_SIZE;
3085	}
3086	/* won't ever get here. */
3087	}
3088
3089
3090	/**
3091	* Write to guest physical memory referenced by GC pointer.
3092	*
3093	* This function uses the current CR3/CR0/CR4 of the guest and will
3094	* bypass access handlers and not set dirty or accessed bits.
3095	*
3096	* @returns VBox status.
3097	* @param pVCpu Handle to the current virtual CPU.
3098	* @param GCPtrDst The destination address (GC pointer).
3099	* @param pvSrc The source address.
3100	* @param cb The number of bytes to write.
3101	*/
3102	VMMDECL(int) PGMPhysSimpleWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
3103	{
3104	PVM pVM = pVCpu->CTX_SUFF(pVM);
3105	VMCPU_ASSERT_EMT(pVCpu);
3106
3107	/*
3108	* Treat the first page as a special case.
3109	*/
3110	if (!cb)
3111	return VINF_SUCCESS;
3112
3113	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleWrite));
3114	STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleWriteBytes), cb);
3115
3116	/* map the 1st page */
3117	void *pvDst;
3118	PGMPAGEMAPLOCK Lock;
3119	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3120	if (RT_FAILURE(rc))
3121	return rc;
3122
3123	/* optimize for the case where access is completely within the first page. */
3124	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
3125	if (RT_LIKELY(cb <= cbPage))
3126	{
3127	memcpy(pvDst, pvSrc, cb);
3128	PGMPhysReleasePageMappingLock(pVM, &Lock);
3129	return VINF_SUCCESS;
3130	}
3131
3132	/* copy to the end of the page. */
3133	memcpy(pvDst, pvSrc, cbPage);
3134	PGMPhysReleasePageMappingLock(pVM, &Lock);
3135	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
3136	pvSrc = (const uint8_t *)pvSrc + cbPage;
3137	cb -= cbPage;
3138
3139	/*
3140	* Page by page.
3141	*/
3142	for (;;)
3143	{
3144	/* map the page */
3145	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3146	if (RT_FAILURE(rc))
3147	return rc;
3148
3149	/* last page? */
3150	if (cb <= PAGE_SIZE)
3151	{
3152	memcpy(pvDst, pvSrc, cb);
3153	PGMPhysReleasePageMappingLock(pVM, &Lock);
3154	return VINF_SUCCESS;
3155	}
3156
3157	/* copy the entire page and advance */
3158	memcpy(pvDst, pvSrc, PAGE_SIZE);
3159	PGMPhysReleasePageMappingLock(pVM, &Lock);
3160	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
3161	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
3162	cb -= PAGE_SIZE;
3163	}
3164	/* won't ever get here. */
3165	}
3166
3167
3168	/**
3169	* Write to guest physical memory referenced by GC pointer and update the PTE.
3170	*
3171	* This function uses the current CR3/CR0/CR4 of the guest and will
3172	* bypass access handlers but will set any dirty and accessed bits in the PTE.
3173	*
3174	* If you don't want to set the dirty bit, use PGMPhysSimpleWriteGCPtr().
3175	*
3176	* @returns VBox status.
3177	* @param pVCpu Handle to the current virtual CPU.
3178	* @param GCPtrDst The destination address (GC pointer).
3179	* @param pvSrc The source address.
3180	* @param cb The number of bytes to write.
3181	*/
3182	VMMDECL(int) PGMPhysSimpleDirtyWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
3183	{
3184	PVM pVM = pVCpu->CTX_SUFF(pVM);
3185	VMCPU_ASSERT_EMT(pVCpu);
3186
3187	/*
3188	* Treat the first page as a special case.
3189	* Btw. this is the same code as in PGMPhyssimpleWriteGCPtr excep for the PGMGstModifyPage.
3190	*/
3191	if (!cb)
3192	return VINF_SUCCESS;
3193
3194	/* map the 1st page */
3195	void *pvDst;
3196	PGMPAGEMAPLOCK Lock;
3197	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3198	if (RT_FAILURE(rc))
3199	return rc;
3200
3201	/* optimize for the case where access is completely within the first page. */
3202	size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
3203	if (RT_LIKELY(cb <= cbPage))
3204	{
3205	memcpy(pvDst, pvSrc, cb);
3206	PGMPhysReleasePageMappingLock(pVM, &Lock);
3207	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
3208	return VINF_SUCCESS;
3209	}
3210
3211	/* copy to the end of the page. */
3212	memcpy(pvDst, pvSrc, cbPage);
3213	PGMPhysReleasePageMappingLock(pVM, &Lock);
3214	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
3215	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
3216	pvSrc = (const uint8_t *)pvSrc + cbPage;
3217	cb -= cbPage;
3218
3219	/*
3220	* Page by page.
3221	*/
3222	for (;;)
3223	{
3224	/* map the page */
3225	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3226	if (RT_FAILURE(rc))
3227	return rc;
3228
3229	/* last page? */
3230	if (cb <= PAGE_SIZE)
3231	{
3232	memcpy(pvDst, pvSrc, cb);
3233	PGMPhysReleasePageMappingLock(pVM, &Lock);
3234	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
3235	return VINF_SUCCESS;
3236	}
3237
3238	/* copy the entire page and advance */
3239	memcpy(pvDst, pvSrc, PAGE_SIZE);
3240	PGMPhysReleasePageMappingLock(pVM, &Lock);
3241	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
3242	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
3243	pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
3244	cb -= PAGE_SIZE;
3245	}
3246	/* won't ever get here. */
3247	}
3248
3249
3250	/**
3251	* Read from guest physical memory referenced by GC pointer.
3252	*
3253	* This function uses the current CR3/CR0/CR4 of the guest and will
3254	* respect access handlers and set accessed bits.
3255	*
3256	* @returns VBox status.
3257	* @param pVCpu Handle to the current virtual CPU.
3258	* @param pvDst The destination address.
3259	* @param GCPtrSrc The source address (GC pointer).
3260	* @param cb The number of bytes to read.
3261	* @thread The vCPU EMT.
3262	*/
3263	VMMDECL(int) PGMPhysReadGCPtr(PVMCPU pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
3264	{
3265	RTGCPHYS GCPhys;
3266	uint64_t fFlags;
3267	int rc;
3268	PVM pVM = pVCpu->CTX_SUFF(pVM);
3269	VMCPU_ASSERT_EMT(pVCpu);
3270
3271	/*
3272	* Anything to do?
3273	*/
3274	if (!cb)
3275	return VINF_SUCCESS;
3276
3277	LogFlow(("PGMPhysReadGCPtr: %RGv %zu\n", GCPtrSrc, cb));
3278
3279	/*
3280	* Optimize reads within a single page.
3281	*/
3282	if (((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
3283	{
3284	/* Convert virtual to physical address + flags */
3285	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
3286	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
3287	GCPhys \|= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
3288
3289	/* mark the guest page as accessed. */
3290	if (!(fFlags & X86_PTE_A))
3291	{
3292	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
3293	AssertRC(rc);
3294	}
3295
3296	return PGMPhysRead(pVM, GCPhys, pvDst, cb);
3297	}
3298
3299	/*
3300	* Page by page.
3301	*/
3302	for (;;)
3303	{
3304	/* Convert virtual to physical address + flags */
3305	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
3306	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
3307	GCPhys \|= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
3308
3309	/* mark the guest page as accessed. */
3310	if (!(fFlags & X86_PTE_A))
3311	{
3312	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
3313	AssertRC(rc);
3314	}
3315
3316	/* copy */
3317	size_t cbRead = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
3318	if (cbRead < cb)
3319	{
3320	rc = PGMPhysRead(pVM, GCPhys, pvDst, cbRead);
3321	if (RT_FAILURE(rc))
3322	return rc;
3323	}
3324	else /* Last page (cbRead is PAGE_SIZE, we only need cb!) */
3325	return PGMPhysRead(pVM, GCPhys, pvDst, cb);
3326
3327	/* next */
3328	Assert(cb > cbRead);
3329	cb -= cbRead;
3330	pvDst = (uint8_t *)pvDst + cbRead;
3331	GCPtrSrc += cbRead;
3332	}
3333	}
3334
3335
3336	/**
3337	* Write to guest physical memory referenced by GC pointer.
3338	*
3339	* This function uses the current CR3/CR0/CR4 of the guest and will
3340	* respect access handlers and set dirty and accessed bits.
3341	*
3342	* @returns VBox status.
3343	* @retval VINF_SUCCESS.
3344	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in R0 and GC, NEVER in R3.
3345	*
3346	* @param pVCpu Handle to the current virtual CPU.
3347	* @param GCPtrDst The destination address (GC pointer).
3348	* @param pvSrc The source address.
3349	* @param cb The number of bytes to write.
3350	*/
3351	VMMDECL(int) PGMPhysWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
3352	{
3353	RTGCPHYS GCPhys;
3354	uint64_t fFlags;
3355	int rc;
3356	PVM pVM = pVCpu->CTX_SUFF(pVM);
3357	VMCPU_ASSERT_EMT(pVCpu);
3358
3359	/*
3360	* Anything to do?
3361	*/
3362	if (!cb)
3363	return VINF_SUCCESS;
3364
3365	LogFlow(("PGMPhysWriteGCPtr: %RGv %zu\n", GCPtrDst, cb));
3366
3367	/*
3368	* Optimize writes within a single page.
3369	*/
3370	if (((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
3371	{
3372	/* Convert virtual to physical address + flags */
3373	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
3374	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
3375	GCPhys \|= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
3376
3377	/* Mention when we ignore X86_PTE_RW... */
3378	if (!(fFlags & X86_PTE_RW))
3379	Log(("PGMPhysGCPtr2GCPhys: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
3380
3381	/* Mark the guest page as accessed and dirty if necessary. */
3382	if ((fFlags & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
3383	{
3384	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
3385	AssertRC(rc);
3386	}
3387
3388	return PGMPhysWrite(pVM, GCPhys, pvSrc, cb);
3389	}
3390
3391	/*
3392	* Page by page.
3393	*/
3394	for (;;)
3395	{
3396	/* Convert virtual to physical address + flags */
3397	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
3398	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
3399	GCPhys \|= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
3400
3401	/* Mention when we ignore X86_PTE_RW... */
3402	if (!(fFlags & X86_PTE_RW))
3403	Log(("PGMPhysGCPtr2GCPhys: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
3404
3405	/* Mark the guest page as accessed and dirty if necessary. */
3406	if ((fFlags & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
3407	{
3408	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
3409	AssertRC(rc);
3410	}
3411
3412	/* copy */
3413	size_t cbWrite = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
3414	if (cbWrite < cb)
3415	{
3416	rc = PGMPhysWrite(pVM, GCPhys, pvSrc, cbWrite);
3417	if (RT_FAILURE(rc))
3418	return rc;
3419	}
3420	else /* Last page (cbWrite is PAGE_SIZE, we only need cb!) */
3421	rc = PGMPhysWrite(pVM, GCPhys, pvSrc, cb);
3422
3423	/* next */
3424	Assert(cb > cbWrite);
3425	cb -= cbWrite;
3426	pvSrc = (uint8_t *)pvSrc + cbWrite;
3427	GCPtrDst += cbWrite;
3428	}
3429	}
3430
3431
3432	/**
3433	* Performs a read of guest virtual memory for instruction emulation.
3434	*
3435	* This will check permissions, raise exceptions and update the access bits.
3436	*
3437	* The current implementation will bypass all access handlers. It may later be
3438	* changed to at least respect MMIO.
3439	*
3440	*
3441	* @returns VBox status code suitable to scheduling.
3442	* @retval VINF_SUCCESS if the read was performed successfully.
3443	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3444	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3445	*
3446	* @param pVCpu Handle to the current virtual CPU.
3447	* @param pCtxCore The context core.
3448	* @param pvDst Where to put the bytes we've read.
3449	* @param GCPtrSrc The source address.
3450	* @param cb The number of bytes to read. Not more than a page.
3451	*
3452	* @remark This function will dynamically map physical pages in GC. This may unmap
3453	* mappings done by the caller. Be careful!
3454	*/
3455	VMMDECL(int) PGMPhysInterpretedRead(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb)
3456	{
3457	PVM pVM = pVCpu->CTX_SUFF(pVM);
3458	Assert(cb <= PAGE_SIZE);
3459	VMCPU_ASSERT_EMT(pVCpu);
3460
3461	/** @todo r=bird: This isn't perfect!
3462	* -# It's not checking for reserved bits being 1.
3463	* -# It's not correctly dealing with the access bit.
3464	* -# It's not respecting MMIO memory or any other access handlers.
3465	*/
3466	/*
3467	* 1. Translate virtual to physical. This may fault.
3468	* 2. Map the physical address.
3469	* 3. Do the read operation.
3470	* 4. Set access bits if required.
3471	*/
3472	int rc;
3473	unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
3474	if (cb <= cb1)
3475	{
3476	/*
3477	* Not crossing pages.
3478	*/
3479	RTGCPHYS GCPhys;
3480	uint64_t fFlags;
3481	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
3482	if (RT_SUCCESS(rc))
3483	{
3484	/** @todo we should check reserved bits ... */
3485	PGMPAGEMAPLOCK PgMpLck;
3486	void const *pvSrc;
3487	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvSrc, &PgMpLck);
3488	switch (rc)
3489	{
3490	case VINF_SUCCESS:
3491	Log(("PGMPhysInterpretedRead: pvDst=%p pvSrc=%p cb=%d\n", pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb));
3492	memcpy(pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
3493	PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
3494	break;
3495	case VERR_PGM_PHYS_PAGE_RESERVED:
3496	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3497	memset(pvDst, 0xff, cb);
3498	break;
3499	default:
3500	Assert(RT_FAILURE_NP(rc));
3501	return rc;
3502	}
3503
3504	/** @todo access bit emulation isn't 100% correct. */
3505	if (!(fFlags & X86_PTE_A))
3506	{
3507	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3508	AssertRC(rc);
3509	}
3510	return VINF_SUCCESS;
3511	}
3512	}
3513	else
3514	{
3515	/*
3516	* Crosses pages.
3517	*/
3518	size_t cb2 = cb - cb1;
3519	uint64_t fFlags1;
3520	RTGCPHYS GCPhys1;
3521	uint64_t fFlags2;
3522	RTGCPHYS GCPhys2;
3523	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
3524	if (RT_SUCCESS(rc))
3525	{
3526	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
3527	if (RT_SUCCESS(rc))
3528	{
3529	/** @todo we should check reserved bits ... */
3530	AssertMsgFailed(("cb=%d cb1=%d cb2=%d GCPtrSrc=%RGv\n", cb, cb1, cb2, GCPtrSrc));
3531	PGMPAGEMAPLOCK PgMpLck;
3532	void const *pvSrc1;
3533	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys1, &pvSrc1, &PgMpLck);
3534	switch (rc)
3535	{
3536	case VINF_SUCCESS:
3537	memcpy(pvDst, (uint8_t *)pvSrc1 + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
3538	PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
3539	break;
3540	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3541	memset(pvDst, 0xff, cb1);
3542	break;
3543	default:
3544	Assert(RT_FAILURE_NP(rc));
3545	return rc;
3546	}
3547
3548	void const *pvSrc2;
3549	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys2, &pvSrc2, &PgMpLck);
3550	switch (rc)
3551	{
3552	case VINF_SUCCESS:
3553	memcpy((uint8_t *)pvDst + cb1, pvSrc2, cb2);
3554	PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
3555	break;
3556	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3557	memset((uint8_t *)pvDst + cb1, 0xff, cb2);
3558	break;
3559	default:
3560	Assert(RT_FAILURE_NP(rc));
3561	return rc;
3562	}
3563
3564	if (!(fFlags1 & X86_PTE_A))
3565	{
3566	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3567	AssertRC(rc);
3568	}
3569	if (!(fFlags2 & X86_PTE_A))
3570	{
3571	rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3572	AssertRC(rc);
3573	}
3574	return VINF_SUCCESS;
3575	}
3576	}
3577	}
3578
3579	/*
3580	* Raise a #PF.
3581	*/
3582	uint32_t uErr;
3583
3584	/* Get the current privilege level. */
3585	uint32_t cpl = CPUMGetGuestCPL(pVCpu);
3586	switch (rc)
3587	{
3588	case VINF_SUCCESS:
3589	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3590	break;
3591
3592	case VERR_PAGE_NOT_PRESENT:
3593	case VERR_PAGE_TABLE_NOT_PRESENT:
3594	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3595	break;
3596
3597	default:
3598	AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
3599	return rc;
3600	}
3601	Log(("PGMPhysInterpretedRead: GCPtrSrc=%RGv cb=%#x -> #PF(%#x)\n", GCPtrSrc, cb, uErr));
3602	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrSrc);
3603	}
3604
3605
3606	/**
3607	* Performs a read of guest virtual memory for instruction emulation.
3608	*
3609	* This will check permissions, raise exceptions and update the access bits.
3610	*
3611	* The current implementation will bypass all access handlers. It may later be
3612	* changed to at least respect MMIO.
3613	*
3614	*
3615	* @returns VBox status code suitable to scheduling.
3616	* @retval VINF_SUCCESS if the read was performed successfully.
3617	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3618	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3619	*
3620	* @param pVCpu Handle to the current virtual CPU.
3621	* @param pCtxCore The context core.
3622	* @param pvDst Where to put the bytes we've read.
3623	* @param GCPtrSrc The source address.
3624	* @param cb The number of bytes to read. Not more than a page.
3625	* @param fRaiseTrap If set the trap will be raised on as per spec, if clear
3626	* an appropriate error status will be returned (no
3627	* informational at all).
3628	*
3629	*
3630	* @remarks Takes the PGM lock.
3631	* @remarks A page fault on the 2nd page of the access will be raised without
3632	* writing the bits on the first page since we're ASSUMING that the
3633	* caller is emulating an instruction access.
3634	* @remarks This function will dynamically map physical pages in GC. This may
3635	* unmap mappings done by the caller. Be careful!
3636	*/
3637	VMMDECL(int) PGMPhysInterpretedReadNoHandlers(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb,
3638	bool fRaiseTrap)
3639	{
3640	PVM pVM = pVCpu->CTX_SUFF(pVM);
3641	Assert(cb <= PAGE_SIZE);
3642	VMCPU_ASSERT_EMT(pVCpu);
3643
3644	/*
3645	* 1. Translate virtual to physical. This may fault.
3646	* 2. Map the physical address.
3647	* 3. Do the read operation.
3648	* 4. Set access bits if required.
3649	*/
3650	int rc;
3651	unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
3652	if (cb <= cb1)
3653	{
3654	/*
3655	* Not crossing pages.
3656	*/
3657	RTGCPHYS GCPhys;
3658	uint64_t fFlags;
3659	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
3660	if (RT_SUCCESS(rc))
3661	{
3662	if (1) /** @todo we should check reserved bits ... */
3663	{
3664	const void *pvSrc;
3665	PGMPAGEMAPLOCK Lock;
3666	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvSrc, &Lock);
3667	switch (rc)
3668	{
3669	case VINF_SUCCESS:
3670	Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d\n",
3671	pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb));
3672	memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
3673	PGMPhysReleasePageMappingLock(pVM, &Lock);
3674	break;
3675	case VERR_PGM_PHYS_PAGE_RESERVED:
3676	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3677	memset(pvDst, 0xff, cb);
3678	break;
3679	default:
3680	AssertMsgFailed(("%Rrc\n", rc));
3681	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3682	return rc;
3683	}
3684
3685	if (!(fFlags & X86_PTE_A))
3686	{
3687	/** @todo access bit emulation isn't 100% correct. */
3688	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3689	AssertRC(rc);
3690	}
3691	return VINF_SUCCESS;
3692	}
3693	}
3694	}
3695	else
3696	{
3697	/*
3698	* Crosses pages.
3699	*/
3700	size_t cb2 = cb - cb1;
3701	uint64_t fFlags1;
3702	RTGCPHYS GCPhys1;
3703	uint64_t fFlags2;
3704	RTGCPHYS GCPhys2;
3705	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
3706	if (RT_SUCCESS(rc))
3707	{
3708	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
3709	if (RT_SUCCESS(rc))
3710	{
3711	if (1) /** @todo we should check reserved bits ... */
3712	{
3713	const void *pvSrc;
3714	PGMPAGEMAPLOCK Lock;
3715	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys1, &pvSrc, &Lock);
3716	switch (rc)
3717	{
3718	case VINF_SUCCESS:
3719	Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d [2]\n",
3720	pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb1));
3721	memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
3722	PGMPhysReleasePageMappingLock(pVM, &Lock);
3723	break;
3724	case VERR_PGM_PHYS_PAGE_RESERVED:
3725	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3726	memset(pvDst, 0xff, cb1);
3727	break;
3728	default:
3729	AssertMsgFailed(("%Rrc\n", rc));
3730	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3731	return rc;
3732	}
3733
3734	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys2, &pvSrc, &Lock);
3735	switch (rc)
3736	{
3737	case VINF_SUCCESS:
3738	memcpy((uint8_t *)pvDst + cb1, pvSrc, cb2);
3739	PGMPhysReleasePageMappingLock(pVM, &Lock);
3740	break;
3741	case VERR_PGM_PHYS_PAGE_RESERVED:
3742	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3743	memset((uint8_t *)pvDst + cb1, 0xff, cb2);
3744	break;
3745	default:
3746	AssertMsgFailed(("%Rrc\n", rc));
3747	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3748	return rc;
3749	}
3750
3751	if (!(fFlags1 & X86_PTE_A))
3752	{
3753	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3754	AssertRC(rc);
3755	}
3756	if (!(fFlags2 & X86_PTE_A))
3757	{
3758	rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3759	AssertRC(rc);
3760	}
3761	return VINF_SUCCESS;
3762	}
3763	/* sort out which page */
3764	}
3765	else
3766	GCPtrSrc += cb1; /* fault on 2nd page */
3767	}
3768	}
3769
3770	/*
3771	* Raise a #PF if we're allowed to do that.
3772	*/
3773	/* Calc the error bits. */
3774	uint32_t cpl = CPUMGetGuestCPL(pVCpu);
3775	uint32_t uErr;
3776	switch (rc)
3777	{
3778	case VINF_SUCCESS:
3779	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3780	rc = VERR_ACCESS_DENIED;
3781	break;
3782
3783	case VERR_PAGE_NOT_PRESENT:
3784	case VERR_PAGE_TABLE_NOT_PRESENT:
3785	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3786	break;
3787
3788	default:
3789	AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
3790	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3791	return rc;
3792	}
3793	if (fRaiseTrap)
3794	{
3795	Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrSrc, cb, uErr));
3796	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrSrc);
3797	}
3798	Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrSrc, cb, uErr));
3799	return rc;
3800	}
3801
3802
3803	/**
3804	* Performs a write to guest virtual memory for instruction emulation.
3805	*
3806	* This will check permissions, raise exceptions and update the dirty and access
3807	* bits.
3808	*
3809	* @returns VBox status code suitable to scheduling.
3810	* @retval VINF_SUCCESS if the read was performed successfully.
3811	* @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3812	* @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3813	*
3814	* @param pVCpu Handle to the current virtual CPU.
3815	* @param pCtxCore The context core.
3816	* @param GCPtrDst The destination address.
3817	* @param pvSrc What to write.
3818	* @param cb The number of bytes to write. Not more than a page.
3819	* @param fRaiseTrap If set the trap will be raised on as per spec, if clear
3820	* an appropriate error status will be returned (no
3821	* informational at all).
3822	*
3823	* @remarks Takes the PGM lock.
3824	* @remarks A page fault on the 2nd page of the access will be raised without
3825	* writing the bits on the first page since we're ASSUMING that the
3826	* caller is emulating an instruction access.
3827	* @remarks This function will dynamically map physical pages in GC. This may
3828	* unmap mappings done by the caller. Be careful!
3829	*/
3830	VMMDECL(int) PGMPhysInterpretedWriteNoHandlers(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, RTGCPTR GCPtrDst, const void *pvSrc,
3831	size_t cb, bool fRaiseTrap)
3832	{
3833	Assert(cb <= PAGE_SIZE);
3834	PVM pVM = pVCpu->CTX_SUFF(pVM);
3835	VMCPU_ASSERT_EMT(pVCpu);
3836
3837	/*
3838	* 1. Translate virtual to physical. This may fault.
3839	* 2. Map the physical address.
3840	* 3. Do the write operation.
3841	* 4. Set access bits if required.
3842	*/
3843	/** @todo Since this method is frequently used by EMInterpret or IOM
3844	* upon a write fault to an write access monitored page, we can
3845	* reuse the guest page table walking from the \#PF code. */
3846	int rc;
3847	unsigned cb1 = PAGE_SIZE - (GCPtrDst & PAGE_OFFSET_MASK);
3848	if (cb <= cb1)
3849	{
3850	/*
3851	* Not crossing pages.
3852	*/
3853	RTGCPHYS GCPhys;
3854	uint64_t fFlags;
3855	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst, &fFlags, &GCPhys);
3856	if (RT_SUCCESS(rc))
3857	{
3858	if ( (fFlags & X86_PTE_RW) /** @todo Also check reserved bits. */
3859	\|\| ( !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
3860	&& CPUMGetGuestCPL(pVCpu) <= 2) ) /** @todo it's 2, right? Check cpl check below as well. */
3861	{
3862	void *pvDst;
3863	PGMPAGEMAPLOCK Lock;
3864	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, &pvDst, &Lock);
3865	switch (rc)
3866	{
3867	case VINF_SUCCESS:
3868	Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
3869	(uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc, cb));
3870	memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb);
3871	PGMPhysReleasePageMappingLock(pVM, &Lock);
3872	break;
3873	case VERR_PGM_PHYS_PAGE_RESERVED:
3874	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3875	/* bit bucket */
3876	break;
3877	default:
3878	AssertMsgFailed(("%Rrc\n", rc));
3879	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3880	return rc;
3881	}
3882
3883	if (!(fFlags & (X86_PTE_A \| X86_PTE_D)))
3884	{
3885	/** @todo dirty & access bit emulation isn't 100% correct. */
3886	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
3887	AssertRC(rc);
3888	}
3889	return VINF_SUCCESS;
3890	}
3891	rc = VERR_ACCESS_DENIED;
3892	}
3893	}
3894	else
3895	{
3896	/*
3897	* Crosses pages.
3898	*/
3899	size_t cb2 = cb - cb1;
3900	uint64_t fFlags1;
3901	RTGCPHYS GCPhys1;
3902	uint64_t fFlags2;
3903	RTGCPHYS GCPhys2;
3904	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst, &fFlags1, &GCPhys1);
3905	if (RT_SUCCESS(rc))
3906	{
3907	rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst + cb1, &fFlags2, &GCPhys2);
3908	if (RT_SUCCESS(rc))
3909	{
3910	if ( ( (fFlags1 & X86_PTE_RW) /** @todo Also check reserved bits. */
3911	&& (fFlags2 & X86_PTE_RW))
3912	\|\| ( !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
3913	&& CPUMGetGuestCPL(pVCpu) <= 2) )
3914	{
3915	void *pvDst;
3916	PGMPAGEMAPLOCK Lock;
3917	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys1, &pvDst, &Lock);
3918	switch (rc)
3919	{
3920	case VINF_SUCCESS:
3921	Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
3922	(uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc, cb1));
3923	memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb1);
3924	PGMPhysReleasePageMappingLock(pVM, &Lock);
3925	break;
3926	case VERR_PGM_PHYS_PAGE_RESERVED:
3927	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3928	/* bit bucket */
3929	break;
3930	default:
3931	AssertMsgFailed(("%Rrc\n", rc));
3932	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3933	return rc;
3934	}
3935
3936	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys2, &pvDst, &Lock);
3937	switch (rc)
3938	{
3939	case VINF_SUCCESS:
3940	memcpy(pvDst, (const uint8_t *)pvSrc + cb1, cb2);
3941	PGMPhysReleasePageMappingLock(pVM, &Lock);
3942	break;
3943	case VERR_PGM_PHYS_PAGE_RESERVED:
3944	case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3945	/* bit bucket */
3946	break;
3947	default:
3948	AssertMsgFailed(("%Rrc\n", rc));
3949	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3950	return rc;
3951	}
3952
3953	if (!(fFlags1 & (X86_PTE_A \| X86_PTE_RW)))
3954	{
3955	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, (X86_PTE_A \| X86_PTE_RW), ~(uint64_t)(X86_PTE_A \| X86_PTE_RW));
3956	AssertRC(rc);
3957	}
3958	if (!(fFlags2 & (X86_PTE_A \| X86_PTE_RW)))
3959	{
3960	rc = PGMGstModifyPage(pVCpu, GCPtrDst + cb1, 1, (X86_PTE_A \| X86_PTE_RW), ~(uint64_t)(X86_PTE_A \| X86_PTE_RW));
3961	AssertRC(rc);
3962	}
3963	return VINF_SUCCESS;
3964	}
3965	if ((fFlags1 & (X86_PTE_RW)) == X86_PTE_RW)
3966	GCPtrDst += cb1; /* fault on the 2nd page. */
3967	rc = VERR_ACCESS_DENIED;
3968	}
3969	else
3970	GCPtrDst += cb1; /* fault on the 2nd page. */
3971	}
3972	}
3973
3974	/*
3975	* Raise a #PF if we're allowed to do that.
3976	*/
3977	/* Calc the error bits. */
3978	uint32_t uErr;
3979	uint32_t cpl = CPUMGetGuestCPL(pVCpu);
3980	switch (rc)
3981	{
3982	case VINF_SUCCESS:
3983	uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD \| X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3984	rc = VERR_ACCESS_DENIED;
3985	break;
3986
3987	case VERR_ACCESS_DENIED:
3988	uErr = (cpl >= 2) ? X86_TRAP_PF_RW \| X86_TRAP_PF_US : X86_TRAP_PF_RW;
3989	break;
3990
3991	case VERR_PAGE_NOT_PRESENT:
3992	case VERR_PAGE_TABLE_NOT_PRESENT:
3993	uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3994	break;
3995
3996	default:
3997	AssertMsgFailed(("rc=%Rrc GCPtrDst=%RGv cb=%#x\n", rc, GCPtrDst, cb));
3998	AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
3999	return rc;
4000	}
4001	if (fRaiseTrap)
4002	{
4003	Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrDst, cb, uErr));
4004	return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrDst);
4005	}
4006	Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrDst, cb, uErr));
4007	return rc;
4008	}
4009
4010
4011	/**
4012	* Return the page type of the specified physical address.
4013	*
4014	* @returns The page type.
4015	* @param pVM Pointer to the VM.
4016	* @param GCPhys Guest physical address
4017	*/
4018	VMMDECL(PGMPAGETYPE) PGMPhysGetPageType(PVM pVM, RTGCPHYS GCPhys)
4019	{
4020	pgmLock(pVM);
4021	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
4022	PGMPAGETYPE enmPgType = pPage ? (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage) : PGMPAGETYPE_INVALID;
4023	pgmUnlock(pVM);
4024
4025	return enmPgType;
4026	}
4027
4028
4029
4030
4031	/**
4032	* Converts a GC physical address to a HC ring-3 pointer, with some
4033	* additional checks.
4034	*
4035	* @returns VBox status code (no informational statuses).
4036	* @retval VINF_SUCCESS on success.
4037	* @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4038	* access handler of some kind.
4039	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4040	* accesses or is odd in any way.
4041	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4042	*
4043	* @param pVM Pointer to the cross context VM structure.
4044	* @param pVCpu Pointer to the cross context virtual CPU structure of
4045	* the calling EMT.
4046	* @param GCPhys The GC physical address to convert. This API mask the
4047	* A20 line when necessary.
4048	* @param fWritable Whether write access is required.
4049	* @param ppv Where to store the pointer corresponding to GCPhys on
4050	* success.
4051	* @param pLock
4052	*
4053	* @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr.
4054	* @thread EMT(pVCpu).
4055	*/
4056	VMM_INT_DECL(int) PGMPhysIemGCPhys2Ptr(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers,
4057	void **ppv, PPGMPAGEMAPLOCK pLock)
4058	{
4059	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
4060
4061	pgmLock(pVM);
4062
4063	PPGMRAMRANGE pRam;
4064	PPGMPAGE pPage;
4065	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4066	if (RT_SUCCESS(rc))
4067	{
4068	if (PGM_PAGE_IS_BALLOONED(pPage))
4069	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4070	else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
4071	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4072	else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
4073	\|\| (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
4074	rc = VINF_SUCCESS;
4075	else
4076	{
4077	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4078	{
4079	Assert(!fByPassHandlers \|\| PGM_PAGE_IS_MMIO(pPage));
4080	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4081	}
4082	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
4083	{
4084	Assert(!fByPassHandlers);
4085	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4086	}
4087	}
4088	if (RT_SUCCESS(rc))
4089	{
4090	int rc2;
4091
4092	/* Make sure what we return is writable. */
4093	if (fWritable)
4094	switch (PGM_PAGE_GET_STATE(pPage))
4095	{
4096	case PGM_PAGE_STATE_ALLOCATED:
4097	break;
4098	case PGM_PAGE_STATE_BALLOONED:
4099	AssertFailed();
4100	case PGM_PAGE_STATE_ZERO:
4101	case PGM_PAGE_STATE_SHARED:
4102	case PGM_PAGE_STATE_WRITE_MONITORED:
4103	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
4104	AssertLogRelRCReturn(rc2, rc2);
4105	break;
4106	}
4107
4108	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4109	void *pv;
4110	rc = pgmRZDynMapHCPageInlined(pVCpu,
4111	PGM_PAGE_GET_HCPHYS(pPage),
4112	&pv
4113	RTLOG_COMMA_SRC_POS);
4114	if (RT_FAILURE(rc))
4115	return rc;
4116	ppv = (void )((uintptr_t)pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4117	pLock->pvPage = pv;
4118	pLock->pVCpu = pVCpu;
4119
4120	#else
4121	/* Get a ring-3 mapping of the address. */
4122	PPGMPAGER3MAPTLBE pTlbe;
4123	rc2 = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
4124	AssertLogRelRCReturn(rc2, rc2);
4125
4126	/* Lock it and calculate the address. */
4127	if (fWritable)
4128	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
4129	else
4130	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
4131	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4132	#endif
4133
4134	Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
4135	}
4136	else
4137	Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
4138
4139	/* else: handler catching all access, no pointer returned. */
4140	}
4141	else
4142	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
4143
4144	pgmUnlock(pVM);
4145	return rc;
4146	}
4147
4148
4149	/**
4150	* Checks if the give GCPhys page requires special handling for the given access
4151	* because it's MMIO or otherwise monitored.
4152	*
4153	* @returns VBox status code (no informational statuses).
4154	* @retval VINF_SUCCESS on success.
4155	* @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4156	* access handler of some kind.
4157	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4158	* accesses or is odd in any way.
4159	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4160	*
4161	* @param pVM Pointer to the VM.
4162	* @param GCPhys The GC physical address to convert. Since this is only
4163	* used for filling the REM TLB, the A20 mask must be
4164	* applied before calling this API.
4165	* @param fWritable Whether write access is required.
4166	*
4167	* @remarks This is a watered down version PGMPhysIemGCPhys2Ptr and really just
4168	* a stop gap thing that should be removed once there is a better TLB
4169	* for virtual address accesses.
4170	*/
4171	VMM_INT_DECL(int) PGMPhysIemQueryAccess(PVM pVM, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers)
4172	{
4173	pgmLock(pVM);
4174	PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
4175
4176	PPGMRAMRANGE pRam;
4177	PPGMPAGE pPage;
4178	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4179	if (RT_SUCCESS(rc))
4180	{
4181	if (PGM_PAGE_IS_BALLOONED(pPage))
4182	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4183	else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
4184	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4185	else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
4186	\|\| (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
4187	rc = VINF_SUCCESS;
4188	else
4189	{
4190	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4191	{
4192	Assert(!fByPassHandlers \|\| PGM_PAGE_IS_MMIO(pPage));
4193	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4194	}
4195	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
4196	{
4197	Assert(!fByPassHandlers);
4198	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4199	}
4200	}
4201	}
4202
4203	pgmUnlock(pVM);
4204	return rc;
4205	}
4206

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp@ 55747

Download in other formats: