PGMPhys.cpp@ 55705

Last change on this file since 55705 was 55493, checked in by vboxsync, 10 years ago
PGM,++: Separated physical access handler callback function pointers from the access handler registrations to reduce footprint and simplify adding a couple of more callbacks.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 168.8 KB

Line
1	/* $Id: PGMPhys.cpp 55493 2015-04-28 16:51:35Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2014 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_PHYS
23	#include <VBox/vmm/pgm.h>
24	#include <VBox/vmm/iom.h>
25	#include <VBox/vmm/mm.h>
26	#include <VBox/vmm/stam.h>
27	#ifdef VBOX_WITH_REM
28	# include <VBox/vmm/rem.h>
29	#endif
30	#include <VBox/vmm/pdmdev.h>
31	#include "PGMInternal.h"
32	#include <VBox/vmm/vm.h>
33	#include <VBox/vmm/uvm.h>
34	#include "PGMInline.h"
35	#include <VBox/sup.h>
36	#include <VBox/param.h>
37	#include <VBox/err.h>
38	#include <VBox/log.h>
39	#include <iprt/assert.h>
40	#include <iprt/alloc.h>
41	#include <iprt/asm.h>
42	#ifdef VBOX_STRICT
43	# include <iprt/crc.h>
44	#endif
45	#include <iprt/thread.h>
46	#include <iprt/string.h>
47	#include <iprt/system.h>
48
49
50	/*******************************************************************************
51	* Defined Constants And Macros *
52	*******************************************************************************/
53	/** The number of pages to free in one batch. */
54	#define PGMPHYS_FREE_PAGE_BATCH_SIZE 128
55
56
57	/*
58	* PGMR3PhysReadU8-64
59	* PGMR3PhysWriteU8-64
60	*/
61	#define PGMPHYSFN_READNAME PGMR3PhysReadU8
62	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
63	#define PGMPHYS_DATASIZE 1
64	#define PGMPHYS_DATATYPE uint8_t
65	#include "PGMPhysRWTmpl.h"
66
67	#define PGMPHYSFN_READNAME PGMR3PhysReadU16
68	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
69	#define PGMPHYS_DATASIZE 2
70	#define PGMPHYS_DATATYPE uint16_t
71	#include "PGMPhysRWTmpl.h"
72
73	#define PGMPHYSFN_READNAME PGMR3PhysReadU32
74	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
75	#define PGMPHYS_DATASIZE 4
76	#define PGMPHYS_DATATYPE uint32_t
77	#include "PGMPhysRWTmpl.h"
78
79	#define PGMPHYSFN_READNAME PGMR3PhysReadU64
80	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
81	#define PGMPHYS_DATASIZE 8
82	#define PGMPHYS_DATATYPE uint64_t
83	#include "PGMPhysRWTmpl.h"
84
85
86	/**
87	* EMT worker for PGMR3PhysReadExternal.
88	*/
89	static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead)
90	{
91	PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead);
92	return VINF_SUCCESS;
93	}
94
95
96	/**
97	* Read from physical memory, external users.
98	*
99	* @returns VBox status code.
100	* @retval VINF_SUCCESS.
101	*
102	* @param pVM Pointer to the VM.
103	* @param GCPhys Physical address to read from.
104	* @param pvBuf Where to read into.
105	* @param cbRead How many bytes to read.
106	*
107	* @thread Any but EMTs.
108	*/
109	VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
110	{
111	VM_ASSERT_OTHER_THREAD(pVM);
112
113	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
114	LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
115
116	pgmLock(pVM);
117
118	/*
119	* Copy loop on ram ranges.
120	*/
121	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
122	for (;;)
123	{
124	/* Inside range or not? */
125	if (pRam && GCPhys >= pRam->GCPhys)
126	{
127	/*
128	* Must work our way thru this page by page.
129	*/
130	RTGCPHYS off = GCPhys - pRam->GCPhys;
131	while (off < pRam->cb)
132	{
133	unsigned iPage = off >> PAGE_SHIFT;
134	PPGMPAGE pPage = &pRam->aPages[iPage];
135
136	/*
137	* If the page has an ALL access handler, we'll have to
138	* delegate the job to EMT.
139	*/
140	if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
141	\|\| PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
142	{
143	pgmUnlock(pVM);
144
145	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 4,
146	pVM, &GCPhys, pvBuf, cbRead);
147	}
148	Assert(!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage));
149
150	/*
151	* Simple stuff, go ahead.
152	*/
153	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
154	if (cb > cbRead)
155	cb = cbRead;
156	PGMPAGEMAPLOCK PgMpLck;
157	const void *pvSrc;
158	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
159	if (RT_SUCCESS(rc))
160	{
161	memcpy(pvBuf, pvSrc, cb);
162	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
163	}
164	else
165	{
166	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
167	pRam->GCPhys + off, pPage, rc));
168	memset(pvBuf, 0xff, cb);
169	}
170
171	/* next page */
172	if (cb >= cbRead)
173	{
174	pgmUnlock(pVM);
175	return VINF_SUCCESS;
176	}
177	cbRead -= cb;
178	off += cb;
179	GCPhys += cb;
180	pvBuf = (char *)pvBuf + cb;
181	} /* walk pages in ram range. */
182	}
183	else
184	{
185	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
186
187	/*
188	* Unassigned address space.
189	*/
190	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
191	if (cb >= cbRead)
192	{
193	memset(pvBuf, 0xff, cbRead);
194	break;
195	}
196	memset(pvBuf, 0xff, cb);
197
198	cbRead -= cb;
199	pvBuf = (char *)pvBuf + cb;
200	GCPhys += cb;
201	}
202
203	/* Advance range if necessary. */
204	while (pRam && GCPhys > pRam->GCPhysLast)
205	pRam = pRam->CTX_SUFF(pNext);
206	} /* Ram range walk */
207
208	pgmUnlock(pVM);
209
210	return VINF_SUCCESS;
211	}
212
213
214	/**
215	* EMT worker for PGMR3PhysWriteExternal.
216	*/
217	static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite)
218	{
219	/** @todo VERR_EM_NO_MEMORY */
220	PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite);
221	return VINF_SUCCESS;
222	}
223
224
225	/**
226	* Write to physical memory, external users.
227	*
228	* @returns VBox status code.
229	* @retval VINF_SUCCESS.
230	* @retval VERR_EM_NO_MEMORY.
231	*
232	* @param pVM Pointer to the VM.
233	* @param GCPhys Physical address to write to.
234	* @param pvBuf What to write.
235	* @param cbWrite How many bytes to write.
236	* @param pszWho Who is writing. For tracking down who is writing
237	* after we've saved the state.
238	*
239	* @thread Any but EMTs.
240	*/
241	VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void pvBuf, size_t cbWrite, const char pszWho)
242	{
243	VM_ASSERT_OTHER_THREAD(pVM);
244
245	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
246	("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x pszWho=%s\n",
247	GCPhys, cbWrite, pszWho));
248	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
249	LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
250
251	pgmLock(pVM);
252
253	/*
254	* Copy loop on ram ranges, stop when we hit something difficult.
255	*/
256	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
257	for (;;)
258	{
259	/* Inside range or not? */
260	if (pRam && GCPhys >= pRam->GCPhys)
261	{
262	/*
263	* Must work our way thru this page by page.
264	*/
265	RTGCPTR off = GCPhys - pRam->GCPhys;
266	while (off < pRam->cb)
267	{
268	RTGCPTR iPage = off >> PAGE_SHIFT;
269	PPGMPAGE pPage = &pRam->aPages[iPage];
270
271	/*
272	* Is the page problematic, we have to do the work on the EMT.
273	*
274	* Allocating writable pages and access handlers are
275	* problematic, write monitored pages are simple and can be
276	* dealt with here.
277	*/
278	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
279	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
280	\|\| PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
281	{
282	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
283	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
284	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
285	else
286	{
287	pgmUnlock(pVM);
288
289	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 4,
290	pVM, &GCPhys, pvBuf, cbWrite);
291	}
292	}
293	Assert(!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage));
294
295	/*
296	* Simple stuff, go ahead.
297	*/
298	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
299	if (cb > cbWrite)
300	cb = cbWrite;
301	PGMPAGEMAPLOCK PgMpLck;
302	void *pvDst;
303	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
304	if (RT_SUCCESS(rc))
305	{
306	memcpy(pvDst, pvBuf, cb);
307	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
308	}
309	else
310	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
311	pRam->GCPhys + off, pPage, rc));
312
313	/* next page */
314	if (cb >= cbWrite)
315	{
316	pgmUnlock(pVM);
317	return VINF_SUCCESS;
318	}
319
320	cbWrite -= cb;
321	off += cb;
322	GCPhys += cb;
323	pvBuf = (const char *)pvBuf + cb;
324	} /* walk pages in ram range */
325	}
326	else
327	{
328	/*
329	* Unassigned address space, skip it.
330	*/
331	if (!pRam)
332	break;
333	size_t cb = pRam->GCPhys - GCPhys;
334	if (cb >= cbWrite)
335	break;
336	cbWrite -= cb;
337	pvBuf = (const char *)pvBuf + cb;
338	GCPhys += cb;
339	}
340
341	/* Advance range if necessary. */
342	while (pRam && GCPhys > pRam->GCPhysLast)
343	pRam = pRam->CTX_SUFF(pNext);
344	} /* Ram range walk */
345
346	pgmUnlock(pVM);
347	return VINF_SUCCESS;
348	}
349
350
351	/**
352	* VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
353	*
354	* @returns see PGMR3PhysGCPhys2CCPtrExternal
355	* @param pVM Pointer to the VM.
356	* @param pGCPhys Pointer to the guest physical address.
357	* @param ppv Where to store the mapping address.
358	* @param pLock Where to store the lock.
359	*/
360	static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
361	{
362	/*
363	* Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
364	* an access handler after it succeeds.
365	*/
366	int rc = pgmLock(pVM);
367	AssertRCReturn(rc, rc);
368
369	rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
370	if (RT_SUCCESS(rc))
371	{
372	PPGMPAGEMAPTLBE pTlbe;
373	int rc2 = pgmPhysPageQueryTlbe(pVM, *pGCPhys, &pTlbe);
374	AssertFatalRC(rc2);
375	PPGMPAGE pPage = pTlbe->pPage;
376	if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
377	{
378	PGMPhysReleasePageMappingLock(pVM, pLock);
379	rc = VERR_PGM_PHYS_PAGE_RESERVED;
380	}
381	else if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
382	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
383	\|\| pgmPoolIsDirtyPage(pVM, *pGCPhys)
384	#endif
385	)
386	{
387	/* We must flush any corresponding pgm pool page here, otherwise we'll
388	* not be informed about writes and keep bogus gst->shw mappings around.
389	*/
390	pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
391	Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
392	/** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
393	* active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
394	}
395	}
396
397	pgmUnlock(pVM);
398	return rc;
399	}
400
401
402	/**
403	* Requests the mapping of a guest page into ring-3, external threads.
404	*
405	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
406	* release it.
407	*
408	* This API will assume your intention is to write to the page, and will
409	* therefore replace shared and zero pages. If you do not intend to modify the
410	* page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
411	*
412	* @returns VBox status code.
413	* @retval VINF_SUCCESS on success.
414	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
415	* backing or if the page has any active access handlers. The caller
416	* must fall back on using PGMR3PhysWriteExternal.
417	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
418	*
419	* @param pVM Pointer to the VM.
420	* @param GCPhys The guest physical address of the page that should be mapped.
421	* @param ppv Where to store the address corresponding to GCPhys.
422	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
423	*
424	* @remark Avoid calling this API from within critical sections (other than the
425	* PGM one) because of the deadlock risk when we have to delegating the
426	* task to an EMT.
427	* @thread Any.
428	*/
429	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
430	{
431	AssertPtr(ppv);
432	AssertPtr(pLock);
433
434	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
435
436	int rc = pgmLock(pVM);
437	AssertRCReturn(rc, rc);
438
439	/*
440	* Query the Physical TLB entry for the page (may fail).
441	*/
442	PPGMPAGEMAPTLBE pTlbe;
443	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
444	if (RT_SUCCESS(rc))
445	{
446	PPGMPAGE pPage = pTlbe->pPage;
447	if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
448	rc = VERR_PGM_PHYS_PAGE_RESERVED;
449	else
450	{
451	/*
452	* If the page is shared, the zero page, or being write monitored
453	* it must be converted to an page that's writable if possible.
454	* We can only deal with write monitored pages here, the rest have
455	* to be on an EMT.
456	*/
457	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
458	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
459	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
460	\|\| pgmPoolIsDirtyPage(pVM, GCPhys)
461	#endif
462	)
463	{
464	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
465	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
466	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
467	&& !pgmPoolIsDirtyPage(pVM, GCPhys)
468	#endif
469	)
470	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
471	else
472	{
473	pgmUnlock(pVM);
474
475	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
476	pVM, &GCPhys, ppv, pLock);
477	}
478	}
479
480	/*
481	* Now, just perform the locking and calculate the return address.
482	*/
483	PPGMPAGEMAP pMap = pTlbe->pMap;
484	if (pMap)
485	pMap->cRefs++;
486
487	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
488	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
489	{
490	if (cLocks == 0)
491	pVM->pgm.s.cWriteLockedPages++;
492	PGM_PAGE_INC_WRITE_LOCKS(pPage);
493	}
494	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
495	{
496	PGM_PAGE_INC_WRITE_LOCKS(pPage);
497	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
498	if (pMap)
499	pMap->cRefs++; /* Extra ref to prevent it from going away. */
500	}
501
502	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
503	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
504	pLock->pvMap = pMap;
505	}
506	}
507
508	pgmUnlock(pVM);
509	return rc;
510	}
511
512
513	/**
514	* Requests the mapping of a guest page into ring-3, external threads.
515	*
516	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
517	* release it.
518	*
519	* @returns VBox status code.
520	* @retval VINF_SUCCESS on success.
521	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
522	* backing or if the page as an active ALL access handler. The caller
523	* must fall back on using PGMPhysRead.
524	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
525	*
526	* @param pVM Pointer to the VM.
527	* @param GCPhys The guest physical address of the page that should be mapped.
528	* @param ppv Where to store the address corresponding to GCPhys.
529	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
530	*
531	* @remark Avoid calling this API from within critical sections (other than
532	* the PGM one) because of the deadlock risk.
533	* @thread Any.
534	*/
535	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
536	{
537	int rc = pgmLock(pVM);
538	AssertRCReturn(rc, rc);
539
540	/*
541	* Query the Physical TLB entry for the page (may fail).
542	*/
543	PPGMPAGEMAPTLBE pTlbe;
544	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
545	if (RT_SUCCESS(rc))
546	{
547	PPGMPAGE pPage = pTlbe->pPage;
548	#if 1
549	/* MMIO pages doesn't have any readable backing. */
550	if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
551	rc = VERR_PGM_PHYS_PAGE_RESERVED;
552	#else
553	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
554	rc = VERR_PGM_PHYS_PAGE_RESERVED;
555	#endif
556	else
557	{
558	/*
559	* Now, just perform the locking and calculate the return address.
560	*/
561	PPGMPAGEMAP pMap = pTlbe->pMap;
562	if (pMap)
563	pMap->cRefs++;
564
565	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
566	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
567	{
568	if (cLocks == 0)
569	pVM->pgm.s.cReadLockedPages++;
570	PGM_PAGE_INC_READ_LOCKS(pPage);
571	}
572	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
573	{
574	PGM_PAGE_INC_READ_LOCKS(pPage);
575	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
576	if (pMap)
577	pMap->cRefs++; /* Extra ref to prevent it from going away. */
578	}
579
580	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
581	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
582	pLock->pvMap = pMap;
583	}
584	}
585
586	pgmUnlock(pVM);
587	return rc;
588	}
589
590
591	#define MAKE_LEAF(a_pNode) \
592	do { \
593	(a_pNode)->pLeftR3 = NIL_RTR3PTR; \
594	(a_pNode)->pRightR3 = NIL_RTR3PTR; \
595	(a_pNode)->pLeftR0 = NIL_RTR0PTR; \
596	(a_pNode)->pRightR0 = NIL_RTR0PTR; \
597	(a_pNode)->pLeftRC = NIL_RTRCPTR; \
598	(a_pNode)->pRightRC = NIL_RTRCPTR; \
599	} while (0)
600
601	#define INSERT_LEFT(a_pParent, a_pNode) \
602	do { \
603	(a_pParent)->pLeftR3 = (a_pNode); \
604	(a_pParent)->pLeftR0 = (a_pNode)->pSelfR0; \
605	(a_pParent)->pLeftRC = (a_pNode)->pSelfRC; \
606	} while (0)
607	#define INSERT_RIGHT(a_pParent, a_pNode) \
608	do { \
609	(a_pParent)->pRightR3 = (a_pNode); \
610	(a_pParent)->pRightR0 = (a_pNode)->pSelfR0; \
611	(a_pParent)->pRightRC = (a_pNode)->pSelfRC; \
612	} while (0)
613
614
615	/**
616	* Recursive tree builder.
617	*
618	* @param ppRam Pointer to the iterator variable.
619	* @param iHeight The hight about normal leaf nodes. Inserts a leaf
620	* node if 0.
621	*/
622	static PPGMRAMRANGE pgmR3PhysRebuildRamRangeSearchTreesRecursively(PPGMRAMRANGE *ppRam, int iDepth)
623	{
624	PPGMRAMRANGE pRam;
625	if (iDepth <= 0)
626	{
627	/*
628	* Leaf node.
629	*/
630	pRam = *ppRam;
631	if (pRam)
632	{
633	*ppRam = pRam->pNextR3;
634	MAKE_LEAF(pRam);
635	}
636	}
637	else
638	{
639
640	/*
641	* Intermediate node.
642	*/
643	PPGMRAMRANGE pLeft = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
644
645	pRam = *ppRam;
646	if (!pRam)
647	return pLeft;
648	*ppRam = pRam->pNextR3;
649	MAKE_LEAF(pRam);
650	INSERT_LEFT(pRam, pLeft);
651
652	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
653	if (pRight)
654	INSERT_RIGHT(pRam, pRight);
655	}
656	return pRam;
657	}
658
659
660	/**
661	* Rebuilds the RAM range search trees.
662	*
663	* @param pVM Pointer to the VM.
664	*/
665	static void pgmR3PhysRebuildRamRangeSearchTrees(PVM pVM)
666	{
667
668	/*
669	* Create the reasonably balanced tree in a sequential fashion.
670	* For simplicity (laziness) we use standard recursion here.
671	*/
672	int iDepth = 0;
673	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
674	PPGMRAMRANGE pRoot = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, 0);
675	while (pRam)
676	{
677	PPGMRAMRANGE pLeft = pRoot;
678
679	pRoot = pRam;
680	pRam = pRam->pNextR3;
681	MAKE_LEAF(pRoot);
682	INSERT_LEFT(pRoot, pLeft);
683
684	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, iDepth);
685	if (pRight)
686	INSERT_RIGHT(pRoot, pRight);
687	/** @todo else: rotate the tree. */
688
689	iDepth++;
690	}
691
692	pVM->pgm.s.pRamRangeTreeR3 = pRoot;
693	pVM->pgm.s.pRamRangeTreeR0 = pRoot ? pRoot->pSelfR0 : NIL_RTR0PTR;
694	pVM->pgm.s.pRamRangeTreeRC = pRoot ? pRoot->pSelfRC : NIL_RTRCPTR;
695
696	#ifdef VBOX_STRICT
697	/*
698	* Verify that the above code works.
699	*/
700	unsigned cRanges = 0;
701	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
702	cRanges++;
703	Assert(cRanges > 0);
704
705	unsigned cMaxDepth = ASMBitLastSetU32(cRanges);
706	if ((1U << cMaxDepth) < cRanges)
707	cMaxDepth++;
708
709	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
710	{
711	unsigned cDepth = 0;
712	PPGMRAMRANGE pRam2 = pVM->pgm.s.pRamRangeTreeR3;
713	for (;;)
714	{
715	if (pRam == pRam2)
716	break;
717	Assert(pRam2);
718	if (pRam->GCPhys < pRam2->GCPhys)
719	pRam2 = pRam2->pLeftR3;
720	else
721	pRam2 = pRam2->pRightR3;
722	}
723	AssertMsg(cDepth <= cMaxDepth, ("cDepth=%d cMaxDepth=%d\n", cDepth, cMaxDepth));
724	}
725	#endif /* VBOX_STRICT */
726	}
727
728	#undef MAKE_LEAF
729	#undef INSERT_LEFT
730	#undef INSERT_RIGHT
731
732	/**
733	* Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
734	*
735	* Called when anything was relocated.
736	*
737	* @param pVM Pointer to the VM.
738	*/
739	void pgmR3PhysRelinkRamRanges(PVM pVM)
740	{
741	PPGMRAMRANGE pCur;
742
743	#ifdef VBOX_STRICT
744	for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
745	{
746	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
747	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfRC == MMHyperCCToRC(pVM, pCur));
748	Assert((pCur->GCPhys & PAGE_OFFSET_MASK) == 0);
749	Assert((pCur->GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
750	Assert((pCur->cb & PAGE_OFFSET_MASK) == 0);
751	Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
752	for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesXR3; pCur2; pCur2 = pCur2->pNextR3)
753	Assert( pCur2 == pCur
754	\|\| strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
755	}
756	#endif
757
758	pCur = pVM->pgm.s.pRamRangesXR3;
759	if (pCur)
760	{
761	pVM->pgm.s.pRamRangesXR0 = pCur->pSelfR0;
762	pVM->pgm.s.pRamRangesXRC = pCur->pSelfRC;
763
764	for (; pCur->pNextR3; pCur = pCur->pNextR3)
765	{
766	pCur->pNextR0 = pCur->pNextR3->pSelfR0;
767	pCur->pNextRC = pCur->pNextR3->pSelfRC;
768	}
769
770	Assert(pCur->pNextR0 == NIL_RTR0PTR);
771	Assert(pCur->pNextRC == NIL_RTRCPTR);
772	}
773	else
774	{
775	Assert(pVM->pgm.s.pRamRangesXR0 == NIL_RTR0PTR);
776	Assert(pVM->pgm.s.pRamRangesXRC == NIL_RTRCPTR);
777	}
778	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
779
780	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
781	}
782
783
784	/**
785	* Links a new RAM range into the list.
786	*
787	* @param pVM Pointer to the VM.
788	* @param pNew Pointer to the new list entry.
789	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
790	*/
791	static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
792	{
793	AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
794	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfR0 == MMHyperCCToR0(pVM, pNew));
795	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfRC == MMHyperCCToRC(pVM, pNew));
796
797	pgmLock(pVM);
798
799	PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesXR3;
800	pNew->pNextR3 = pRam;
801	pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
802	pNew->pNextRC = pRam ? pRam->pSelfRC : NIL_RTRCPTR;
803
804	if (pPrev)
805	{
806	pPrev->pNextR3 = pNew;
807	pPrev->pNextR0 = pNew->pSelfR0;
808	pPrev->pNextRC = pNew->pSelfRC;
809	}
810	else
811	{
812	pVM->pgm.s.pRamRangesXR3 = pNew;
813	pVM->pgm.s.pRamRangesXR0 = pNew->pSelfR0;
814	pVM->pgm.s.pRamRangesXRC = pNew->pSelfRC;
815	}
816	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
817
818	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
819	pgmUnlock(pVM);
820	}
821
822
823	/**
824	* Unlink an existing RAM range from the list.
825	*
826	* @param pVM Pointer to the VM.
827	* @param pRam Pointer to the new list entry.
828	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
829	*/
830	static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
831	{
832	Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesXR3 == pRam);
833	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
834	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfRC == MMHyperCCToRC(pVM, pRam));
835
836	pgmLock(pVM);
837
838	PPGMRAMRANGE pNext = pRam->pNextR3;
839	if (pPrev)
840	{
841	pPrev->pNextR3 = pNext;
842	pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
843	pPrev->pNextRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
844	}
845	else
846	{
847	Assert(pVM->pgm.s.pRamRangesXR3 == pRam);
848	pVM->pgm.s.pRamRangesXR3 = pNext;
849	pVM->pgm.s.pRamRangesXR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
850	pVM->pgm.s.pRamRangesXRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
851	}
852	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
853
854	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
855	pgmUnlock(pVM);
856	}
857
858
859	/**
860	* Unlink an existing RAM range from the list.
861	*
862	* @param pVM Pointer to the VM.
863	* @param pRam Pointer to the new list entry.
864	*/
865	static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
866	{
867	pgmLock(pVM);
868
869	/* find prev. */
870	PPGMRAMRANGE pPrev = NULL;
871	PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesXR3;
872	while (pCur != pRam)
873	{
874	pPrev = pCur;
875	pCur = pCur->pNextR3;
876	}
877	AssertFatal(pCur);
878
879	pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
880	pgmUnlock(pVM);
881	}
882
883
884	/**
885	* Frees a range of pages, replacing them with ZERO pages of the specified type.
886	*
887	* @returns VBox status code.
888	* @param pVM Pointer to the VM.
889	* @param pRam The RAM range in which the pages resides.
890	* @param GCPhys The address of the first page.
891	* @param GCPhysLast The address of the last page.
892	* @param uType The page type to replace then with.
893	*/
894	static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint8_t uType)
895	{
896	PGM_LOCK_ASSERT_OWNER(pVM);
897	uint32_t cPendingPages = 0;
898	PGMMFREEPAGESREQ pReq;
899	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
900	AssertLogRelRCReturn(rc, rc);
901
902	/* Iterate the pages. */
903	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
904	uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
905	while (cPagesLeft-- > 0)
906	{
907	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
908	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
909
910	PGM_PAGE_SET_TYPE(pVM, pPageDst, uType);
911
912	GCPhys += PAGE_SIZE;
913	pPageDst++;
914	}
915
916	if (cPendingPages)
917	{
918	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
919	AssertLogRelRCReturn(rc, rc);
920	}
921	GMMR3FreePagesCleanup(pReq);
922
923	return rc;
924	}
925
926	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
927
928	/**
929	* Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
930	*
931	* This is only called on one of the EMTs while the other ones are waiting for
932	* it to complete this function.
933	*
934	* @returns VINF_SUCCESS (VBox strict status code).
935	* @param pVM Pointer to the VM.
936	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
937	* @param pvUser User parameter
938	*/
939	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
940	{
941	uintptr_t paUser = (uintptr_t )pvUser;
942	bool fInflate = !!paUser[0];
943	unsigned cPages = paUser[1];
944	RTGCPHYS paPhysPage = (RTGCPHYS )paUser[2];
945	uint32_t cPendingPages = 0;
946	PGMMFREEPAGESREQ pReq;
947	int rc;
948
949	Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
950	pgmLock(pVM);
951
952	if (fInflate)
953	{
954	/* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
955	pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
956
957	/* Replace pages with ZERO pages. */
958	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
959	if (RT_FAILURE(rc))
960	{
961	pgmUnlock(pVM);
962	AssertLogRelRC(rc);
963	return rc;
964	}
965
966	/* Iterate the pages. */
967	for (unsigned i = 0; i < cPages; i++)
968	{
969	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
970	if ( pPage == NULL
971	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM)
972	{
973	Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], pPage ? PGM_PAGE_GET_TYPE(pPage) : 0));
974	break;
975	}
976
977	LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
978
979	/* Flush the shadow PT if this page was previously used as a guest page table. */
980	pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
981
982	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i]);
983	if (RT_FAILURE(rc))
984	{
985	pgmUnlock(pVM);
986	AssertLogRelRC(rc);
987	return rc;
988	}
989	Assert(PGM_PAGE_IS_ZERO(pPage));
990	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_BALLOONED);
991	}
992
993	if (cPendingPages)
994	{
995	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
996	if (RT_FAILURE(rc))
997	{
998	pgmUnlock(pVM);
999	AssertLogRelRC(rc);
1000	return rc;
1001	}
1002	}
1003	GMMR3FreePagesCleanup(pReq);
1004	}
1005	else
1006	{
1007	/* Iterate the pages. */
1008	for (unsigned i = 0; i < cPages; i++)
1009	{
1010	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
1011	AssertBreak(pPage && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
1012
1013	LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
1014
1015	Assert(PGM_PAGE_IS_BALLOONED(pPage));
1016
1017	/* Change back to zero page. */
1018	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1019	}
1020
1021	/* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
1022	}
1023
1024	/* Notify GMM about the balloon change. */
1025	rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
1026	if (RT_SUCCESS(rc))
1027	{
1028	if (!fInflate)
1029	{
1030	Assert(pVM->pgm.s.cBalloonedPages >= cPages);
1031	pVM->pgm.s.cBalloonedPages -= cPages;
1032	}
1033	else
1034	pVM->pgm.s.cBalloonedPages += cPages;
1035	}
1036
1037	pgmUnlock(pVM);
1038
1039	/* Flush the recompiler's TLB as well. */
1040	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1041	CPUMSetChangedFlags(&pVM->aCpus[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1042
1043	AssertLogRelRC(rc);
1044	return rc;
1045	}
1046
1047
1048	/**
1049	* Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
1050	*
1051	* @returns VBox status code.
1052	* @param pVM Pointer to the VM.
1053	* @param fInflate Inflate or deflate memory balloon
1054	* @param cPages Number of pages to free
1055	* @param paPhysPage Array of guest physical addresses
1056	*/
1057	static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1058	{
1059	uintptr_t paUser[3];
1060
1061	paUser[0] = fInflate;
1062	paUser[1] = cPages;
1063	paUser[2] = (uintptr_t)paPhysPage;
1064	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1065	AssertRC(rc);
1066
1067	/* Made a copy in PGMR3PhysFreeRamPages; free it here. */
1068	RTMemFree(paPhysPage);
1069	}
1070
1071	#endif /* 64-bit host && (Windows \|\| Solaris \|\| Linux \|\| FreeBSD) */
1072
1073	/**
1074	* Inflate or deflate a memory balloon
1075	*
1076	* @returns VBox status code.
1077	* @param pVM Pointer to the VM.
1078	* @param fInflate Inflate or deflate memory balloon
1079	* @param cPages Number of pages to free
1080	* @param paPhysPage Array of guest physical addresses
1081	*/
1082	VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1083	{
1084	/* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
1085	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
1086	int rc;
1087
1088	/* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
1089	AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
1090
1091	/* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
1092	* In the SMP case we post a request packet to postpone the job.
1093	*/
1094	if (pVM->cCpus > 1)
1095	{
1096	unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
1097	RTGCPHYS paPhysPageCopy = (RTGCPHYS )RTMemAlloc(cbPhysPage);
1098	AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
1099
1100	memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
1101
1102	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
1103	AssertRC(rc);
1104	}
1105	else
1106	{
1107	uintptr_t paUser[3];
1108
1109	paUser[0] = fInflate;
1110	paUser[1] = cPages;
1111	paUser[2] = (uintptr_t)paPhysPage;
1112	rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1113	AssertRC(rc);
1114	}
1115	return rc;
1116
1117	#else
1118	NOREF(pVM); NOREF(fInflate); NOREF(cPages); NOREF(paPhysPage);
1119	return VERR_NOT_IMPLEMENTED;
1120	#endif
1121	}
1122
1123
1124	/**
1125	* Rendezvous callback used by PGMR3WriteProtectRAM that write protects all
1126	* physical RAM.
1127	*
1128	* This is only called on one of the EMTs while the other ones are waiting for
1129	* it to complete this function.
1130	*
1131	* @returns VINF_SUCCESS (VBox strict status code).
1132	* @param pVM Pointer to the VM.
1133	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
1134	* @param pvUser User parameter, unused.
1135	*/
1136	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1137	{
1138	int rc = VINF_SUCCESS;
1139	NOREF(pvUser); NOREF(pVCpu);
1140
1141	pgmLock(pVM);
1142	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1143	pgmPoolResetDirtyPages(pVM);
1144	#endif
1145
1146	/** @todo pointless to write protect the physical page pointed to by RSP. */
1147
1148	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1149	pRam;
1150	pRam = pRam->CTX_SUFF(pNext))
1151	{
1152	uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1153	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1154	{
1155	PPGMPAGE pPage = &pRam->aPages[iPage];
1156	PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1157
1158	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1159	\|\| enmPageType == PGMPAGETYPE_MMIO2)
1160	{
1161	/*
1162	* A RAM page.
1163	*/
1164	switch (PGM_PAGE_GET_STATE(pPage))
1165	{
1166	case PGM_PAGE_STATE_ALLOCATED:
1167	/** @todo Optimize this: Don't always re-enable write
1168	* monitoring if the page is known to be very busy. */
1169	if (PGM_PAGE_IS_WRITTEN_TO(pPage))
1170	{
1171	PGM_PAGE_CLEAR_WRITTEN_TO(pVM, pPage);
1172	/* Remember this dirty page for the next (memory) sync. */
1173	PGM_PAGE_SET_FT_DIRTY(pPage);
1174	}
1175
1176	pgmPhysPageWriteMonitor(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1177	break;
1178
1179	case PGM_PAGE_STATE_SHARED:
1180	AssertFailed();
1181	break;
1182
1183	case PGM_PAGE_STATE_WRITE_MONITORED: /* nothing to change. */
1184	default:
1185	break;
1186	}
1187	}
1188	}
1189	}
1190	pgmR3PoolWriteProtectPages(pVM);
1191	PGM_INVL_ALL_VCPU_TLBS(pVM);
1192	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1193	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1194
1195	pgmUnlock(pVM);
1196	return rc;
1197	}
1198
1199	/**
1200	* Protect all physical RAM to monitor writes
1201	*
1202	* @returns VBox status code.
1203	* @param pVM Pointer to the VM.
1204	*/
1205	VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
1206	{
1207	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1208
1209	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
1210	AssertRC(rc);
1211	return rc;
1212	}
1213
1214	/**
1215	* Enumerate all dirty FT pages.
1216	*
1217	* @returns VBox status code.
1218	* @param pVM Pointer to the VM.
1219	* @param pfnEnum Enumerate callback handler.
1220	* @param pvUser Enumerate callback handler parameter.
1221	*/
1222	VMMR3DECL(int) PGMR3PhysEnumDirtyFTPages(PVM pVM, PFNPGMENUMDIRTYFTPAGES pfnEnum, void *pvUser)
1223	{
1224	int rc = VINF_SUCCESS;
1225
1226	pgmLock(pVM);
1227	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1228	pRam;
1229	pRam = pRam->CTX_SUFF(pNext))
1230	{
1231	uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1232	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1233	{
1234	PPGMPAGE pPage = &pRam->aPages[iPage];
1235	PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1236
1237	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1238	\|\| enmPageType == PGMPAGETYPE_MMIO2)
1239	{
1240	/*
1241	* A RAM page.
1242	*/
1243	switch (PGM_PAGE_GET_STATE(pPage))
1244	{
1245	case PGM_PAGE_STATE_ALLOCATED:
1246	case PGM_PAGE_STATE_WRITE_MONITORED:
1247	if ( !PGM_PAGE_IS_WRITTEN_TO(pPage) /* not very recently updated? */
1248	&& PGM_PAGE_IS_FT_DIRTY(pPage))
1249	{
1250	unsigned cbPageRange = PAGE_SIZE;
1251	unsigned iPageClean = iPage + 1;
1252	RTGCPHYS GCPhysPage = pRam->GCPhys + iPage * PAGE_SIZE;
1253	uint8_t *pu8Page = NULL;
1254	PGMPAGEMAPLOCK Lock;
1255
1256	/* Find the next clean page, so we can merge adjacent dirty pages. */
1257	for (; iPageClean < cPages; iPageClean++)
1258	{
1259	PPGMPAGE pPageNext = &pRam->aPages[iPageClean];
1260	if ( RT_UNLIKELY(PGM_PAGE_GET_TYPE(pPageNext) != PGMPAGETYPE_RAM)
1261	\|\| PGM_PAGE_GET_STATE(pPageNext) != PGM_PAGE_STATE_ALLOCATED
1262	\|\| PGM_PAGE_IS_WRITTEN_TO(pPageNext)
1263	\|\| !PGM_PAGE_IS_FT_DIRTY(pPageNext)
1264	/* Crossing a chunk boundary? */
1265	\|\| (GCPhysPage & GMM_PAGEID_IDX_MASK) != ((GCPhysPage + cbPageRange) & GMM_PAGEID_IDX_MASK)
1266	)
1267	break;
1268
1269	cbPageRange += PAGE_SIZE;
1270	}
1271
1272	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysPage, (const void **)&pu8Page, &Lock);
1273	if (RT_SUCCESS(rc))
1274	{
1275	/** @todo this is risky; the range might be changed, but little choice as the sync
1276	* costs a lot of time. */
1277	pgmUnlock(pVM);
1278	pfnEnum(pVM, GCPhysPage, pu8Page, cbPageRange, pvUser);
1279	pgmLock(pVM);
1280	PGMPhysReleasePageMappingLock(pVM, &Lock);
1281	}
1282
1283	for (iPage; iPage < iPageClean; iPage++)
1284	PGM_PAGE_CLEAR_FT_DIRTY(&pRam->aPages[iPage]);
1285
1286	iPage = iPageClean - 1;
1287	}
1288	break;
1289	}
1290	}
1291	}
1292	}
1293	pgmUnlock(pVM);
1294	return rc;
1295	}
1296
1297
1298	/**
1299	* Gets the number of ram ranges.
1300	*
1301	* @returns Number of ram ranges. Returns UINT32_MAX if @a pVM is invalid.
1302	* @param pVM Pointer to the VM.
1303	*/
1304	VMMR3DECL(uint32_t) PGMR3PhysGetRamRangeCount(PVM pVM)
1305	{
1306	VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
1307
1308	pgmLock(pVM);
1309	uint32_t cRamRanges = 0;
1310	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext))
1311	cRamRanges++;
1312	pgmUnlock(pVM);
1313	return cRamRanges;
1314	}
1315
1316
1317	/**
1318	* Get information about a range.
1319	*
1320	* @returns VINF_SUCCESS or VERR_OUT_OF_RANGE.
1321	* @param pVM Pointer to the VM.
1322	* @param iRange The ordinal of the range.
1323	* @param pGCPhysStart Where to return the start of the range. Optional.
1324	* @param pGCPhysLast Where to return the address of the last byte in the
1325	* range. Optional.
1326	* @param pfIsMmio Where to indicate that this is a pure MMIO range.
1327	* Optional.
1328	*/
1329	VMMR3DECL(int) PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
1330	const char *ppszDesc, bool pfIsMmio)
1331	{
1332	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1333
1334	pgmLock(pVM);
1335	uint32_t iCurRange = 0;
1336	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext), iCurRange++)
1337	if (iCurRange == iRange)
1338	{
1339	if (pGCPhysStart)
1340	*pGCPhysStart = pCur->GCPhys;
1341	if (pGCPhysLast)
1342	*pGCPhysLast = pCur->GCPhysLast;
1343	if (ppszDesc)
1344	*ppszDesc = pCur->pszDesc;
1345	if (pfIsMmio)
1346	*pfIsMmio = !!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO);
1347
1348	pgmUnlock(pVM);
1349	return VINF_SUCCESS;
1350	}
1351	pgmUnlock(pVM);
1352	return VERR_OUT_OF_RANGE;
1353	}
1354
1355
1356	/**
1357	* Query the amount of free memory inside VMMR0
1358	*
1359	* @returns VBox status code.
1360	* @param pUVM The user mode VM handle.
1361	* @param pcbAllocMem Where to return the amount of memory allocated
1362	* by VMs.
1363	* @param pcbFreeMem Where to return the amount of memory that is
1364	* allocated from the host but not currently used
1365	* by any VMs.
1366	* @param pcbBallonedMem Where to return the sum of memory that is
1367	* currently ballooned by the VMs.
1368	* @param pcbSharedMem Where to return the amount of memory that is
1369	* currently shared.
1370	*/
1371	VMMR3DECL(int) PGMR3QueryGlobalMemoryStats(PUVM pUVM, uint64_t pcbAllocMem, uint64_t pcbFreeMem,
1372	uint64_t pcbBallonedMem, uint64_t pcbSharedMem)
1373	{
1374	UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
1375	VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
1376
1377	uint64_t cAllocPages = 0;
1378	uint64_t cFreePages = 0;
1379	uint64_t cBalloonPages = 0;
1380	uint64_t cSharedPages = 0;
1381	int rc = GMMR3QueryHypervisorMemoryStats(pUVM->pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
1382	AssertRCReturn(rc, rc);
1383
1384	if (pcbAllocMem)
1385	pcbAllocMem = cAllocPages _4K;
1386
1387	if (pcbFreeMem)
1388	pcbFreeMem = cFreePages _4K;
1389
1390	if (pcbBallonedMem)
1391	pcbBallonedMem = cBalloonPages _4K;
1392
1393	if (pcbSharedMem)
1394	pcbSharedMem = cSharedPages _4K;
1395
1396	Log(("PGMR3QueryVMMMemoryStats: all=%llx free=%llx ballooned=%llx shared=%llx\n",
1397	cAllocPages, cFreePages, cBalloonPages, cSharedPages));
1398	return VINF_SUCCESS;
1399	}
1400
1401
1402	/**
1403	* Query memory stats for the VM.
1404	*
1405	* @returns VBox status code.
1406	* @param pUVM The user mode VM handle.
1407	* @param pcbTotalMem Where to return total amount memory the VM may
1408	* possibly use.
1409	* @param pcbPrivateMem Where to return the amount of private memory
1410	* currently allocated.
1411	* @param pcbSharedMem Where to return the amount of actually shared
1412	* memory currently used by the VM.
1413	* @param pcbZeroMem Where to return the amount of memory backed by
1414	* zero pages.
1415	*
1416	* @remarks The total mem is normally larger than the sum of the three
1417	* components. There are two reasons for this, first the amount of
1418	* shared memory is what we're sure is shared instead of what could
1419	* possibly be shared with someone. Secondly, because the total may
1420	* include some pure MMIO pages that doesn't go into any of the three
1421	* sub-counts.
1422	*
1423	* @todo Why do we return reused shared pages instead of anything that could
1424	* potentially be shared? Doesn't this mean the first VM gets a much
1425	* lower number of shared pages?
1426	*/
1427	VMMR3DECL(int) PGMR3QueryMemoryStats(PUVM pUVM, uint64_t pcbTotalMem, uint64_t pcbPrivateMem,
1428	uint64_t pcbSharedMem, uint64_t pcbZeroMem)
1429	{
1430	UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
1431	PVM pVM = pUVM->pVM;
1432	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1433
1434	if (pcbTotalMem)
1435	pcbTotalMem = (uint64_t)pVM->pgm.s.cAllPages PAGE_SIZE;
1436
1437	if (pcbPrivateMem)
1438	pcbPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages PAGE_SIZE;
1439
1440	if (pcbSharedMem)
1441	pcbSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages PAGE_SIZE;
1442
1443	if (pcbZeroMem)
1444	pcbZeroMem = (uint64_t)pVM->pgm.s.cZeroPages PAGE_SIZE;
1445
1446	Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
1447	return VINF_SUCCESS;
1448	}
1449
1450
1451	/**
1452	* PGMR3PhysRegisterRam worker that initializes and links a RAM range.
1453	*
1454	* @param pVM Pointer to the VM.
1455	* @param pNew The new RAM range.
1456	* @param GCPhys The address of the RAM range.
1457	* @param GCPhysLast The last address of the RAM range.
1458	* @param RCPtrNew The RC address if the range is floating. NIL_RTRCPTR
1459	* if in HMA.
1460	* @param R0PtrNew Ditto for R0.
1461	* @param pszDesc The description.
1462	* @param pPrev The previous RAM range (for linking).
1463	*/
1464	static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
1465	RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
1466	{
1467	/*
1468	* Initialize the range.
1469	*/
1470	pNew->pSelfR0 = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
1471	pNew->pSelfRC = RCPtrNew != NIL_RTRCPTR ? RCPtrNew : MMHyperCCToRC(pVM, pNew);
1472	pNew->GCPhys = GCPhys;
1473	pNew->GCPhysLast = GCPhysLast;
1474	pNew->cb = GCPhysLast - GCPhys + 1;
1475	pNew->pszDesc = pszDesc;
1476	pNew->fFlags = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
1477	pNew->pvR3 = NULL;
1478	pNew->paLSPages = NULL;
1479
1480	uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
1481	RTGCPHYS iPage = cPages;
1482	while (iPage-- > 0)
1483	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
1484
1485	/* Update the page count stats. */
1486	pVM->pgm.s.cZeroPages += cPages;
1487	pVM->pgm.s.cAllPages += cPages;
1488
1489	/*
1490	* Link it.
1491	*/
1492	pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
1493	}
1494
1495
1496	/**
1497	* Relocate a floating RAM range.
1498	*
1499	* @copydoc FNPGMRELOCATE.
1500	*/
1501	static DECLCALLBACK(bool) pgmR3PhysRamRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser)
1502	{
1503	PPGMRAMRANGE pRam = (PPGMRAMRANGE)pvUser;
1504	Assert(pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
1505	Assert(pRam->pSelfRC == GCPtrOld + PAGE_SIZE);
1506
1507	switch (enmMode)
1508	{
1509	case PGMRELOCATECALL_SUGGEST:
1510	return true;
1511
1512	case PGMRELOCATECALL_RELOCATE:
1513	{
1514	/*
1515	* Update myself, then relink all the ranges and flush the RC TLB.
1516	*/
1517	pgmLock(pVM);
1518
1519	pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
1520
1521	pgmR3PhysRelinkRamRanges(pVM);
1522	for (unsigned i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
1523	pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
1524
1525	pgmUnlock(pVM);
1526	return true;
1527	}
1528
1529	default:
1530	AssertFailedReturn(false);
1531	}
1532	}
1533
1534
1535	/**
1536	* PGMR3PhysRegisterRam worker that registers a high chunk.
1537	*
1538	* @returns VBox status code.
1539	* @param pVM Pointer to the VM.
1540	* @param GCPhys The address of the RAM.
1541	* @param cRamPages The number of RAM pages to register.
1542	* @param cbChunk The size of the PGMRAMRANGE guest mapping.
1543	* @param iChunk The chunk number.
1544	* @param pszDesc The RAM range description.
1545	* @param ppPrev Previous RAM range pointer. In/Out.
1546	*/
1547	static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages,
1548	uint32_t cbChunk, uint32_t iChunk, const char *pszDesc,
1549	PPGMRAMRANGE *ppPrev)
1550	{
1551	const char *pszDescChunk = iChunk == 0
1552	? pszDesc
1553	: MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
1554	AssertReturn(pszDescChunk, VERR_NO_MEMORY);
1555
1556	/*
1557	* Allocate memory for the new chunk.
1558	*/
1559	size_t const cChunkPages = RT_ALIGN_Z(RT_UOFFSETOF(PGMRAMRANGE, aPages[cRamPages]), PAGE_SIZE) >> PAGE_SHIFT;
1560	PSUPPAGE paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
1561	AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
1562	RTR0PTR R0PtrChunk = NIL_RTR0PTR;
1563	void *pvChunk = NULL;
1564	int rc = SUPR3PageAllocEx(cChunkPages, 0 /fFlags/, &pvChunk,
1565	#if defined(VBOX_WITH_MORE_RING0_MEM_MAPPINGS)
1566	&R0PtrChunk,
1567	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE)
1568	HMIsEnabled(pVM) ? &R0PtrChunk : NULL,
1569	#else
1570	NULL,
1571	#endif
1572	paChunkPages);
1573	if (RT_SUCCESS(rc))
1574	{
1575	#if defined(VBOX_WITH_MORE_RING0_MEM_MAPPINGS)
1576	Assert(R0PtrChunk != NIL_RTR0PTR);
1577	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE)
1578	if (!HMIsEnabled(pVM))
1579	R0PtrChunk = NIL_RTR0PTR;
1580	#else
1581	R0PtrChunk = (uintptr_t)pvChunk;
1582	#endif
1583	memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
1584
1585	PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
1586
1587	/*
1588	* Create a mapping and map the pages into it.
1589	* We push these in below the HMA.
1590	*/
1591	RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
1592	rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /fFlags/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
1593	if (RT_SUCCESS(rc))
1594	{
1595	pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
1596
1597	RTGCPTR const GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
1598	RTGCPTR GCPtrPage = GCPtrChunk;
1599	for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
1600	rc = PGMMap(pVM, GCPtrPage, paChunkPages[iPage].Phys, PAGE_SIZE, 0);
1601	if (RT_SUCCESS(rc))
1602	{
1603	/*
1604	* Ok, init and link the range.
1605	*/
1606	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
1607	(RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
1608	*ppPrev = pNew;
1609	}
1610	}
1611
1612	if (RT_FAILURE(rc))
1613	SUPR3PageFreeEx(pvChunk, cChunkPages);
1614	}
1615
1616	RTMemTmpFree(paChunkPages);
1617	return rc;
1618	}
1619
1620
1621	/**
1622	* Sets up a range RAM.
1623	*
1624	* This will check for conflicting registrations, make a resource
1625	* reservation for the memory (with GMM), and setup the per-page
1626	* tracking structures (PGMPAGE).
1627	*
1628	* @returns VBox status code.
1629	* @param pVM Pointer to the VM.
1630	* @param GCPhys The physical address of the RAM.
1631	* @param cb The size of the RAM.
1632	* @param pszDesc The description - not copied, so, don't free or change it.
1633	*/
1634	VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
1635	{
1636	/*
1637	* Validate input.
1638	*/
1639	Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
1640	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
1641	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
1642	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
1643	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1644	AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
1645	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1646	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1647
1648	pgmLock(pVM);
1649
1650	/*
1651	* Find range location and check for conflicts.
1652	* (We don't lock here because the locking by EMT is only required on update.)
1653	*/
1654	PPGMRAMRANGE pPrev = NULL;
1655	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
1656	while (pRam && GCPhysLast >= pRam->GCPhys)
1657	{
1658	if ( GCPhysLast >= pRam->GCPhys
1659	&& GCPhys <= pRam->GCPhysLast)
1660	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
1661	GCPhys, GCPhysLast, pszDesc,
1662	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1663	VERR_PGM_RAM_CONFLICT);
1664
1665	/* next */
1666	pPrev = pRam;
1667	pRam = pRam->pNextR3;
1668	}
1669
1670	/*
1671	* Register it with GMM (the API bitches).
1672	*/
1673	const RTGCPHYS cPages = cb >> PAGE_SHIFT;
1674	int rc = MMR3IncreaseBaseReservation(pVM, cPages);
1675	if (RT_FAILURE(rc))
1676	{
1677	pgmUnlock(pVM);
1678	return rc;
1679	}
1680
1681	if ( GCPhys >= _4G
1682	&& cPages > 256)
1683	{
1684	/*
1685	* The PGMRAMRANGE structures for the high memory can get very big.
1686	* In order to avoid SUPR3PageAllocEx allocation failures due to the
1687	* allocation size limit there and also to avoid being unable to find
1688	* guest mapping space for them, we split this memory up into 4MB in
1689	* (potential) raw-mode configs and 16MB chunks in forced AMD-V/VT-x
1690	* mode.
1691	*
1692	* The first and last page of each mapping are guard pages and marked
1693	* not-present. So, we've got 4186112 and 16769024 bytes available for
1694	* the PGMRAMRANGE structure.
1695	*
1696	* Note! The sizes used here will influence the saved state.
1697	*/
1698	uint32_t cbChunk;
1699	uint32_t cPagesPerChunk;
1700	if (HMIsEnabled(pVM))
1701	{
1702	cbChunk = 16U*_1M;
1703	cPagesPerChunk = 1048048; /* max ~1048059 */
1704	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 1048048 < 16U_1M - PAGE_SIZE 2);
1705	}
1706	else
1707	{
1708	cbChunk = 4U*_1M;
1709	cPagesPerChunk = 261616; /* max ~261627 */
1710	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 261616 < 4U_1M - PAGE_SIZE 2);
1711	}
1712	AssertRelease(RT_UOFFSETOF(PGMRAMRANGE, aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
1713
1714	RTGCPHYS cPagesLeft = cPages;
1715	RTGCPHYS GCPhysChunk = GCPhys;
1716	uint32_t iChunk = 0;
1717	while (cPagesLeft > 0)
1718	{
1719	uint32_t cPagesInChunk = cPagesLeft;
1720	if (cPagesInChunk > cPagesPerChunk)
1721	cPagesInChunk = cPagesPerChunk;
1722
1723	rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, cbChunk, iChunk, pszDesc, &pPrev);
1724	AssertRCReturn(rc, rc);
1725
1726	/* advance */
1727	GCPhysChunk += (RTGCPHYS)cPagesInChunk << PAGE_SHIFT;
1728	cPagesLeft -= cPagesInChunk;
1729	iChunk++;
1730	}
1731	}
1732	else
1733	{
1734	/*
1735	* Allocate, initialize and link the new RAM range.
1736	*/
1737	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
1738	PPGMRAMRANGE pNew;
1739	rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
1740	AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
1741
1742	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
1743	}
1744	pgmPhysInvalidatePageMapTLB(pVM);
1745	pgmUnlock(pVM);
1746
1747	#ifdef VBOX_WITH_REM
1748	/*
1749	* Notify REM.
1750	*/
1751	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_RAM);
1752	#endif
1753
1754	return VINF_SUCCESS;
1755	}
1756
1757
1758	/**
1759	* Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
1760	*
1761	* We do this late in the init process so that all the ROM and MMIO ranges have
1762	* been registered already and we don't go wasting memory on them.
1763	*
1764	* @returns VBox status code.
1765	*
1766	* @param pVM Pointer to the VM.
1767	*/
1768	int pgmR3PhysRamPreAllocate(PVM pVM)
1769	{
1770	Assert(pVM->pgm.s.fRamPreAlloc);
1771	Log(("pgmR3PhysRamPreAllocate: enter\n"));
1772
1773	/*
1774	* Walk the RAM ranges and allocate all RAM pages, halt at
1775	* the first allocation error.
1776	*/
1777	uint64_t cPages = 0;
1778	uint64_t NanoTS = RTTimeNanoTS();
1779	pgmLock(pVM);
1780	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1781	{
1782	PPGMPAGE pPage = &pRam->aPages[0];
1783	RTGCPHYS GCPhys = pRam->GCPhys;
1784	uint32_t cLeft = pRam->cb >> PAGE_SHIFT;
1785	while (cLeft-- > 0)
1786	{
1787	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1788	{
1789	switch (PGM_PAGE_GET_STATE(pPage))
1790	{
1791	case PGM_PAGE_STATE_ZERO:
1792	{
1793	int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
1794	if (RT_FAILURE(rc))
1795	{
1796	LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
1797	pgmUnlock(pVM);
1798	return rc;
1799	}
1800	cPages++;
1801	break;
1802	}
1803
1804	case PGM_PAGE_STATE_BALLOONED:
1805	case PGM_PAGE_STATE_ALLOCATED:
1806	case PGM_PAGE_STATE_WRITE_MONITORED:
1807	case PGM_PAGE_STATE_SHARED:
1808	/* nothing to do here. */
1809	break;
1810	}
1811	}
1812
1813	/* next */
1814	pPage++;
1815	GCPhys += PAGE_SIZE;
1816	}
1817	}
1818	pgmUnlock(pVM);
1819	NanoTS = RTTimeNanoTS() - NanoTS;
1820
1821	LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
1822	Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
1823	return VINF_SUCCESS;
1824	}
1825
1826
1827	/**
1828	* Checks shared page checksums.
1829	*
1830	* @param pVM Pointer to the shared VM structure.
1831	*/
1832	void pgmR3PhysAssertSharedPageChecksums(PVM pVM)
1833	{
1834	#ifdef VBOX_STRICT
1835	pgmLock(pVM);
1836
1837	if (pVM->pgm.s.cSharedPages > 0)
1838	{
1839	/*
1840	* Walk the ram ranges.
1841	*/
1842	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1843	{
1844	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1845	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1846
1847	while (iPage-- > 0)
1848	{
1849	PPGMPAGE pPage = &pRam->aPages[iPage];
1850	if (PGM_PAGE_IS_SHARED(pPage))
1851	{
1852	uint32_t u32Checksum = pPage->s.u2Unused0 \| ((uint32_t)pPage->s.u2Unused1 << 8);
1853	if (!u32Checksum)
1854	{
1855	RTGCPHYS GCPhysPage = pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
1856	void const *pvPage;
1857	int rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhysPage, &pvPage);
1858	if (RT_SUCCESS(rc))
1859	{
1860	uint32_t u32Checksum2 = RTCrc32(pvPage, PAGE_SIZE);
1861	# if 0
1862	AssertMsg((u32Checksum2 & UINT32_C(0x00000303)) == u32Checksum, ("GCPhysPage=%RGp\n", GCPhysPage));
1863	# else
1864	if ((u32Checksum2 & UINT32_C(0x00000303)) == u32Checksum)
1865	LogFlow(("shpg %#x @ %RGp %#x [OK]\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
1866	else
1867	AssertMsgFailed(("shpg %#x @ %RGp %#x\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
1868	# endif
1869	}
1870	else
1871	AssertRC(rc);
1872	}
1873	}
1874
1875	} /* for each page */
1876
1877	} /* for each ram range */
1878	}
1879
1880	pgmUnlock(pVM);
1881	#endif /* VBOX_STRICT */
1882	NOREF(pVM);
1883	}
1884
1885
1886	/**
1887	* Resets the physical memory state.
1888	*
1889	* ASSUMES that the caller owns the PGM lock.
1890	*
1891	* @returns VBox status code.
1892	* @param pVM Pointer to the VM.
1893	*/
1894	int pgmR3PhysRamReset(PVM pVM)
1895	{
1896	PGM_LOCK_ASSERT_OWNER(pVM);
1897
1898	/* Reset the memory balloon. */
1899	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1900	AssertRC(rc);
1901
1902	#ifdef VBOX_WITH_PAGE_SHARING
1903	/* Clear all registered shared modules. */
1904	pgmR3PhysAssertSharedPageChecksums(pVM);
1905	rc = GMMR3ResetSharedModules(pVM);
1906	AssertRC(rc);
1907	#endif
1908	/* Reset counters. */
1909	pVM->pgm.s.cReusedSharedPages = 0;
1910	pVM->pgm.s.cBalloonedPages = 0;
1911
1912	return VINF_SUCCESS;
1913	}
1914
1915
1916	/**
1917	* Resets (zeros) the RAM after all devices and components have been reset.
1918	*
1919	* ASSUMES that the caller owns the PGM lock.
1920	*
1921	* @returns VBox status code.
1922	* @param pVM Pointer to the VM.
1923	*/
1924	int pgmR3PhysRamZeroAll(PVM pVM)
1925	{
1926	PGM_LOCK_ASSERT_OWNER(pVM);
1927
1928	/*
1929	* We batch up pages that should be freed instead of calling GMM for
1930	* each and every one of them.
1931	*/
1932	uint32_t cPendingPages = 0;
1933	PGMMFREEPAGESREQ pReq;
1934	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1935	AssertLogRelRCReturn(rc, rc);
1936
1937	/*
1938	* Walk the ram ranges.
1939	*/
1940	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1941	{
1942	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1943	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1944
1945	#ifndef NO_RAM_RESET
1946	if (!pVM->pgm.s.fRamPreAlloc)
1947	#else
1948	if (0)
1949	#endif
1950	{
1951	/* Replace all RAM pages by ZERO pages. */
1952	while (iPage-- > 0)
1953	{
1954	PPGMPAGE pPage = &pRam->aPages[iPage];
1955	switch (PGM_PAGE_GET_TYPE(pPage))
1956	{
1957	case PGMPAGETYPE_RAM:
1958	/* Do not replace pages part of a 2 MB continuous range
1959	with zero pages, but zero them instead. */
1960	if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE
1961	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
1962	{
1963	void *pvPage;
1964	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1965	AssertLogRelRCReturn(rc, rc);
1966	ASMMemZeroPage(pvPage);
1967	}
1968	else if (PGM_PAGE_IS_BALLOONED(pPage))
1969	{
1970	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
1971	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1972	}
1973	else if (!PGM_PAGE_IS_ZERO(pPage))
1974	{
1975	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1976	AssertLogRelRCReturn(rc, rc);
1977	}
1978	break;
1979
1980	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1981	case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone? I don't think VT-x copes with this code. */
1982	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
1983	true /fDoAccounting/);
1984	break;
1985
1986	case PGMPAGETYPE_MMIO2:
1987	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
1988	case PGMPAGETYPE_ROM:
1989	case PGMPAGETYPE_MMIO:
1990	break;
1991	default:
1992	AssertFailed();
1993	}
1994	} /* for each page */
1995	}
1996	else
1997	{
1998	/* Zero the memory. */
1999	while (iPage-- > 0)
2000	{
2001	PPGMPAGE pPage = &pRam->aPages[iPage];
2002	switch (PGM_PAGE_GET_TYPE(pPage))
2003	{
2004	case PGMPAGETYPE_RAM:
2005	switch (PGM_PAGE_GET_STATE(pPage))
2006	{
2007	case PGM_PAGE_STATE_ZERO:
2008	break;
2009
2010	case PGM_PAGE_STATE_BALLOONED:
2011	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
2012	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
2013	break;
2014
2015	case PGM_PAGE_STATE_SHARED:
2016	case PGM_PAGE_STATE_WRITE_MONITORED:
2017	rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
2018	AssertLogRelRCReturn(rc, rc);
2019	/* no break */
2020
2021	case PGM_PAGE_STATE_ALLOCATED:
2022	{
2023	void *pvPage;
2024	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
2025	AssertLogRelRCReturn(rc, rc);
2026	#ifndef NO_RAM_RESET
2027	ASMMemZeroPage(pvPage);
2028	#endif
2029	break;
2030	}
2031	}
2032	break;
2033
2034	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2035	case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone? I don't think VT-x copes with this code. */
2036	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
2037	true /fDoAccounting/);
2038	break;
2039
2040	case PGMPAGETYPE_MMIO2:
2041	case PGMPAGETYPE_ROM_SHADOW:
2042	case PGMPAGETYPE_ROM:
2043	case PGMPAGETYPE_MMIO:
2044	break;
2045	default:
2046	AssertFailed();
2047
2048	}
2049	} /* for each page */
2050	}
2051
2052	}
2053
2054	/*
2055	* Finish off any pages pending freeing.
2056	*/
2057	if (cPendingPages)
2058	{
2059	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2060	AssertLogRelRCReturn(rc, rc);
2061	}
2062	GMMR3FreePagesCleanup(pReq);
2063	return VINF_SUCCESS;
2064	}
2065
2066
2067	/**
2068	* Frees all RAM during VM termination
2069	*
2070	* ASSUMES that the caller owns the PGM lock.
2071	*
2072	* @returns VBox status code.
2073	* @param pVM Pointer to the VM.
2074	*/
2075	int pgmR3PhysRamTerm(PVM pVM)
2076	{
2077	PGM_LOCK_ASSERT_OWNER(pVM);
2078
2079	/* Reset the memory balloon. */
2080	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
2081	AssertRC(rc);
2082
2083	#ifdef VBOX_WITH_PAGE_SHARING
2084	/*
2085	* Clear all registered shared modules.
2086	*/
2087	pgmR3PhysAssertSharedPageChecksums(pVM);
2088	rc = GMMR3ResetSharedModules(pVM);
2089	AssertRC(rc);
2090
2091	/*
2092	* Flush the handy pages updates to make sure no shared pages are hiding
2093	* in there. (No unlikely if the VM shuts down, apparently.)
2094	*/
2095	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_FLUSH_HANDY_PAGES, 0, NULL);
2096	#endif
2097
2098	/*
2099	* We batch up pages that should be freed instead of calling GMM for
2100	* each and every one of them.
2101	*/
2102	uint32_t cPendingPages = 0;
2103	PGMMFREEPAGESREQ pReq;
2104	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2105	AssertLogRelRCReturn(rc, rc);
2106
2107	/*
2108	* Walk the ram ranges.
2109	*/
2110	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
2111	{
2112	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
2113	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
2114
2115	while (iPage-- > 0)
2116	{
2117	PPGMPAGE pPage = &pRam->aPages[iPage];
2118	switch (PGM_PAGE_GET_TYPE(pPage))
2119	{
2120	case PGMPAGETYPE_RAM:
2121	/* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
2122	/** @todo change this to explicitly free private pages here. */
2123	if (PGM_PAGE_IS_SHARED(pPage))
2124	{
2125	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
2126	AssertLogRelRCReturn(rc, rc);
2127	}
2128	break;
2129
2130	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2131	case PGMPAGETYPE_SPECIAL_ALIAS_MMIO:
2132	case PGMPAGETYPE_MMIO2:
2133	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2134	case PGMPAGETYPE_ROM:
2135	case PGMPAGETYPE_MMIO:
2136	break;
2137	default:
2138	AssertFailed();
2139	}
2140	} /* for each page */
2141	}
2142
2143	/*
2144	* Finish off any pages pending freeing.
2145	*/
2146	if (cPendingPages)
2147	{
2148	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2149	AssertLogRelRCReturn(rc, rc);
2150	}
2151	GMMR3FreePagesCleanup(pReq);
2152	return VINF_SUCCESS;
2153	}
2154
2155
2156	/**
2157	* This is the interface IOM is using to register an MMIO region.
2158	*
2159	* It will check for conflicts and ensure that a RAM range structure
2160	* is present before calling the PGMR3HandlerPhysicalRegister API to
2161	* register the callbacks.
2162	*
2163	* @returns VBox status code.
2164	*
2165	* @param pVM Pointer to the VM.
2166	* @param GCPhys The start of the MMIO region.
2167	* @param cb The size of the MMIO region.
2168	* @param hType The physical access handler type registration.
2169	* @param pvUserR3 The user argument for R3.
2170	* @param pvUserR0 The user argument for R0.
2171	* @param pvUserRC The user argument for RC.
2172	* @param pszDesc The description of the MMIO region.
2173	*/
2174	VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMPHYSHANDLERTYPE hType,
2175	RTR3PTR pvUserR3, RTR0PTR pvUserR0, RTRCPTR pvUserRC, const char *pszDesc)
2176	{
2177	/*
2178	* Assert on some assumption.
2179	*/
2180	VM_ASSERT_EMT(pVM);
2181	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2182	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2183	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2184	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2185	Assert(((PPGMPHYSHANDLERTYPEINT)MMHyperHeapOffsetToPtr(pVM, hType))->enmKind == PGMPHYSHANDLERKIND_MMIO);
2186
2187	int rc = pgmLock(pVM);
2188	AssertRCReturn(rc, rc);
2189
2190	/*
2191	* Make sure there's a RAM range structure for the region.
2192	*/
2193	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2194	bool fRamExists = false;
2195	PPGMRAMRANGE pRamPrev = NULL;
2196	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2197	while (pRam && GCPhysLast >= pRam->GCPhys)
2198	{
2199	if ( GCPhysLast >= pRam->GCPhys
2200	&& GCPhys <= pRam->GCPhysLast)
2201	{
2202	/* Simplification: all within the same range. */
2203	AssertLogRelMsgReturnStmt( GCPhys >= pRam->GCPhys
2204	&& GCPhysLast <= pRam->GCPhysLast,
2205	("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
2206	GCPhys, GCPhysLast, pszDesc,
2207	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2208	pgmUnlock(pVM),
2209	VERR_PGM_RAM_CONFLICT);
2210
2211	/* Check that it's all RAM or MMIO pages. */
2212	PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2213	uint32_t cLeft = cb >> PAGE_SHIFT;
2214	while (cLeft-- > 0)
2215	{
2216	AssertLogRelMsgReturnStmt( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2217	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
2218	("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
2219	GCPhys, GCPhysLast, pszDesc, pRam->GCPhys, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
2220	pgmUnlock(pVM),
2221	VERR_PGM_RAM_CONFLICT);
2222	pPage++;
2223	}
2224
2225	/* Looks good. */
2226	fRamExists = true;
2227	break;
2228	}
2229
2230	/* next */
2231	pRamPrev = pRam;
2232	pRam = pRam->pNextR3;
2233	}
2234	PPGMRAMRANGE pNew;
2235	if (fRamExists)
2236	{
2237	pNew = NULL;
2238
2239	/*
2240	* Make all the pages in the range MMIO/ZERO pages, freeing any
2241	* RAM pages currently mapped here. This might not be 100% correct
2242	* for PCI memory, but we're doing the same thing for MMIO2 pages.
2243	*/
2244	rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
2245	AssertRCReturnStmt(rc, pgmUnlock(pVM), rc);
2246
2247	/* Force a PGM pool flush as guest ram references have been changed. */
2248	/** @todo not entirely SMP safe; assuming for now the guest takes
2249	* care of this internally (not touch mapped mmio while changing the
2250	* mapping). */
2251	PVMCPU pVCpu = VMMGetCpu(pVM);
2252	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2253	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2254	}
2255	else
2256	{
2257
2258	/*
2259	* No RAM range, insert an ad hoc one.
2260	*
2261	* Note that we don't have to tell REM about this range because
2262	* PGMHandlerPhysicalRegisterEx will do that for us.
2263	*/
2264	Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
2265
2266	const uint32_t cPages = cb >> PAGE_SHIFT;
2267	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
2268	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
2269	AssertLogRelMsgRCReturnStmt(rc, ("cbRamRange=%zu\n", cbRamRange), pgmUnlock(pVM), rc);
2270
2271	/* Initialize the range. */
2272	pNew->pSelfR0 = MMHyperCCToR0(pVM, pNew);
2273	pNew->pSelfRC = MMHyperCCToRC(pVM, pNew);
2274	pNew->GCPhys = GCPhys;
2275	pNew->GCPhysLast = GCPhysLast;
2276	pNew->cb = cb;
2277	pNew->pszDesc = pszDesc;
2278	pNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
2279	pNew->pvR3 = NULL;
2280	pNew->paLSPages = NULL;
2281
2282	uint32_t iPage = cPages;
2283	while (iPage-- > 0)
2284	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
2285	Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
2286
2287	/* update the page count stats. */
2288	pVM->pgm.s.cPureMmioPages += cPages;
2289	pVM->pgm.s.cAllPages += cPages;
2290
2291	/* link it */
2292	pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
2293	}
2294
2295	/*
2296	* Register the access handler.
2297	*/
2298	rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, hType, pvUserR3, pvUserR0, pvUserRC, pszDesc);
2299	if ( RT_FAILURE(rc)
2300	&& !fRamExists)
2301	{
2302	pVM->pgm.s.cPureMmioPages -= cb >> PAGE_SHIFT;
2303	pVM->pgm.s.cAllPages -= cb >> PAGE_SHIFT;
2304
2305	/* remove the ad hoc range. */
2306	pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
2307	pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
2308	MMHyperFree(pVM, pRam);
2309	}
2310	pgmPhysInvalidatePageMapTLB(pVM);
2311
2312	pgmUnlock(pVM);
2313	return rc;
2314	}
2315
2316
2317	/**
2318	* This is the interface IOM is using to register an MMIO region.
2319	*
2320	* It will take care of calling PGMHandlerPhysicalDeregister and clean up
2321	* any ad hoc PGMRAMRANGE left behind.
2322	*
2323	* @returns VBox status code.
2324	* @param pVM Pointer to the VM.
2325	* @param GCPhys The start of the MMIO region.
2326	* @param cb The size of the MMIO region.
2327	*/
2328	VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
2329	{
2330	VM_ASSERT_EMT(pVM);
2331
2332	int rc = pgmLock(pVM);
2333	AssertRCReturn(rc, rc);
2334
2335	/*
2336	* First deregister the handler, then check if we should remove the ram range.
2337	*/
2338	rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
2339	if (RT_SUCCESS(rc))
2340	{
2341	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2342	PPGMRAMRANGE pRamPrev = NULL;
2343	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2344	while (pRam && GCPhysLast >= pRam->GCPhys)
2345	{
2346	/** @todo We're being a bit too careful here. rewrite. */
2347	if ( GCPhysLast == pRam->GCPhysLast
2348	&& GCPhys == pRam->GCPhys)
2349	{
2350	Assert(pRam->cb == cb);
2351
2352	/*
2353	* See if all the pages are dead MMIO pages.
2354	*/
2355	uint32_t const cPages = cb >> PAGE_SHIFT;
2356	bool fAllMMIO = true;
2357	uint32_t iPage = 0;
2358	uint32_t cLeft = cPages;
2359	while (cLeft-- > 0)
2360	{
2361	PPGMPAGE pPage = &pRam->aPages[iPage];
2362	if ( !PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)
2363	/\|\| not-out-of-action later /)
2364	{
2365	fAllMMIO = false;
2366	AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2367	break;
2368	}
2369	Assert( PGM_PAGE_IS_ZERO(pPage)
2370	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
2371	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO);
2372	pPage++;
2373	}
2374	if (fAllMMIO)
2375	{
2376	/*
2377	* Ad-hoc range, unlink and free it.
2378	*/
2379	Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
2380	GCPhys, GCPhysLast, pRam->pszDesc));
2381
2382	pVM->pgm.s.cAllPages -= cPages;
2383	pVM->pgm.s.cPureMmioPages -= cPages;
2384
2385	pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
2386	pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
2387	MMHyperFree(pVM, pRam);
2388	break;
2389	}
2390	}
2391
2392	/*
2393	* Range match? It will all be within one range (see PGMAllHandler.cpp).
2394	*/
2395	if ( GCPhysLast >= pRam->GCPhys
2396	&& GCPhys <= pRam->GCPhysLast)
2397	{
2398	Assert(GCPhys >= pRam->GCPhys);
2399	Assert(GCPhysLast <= pRam->GCPhysLast);
2400
2401	/*
2402	* Turn the pages back into RAM pages.
2403	*/
2404	uint32_t iPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2405	uint32_t cLeft = cb >> PAGE_SHIFT;
2406	while (cLeft--)
2407	{
2408	PPGMPAGE pPage = &pRam->aPages[iPage];
2409	AssertMsg( (PGM_PAGE_IS_MMIO(pPage) && PGM_PAGE_IS_ZERO(pPage))
2410	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
2411	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO,
2412	("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2413	if (PGM_PAGE_IS_MMIO_OR_ALIAS(pPage))
2414	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_RAM);
2415	}
2416	break;
2417	}
2418
2419	/* next */
2420	pRamPrev = pRam;
2421	pRam = pRam->pNextR3;
2422	}
2423	}
2424
2425	/* Force a PGM pool flush as guest ram references have been changed. */
2426	/** @todo Not entirely SMP safe; assuming for now the guest takes care of
2427	* this internally (not touch mapped mmio while changing the mapping). */
2428	PVMCPU pVCpu = VMMGetCpu(pVM);
2429	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2430	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2431
2432	pgmPhysInvalidatePageMapTLB(pVM);
2433	pgmPhysInvalidRamRangeTlbs(pVM);
2434	pgmUnlock(pVM);
2435	return rc;
2436	}
2437
2438
2439	/**
2440	* Locate a MMIO2 range.
2441	*
2442	* @returns Pointer to the MMIO2 range.
2443	* @param pVM Pointer to the VM.
2444	* @param pDevIns The device instance owning the region.
2445	* @param iRegion The region.
2446	*/
2447	DECLINLINE(PPGMMMIO2RANGE) pgmR3PhysMMIO2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2448	{
2449	/*
2450	* Search the list.
2451	*/
2452	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2453	if ( pCur->pDevInsR3 == pDevIns
2454	&& pCur->iRegion == iRegion)
2455	return pCur;
2456	return NULL;
2457	}
2458
2459
2460	/**
2461	* Allocate and register an MMIO2 region.
2462	*
2463	* As mentioned elsewhere, MMIO2 is just RAM spelled differently. It's RAM
2464	* associated with a device. It is also non-shared memory with a permanent
2465	* ring-3 mapping and page backing (presently).
2466	*
2467	* A MMIO2 range may overlap with base memory if a lot of RAM is configured for
2468	* the VM, in which case we'll drop the base memory pages. Presently we will
2469	* make no attempt to preserve anything that happens to be present in the base
2470	* memory that is replaced, this is of course incorrect but it's too much
2471	* effort.
2472	*
2473	* @returns VBox status code.
2474	* @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
2475	* memory.
2476	* @retval VERR_ALREADY_EXISTS if the region already exists.
2477	*
2478	* @param pVM Pointer to the VM.
2479	* @param pDevIns The device instance owning the region.
2480	* @param iRegion The region number. If the MMIO2 memory is a PCI
2481	* I/O region this number has to be the number of that
2482	* region. Otherwise it can be any number safe
2483	* UINT8_MAX.
2484	* @param cb The size of the region. Must be page aligned.
2485	* @param fFlags Reserved for future use, must be zero.
2486	* @param ppv Where to store the pointer to the ring-3 mapping of
2487	* the memory.
2488	* @param pszDesc The description.
2489	*/
2490	VMMR3DECL(int) PGMR3PhysMMIO2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags,
2491	void *ppv, const char pszDesc)
2492	{
2493	/*
2494	* Validate input.
2495	*/
2496	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2497	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2498	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2499	AssertPtrReturn(ppv, VERR_INVALID_POINTER);
2500	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2501	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2502	AssertReturn(pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion) == NULL, VERR_ALREADY_EXISTS);
2503	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2504	AssertReturn(cb, VERR_INVALID_PARAMETER);
2505	AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
2506
2507	const uint32_t cPages = cb >> PAGE_SHIFT;
2508	AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
2509	AssertLogRelReturn(cPages <= PGM_MMIO2_MAX_PAGE_COUNT, VERR_NO_MEMORY);
2510
2511	/*
2512	* For the 2nd+ instance, mangle the description string so it's unique.
2513	*/
2514	if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
2515	{
2516	pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
2517	if (!pszDesc)
2518	return VERR_NO_MEMORY;
2519	}
2520
2521	/*
2522	* Allocate an MMIO2 range ID (not freed on failure).
2523	* The zero ID is not used as it could be confused with NIL_GMM_PAGEID.
2524	*/
2525	pgmLock(pVM);
2526	uint8_t idMmio2 = pVM->pgm.s.cMmio2Regions + 1;
2527	if (idMmio2 > PGM_MMIO2_MAX_RANGES)
2528	{
2529	pgmUnlock(pVM);
2530	AssertLogRelFailedReturn(VERR_PGM_TOO_MANY_MMIO2_RANGES);
2531	}
2532	pVM->pgm.s.cMmio2Regions = idMmio2;
2533	pgmUnlock(pVM);
2534
2535	/*
2536	* Try reserve and allocate the backing memory first as this is what is
2537	* most likely to fail.
2538	*/
2539	int rc = MMR3AdjustFixedReservation(pVM, cPages, pszDesc);
2540	if (RT_SUCCESS(rc))
2541	{
2542	void *pvPages;
2543	PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(SUPPAGE));
2544	if (RT_SUCCESS(rc))
2545	rc = SUPR3PageAllocEx(cPages, 0 /fFlags/, &pvPages, NULL /pR0Ptr/, paPages);
2546	if (RT_SUCCESS(rc))
2547	{
2548	memset(pvPages, 0, cPages * PAGE_SIZE);
2549
2550	/*
2551	* Create the MMIO2 range record for it.
2552	*/
2553	const size_t cbRange = RT_OFFSETOF(PGMMMIO2RANGE, RamRange.aPages[cPages]);
2554	PPGMMMIO2RANGE pNew;
2555	rc = MMR3HyperAllocOnceNoRel(pVM, cbRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
2556	AssertLogRelMsgRC(rc, ("cbRamRange=%zu\n", cbRange));
2557	if (RT_SUCCESS(rc))
2558	{
2559	pNew->pDevInsR3 = pDevIns;
2560	pNew->pvR3 = pvPages;
2561	//pNew->pNext = NULL;
2562	//pNew->fMapped = false;
2563	//pNew->fOverlapping = false;
2564	pNew->iRegion = iRegion;
2565	pNew->idSavedState = UINT8_MAX;
2566	pNew->idMmio2 = idMmio2;
2567	pNew->RamRange.pSelfR0 = MMHyperCCToR0(pVM, &pNew->RamRange);
2568	pNew->RamRange.pSelfRC = MMHyperCCToRC(pVM, &pNew->RamRange);
2569	pNew->RamRange.GCPhys = NIL_RTGCPHYS;
2570	pNew->RamRange.GCPhysLast = NIL_RTGCPHYS;
2571	pNew->RamRange.pszDesc = pszDesc;
2572	pNew->RamRange.cb = cb;
2573	pNew->RamRange.fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2;
2574	pNew->RamRange.pvR3 = pvPages;
2575	//pNew->RamRange.paLSPages = NULL;
2576
2577	uint32_t iPage = cPages;
2578	while (iPage-- > 0)
2579	{
2580	PGM_PAGE_INIT(&pNew->RamRange.aPages[iPage],
2581	paPages[iPage].Phys,
2582	PGM_MMIO2_PAGEID_MAKE(idMmio2, iPage),
2583	PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
2584	}
2585
2586	/* update page count stats */
2587	pVM->pgm.s.cAllPages += cPages;
2588	pVM->pgm.s.cPrivatePages += cPages;
2589
2590	/*
2591	* Link it into the list.
2592	* Since there is no particular order, just push it.
2593	*/
2594	/** @todo we can save us the linked list now, just search the lookup table... */
2595	pgmLock(pVM);
2596	Assert(pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] == NULL);
2597	Assert(pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] == NIL_RTR0PTR);
2598	pNew->pNextR3 = pVM->pgm.s.pMmio2RangesR3;
2599	pVM->pgm.s.pMmio2RangesR3 = pNew;
2600	pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] = pNew;
2601	pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] = MMHyperCCToR0(pVM, pNew);
2602	pgmUnlock(pVM);
2603
2604	*ppv = pvPages;
2605	RTMemTmpFree(paPages);
2606	pgmPhysInvalidatePageMapTLB(pVM);
2607	return VINF_SUCCESS;
2608	}
2609
2610	SUPR3PageFreeEx(pvPages, cPages);
2611	}
2612	RTMemTmpFree(paPages);
2613	MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pszDesc);
2614	}
2615	if (pDevIns->iInstance > 0)
2616	MMR3HeapFree((void *)pszDesc);
2617	return rc;
2618	}
2619
2620
2621	/**
2622	* Deregisters and frees an MMIO2 region.
2623	*
2624	* Any physical (and virtual) access handlers registered for the region must
2625	* be deregistered before calling this function.
2626	*
2627	* @returns VBox status code.
2628	* @param pVM Pointer to the VM.
2629	* @param pDevIns The device instance owning the region.
2630	* @param iRegion The region. If it's UINT32_MAX it'll be a wildcard match.
2631	*/
2632	VMMR3DECL(int) PGMR3PhysMMIO2Deregister(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2633	{
2634	/*
2635	* Validate input.
2636	*/
2637	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2638	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2639	AssertReturn(iRegion <= UINT8_MAX \|\| iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
2640
2641	pgmLock(pVM);
2642	int rc = VINF_SUCCESS;
2643	unsigned cFound = 0;
2644	PPGMMMIO2RANGE pPrev = NULL;
2645	PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3;
2646	while (pCur)
2647	{
2648	if ( pCur->pDevInsR3 == pDevIns
2649	&& ( iRegion == UINT32_MAX
2650	\|\| pCur->iRegion == iRegion))
2651	{
2652	cFound++;
2653
2654	/*
2655	* Unmap it if it's mapped.
2656	*/
2657	if (pCur->fMapped)
2658	{
2659	int rc2 = PGMR3PhysMMIO2Unmap(pVM, pCur->pDevInsR3, pCur->iRegion, pCur->RamRange.GCPhys);
2660	AssertRC(rc2);
2661	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2662	rc = rc2;
2663	}
2664
2665	/*
2666	* Unlink it
2667	*/
2668	PPGMMMIO2RANGE pNext = pCur->pNextR3;
2669	if (pPrev)
2670	pPrev->pNextR3 = pNext;
2671	else
2672	pVM->pgm.s.pMmio2RangesR3 = pNext;
2673	pCur->pNextR3 = NULL;
2674
2675	uint8_t idMmio2 = pCur->idMmio2;
2676	Assert(pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] == pCur);
2677	pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] = NULL;
2678	pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] = NIL_RTR0PTR;
2679
2680	/*
2681	* Free the memory.
2682	*/
2683	int rc2 = SUPR3PageFreeEx(pCur->pvR3, pCur->RamRange.cb >> PAGE_SHIFT);
2684	AssertRC(rc2);
2685	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2686	rc = rc2;
2687
2688	uint32_t const cPages = pCur->RamRange.cb >> PAGE_SHIFT;
2689	rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pCur->RamRange.pszDesc);
2690	AssertRC(rc2);
2691	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2692	rc = rc2;
2693
2694	/* we're leaking hyper memory here if done at runtime. */
2695	#ifdef VBOX_STRICT
2696	VMSTATE const enmState = VMR3GetState(pVM);
2697	AssertMsg( enmState == VMSTATE_POWERING_OFF
2698	\|\| enmState == VMSTATE_POWERING_OFF_LS
2699	\|\| enmState == VMSTATE_OFF
2700	\|\| enmState == VMSTATE_OFF_LS
2701	\|\| enmState == VMSTATE_DESTROYING
2702	\|\| enmState == VMSTATE_TERMINATED
2703	\|\| enmState == VMSTATE_CREATING
2704	, ("%s\n", VMR3GetStateName(enmState)));
2705	#endif
2706	/*rc = MMHyperFree(pVM, pCur);
2707	AssertRCReturn(rc, rc); - not safe, see the alloc call. */
2708
2709
2710	/* update page count stats */
2711	pVM->pgm.s.cAllPages -= cPages;
2712	pVM->pgm.s.cPrivatePages -= cPages;
2713
2714	/* next */
2715	pCur = pNext;
2716	}
2717	else
2718	{
2719	pPrev = pCur;
2720	pCur = pCur->pNextR3;
2721	}
2722	}
2723	pgmPhysInvalidatePageMapTLB(pVM);
2724	pgmUnlock(pVM);
2725	return !cFound && iRegion != UINT32_MAX ? VERR_NOT_FOUND : rc;
2726	}
2727
2728
2729	/**
2730	* Maps a MMIO2 region.
2731	*
2732	* This is done when a guest / the bios / state loading changes the
2733	* PCI config. The replacing of base memory has the same restrictions
2734	* as during registration, of course.
2735	*
2736	* @returns VBox status code.
2737	*
2738	* @param pVM Pointer to the VM.
2739	* @param pDevIns The device instance owning the region.
2740	* @param iRegion The index of the registered region.
2741	* @param GCPhys The guest-physical address to be remapped.
2742	*/
2743	VMMR3DECL(int) PGMR3PhysMMIO2Map(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2744	{
2745	/*
2746	* Validate input
2747	*/
2748	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2749	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2750	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2751	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2752	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2753	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2754
2755	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2756	AssertReturn(pCur, VERR_NOT_FOUND);
2757	AssertReturn(!pCur->fMapped, VERR_WRONG_ORDER);
2758	Assert(pCur->RamRange.GCPhys == NIL_RTGCPHYS);
2759	Assert(pCur->RamRange.GCPhysLast == NIL_RTGCPHYS);
2760
2761	const RTGCPHYS GCPhysLast = GCPhys + pCur->RamRange.cb - 1;
2762	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2763
2764	/*
2765	* Find our location in the ram range list, checking for
2766	* restriction we don't bother implementing yet (partially overlapping).
2767	*/
2768	bool fRamExists = false;
2769	PPGMRAMRANGE pRamPrev = NULL;
2770	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2771	while (pRam && GCPhysLast >= pRam->GCPhys)
2772	{
2773	if ( GCPhys <= pRam->GCPhysLast
2774	&& GCPhysLast >= pRam->GCPhys)
2775	{
2776	/* completely within? */
2777	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
2778	&& GCPhysLast <= pRam->GCPhysLast,
2779	("%RGp-%RGp (MMIO2/%s) falls partly outside %RGp-%RGp (%s)\n",
2780	GCPhys, GCPhysLast, pCur->RamRange.pszDesc,
2781	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2782	VERR_PGM_RAM_CONFLICT);
2783	fRamExists = true;
2784	break;
2785	}
2786
2787	/* next */
2788	pRamPrev = pRam;
2789	pRam = pRam->pNextR3;
2790	}
2791	if (fRamExists)
2792	{
2793	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2794	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2795	while (cPagesLeft-- > 0)
2796	{
2797	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
2798	("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
2799	GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pCur->RamRange.pszDesc),
2800	VERR_PGM_RAM_CONFLICT);
2801	pPage++;
2802	}
2803	}
2804	Log(("PGMR3PhysMMIO2Map: %RGp-%RGp fRamExists=%RTbool %s\n",
2805	GCPhys, GCPhysLast, fRamExists, pCur->RamRange.pszDesc));
2806
2807	/*
2808	* Make the changes.
2809	*/
2810	pgmLock(pVM);
2811
2812	pCur->RamRange.GCPhys = GCPhys;
2813	pCur->RamRange.GCPhysLast = GCPhysLast;
2814	pCur->fMapped = true;
2815	pCur->fOverlapping = fRamExists;
2816
2817	if (fRamExists)
2818	{
2819	/** @todo use pgmR3PhysFreePageRange here. */
2820	uint32_t cPendingPages = 0;
2821	PGMMFREEPAGESREQ pReq;
2822	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2823	AssertLogRelRCReturn(rc, rc);
2824
2825	/* replace the pages, freeing all present RAM pages. */
2826	PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2827	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2828	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2829	while (cPagesLeft-- > 0)
2830	{
2831	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
2832	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
2833
2834	RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
2835	uint32_t const idPage = PGM_PAGE_GET_PAGEID(pPageSrc);
2836	PGM_PAGE_SET_PAGEID(pVM, pPageDst, idPage);
2837	PGM_PAGE_SET_HCPHYS(pVM, pPageDst, HCPhys);
2838	PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO2);
2839	PGM_PAGE_SET_STATE(pVM, pPageDst, PGM_PAGE_STATE_ALLOCATED);
2840	PGM_PAGE_SET_PDE_TYPE(pVM, pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
2841	PGM_PAGE_SET_PTE_INDEX(pVM, pPageDst, 0);
2842	PGM_PAGE_SET_TRACKING(pVM, pPageDst, 0);
2843
2844	pVM->pgm.s.cZeroPages--;
2845	GCPhys += PAGE_SIZE;
2846	pPageSrc++;
2847	pPageDst++;
2848	}
2849
2850	/* Flush physical page map TLB. */
2851	pgmPhysInvalidatePageMapTLB(pVM);
2852
2853	if (cPendingPages)
2854	{
2855	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2856	AssertLogRelRCReturn(rc, rc);
2857	}
2858	GMMR3FreePagesCleanup(pReq);
2859
2860	/* Force a PGM pool flush as guest ram references have been changed. */
2861	/** @todo not entirely SMP safe; assuming for now the guest takes care of
2862	* this internally (not touch mapped mmio while changing the mapping). */
2863	PVMCPU pVCpu = VMMGetCpu(pVM);
2864	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2865	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2866
2867	pgmUnlock(pVM);
2868	}
2869	else
2870	{
2871	RTGCPHYS cb = pCur->RamRange.cb;
2872
2873	/* Clear the tracking data of pages we're going to reactivate. */
2874	PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2875	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2876	while (cPagesLeft-- > 0)
2877	{
2878	PGM_PAGE_SET_TRACKING(pVM, pPageSrc, 0);
2879	PGM_PAGE_SET_PTE_INDEX(pVM, pPageSrc, 0);
2880	pPageSrc++;
2881	}
2882
2883	/* link in the ram range */
2884	pgmR3PhysLinkRamRange(pVM, &pCur->RamRange, pRamPrev);
2885	pgmUnlock(pVM);
2886
2887	#ifdef VBOX_WITH_REM
2888	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_MMIO2);
2889	#endif
2890	}
2891
2892	pgmPhysInvalidatePageMapTLB(pVM);
2893	return VINF_SUCCESS;
2894	}
2895
2896
2897	/**
2898	* Unmaps a MMIO2 region.
2899	*
2900	* This is done when a guest / the bios / state loading changes the
2901	* PCI config. The replacing of base memory has the same restrictions
2902	* as during registration, of course.
2903	*/
2904	VMMR3DECL(int) PGMR3PhysMMIO2Unmap(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2905	{
2906	/*
2907	* Validate input
2908	*/
2909	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2910	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2911	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2912	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2913	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2914	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2915
2916	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2917	AssertReturn(pCur, VERR_NOT_FOUND);
2918	AssertReturn(pCur->fMapped, VERR_WRONG_ORDER);
2919	AssertReturn(pCur->RamRange.GCPhys == GCPhys, VERR_INVALID_PARAMETER);
2920	Assert(pCur->RamRange.GCPhysLast != NIL_RTGCPHYS);
2921
2922	Log(("PGMR3PhysMMIO2Unmap: %RGp-%RGp %s\n",
2923	pCur->RamRange.GCPhys, pCur->RamRange.GCPhysLast, pCur->RamRange.pszDesc));
2924
2925	/*
2926	* Unmap it.
2927	*/
2928	pgmLock(pVM);
2929
2930	#ifdef VBOX_WITH_REM
2931	RTGCPHYS GCPhysRangeREM;
2932	RTGCPHYS cbRangeREM;
2933	bool fInformREM;
2934	#endif
2935	if (pCur->fOverlapping)
2936	{
2937	/* Restore the RAM pages we've replaced. */
2938	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2939	while (pRam->GCPhys > pCur->RamRange.GCPhysLast)
2940	pRam = pRam->pNextR3;
2941
2942	PPGMPAGE pPageDst = &pRam->aPages[(pCur->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2943	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2944	while (cPagesLeft-- > 0)
2945	{
2946	PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
2947	pVM->pgm.s.cZeroPages++;
2948	pPageDst++;
2949	}
2950
2951	/* Flush physical page map TLB. */
2952	pgmPhysInvalidatePageMapTLB(pVM);
2953	#ifdef VBOX_WITH_REM
2954	GCPhysRangeREM = NIL_RTGCPHYS; /* shuts up gcc */
2955	cbRangeREM = RTGCPHYS_MAX; /* ditto */
2956	fInformREM = false;
2957	#endif
2958	}
2959	else
2960	{
2961	#ifdef VBOX_WITH_REM
2962	GCPhysRangeREM = pCur->RamRange.GCPhys;
2963	cbRangeREM = pCur->RamRange.cb;
2964	fInformREM = true;
2965	#endif
2966	pgmR3PhysUnlinkRamRange(pVM, &pCur->RamRange);
2967	}
2968
2969	pCur->RamRange.GCPhys = NIL_RTGCPHYS;
2970	pCur->RamRange.GCPhysLast = NIL_RTGCPHYS;
2971	pCur->fOverlapping = false;
2972	pCur->fMapped = false;
2973
2974	/* Force a PGM pool flush as guest ram references have been changed. */
2975	/** @todo not entirely SMP safe; assuming for now the guest takes care
2976	* of this internally (not touch mapped mmio while changing the
2977	* mapping). */
2978	PVMCPU pVCpu = VMMGetCpu(pVM);
2979	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2980	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2981
2982	pgmPhysInvalidatePageMapTLB(pVM);
2983	pgmPhysInvalidRamRangeTlbs(pVM);
2984	pgmUnlock(pVM);
2985
2986	#ifdef VBOX_WITH_REM
2987	if (fInformREM)
2988	REMR3NotifyPhysRamDeregister(pVM, GCPhysRangeREM, cbRangeREM);
2989	#endif
2990
2991	return VINF_SUCCESS;
2992	}
2993
2994
2995	/**
2996	* Checks if the given address is an MMIO2 base address or not.
2997	*
2998	* @returns true/false accordingly.
2999	* @param pVM Pointer to the VM.
3000	* @param pDevIns The owner of the memory, optional.
3001	* @param GCPhys The address to check.
3002	*/
3003	VMMR3DECL(bool) PGMR3PhysMMIO2IsBase(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
3004	{
3005	/*
3006	* Validate input
3007	*/
3008	VM_ASSERT_EMT_RETURN(pVM, false);
3009	AssertPtrReturn(pDevIns, false);
3010	AssertReturn(GCPhys != NIL_RTGCPHYS, false);
3011	AssertReturn(GCPhys != 0, false);
3012	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), false);
3013
3014	/*
3015	* Search the list.
3016	*/
3017	pgmLock(pVM);
3018	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
3019	if (pCur->RamRange.GCPhys == GCPhys)
3020	{
3021	Assert(pCur->fMapped);
3022	pgmUnlock(pVM);
3023	return true;
3024	}
3025	pgmUnlock(pVM);
3026	return false;
3027	}
3028
3029
3030	/**
3031	* Gets the HC physical address of a page in the MMIO2 region.
3032	*
3033	* This is API is intended for MMHyper and shouldn't be called
3034	* by anyone else...
3035	*
3036	* @returns VBox status code.
3037	* @param pVM Pointer to the VM.
3038	* @param pDevIns The owner of the memory, optional.
3039	* @param iRegion The region.
3040	* @param off The page expressed an offset into the MMIO2 region.
3041	* @param pHCPhys Where to store the result.
3042	*/
3043	VMMR3DECL(int) PGMR3PhysMMIO2GetHCPhys(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, PRTHCPHYS pHCPhys)
3044	{
3045	/*
3046	* Validate input
3047	*/
3048	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3049	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3050	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
3051
3052	pgmLock(pVM);
3053	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
3054	AssertReturn(pCur, VERR_NOT_FOUND);
3055	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
3056
3057	PCPGMPAGE pPage = &pCur->RamRange.aPages[off >> PAGE_SHIFT];
3058	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3059	pgmUnlock(pVM);
3060	return VINF_SUCCESS;
3061	}
3062
3063
3064	/**
3065	* Maps a portion of an MMIO2 region into kernel space (host).
3066	*
3067	* The kernel mapping will become invalid when the MMIO2 memory is deregistered
3068	* or the VM is terminated.
3069	*
3070	* @return VBox status code.
3071	*
3072	* @param pVM Pointer to the VM.
3073	* @param pDevIns The device owning the MMIO2 memory.
3074	* @param iRegion The region.
3075	* @param off The offset into the region. Must be page aligned.
3076	* @param cb The number of bytes to map. Must be page aligned.
3077	* @param pszDesc Mapping description.
3078	* @param pR0Ptr Where to store the R0 address.
3079	*/
3080	VMMR3DECL(int) PGMR3PhysMMIO2MapKernel(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
3081	const char *pszDesc, PRTR0PTR pR0Ptr)
3082	{
3083	/*
3084	* Validate input.
3085	*/
3086	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3087	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3088	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
3089
3090	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
3091	AssertReturn(pCur, VERR_NOT_FOUND);
3092	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
3093	AssertReturn(cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
3094	AssertReturn(off + cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
3095	NOREF(pszDesc);
3096
3097	/*
3098	* Pass the request on to the support library/driver.
3099	*/
3100	int rc = SUPR3PageMapKernel(pCur->pvR3, off, cb, 0, pR0Ptr);
3101
3102	return rc;
3103	}
3104
3105
3106	/**
3107	* Worker for PGMR3PhysRomRegister.
3108	*
3109	* This is here to simplify lock management, i.e. the caller does all the
3110	* locking and we can simply return without needing to remember to unlock
3111	* anything first.
3112	*
3113	* @returns VBox status.
3114	* @param pVM Pointer to the VM.
3115	* @param pDevIns The device instance owning the ROM.
3116	* @param GCPhys First physical address in the range.
3117	* Must be page aligned!
3118	* @param cb The size of the range (in bytes).
3119	* Must be page aligned!
3120	* @param pvBinary Pointer to the binary data backing the ROM image.
3121	* @param cbBinary The size of the binary data pvBinary points to.
3122	* This must be less or equal to @a cb.
3123	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
3124	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
3125	* @param pszDesc Pointer to description string. This must not be freed.
3126	*/
3127	static int pgmR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
3128	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
3129	{
3130	/*
3131	* Validate input.
3132	*/
3133	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3134	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
3135	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
3136	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
3137	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3138	AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
3139	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
3140	AssertReturn(!(fFlags & ~(PGMPHYS_ROM_FLAGS_SHADOWED \| PGMPHYS_ROM_FLAGS_PERMANENT_BINARY)), VERR_INVALID_PARAMETER);
3141	VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
3142
3143	const uint32_t cPages = cb >> PAGE_SHIFT;
3144
3145	/*
3146	* Find the ROM location in the ROM list first.
3147	*/
3148	PPGMROMRANGE pRomPrev = NULL;
3149	PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3;
3150	while (pRom && GCPhysLast >= pRom->GCPhys)
3151	{
3152	if ( GCPhys <= pRom->GCPhysLast
3153	&& GCPhysLast >= pRom->GCPhys)
3154	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
3155	GCPhys, GCPhysLast, pszDesc,
3156	pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
3157	VERR_PGM_RAM_CONFLICT);
3158	/* next */
3159	pRomPrev = pRom;
3160	pRom = pRom->pNextR3;
3161	}
3162
3163	/*
3164	* Find the RAM location and check for conflicts.
3165	*
3166	* Conflict detection is a bit different than for RAM
3167	* registration since a ROM can be located within a RAM
3168	* range. So, what we have to check for is other memory
3169	* types (other than RAM that is) and that we don't span
3170	* more than one RAM range (layz).
3171	*/
3172	bool fRamExists = false;
3173	PPGMRAMRANGE pRamPrev = NULL;
3174	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
3175	while (pRam && GCPhysLast >= pRam->GCPhys)
3176	{
3177	if ( GCPhys <= pRam->GCPhysLast
3178	&& GCPhysLast >= pRam->GCPhys)
3179	{
3180	/* completely within? */
3181	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
3182	&& GCPhysLast <= pRam->GCPhysLast,
3183	("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
3184	GCPhys, GCPhysLast, pszDesc,
3185	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
3186	VERR_PGM_RAM_CONFLICT);
3187	fRamExists = true;
3188	break;
3189	}
3190
3191	/* next */
3192	pRamPrev = pRam;
3193	pRam = pRam->pNextR3;
3194	}
3195	if (fRamExists)
3196	{
3197	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3198	uint32_t cPagesLeft = cPages;
3199	while (cPagesLeft-- > 0)
3200	{
3201	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
3202	("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
3203	pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
3204	pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
3205	Assert(PGM_PAGE_IS_ZERO(pPage));
3206	pPage++;
3207	}
3208	}
3209
3210	/*
3211	* Update the base memory reservation if necessary.
3212	*/
3213	uint32_t cExtraBaseCost = fRamExists ? 0 : cPages;
3214	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3215	cExtraBaseCost += cPages;
3216	if (cExtraBaseCost)
3217	{
3218	int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
3219	if (RT_FAILURE(rc))
3220	return rc;
3221	}
3222
3223	/*
3224	* Allocate memory for the virgin copy of the RAM.
3225	*/
3226	PGMMALLOCATEPAGESREQ pReq;
3227	int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
3228	AssertRCReturn(rc, rc);
3229
3230	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3231	{
3232	pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
3233	pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
3234	pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
3235	}
3236
3237	rc = GMMR3AllocatePagesPerform(pVM, pReq);
3238	if (RT_FAILURE(rc))
3239	{
3240	GMMR3AllocatePagesCleanup(pReq);
3241	return rc;
3242	}
3243
3244	/*
3245	* Allocate the new ROM range and RAM range (if necessary).
3246	*/
3247	PPGMROMRANGE pRomNew;
3248	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
3249	if (RT_SUCCESS(rc))
3250	{
3251	PPGMRAMRANGE pRamNew = NULL;
3252	if (!fRamExists)
3253	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), sizeof(PGMPAGE), MM_TAG_PGM_PHYS, (void **)&pRamNew);
3254	if (RT_SUCCESS(rc))
3255	{
3256	/*
3257	* Initialize and insert the RAM range (if required).
3258	*/
3259	PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
3260	if (!fRamExists)
3261	{
3262	pRamNew->pSelfR0 = MMHyperCCToR0(pVM, pRamNew);
3263	pRamNew->pSelfRC = MMHyperCCToRC(pVM, pRamNew);
3264	pRamNew->GCPhys = GCPhys;
3265	pRamNew->GCPhysLast = GCPhysLast;
3266	pRamNew->cb = cb;
3267	pRamNew->pszDesc = pszDesc;
3268	pRamNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
3269	pRamNew->pvR3 = NULL;
3270	pRamNew->paLSPages = NULL;
3271
3272	PPGMPAGE pPage = &pRamNew->aPages[0];
3273	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3274	{
3275	PGM_PAGE_INIT(pPage,
3276	pReq->aPages[iPage].HCPhysGCPhys,
3277	pReq->aPages[iPage].idPage,
3278	PGMPAGETYPE_ROM,
3279	PGM_PAGE_STATE_ALLOCATED);
3280
3281	pRomPage->Virgin = *pPage;
3282	}
3283
3284	pVM->pgm.s.cAllPages += cPages;
3285	pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
3286	}
3287	else
3288	{
3289	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3290	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3291	{
3292	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_ROM);
3293	PGM_PAGE_SET_HCPHYS(pVM, pPage, pReq->aPages[iPage].HCPhysGCPhys);
3294	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
3295	PGM_PAGE_SET_PAGEID(pVM, pPage, pReq->aPages[iPage].idPage);
3296	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
3297	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
3298	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
3299
3300	pRomPage->Virgin = *pPage;
3301	}
3302
3303	pRamNew = pRam;
3304
3305	pVM->pgm.s.cZeroPages -= cPages;
3306	}
3307	pVM->pgm.s.cPrivatePages += cPages;
3308
3309	/* Flush physical page map TLB. */
3310	pgmPhysInvalidatePageMapTLB(pVM);
3311
3312
3313	/*
3314	* !HACK ALERT! REM + (Shadowed) ROM ==> mess.
3315	*
3316	* If it's shadowed we'll register the handler after the ROM notification
3317	* so we get the access handler callbacks that we should. If it isn't
3318	* shadowed we'll do it the other way around to make REM use the built-in
3319	* ROM behavior and not the handler behavior (which is to route all access
3320	* to PGM atm).
3321	*/
3322	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3323	{
3324	#ifdef VBOX_WITH_REM
3325	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, true /* fShadowed */);
3326	#endif
3327	rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, pVM->pgm.s.hRomPhysHandlerType,
3328	pRomNew, MMHyperCCToR0(pVM, pRomNew), MMHyperCCToRC(pVM, pRomNew),
3329	pszDesc);
3330	}
3331	else
3332	{
3333	rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, pVM->pgm.s.hRomPhysHandlerType,
3334	pRomNew, MMHyperCCToR0(pVM, pRomNew), MMHyperCCToRC(pVM, pRomNew),
3335	pszDesc);
3336	#ifdef VBOX_WITH_REM
3337	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, false /* fShadowed */);
3338	#endif
3339	}
3340	if (RT_SUCCESS(rc))
3341	{
3342	/*
3343	* Copy the image over to the virgin pages.
3344	* This must be done after linking in the RAM range.
3345	*/
3346	size_t cbBinaryLeft = cbBinary;
3347	PPGMPAGE pRamPage = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
3348	for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
3349	{
3350	void *pvDstPage;
3351	rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << PAGE_SHIFT), &pvDstPage);
3352	if (RT_FAILURE(rc))
3353	{
3354	VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
3355	break;
3356	}
3357	if (cbBinaryLeft >= PAGE_SIZE)
3358	{
3359	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), PAGE_SIZE);
3360	cbBinaryLeft -= PAGE_SIZE;
3361	}
3362	else
3363	{
3364	ASMMemZeroPage(pvDstPage); /* (shouldn't be necessary, but can't hurt either) */
3365	if (cbBinaryLeft > 0)
3366	{
3367	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), cbBinaryLeft);
3368	cbBinaryLeft = 0;
3369	}
3370	}
3371	}
3372	if (RT_SUCCESS(rc))
3373	{
3374	/*
3375	* Initialize the ROM range.
3376	* Note that the Virgin member of the pages has already been initialized above.
3377	*/
3378	pRomNew->GCPhys = GCPhys;
3379	pRomNew->GCPhysLast = GCPhysLast;
3380	pRomNew->cb = cb;
3381	pRomNew->fFlags = fFlags;
3382	pRomNew->idSavedState = UINT8_MAX;
3383	pRomNew->cbOriginal = cbBinary;
3384	#ifdef VBOX_STRICT
3385	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
3386	? pvBinary : RTMemDup(pvBinary, cbBinary);
3387	#else
3388	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY ? pvBinary : NULL;
3389	#endif
3390	pRomNew->pszDesc = pszDesc;
3391
3392	for (unsigned iPage = 0; iPage < cPages; iPage++)
3393	{
3394	PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
3395	pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
3396	PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
3397	}
3398
3399	/* update the page count stats for the shadow pages. */
3400	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3401	{
3402	pVM->pgm.s.cZeroPages += cPages;
3403	pVM->pgm.s.cAllPages += cPages;
3404	}
3405
3406	/*
3407	* Insert the ROM range, tell REM and return successfully.
3408	*/
3409	pRomNew->pNextR3 = pRom;
3410	pRomNew->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
3411	pRomNew->pNextRC = pRom ? MMHyperCCToRC(pVM, pRom) : NIL_RTRCPTR;
3412
3413	if (pRomPrev)
3414	{
3415	pRomPrev->pNextR3 = pRomNew;
3416	pRomPrev->pNextR0 = MMHyperCCToR0(pVM, pRomNew);
3417	pRomPrev->pNextRC = MMHyperCCToRC(pVM, pRomNew);
3418	}
3419	else
3420	{
3421	pVM->pgm.s.pRomRangesR3 = pRomNew;
3422	pVM->pgm.s.pRomRangesR0 = MMHyperCCToR0(pVM, pRomNew);
3423	pVM->pgm.s.pRomRangesRC = MMHyperCCToRC(pVM, pRomNew);
3424	}
3425
3426	pgmPhysInvalidatePageMapTLB(pVM);
3427	GMMR3AllocatePagesCleanup(pReq);
3428	return VINF_SUCCESS;
3429	}
3430
3431	/* bail out */
3432
3433	int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
3434	AssertRC(rc2);
3435	}
3436
3437	if (!fRamExists)
3438	{
3439	pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
3440	MMHyperFree(pVM, pRamNew);
3441	}
3442	}
3443	MMHyperFree(pVM, pRomNew);
3444	}
3445
3446	/** @todo Purge the mapping cache or something... */
3447	GMMR3FreeAllocatedPages(pVM, pReq);
3448	GMMR3AllocatePagesCleanup(pReq);
3449	return rc;
3450	}
3451
3452
3453	/**
3454	* Registers a ROM image.
3455	*
3456	* Shadowed ROM images requires double the amount of backing memory, so,
3457	* don't use that unless you have to. Shadowing of ROM images is process
3458	* where we can select where the reads go and where the writes go. On real
3459	* hardware the chipset provides means to configure this. We provide
3460	* PGMR3PhysProtectROM() for this purpose.
3461	*
3462	* A read-only copy of the ROM image will always be kept around while we
3463	* will allocate RAM pages for the changes on demand (unless all memory
3464	* is configured to be preallocated).
3465	*
3466	* @returns VBox status.
3467	* @param pVM Pointer to the VM.
3468	* @param pDevIns The device instance owning the ROM.
3469	* @param GCPhys First physical address in the range.
3470	* Must be page aligned!
3471	* @param cb The size of the range (in bytes).
3472	* Must be page aligned!
3473	* @param pvBinary Pointer to the binary data backing the ROM image.
3474	* @param cbBinary The size of the binary data pvBinary points to.
3475	* This must be less or equal to @a cb.
3476	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
3477	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
3478	* @param pszDesc Pointer to description string. This must not be freed.
3479	*
3480	* @remark There is no way to remove the rom, automatically on device cleanup or
3481	* manually from the device yet. This isn't difficult in any way, it's
3482	* just not something we expect to be necessary for a while.
3483	*/
3484	VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
3485	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
3486	{
3487	Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
3488	pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
3489	pgmLock(pVM);
3490	int rc = pgmR3PhysRomRegister(pVM, pDevIns, GCPhys, cb, pvBinary, cbBinary, fFlags, pszDesc);
3491	pgmUnlock(pVM);
3492	return rc;
3493	}
3494
3495
3496	/**
3497	* \#PF Handler callback for ROM write accesses.
3498	*
3499	* @returns VINF_SUCCESS if the handler have carried out the operation.
3500	* @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
3501	* @param pVM Pointer to the VM.
3502	* @param GCPhys The physical address the guest is writing to.
3503	* @param pvPhys The HC mapping of that address.
3504	* @param pvBuf What the guest is reading/writing.
3505	* @param cbBuf How much it's reading/writing.
3506	* @param enmAccessType The access type.
3507	* @param pvUser User argument.
3508	*/
3509	DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
3510	PGMACCESSTYPE enmAccessType, void *pvUser)
3511	{
3512	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
3513	const uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3514	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
3515	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3516	Log5(("pgmR3PhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
3517	NOREF(pvPhys);
3518
3519	if (enmAccessType == PGMACCESSTYPE_READ)
3520	{
3521	switch (pRomPage->enmProt)
3522	{
3523	/*
3524	* Take the default action.
3525	*/
3526	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3527	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3528	case PGMROMPROT_READ_ROM_WRITE_RAM:
3529	case PGMROMPROT_READ_RAM_WRITE_RAM:
3530	return VINF_PGM_HANDLER_DO_DEFAULT;
3531
3532	default:
3533	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3534	pRom->aPages[iPage].enmProt, iPage, GCPhys),
3535	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
3536	}
3537	}
3538	else
3539	{
3540	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
3541	switch (pRomPage->enmProt)
3542	{
3543	/*
3544	* Ignore writes.
3545	*/
3546	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3547	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3548	return VINF_SUCCESS;
3549
3550	/*
3551	* Write to the RAM page.
3552	*/
3553	case PGMROMPROT_READ_ROM_WRITE_RAM:
3554	case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's way simpler that way. */
3555	{
3556	/* This should be impossible now, pvPhys doesn't work cross page anylonger. */
3557	Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
3558
3559	/*
3560	* Take the lock, do lazy allocation, map the page and copy the data.
3561	*
3562	* Note that we have to bypass the mapping TLB since it works on
3563	* guest physical addresses and entering the shadow page would
3564	* kind of screw things up...
3565	*/
3566	int rc = pgmLock(pVM);
3567	AssertRC(rc);
3568
3569	PPGMPAGE pShadowPage = &pRomPage->Shadow;
3570	if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
3571	{
3572	pShadowPage = pgmPhysGetPage(pVM, GCPhys);
3573	AssertLogRelReturn(pShadowPage, VERR_PGM_PHYS_PAGE_GET_IPE);
3574	}
3575
3576	void *pvDstPage;
3577	rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
3578	if (RT_SUCCESS(rc))
3579	{
3580	memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
3581	pRomPage->LiveSave.fWrittenTo = true;
3582	}
3583
3584	pgmUnlock(pVM);
3585	return rc;
3586	}
3587
3588	default:
3589	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3590	pRom->aPages[iPage].enmProt, iPage, GCPhys),
3591	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
3592	}
3593	}
3594	}
3595
3596
3597	/**
3598	* Called by PGMR3MemSetup to reset the shadow, switch to the virgin, and verify
3599	* that the virgin part is untouched.
3600	*
3601	* This is done after the normal memory has been cleared.
3602	*
3603	* ASSUMES that the caller owns the PGM lock.
3604	*
3605	* @param pVM Pointer to the VM.
3606	*/
3607	int pgmR3PhysRomReset(PVM pVM)
3608	{
3609	PGM_LOCK_ASSERT_OWNER(pVM);
3610	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3611	{
3612	const uint32_t cPages = pRom->cb >> PAGE_SHIFT;
3613
3614	if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3615	{
3616	/*
3617	* Reset the physical handler.
3618	*/
3619	int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
3620	AssertRCReturn(rc, rc);
3621
3622	/*
3623	* What we do with the shadow pages depends on the memory
3624	* preallocation option. If not enabled, we'll just throw
3625	* out all the dirty pages and replace them by the zero page.
3626	*/
3627	if (!pVM->pgm.s.fRamPreAlloc)
3628	{
3629	/* Free the dirty pages. */
3630	uint32_t cPendingPages = 0;
3631	PGMMFREEPAGESREQ pReq;
3632	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
3633	AssertRCReturn(rc, rc);
3634
3635	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3636	if ( !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
3637	&& !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
3638	{
3639	Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
3640	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
3641	pRom->GCPhys + (iPage << PAGE_SHIFT));
3642	AssertLogRelRCReturn(rc, rc);
3643	}
3644
3645	if (cPendingPages)
3646	{
3647	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
3648	AssertLogRelRCReturn(rc, rc);
3649	}
3650	GMMR3FreePagesCleanup(pReq);
3651	}
3652	else
3653	{
3654	/* clear all the shadow pages. */
3655	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3656	{
3657	if (PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow))
3658	continue;
3659	Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
3660	void *pvDstPage;
3661	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3662	rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
3663	if (RT_FAILURE(rc))
3664	break;
3665	ASMMemZeroPage(pvDstPage);
3666	}
3667	AssertRCReturn(rc, rc);
3668	}
3669	}
3670
3671	#ifdef VBOX_STRICT
3672	/*
3673	* Verify that the virgin page is unchanged if possible.
3674	*/
3675	if (pRom->pvOriginal)
3676	{
3677	size_t cbSrcLeft = pRom->cbOriginal;
3678	uint8_t const pbSrcPage = (uint8_t const )pRom->pvOriginal;
3679	for (uint32_t iPage = 0; iPage < cPages && cbSrcLeft > 0; iPage++, pbSrcPage += PAGE_SIZE)
3680	{
3681	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3682	void const *pvDstPage;
3683	int rc = pgmPhysPageMapReadOnly(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPage);
3684	if (RT_FAILURE(rc))
3685	break;
3686
3687	if (memcmp(pvDstPage, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE)))
3688	{
3689	# ifdef DEBUG_bird /* This is darn handy for EFI debugging w/ snapshots, should be made default later. */
3690	void *pvDstPageW;
3691	rc = pgmPhysPageMap(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPageW);
3692	AssertRCReturn(rc, rc);
3693	memcpy(pvDstPageW, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE));
3694	# else
3695	LogRel(("pgmR3PhysRomReset: %RGp rom page changed (%s) - loaded saved state?\n",
3696	GCPhys, pRom->pszDesc));
3697	# endif
3698	}
3699	cbSrcLeft -= RT_MIN(cbSrcLeft, PAGE_SIZE);
3700	}
3701	}
3702	#endif
3703	}
3704
3705	return VINF_SUCCESS;
3706	}
3707
3708
3709	/**
3710	* Called by PGMR3Term to free resources.
3711	*
3712	* ASSUMES that the caller owns the PGM lock.
3713	*
3714	* @param pVM Pointer to the VM.
3715	*/
3716	void pgmR3PhysRomTerm(PVM pVM)
3717	{
3718	#ifdef RT_STRICT
3719	/*
3720	* Free the heap copy of the original bits.
3721	*/
3722	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3723	{
3724	if ( pRom->pvOriginal
3725	&& !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
3726	{
3727	RTMemFree((void *)pRom->pvOriginal);
3728	pRom->pvOriginal = NULL;
3729	}
3730	}
3731	#endif
3732	}
3733
3734
3735	/**
3736	* Change the shadowing of a range of ROM pages.
3737	*
3738	* This is intended for implementing chipset specific memory registers
3739	* and will not be very strict about the input. It will silently ignore
3740	* any pages that are not the part of a shadowed ROM.
3741	*
3742	* @returns VBox status code.
3743	* @retval VINF_PGM_SYNC_CR3
3744	*
3745	* @param pVM Pointer to the VM.
3746	* @param GCPhys Where to start. Page aligned.
3747	* @param cb How much to change. Page aligned.
3748	* @param enmProt The new ROM protection.
3749	*/
3750	VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
3751	{
3752	/*
3753	* Check input
3754	*/
3755	if (!cb)
3756	return VINF_SUCCESS;
3757	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3758	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3759	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
3760	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3761	AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
3762
3763	/*
3764	* Process the request.
3765	*/
3766	pgmLock(pVM);
3767	int rc = VINF_SUCCESS;
3768	bool fFlushTLB = false;
3769	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3770	{
3771	if ( GCPhys <= pRom->GCPhysLast
3772	&& GCPhysLast >= pRom->GCPhys
3773	&& (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
3774	{
3775	/*
3776	* Iterate the relevant pages and make necessary the changes.
3777	*/
3778	bool fChanges = false;
3779	uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
3780	? pRom->cb >> PAGE_SHIFT
3781	: (GCPhysLast - pRom->GCPhys + 1) >> PAGE_SHIFT;
3782	for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3783	iPage < cPages;
3784	iPage++)
3785	{
3786	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3787	if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
3788	{
3789	fChanges = true;
3790
3791	/* flush references to the page. */
3792	PPGMPAGE pRamPage = pgmPhysGetPage(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT));
3793	int rc2 = pgmPoolTrackUpdateGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage,
3794	true /fFlushPTEs/, &fFlushTLB);
3795	if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS \|\| RT_FAILURE(rc2)))
3796	rc = rc2;
3797
3798	PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
3799	PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
3800
3801	pOld = pRamPage;
3802	pRamPage = pNew;
3803	/** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
3804	}
3805	pRomPage->enmProt = enmProt;
3806	}
3807
3808	/*
3809	* Reset the access handler if we made changes, no need
3810	* to optimize this.
3811	*/
3812	if (fChanges)
3813	{
3814	int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
3815	if (RT_FAILURE(rc2))
3816	{
3817	pgmUnlock(pVM);
3818	AssertRC(rc);
3819	return rc2;
3820	}
3821	}
3822
3823	/* Advance - cb isn't updated. */
3824	GCPhys = pRom->GCPhys + (cPages << PAGE_SHIFT);
3825	}
3826	}
3827	pgmUnlock(pVM);
3828	if (fFlushTLB)
3829	PGM_INVL_ALL_VCPU_TLBS(pVM);
3830
3831	return rc;
3832	}
3833
3834
3835	/**
3836	* Sets the Address Gate 20 state.
3837	*
3838	* @param pVCpu Pointer to the VMCPU.
3839	* @param fEnable True if the gate should be enabled.
3840	* False if the gate should be disabled.
3841	*/
3842	VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
3843	{
3844	LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
3845	if (pVCpu->pgm.s.fA20Enabled != fEnable)
3846	{
3847	pVCpu->pgm.s.fA20Enabled = fEnable;
3848	pVCpu->pgm.s.GCPhysA20Mask = ~((RTGCPHYS)!fEnable << 20);
3849	#ifdef VBOX_WITH_REM
3850	REMR3A20Set(pVCpu->pVMR3, pVCpu, fEnable);
3851	#endif
3852	#ifdef PGM_WITH_A20
3853	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL;
3854	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3855	pgmR3RefreshShadowModeAfterA20Change(pVCpu);
3856	HMFlushTLB(pVCpu);
3857	#endif
3858	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cA20Changes);
3859	}
3860	}
3861
3862
3863	/**
3864	* Tree enumeration callback for dealing with age rollover.
3865	* It will perform a simple compression of the current age.
3866	*/
3867	static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
3868	{
3869	/* Age compression - ASSUMES iNow == 4. */
3870	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3871	if (pChunk->iLastUsed >= UINT32_C(0xffffff00))
3872	pChunk->iLastUsed = 3;
3873	else if (pChunk->iLastUsed >= UINT32_C(0xfffff000))
3874	pChunk->iLastUsed = 2;
3875	else if (pChunk->iLastUsed)
3876	pChunk->iLastUsed = 1;
3877	else /* iLastUsed = 0 */
3878	pChunk->iLastUsed = 4;
3879
3880	NOREF(pvUser);
3881	return 0;
3882	}
3883
3884
3885	/**
3886	* The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
3887	*/
3888	typedef struct PGMR3PHYSCHUNKUNMAPCB
3889	{
3890	PVM pVM; /*< Pointer to the VM. /
3891	PPGMCHUNKR3MAP pChunk; /*< The chunk to unmap. /
3892	} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
3893
3894
3895	/**
3896	* Callback used to find the mapping that's been unused for
3897	* the longest time.
3898	*/
3899	static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
3900	{
3901	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3902	PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
3903
3904	/*
3905	* Check for locks and compare when last used.
3906	*/
3907	if (pChunk->cRefs)
3908	return 0;
3909	if (pChunk->cPermRefs)
3910	return 0;
3911	if ( pArg->pChunk
3912	&& pChunk->iLastUsed >= pArg->pChunk->iLastUsed)
3913	return 0;
3914
3915	/*
3916	* Check that it's not in any of the TLBs.
3917	*/
3918	PVM pVM = pArg->pVM;
3919	if ( pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(pChunk->Core.Key)].idChunk
3920	== pChunk->Core.Key)
3921	{
3922	pChunk = NULL;
3923	return 0;
3924	}
3925	#ifdef VBOX_STRICT
3926	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
3927	{
3928	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk != pChunk);
3929	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk != pChunk->Core.Key);
3930	}
3931	#endif
3932
3933	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
3934	if (pVM->pgm.s.PhysTlbHC.aEntries[i].pMap == pChunk)
3935	return 0;
3936
3937	pArg->pChunk = pChunk;
3938	return 0;
3939	}
3940
3941
3942	/**
3943	* Finds a good candidate for unmapping when the ring-3 mapping cache is full.
3944	*
3945	* The candidate will not be part of any TLBs, so no need to flush
3946	* anything afterwards.
3947	*
3948	* @returns Chunk id.
3949	* @param pVM Pointer to the VM.
3950	*/
3951	static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
3952	{
3953	PGM_LOCK_ASSERT_OWNER(pVM);
3954
3955	/*
3956	* Enumerate the age tree starting with the left most node.
3957	*/
3958	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3959	PGMR3PHYSCHUNKUNMAPCB Args;
3960	Args.pVM = pVM;
3961	Args.pChunk = NULL;
3962	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
3963	Assert(Args.pChunk);
3964	if (Args.pChunk)
3965	{
3966	Assert(Args.pChunk->cRefs == 0);
3967	Assert(Args.pChunk->cPermRefs == 0);
3968	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3969	return Args.pChunk->Core.Key;
3970	}
3971
3972	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3973	return INT32_MAX;
3974	}
3975
3976
3977	/**
3978	* Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
3979	*
3980	* This is only called on one of the EMTs while the other ones are waiting for
3981	* it to complete this function.
3982	*
3983	* @returns VINF_SUCCESS (VBox strict status code).
3984	* @param pVM Pointer to the VM.
3985	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
3986	* @param pvUser User pointer. Unused
3987	*
3988	*/
3989	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
3990	{
3991	int rc = VINF_SUCCESS;
3992	pgmLock(pVM);
3993	NOREF(pVCpu); NOREF(pvUser);
3994
3995	if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
3996	{
3997	/* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
3998	/** @todo also not really efficient to unmap a chunk that contains PD
3999	* or PT pages. */
4000	pgmR3PoolClearAllRendezvous(pVM, &pVM->aCpus[0], NULL /* no need to flush the REM TLB as we already did that above */);
4001
4002	/*
4003	* Request the ring-0 part to unmap a chunk to make space in the mapping cache.
4004	*/
4005	GMMMAPUNMAPCHUNKREQ Req;
4006	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
4007	Req.Hdr.cbReq = sizeof(Req);
4008	Req.pvR3 = NULL;
4009	Req.idChunkMap = NIL_GMM_CHUNKID;
4010	Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
4011	if (Req.idChunkUnmap != INT32_MAX)
4012	{
4013	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
4014	rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
4015	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
4016	if (RT_SUCCESS(rc))
4017	{
4018	/*
4019	* Remove the unmapped one.
4020	*/
4021	PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
4022	AssertRelease(pUnmappedChunk);
4023	AssertRelease(!pUnmappedChunk->cRefs);
4024	AssertRelease(!pUnmappedChunk->cPermRefs);
4025	pUnmappedChunk->pv = NULL;
4026	pUnmappedChunk->Core.Key = UINT32_MAX;
4027	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4028	MMR3HeapFree(pUnmappedChunk);
4029	#else
4030	MMR3UkHeapFree(pVM, pUnmappedChunk, MM_TAG_PGM_CHUNK_MAPPING);
4031	#endif
4032	pVM->pgm.s.ChunkR3Map.c--;
4033	pVM->pgm.s.cUnmappedChunks++;
4034
4035	/*
4036	* Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses).
4037	*/
4038	/** @todo We should not flush chunks which include cr3 mappings. */
4039	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4040	{
4041	PPGMCPU pPGM = &pVM->aCpus[idCpu].pgm.s;
4042
4043	pPGM->pGst32BitPdR3 = NULL;
4044	pPGM->pGstPaePdptR3 = NULL;
4045	pPGM->pGstAmd64Pml4R3 = NULL;
4046	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4047	pPGM->pGst32BitPdR0 = NIL_RTR0PTR;
4048	pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
4049	pPGM->pGstAmd64Pml4R0 = NIL_RTR0PTR;
4050	#endif
4051	for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
4052	{
4053	pPGM->apGstPaePDsR3[i] = NULL;
4054	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4055	pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
4056	#endif
4057	}
4058
4059	/* Flush REM TLBs. */
4060	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
4061	}
4062	#ifdef VBOX_WITH_REM
4063	/* Flush REM translation blocks. */
4064	REMFlushTBs(pVM);
4065	#endif
4066	}
4067	}
4068	}
4069	pgmUnlock(pVM);
4070	return rc;
4071	}
4072
4073	/**
4074	* Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
4075	*
4076	* @returns VBox status code.
4077	* @param pVM Pointer to the VM.
4078	*/
4079	void pgmR3PhysUnmapChunk(PVM pVM)
4080	{
4081	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
4082	AssertRC(rc);
4083	}
4084
4085
4086	/**
4087	* Maps the given chunk into the ring-3 mapping cache.
4088	*
4089	* This will call ring-0.
4090	*
4091	* @returns VBox status code.
4092	* @param pVM Pointer to the VM.
4093	* @param idChunk The chunk in question.
4094	* @param ppChunk Where to store the chunk tracking structure.
4095	*
4096	* @remarks Called from within the PGM critical section.
4097	* @remarks Can be called from any thread!
4098	*/
4099	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
4100	{
4101	int rc;
4102
4103	PGM_LOCK_ASSERT_OWNER(pVM);
4104
4105	/*
4106	* Move the chunk time forward.
4107	*/
4108	pVM->pgm.s.ChunkR3Map.iNow++;
4109	if (pVM->pgm.s.ChunkR3Map.iNow == 0)
4110	{
4111	pVM->pgm.s.ChunkR3Map.iNow = 4;
4112	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkAgeingRolloverCallback, NULL);
4113	}
4114
4115	/*
4116	* Allocate a new tracking structure first.
4117	*/
4118	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4119	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
4120	#else
4121	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3UkHeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk), NULL);
4122	#endif
4123	AssertReturn(pChunk, VERR_NO_MEMORY);
4124	pChunk->Core.Key = idChunk;
4125	pChunk->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
4126
4127	/*
4128	* Request the ring-0 part to map the chunk in question.
4129	*/
4130	GMMMAPUNMAPCHUNKREQ Req;
4131	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
4132	Req.Hdr.cbReq = sizeof(Req);
4133	Req.pvR3 = NULL;
4134	Req.idChunkMap = idChunk;
4135	Req.idChunkUnmap = NIL_GMM_CHUNKID;
4136
4137	/* Must be callable from any thread, so can't use VMMR3CallR0. */
4138	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
4139	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
4140	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
4141	if (RT_SUCCESS(rc))
4142	{
4143	pChunk->pv = Req.pvR3;
4144
4145	/*
4146	* If we're running out of virtual address space, then we should
4147	* unmap another chunk.
4148	*
4149	* Currently, an unmap operation requires that all other virtual CPUs
4150	* are idling and not by chance making use of the memory we're
4151	* unmapping. So, we create an async unmap operation here.
4152	*
4153	* Now, when creating or restoring a saved state this wont work very
4154	* well since we may want to restore all guest RAM + a little something.
4155	* So, we have to do the unmap synchronously. Fortunately for us
4156	* though, during these operations the other virtual CPUs are inactive
4157	* and it should be safe to do this.
4158	*/
4159	/** @todo Eventually we should lock all memory when used and do
4160	* map+unmap as one kernel call without any rendezvous or
4161	* other precautions. */
4162	if (pVM->pgm.s.ChunkR3Map.c + 1 >= pVM->pgm.s.ChunkR3Map.cMax)
4163	{
4164	switch (VMR3GetState(pVM))
4165	{
4166	case VMSTATE_LOADING:
4167	case VMSTATE_SAVING:
4168	{
4169	PVMCPU pVCpu = VMMGetCpu(pVM);
4170	if ( pVCpu
4171	&& pVM->pgm.s.cDeprecatedPageLocks == 0)
4172	{
4173	pgmR3PhysUnmapChunkRendezvous(pVM, pVCpu, NULL);
4174	break;
4175	}
4176	/* fall thru */
4177	}
4178	default:
4179	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
4180	AssertRC(rc);
4181	break;
4182	}
4183	}
4184
4185	/*
4186	* Update the tree. We must do this after any unmapping to make sure
4187	* the chunk we're going to return isn't unmapped by accident.
4188	*/
4189	AssertPtr(Req.pvR3);
4190	bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
4191	AssertRelease(fRc);
4192	pVM->pgm.s.ChunkR3Map.c++;
4193	pVM->pgm.s.cMappedChunks++;
4194	}
4195	else
4196	{
4197	/** @todo this may fail because of /proc/sys/vm/max_map_count, so we
4198	* should probably restrict ourselves on linux. */
4199	AssertRC(rc);
4200	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4201	MMR3HeapFree(pChunk);
4202	#else
4203	MMR3UkHeapFree(pVM, pChunk, MM_TAG_PGM_CHUNK_MAPPING);
4204	#endif
4205	pChunk = NULL;
4206	}
4207
4208	*ppChunk = pChunk;
4209	return rc;
4210	}
4211
4212
4213	/**
4214	* For VMMCALLRING3_PGM_MAP_CHUNK, considered internal.
4215	*
4216	* @returns see pgmR3PhysChunkMap.
4217	* @param pVM Pointer to the VM.
4218	* @param idChunk The chunk to map.
4219	*/
4220	VMMR3DECL(int) PGMR3PhysChunkMap(PVM pVM, uint32_t idChunk)
4221	{
4222	PPGMCHUNKR3MAP pChunk;
4223	int rc;
4224
4225	pgmLock(pVM);
4226	rc = pgmR3PhysChunkMap(pVM, idChunk, &pChunk);
4227	pgmUnlock(pVM);
4228	return rc;
4229	}
4230
4231
4232	/**
4233	* Invalidates the TLB for the ring-3 mapping cache.
4234	*
4235	* @param pVM Pointer to the VM.
4236	*/
4237	VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
4238	{
4239	pgmLock(pVM);
4240	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
4241	{
4242	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
4243	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
4244	}
4245	/* The page map TLB references chunks, so invalidate that one too. */
4246	pgmPhysInvalidatePageMapTLB(pVM);
4247	pgmUnlock(pVM);
4248	}
4249
4250
4251	/**
4252	* Response to VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE to allocate a large
4253	* (2MB) page for use with a nested paging PDE.
4254	*
4255	* @returns The following VBox status codes.
4256	* @retval VINF_SUCCESS on success.
4257	* @retval VINF_EM_NO_MEMORY if we're out of memory.
4258	*
4259	* @param pVM Pointer to the VM.
4260	* @param GCPhys GC physical start address of the 2 MB range
4261	*/
4262	VMMR3DECL(int) PGMR3PhysAllocateLargeHandyPage(PVM pVM, RTGCPHYS GCPhys)
4263	{
4264	#ifdef PGM_WITH_LARGE_PAGES
4265	uint64_t u64TimeStamp1, u64TimeStamp2;
4266
4267	pgmLock(pVM);
4268
4269	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4270	u64TimeStamp1 = RTTimeMilliTS();
4271	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE, 0, NULL);
4272	u64TimeStamp2 = RTTimeMilliTS();
4273	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4274	if (RT_SUCCESS(rc))
4275	{
4276	Assert(pVM->pgm.s.cLargeHandyPages == 1);
4277
4278	uint32_t idPage = pVM->pgm.s.aLargeHandyPage[0].idPage;
4279	RTHCPHYS HCPhys = pVM->pgm.s.aLargeHandyPage[0].HCPhysGCPhys;
4280
4281	void *pv;
4282
4283	/* Map the large page into our address space.
4284	*
4285	* Note: assuming that within the 2 MB range:
4286	* - GCPhys + PAGE_SIZE = HCPhys + PAGE_SIZE (whole point of this exercise)
4287	* - user space mapping is continuous as well
4288	* - page id (GCPhys) + 1 = page id (GCPhys + PAGE_SIZE)
4289	*/
4290	rc = pgmPhysPageMapByPageID(pVM, idPage, HCPhys, &pv);
4291	AssertLogRelMsg(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", idPage, HCPhys, rc));
4292
4293	if (RT_SUCCESS(rc))
4294	{
4295	/*
4296	* Clear the pages.
4297	*/
4298	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4299	for (unsigned i = 0; i < _2M/PAGE_SIZE; i++)
4300	{
4301	ASMMemZeroPage(pv);
4302
4303	PPGMPAGE pPage;
4304	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
4305	AssertRC(rc);
4306
4307	Assert(PGM_PAGE_IS_ZERO(pPage));
4308	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
4309	pVM->pgm.s.cZeroPages--;
4310
4311	/*
4312	* Do the PGMPAGE modifications.
4313	*/
4314	pVM->pgm.s.cPrivatePages++;
4315	PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
4316	PGM_PAGE_SET_PAGEID(pVM, pPage, idPage);
4317	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
4318	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE);
4319	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4320	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4321
4322	/* Somewhat dirty assumption that page ids are increasing. */
4323	idPage++;
4324
4325	HCPhys += PAGE_SIZE;
4326	GCPhys += PAGE_SIZE;
4327
4328	pv = (void *)((uintptr_t)pv + PAGE_SIZE);
4329
4330	Log3(("PGMR3PhysAllocateLargePage: idPage=%#x HCPhys=%RGp\n", idPage, HCPhys));
4331	}
4332	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4333
4334	/* Flush all TLBs. */
4335	PGM_INVL_ALL_VCPU_TLBS(pVM);
4336	pgmPhysInvalidatePageMapTLB(pVM);
4337	}
4338	pVM->pgm.s.cLargeHandyPages = 0;
4339	}
4340
4341	if (RT_SUCCESS(rc))
4342	{
4343	static uint32_t cTimeOut = 0;
4344	uint64_t u64TimeStampDelta = u64TimeStamp2 - u64TimeStamp1;
4345
4346	if (u64TimeStampDelta > 100)
4347	{
4348	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatLargePageOverflow);
4349	if ( ++cTimeOut > 10
4350	\|\| u64TimeStampDelta > 1000 /* more than one second forces an early retirement from allocating large pages. */)
4351	{
4352	/* If repeated attempts to allocate a large page takes more than 100 ms, then we fall back to normal 4k pages.
4353	* E.g. Vista 64 tries to move memory around, which takes a huge amount of time.
4354	*/
4355	LogRel(("PGMR3PhysAllocateLargePage: allocating large pages takes too long (last attempt %d ms; nr of timeouts %d); DISABLE\n", u64TimeStampDelta, cTimeOut));
4356	PGMSetLargePageUsage(pVM, false);
4357	}
4358	}
4359	else
4360	if (cTimeOut > 0)
4361	cTimeOut--;
4362	}
4363
4364	pgmUnlock(pVM);
4365	return rc;
4366	#else
4367	return VERR_NOT_IMPLEMENTED;
4368	#endif /* PGM_WITH_LARGE_PAGES */
4369	}
4370
4371
4372	/**
4373	* Response to VM_FF_PGM_NEED_HANDY_PAGES and VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES.
4374	*
4375	* This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
4376	* signal and clear the out of memory condition. When contracted, this API is
4377	* used to try clear the condition when the user wants to resume.
4378	*
4379	* @returns The following VBox status codes.
4380	* @retval VINF_SUCCESS on success. FFs cleared.
4381	* @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
4382	* this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
4383	*
4384	* @param pVM Pointer to the VM.
4385	*
4386	* @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
4387	* in EM.cpp and shouldn't be propagated outside TRPM, HM, EM and
4388	* pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
4389	* handler.
4390	*/
4391	VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
4392	{
4393	pgmLock(pVM);
4394
4395	/*
4396	* Allocate more pages, noting down the index of the first new page.
4397	*/
4398	uint32_t iClear = pVM->pgm.s.cHandyPages;
4399	AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_PGM_HANDY_PAGE_IPE);
4400	Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
4401	int rcAlloc = VINF_SUCCESS;
4402	int rcSeed = VINF_SUCCESS;
4403	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4404	while (rc == VERR_GMM_SEED_ME)
4405	{
4406	void *pvChunk;
4407	rcAlloc = rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
4408	if (RT_SUCCESS(rc))
4409	{
4410	rcSeed = rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
4411	if (RT_FAILURE(rc))
4412	SUPR3PageFree(pvChunk, GMM_CHUNK_SIZE >> PAGE_SHIFT);
4413	}
4414	if (RT_SUCCESS(rc))
4415	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4416	}
4417
4418	/* todo: we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
4419	if ( rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
4420	&& pVM->pgm.s.cHandyPages > 0)
4421	{
4422	/* Still handy pages left, so don't panic. */
4423	rc = VINF_SUCCESS;
4424	}
4425
4426	if (RT_SUCCESS(rc))
4427	{
4428	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
4429	Assert(pVM->pgm.s.cHandyPages > 0);
4430	VM_FF_CLEAR(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4431	VM_FF_CLEAR(pVM, VM_FF_PGM_NO_MEMORY);
4432
4433	#ifdef VBOX_STRICT
4434	uint32_t i;
4435	for (i = iClear; i < pVM->pgm.s.cHandyPages; i++)
4436	if ( pVM->pgm.s.aHandyPages[i].idPage == NIL_GMM_PAGEID
4437	\|\| pVM->pgm.s.aHandyPages[i].idSharedPage != NIL_GMM_PAGEID
4438	\|\| (pVM->pgm.s.aHandyPages[i].HCPhysGCPhys & PAGE_OFFSET_MASK))
4439	break;
4440	if (i != pVM->pgm.s.cHandyPages)
4441	{
4442	RTAssertMsg1Weak(NULL, __LINE__, __FILE__, __FUNCTION__);
4443	RTAssertMsg2Weak("i=%d iClear=%d cHandyPages=%d\n", i, iClear, pVM->pgm.s.cHandyPages);
4444	for (uint32_t j = iClear; j < pVM->pgm.s.cHandyPages; j++)
4445	RTAssertMsg2Add(("%03d: idPage=%d HCPhysGCPhys=%RHp idSharedPage=%d%\n", j,
4446	pVM->pgm.s.aHandyPages[j].idPage,
4447	pVM->pgm.s.aHandyPages[j].HCPhysGCPhys,
4448	pVM->pgm.s.aHandyPages[j].idSharedPage,
4449	j == i ? " <---" : ""));
4450	RTAssertPanic();
4451	}
4452	#endif
4453	/*
4454	* Clear the pages.
4455	*/
4456	while (iClear < pVM->pgm.s.cHandyPages)
4457	{
4458	PGMMPAGEDESC pPage = &pVM->pgm.s.aHandyPages[iClear];
4459	void *pv;
4460	rc = pgmPhysPageMapByPageID(pVM, pPage->idPage, pPage->HCPhysGCPhys, &pv);
4461	AssertLogRelMsgBreak(RT_SUCCESS(rc),
4462	("%u/%u: idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n",
4463	iClear, pVM->pgm.s.cHandyPages, pPage->idPage, pPage->HCPhysGCPhys, rc));
4464	ASMMemZeroPage(pv);
4465	iClear++;
4466	Log3(("PGMR3PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
4467	}
4468	}
4469	else
4470	{
4471	uint64_t cAllocPages, cMaxPages, cBalloonPages;
4472
4473	/*
4474	* We should never get here unless there is a genuine shortage of
4475	* memory (or some internal error). Flag the error so the VM can be
4476	* suspended ASAP and the user informed. If we're totally out of
4477	* handy pages we will return failure.
4478	*/
4479	/* Report the failure. */
4480	LogRel(("PGM: Failed to procure handy pages; rc=%Rrc rcAlloc=%Rrc rcSeed=%Rrc cHandyPages=%#x\n"
4481	" cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
4482	rc, rcAlloc, rcSeed,
4483	pVM->pgm.s.cHandyPages,
4484	pVM->pgm.s.cAllPages,
4485	pVM->pgm.s.cPrivatePages,
4486	pVM->pgm.s.cSharedPages,
4487	pVM->pgm.s.cZeroPages));
4488
4489	if (GMMR3QueryMemoryStats(pVM, &cAllocPages, &cMaxPages, &cBalloonPages) == VINF_SUCCESS)
4490	{
4491	LogRel(("GMM: Statistics:\n"
4492	" Allocated pages: %RX64\n"
4493	" Maximum pages: %RX64\n"
4494	" Ballooned pages: %RX64\n", cAllocPages, cMaxPages, cBalloonPages));
4495	}
4496
4497	if ( rc != VERR_NO_MEMORY
4498	&& rc != VERR_NO_PHYS_MEMORY
4499	&& rc != VERR_LOCK_FAILED)
4500	{
4501	for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4502	{
4503	LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
4504	i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
4505	pVM->pgm.s.aHandyPages[i].idSharedPage));
4506	uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
4507	if (idPage != NIL_GMM_PAGEID)
4508	{
4509	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
4510	pRam;
4511	pRam = pRam->pNextR3)
4512	{
4513	uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
4514	for (uint32_t iPage = 0; iPage < cPages; iPage++)
4515	if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
4516	LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
4517	pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
4518	}
4519	}
4520	}
4521	}
4522
4523	/* Set the FFs and adjust rc. */
4524	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4525	VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
4526	if ( rc == VERR_NO_MEMORY
4527	\|\| rc == VERR_NO_PHYS_MEMORY
4528	\|\| rc == VERR_LOCK_FAILED)
4529	rc = VINF_EM_NO_MEMORY;
4530	}
4531
4532	pgmUnlock(pVM);
4533	return rc;
4534	}
4535
4536
4537	/**
4538	* Frees the specified RAM page and replaces it with the ZERO page.
4539	*
4540	* This is used by ballooning, remapping MMIO2, RAM reset and state loading.
4541	*
4542	* @param pVM Pointer to the VM.
4543	* @param pReq Pointer to the request.
4544	* @param pcPendingPages Where the number of pages waiting to be freed are
4545	* kept. This will normally be incremented.
4546	* @param pPage Pointer to the page structure.
4547	* @param GCPhys The guest physical address of the page, if applicable.
4548	*
4549	* @remarks The caller must own the PGM lock.
4550	*/
4551	int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys)
4552	{
4553	/*
4554	* Assert sanity.
4555	*/
4556	PGM_LOCK_ASSERT_OWNER(pVM);
4557	if (RT_UNLIKELY( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
4558	&& PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
4559	{
4560	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4561	return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
4562	}
4563
4564	/** @todo What about ballooning of large pages??! */
4565	Assert( PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
4566	&& PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED);
4567
4568	if ( PGM_PAGE_IS_ZERO(pPage)
4569	\|\| PGM_PAGE_IS_BALLOONED(pPage))
4570	return VINF_SUCCESS;
4571
4572	const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
4573	Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
4574	if (RT_UNLIKELY( idPage == NIL_GMM_PAGEID
4575	\|\| idPage > GMM_PAGEID_LAST
4576	\|\| PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
4577	{
4578	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4579	return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
4580	}
4581
4582	/* update page count stats. */
4583	if (PGM_PAGE_IS_SHARED(pPage))
4584	pVM->pgm.s.cSharedPages--;
4585	else
4586	pVM->pgm.s.cPrivatePages--;
4587	pVM->pgm.s.cZeroPages++;
4588
4589	/* Deal with write monitored pages. */
4590	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
4591	{
4592	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
4593	pVM->pgm.s.cWrittenToPages++;
4594	}
4595
4596	/*
4597	* pPage = ZERO page.
4598	*/
4599	PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg);
4600	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
4601	PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
4602	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4603	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4604	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4605
4606	/* Flush physical page map TLB entry. */
4607	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
4608
4609	/*
4610	* Make sure it's not in the handy page array.
4611	*/
4612	for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4613	{
4614	if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
4615	{
4616	pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
4617	break;
4618	}
4619	if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
4620	{
4621	pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
4622	break;
4623	}
4624	}
4625
4626	/*
4627	* Push it onto the page array.
4628	*/
4629	uint32_t iPage = *pcPendingPages;
4630	Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
4631	*pcPendingPages += 1;
4632
4633	pReq->aPages[iPage].idPage = idPage;
4634
4635	if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
4636	return VINF_SUCCESS;
4637
4638	/*
4639	* Flush the pages.
4640	*/
4641	int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
4642	if (RT_SUCCESS(rc))
4643	{
4644	GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
4645	*pcPendingPages = 0;
4646	}
4647	return rc;
4648	}
4649
4650
4651	/**
4652	* Converts a GC physical address to a HC ring-3 pointer, with some
4653	* additional checks.
4654	*
4655	* @returns VBox status code.
4656	* @retval VINF_SUCCESS on success.
4657	* @retval VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4658	* access handler of some kind.
4659	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4660	* accesses or is odd in any way.
4661	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4662	*
4663	* @param pVM Pointer to the VM.
4664	* @param GCPhys The GC physical address to convert. Since this is only
4665	* used for filling the REM TLB, the A20 mask must be
4666	* applied before calling this API.
4667	* @param fWritable Whether write access is required.
4668	* @param ppv Where to store the pointer corresponding to GCPhys on
4669	* success.
4670	*/
4671	VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
4672	{
4673	pgmLock(pVM);
4674	PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
4675
4676	PPGMRAMRANGE pRam;
4677	PPGMPAGE pPage;
4678	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4679	if (RT_SUCCESS(rc))
4680	{
4681	if (PGM_PAGE_IS_BALLOONED(pPage))
4682	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4683	else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
4684	rc = VINF_SUCCESS;
4685	else
4686	{
4687	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4688	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4689	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
4690	{
4691	/** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
4692	* in -norawr0 mode. */
4693	if (fWritable)
4694	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4695	}
4696	else
4697	{
4698	/* Temporarily disabled physical handler(s), since the recompiler
4699	doesn't get notified when it's reset we'll have to pretend it's
4700	operating normally. */
4701	if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
4702	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4703	else
4704	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4705	}
4706	}
4707	if (RT_SUCCESS(rc))
4708	{
4709	int rc2;
4710
4711	/* Make sure what we return is writable. */
4712	if (fWritable)
4713	switch (PGM_PAGE_GET_STATE(pPage))
4714	{
4715	case PGM_PAGE_STATE_ALLOCATED:
4716	break;
4717	case PGM_PAGE_STATE_BALLOONED:
4718	AssertFailed();
4719	break;
4720	case PGM_PAGE_STATE_ZERO:
4721	case PGM_PAGE_STATE_SHARED:
4722	if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
4723	break;
4724	case PGM_PAGE_STATE_WRITE_MONITORED:
4725	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
4726	AssertLogRelRCReturn(rc2, rc2);
4727	break;
4728	}
4729
4730	/* Get a ring-3 mapping of the address. */
4731	PPGMPAGER3MAPTLBE pTlbe;
4732	rc2 = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
4733	AssertLogRelRCReturn(rc2, rc2);
4734	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4735	/** @todo mapping/locking hell; this isn't horribly efficient since
4736	* pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
4737
4738	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
4739	}
4740	else
4741	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
4742
4743	/* else: handler catching all access, no pointer returned. */
4744	}
4745	else
4746	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
4747
4748	pgmUnlock(pVM);
4749	return rc;
4750	}
4751

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp@ 55705

Download in other formats: