PGMPhys.cpp@ 38955

Last change on this file since 38955 was 38955, checked in by vboxsync, 13 years ago
pgmR3PhysChunkMap: Make sure we don't unmap the chunk we just added.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 164.7 KB

Line
1	/* $Id: PGMPhys.cpp 38955 2011-10-06 12:23:39Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_PHYS
23	#include <VBox/vmm/pgm.h>
24	#include <VBox/vmm/iom.h>
25	#include <VBox/vmm/mm.h>
26	#include <VBox/vmm/stam.h>
27	#include <VBox/vmm/rem.h>
28	#include <VBox/vmm/pdmdev.h>
29	#include "PGMInternal.h"
30	#include <VBox/vmm/vm.h>
31	#include "PGMInline.h"
32	#include <VBox/sup.h>
33	#include <VBox/param.h>
34	#include <VBox/err.h>
35	#include <VBox/log.h>
36	#include <iprt/assert.h>
37	#include <iprt/alloc.h>
38	#include <iprt/asm.h>
39	#include <iprt/thread.h>
40	#include <iprt/string.h>
41	#include <iprt/system.h>
42
43
44	/*******************************************************************************
45	* Defined Constants And Macros *
46	*******************************************************************************/
47	/** The number of pages to free in one batch. */
48	#define PGMPHYS_FREE_PAGE_BATCH_SIZE 128
49
50
51	/*******************************************************************************
52	* Internal Functions *
53	*******************************************************************************/
54	static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
55
56
57	/*
58	* PGMR3PhysReadU8-64
59	* PGMR3PhysWriteU8-64
60	*/
61	#define PGMPHYSFN_READNAME PGMR3PhysReadU8
62	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
63	#define PGMPHYS_DATASIZE 1
64	#define PGMPHYS_DATATYPE uint8_t
65	#include "PGMPhysRWTmpl.h"
66
67	#define PGMPHYSFN_READNAME PGMR3PhysReadU16
68	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
69	#define PGMPHYS_DATASIZE 2
70	#define PGMPHYS_DATATYPE uint16_t
71	#include "PGMPhysRWTmpl.h"
72
73	#define PGMPHYSFN_READNAME PGMR3PhysReadU32
74	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
75	#define PGMPHYS_DATASIZE 4
76	#define PGMPHYS_DATATYPE uint32_t
77	#include "PGMPhysRWTmpl.h"
78
79	#define PGMPHYSFN_READNAME PGMR3PhysReadU64
80	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
81	#define PGMPHYS_DATASIZE 8
82	#define PGMPHYS_DATATYPE uint64_t
83	#include "PGMPhysRWTmpl.h"
84
85
86	/**
87	* EMT worker for PGMR3PhysReadExternal.
88	*/
89	static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead)
90	{
91	PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead);
92	return VINF_SUCCESS;
93	}
94
95
96	/**
97	* Read from physical memory, external users.
98	*
99	* @returns VBox status code.
100	* @retval VINF_SUCCESS.
101	*
102	* @param pVM VM Handle.
103	* @param GCPhys Physical address to read from.
104	* @param pvBuf Where to read into.
105	* @param cbRead How many bytes to read.
106	*
107	* @thread Any but EMTs.
108	*/
109	VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
110	{
111	VM_ASSERT_OTHER_THREAD(pVM);
112
113	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
114	LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
115
116	pgmLock(pVM);
117
118	/*
119	* Copy loop on ram ranges.
120	*/
121	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
122	for (;;)
123	{
124	/* Inside range or not? */
125	if (pRam && GCPhys >= pRam->GCPhys)
126	{
127	/*
128	* Must work our way thru this page by page.
129	*/
130	RTGCPHYS off = GCPhys - pRam->GCPhys;
131	while (off < pRam->cb)
132	{
133	unsigned iPage = off >> PAGE_SHIFT;
134	PPGMPAGE pPage = &pRam->aPages[iPage];
135
136	/*
137	* If the page has an ALL access handler, we'll have to
138	* delegate the job to EMT.
139	*/
140	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
141	{
142	pgmUnlock(pVM);
143
144	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 4,
145	pVM, &GCPhys, pvBuf, cbRead);
146	}
147	Assert(!PGM_PAGE_IS_MMIO(pPage));
148
149	/*
150	* Simple stuff, go ahead.
151	*/
152	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
153	if (cb > cbRead)
154	cb = cbRead;
155	PGMPAGEMAPLOCK PgMpLck;
156	const void *pvSrc;
157	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
158	if (RT_SUCCESS(rc))
159	{
160	memcpy(pvBuf, pvSrc, cb);
161	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
162	}
163	else
164	{
165	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
166	pRam->GCPhys + off, pPage, rc));
167	memset(pvBuf, 0xff, cb);
168	}
169
170	/* next page */
171	if (cb >= cbRead)
172	{
173	pgmUnlock(pVM);
174	return VINF_SUCCESS;
175	}
176	cbRead -= cb;
177	off += cb;
178	GCPhys += cb;
179	pvBuf = (char *)pvBuf + cb;
180	} /* walk pages in ram range. */
181	}
182	else
183	{
184	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
185
186	/*
187	* Unassigned address space.
188	*/
189	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
190	if (cb >= cbRead)
191	{
192	memset(pvBuf, 0xff, cbRead);
193	break;
194	}
195	memset(pvBuf, 0xff, cb);
196
197	cbRead -= cb;
198	pvBuf = (char *)pvBuf + cb;
199	GCPhys += cb;
200	}
201
202	/* Advance range if necessary. */
203	while (pRam && GCPhys > pRam->GCPhysLast)
204	pRam = pRam->CTX_SUFF(pNext);
205	} /* Ram range walk */
206
207	pgmUnlock(pVM);
208
209	return VINF_SUCCESS;
210	}
211
212
213	/**
214	* EMT worker for PGMR3PhysWriteExternal.
215	*/
216	static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite)
217	{
218	/** @todo VERR_EM_NO_MEMORY */
219	PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite);
220	return VINF_SUCCESS;
221	}
222
223
224	/**
225	* Write to physical memory, external users.
226	*
227	* @returns VBox status code.
228	* @retval VINF_SUCCESS.
229	* @retval VERR_EM_NO_MEMORY.
230	*
231	* @param pVM VM Handle.
232	* @param GCPhys Physical address to write to.
233	* @param pvBuf What to write.
234	* @param cbWrite How many bytes to write.
235	* @param pszWho Who is writing. For tracking down who is writing
236	* after we've saved the state.
237	*
238	* @thread Any but EMTs.
239	*/
240	VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void pvBuf, size_t cbWrite, const char pszWho)
241	{
242	VM_ASSERT_OTHER_THREAD(pVM);
243
244	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
245	("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x pszWho=%s\n",
246	GCPhys, cbWrite, pszWho));
247	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
248	LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
249
250	pgmLock(pVM);
251
252	/*
253	* Copy loop on ram ranges, stop when we hit something difficult.
254	*/
255	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
256	for (;;)
257	{
258	/* Inside range or not? */
259	if (pRam && GCPhys >= pRam->GCPhys)
260	{
261	/*
262	* Must work our way thru this page by page.
263	*/
264	RTGCPTR off = GCPhys - pRam->GCPhys;
265	while (off < pRam->cb)
266	{
267	RTGCPTR iPage = off >> PAGE_SHIFT;
268	PPGMPAGE pPage = &pRam->aPages[iPage];
269
270	/*
271	* Is the page problematic, we have to do the work on the EMT.
272	*
273	* Allocating writable pages and access handlers are
274	* problematic, write monitored pages are simple and can be
275	* dealt with here.
276	*/
277	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
278	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
279	{
280	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
281	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
282	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
283	else
284	{
285	pgmUnlock(pVM);
286
287	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 4,
288	pVM, &GCPhys, pvBuf, cbWrite);
289	}
290	}
291	Assert(!PGM_PAGE_IS_MMIO(pPage));
292
293	/*
294	* Simple stuff, go ahead.
295	*/
296	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
297	if (cb > cbWrite)
298	cb = cbWrite;
299	PGMPAGEMAPLOCK PgMpLck;
300	void *pvDst;
301	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
302	if (RT_SUCCESS(rc))
303	{
304	memcpy(pvDst, pvBuf, cb);
305	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
306	}
307	else
308	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
309	pRam->GCPhys + off, pPage, rc));
310
311	/* next page */
312	if (cb >= cbWrite)
313	{
314	pgmUnlock(pVM);
315	return VINF_SUCCESS;
316	}
317
318	cbWrite -= cb;
319	off += cb;
320	GCPhys += cb;
321	pvBuf = (const char *)pvBuf + cb;
322	} /* walk pages in ram range */
323	}
324	else
325	{
326	/*
327	* Unassigned address space, skip it.
328	*/
329	if (!pRam)
330	break;
331	size_t cb = pRam->GCPhys - GCPhys;
332	if (cb >= cbWrite)
333	break;
334	cbWrite -= cb;
335	pvBuf = (const char *)pvBuf + cb;
336	GCPhys += cb;
337	}
338
339	/* Advance range if necessary. */
340	while (pRam && GCPhys > pRam->GCPhysLast)
341	pRam = pRam->CTX_SUFF(pNext);
342	} /* Ram range walk */
343
344	pgmUnlock(pVM);
345	return VINF_SUCCESS;
346	}
347
348
349	/**
350	* VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
351	*
352	* @returns see PGMR3PhysGCPhys2CCPtrExternal
353	* @param pVM The VM handle.
354	* @param pGCPhys Pointer to the guest physical address.
355	* @param ppv Where to store the mapping address.
356	* @param pLock Where to store the lock.
357	*/
358	static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
359	{
360	/*
361	* Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
362	* an access handler after it succeeds.
363	*/
364	int rc = pgmLock(pVM);
365	AssertRCReturn(rc, rc);
366
367	rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
368	if (RT_SUCCESS(rc))
369	{
370	PPGMPAGEMAPTLBE pTlbe;
371	int rc2 = pgmPhysPageQueryTlbe(pVM, *pGCPhys, &pTlbe);
372	AssertFatalRC(rc2);
373	PPGMPAGE pPage = pTlbe->pPage;
374	if (PGM_PAGE_IS_MMIO(pPage))
375	{
376	PGMPhysReleasePageMappingLock(pVM, pLock);
377	rc = VERR_PGM_PHYS_PAGE_RESERVED;
378	}
379	else if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
380	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
381	\|\| pgmPoolIsDirtyPage(pVM, *pGCPhys)
382	#endif
383	)
384	{
385	/* We must flush any corresponding pgm pool page here, otherwise we'll
386	* not be informed about writes and keep bogus gst->shw mappings around.
387	*/
388	pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
389	Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
390	/** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
391	* active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
392	}
393	}
394
395	pgmUnlock(pVM);
396	return rc;
397	}
398
399
400	/**
401	* Requests the mapping of a guest page into ring-3, external threads.
402	*
403	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
404	* release it.
405	*
406	* This API will assume your intention is to write to the page, and will
407	* therefore replace shared and zero pages. If you do not intend to modify the
408	* page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
409	*
410	* @returns VBox status code.
411	* @retval VINF_SUCCESS on success.
412	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
413	* backing or if the page has any active access handlers. The caller
414	* must fall back on using PGMR3PhysWriteExternal.
415	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
416	*
417	* @param pVM The VM handle.
418	* @param GCPhys The guest physical address of the page that should be mapped.
419	* @param ppv Where to store the address corresponding to GCPhys.
420	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
421	*
422	* @remark Avoid calling this API from within critical sections (other than the
423	* PGM one) because of the deadlock risk when we have to delegating the
424	* task to an EMT.
425	* @thread Any.
426	*/
427	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
428	{
429	AssertPtr(ppv);
430	AssertPtr(pLock);
431
432	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
433
434	int rc = pgmLock(pVM);
435	AssertRCReturn(rc, rc);
436
437	/*
438	* Query the Physical TLB entry for the page (may fail).
439	*/
440	PPGMPAGEMAPTLBE pTlbe;
441	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
442	if (RT_SUCCESS(rc))
443	{
444	PPGMPAGE pPage = pTlbe->pPage;
445	if (PGM_PAGE_IS_MMIO(pPage))
446	rc = VERR_PGM_PHYS_PAGE_RESERVED;
447	else
448	{
449	/*
450	* If the page is shared, the zero page, or being write monitored
451	* it must be converted to an page that's writable if possible.
452	* We can only deal with write monitored pages here, the rest have
453	* to be on an EMT.
454	*/
455	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
456	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
457	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
458	\|\| pgmPoolIsDirtyPage(pVM, GCPhys)
459	#endif
460	)
461	{
462	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
463	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
464	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
465	&& !pgmPoolIsDirtyPage(pVM, GCPhys)
466	#endif
467	)
468	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
469	else
470	{
471	pgmUnlock(pVM);
472
473	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
474	pVM, &GCPhys, ppv, pLock);
475	}
476	}
477
478	/*
479	* Now, just perform the locking and calculate the return address.
480	*/
481	PPGMPAGEMAP pMap = pTlbe->pMap;
482	if (pMap)
483	pMap->cRefs++;
484
485	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
486	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
487	{
488	if (cLocks == 0)
489	pVM->pgm.s.cWriteLockedPages++;
490	PGM_PAGE_INC_WRITE_LOCKS(pPage);
491	}
492	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
493	{
494	PGM_PAGE_INC_WRITE_LOCKS(pPage);
495	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
496	if (pMap)
497	pMap->cRefs++; /* Extra ref to prevent it from going away. */
498	}
499
500	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
501	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
502	pLock->pvMap = pMap;
503	}
504	}
505
506	pgmUnlock(pVM);
507	return rc;
508	}
509
510
511	/**
512	* Requests the mapping of a guest page into ring-3, external threads.
513	*
514	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
515	* release it.
516	*
517	* @returns VBox status code.
518	* @retval VINF_SUCCESS on success.
519	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
520	* backing or if the page as an active ALL access handler. The caller
521	* must fall back on using PGMPhysRead.
522	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
523	*
524	* @param pVM The VM handle.
525	* @param GCPhys The guest physical address of the page that should be mapped.
526	* @param ppv Where to store the address corresponding to GCPhys.
527	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
528	*
529	* @remark Avoid calling this API from within critical sections (other than
530	* the PGM one) because of the deadlock risk.
531	* @thread Any.
532	*/
533	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
534	{
535	int rc = pgmLock(pVM);
536	AssertRCReturn(rc, rc);
537
538	/*
539	* Query the Physical TLB entry for the page (may fail).
540	*/
541	PPGMPAGEMAPTLBE pTlbe;
542	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
543	if (RT_SUCCESS(rc))
544	{
545	PPGMPAGE pPage = pTlbe->pPage;
546	#if 1
547	/* MMIO pages doesn't have any readable backing. */
548	if (PGM_PAGE_IS_MMIO(pPage))
549	rc = VERR_PGM_PHYS_PAGE_RESERVED;
550	#else
551	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
552	rc = VERR_PGM_PHYS_PAGE_RESERVED;
553	#endif
554	else
555	{
556	/*
557	* Now, just perform the locking and calculate the return address.
558	*/
559	PPGMPAGEMAP pMap = pTlbe->pMap;
560	if (pMap)
561	pMap->cRefs++;
562
563	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
564	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
565	{
566	if (cLocks == 0)
567	pVM->pgm.s.cReadLockedPages++;
568	PGM_PAGE_INC_READ_LOCKS(pPage);
569	}
570	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
571	{
572	PGM_PAGE_INC_READ_LOCKS(pPage);
573	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
574	if (pMap)
575	pMap->cRefs++; /* Extra ref to prevent it from going away. */
576	}
577
578	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
579	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
580	pLock->pvMap = pMap;
581	}
582	}
583
584	pgmUnlock(pVM);
585	return rc;
586	}
587
588
589	#define MAKE_LEAF(a_pNode) \
590	do { \
591	(a_pNode)->pLeftR3 = NIL_RTR3PTR; \
592	(a_pNode)->pRightR3 = NIL_RTR3PTR; \
593	(a_pNode)->pLeftR0 = NIL_RTR0PTR; \
594	(a_pNode)->pRightR0 = NIL_RTR0PTR; \
595	(a_pNode)->pLeftRC = NIL_RTRCPTR; \
596	(a_pNode)->pRightRC = NIL_RTRCPTR; \
597	} while (0)
598
599	#define INSERT_LEFT(a_pParent, a_pNode) \
600	do { \
601	(a_pParent)->pLeftR3 = (a_pNode); \
602	(a_pParent)->pLeftR0 = (a_pNode)->pSelfR0; \
603	(a_pParent)->pLeftRC = (a_pNode)->pSelfRC; \
604	} while (0)
605	#define INSERT_RIGHT(a_pParent, a_pNode) \
606	do { \
607	(a_pParent)->pRightR3 = (a_pNode); \
608	(a_pParent)->pRightR0 = (a_pNode)->pSelfR0; \
609	(a_pParent)->pRightRC = (a_pNode)->pSelfRC; \
610	} while (0)
611
612
613	/**
614	* Recursive tree builder.
615	*
616	* @param ppRam Pointer to the iterator variable.
617	* @param iHeight The hight about normal leaf nodes. Inserts a leaf
618	* node if 0.
619	*/
620	static PPGMRAMRANGE pgmR3PhysRebuildRamRangeSearchTreesRecursively(PPGMRAMRANGE *ppRam, int iDepth)
621	{
622	PPGMRAMRANGE pRam;
623	if (iDepth <= 0)
624	{
625	/*
626	* Leaf node.
627	*/
628	pRam = *ppRam;
629	if (pRam)
630	{
631	*ppRam = pRam->pNextR3;
632	MAKE_LEAF(pRam);
633	}
634	}
635	else
636	{
637
638	/*
639	* Intermediate node.
640	*/
641	PPGMRAMRANGE pLeft = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
642
643	pRam = *ppRam;
644	if (!pRam)
645	return pLeft;
646	*ppRam = pRam->pNextR3;
647	MAKE_LEAF(pRam);
648	INSERT_LEFT(pRam, pLeft);
649
650	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
651	if (pRight)
652	INSERT_RIGHT(pRam, pRight);
653	}
654	return pRam;
655	}
656
657
658	/**
659	* Rebuilds the RAM range search trees.
660	*
661	* @param pVM The VM handle.
662	*/
663	static void pgmR3PhysRebuildRamRangeSearchTrees(PVM pVM)
664	{
665
666	/*
667	* Create the reasonably balanced tree in a sequential fashion.
668	* For simplicity (laziness) we use standard recursion here.
669	*/
670	int iDepth = 0;
671	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
672	PPGMRAMRANGE pRoot = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, 0);
673	while (pRam)
674	{
675	PPGMRAMRANGE pLeft = pRoot;
676
677	pRoot = pRam;
678	pRam = pRam->pNextR3;
679	MAKE_LEAF(pRoot);
680	INSERT_LEFT(pRoot, pLeft);
681
682	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, iDepth);
683	if (pRight)
684	INSERT_RIGHT(pRoot, pRight);
685	/** @todo else: rotate the tree. */
686
687	iDepth++;
688	}
689
690	pVM->pgm.s.pRamRangeTreeR3 = pRoot;
691	pVM->pgm.s.pRamRangeTreeR0 = pRoot ? pRoot->pSelfR0 : NIL_RTR0PTR;
692	pVM->pgm.s.pRamRangeTreeRC = pRoot ? pRoot->pSelfRC : NIL_RTRCPTR;
693
694	#ifdef VBOX_STRICT
695	/*
696	* Verify that the above code works.
697	*/
698	unsigned cRanges = 0;
699	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
700	cRanges++;
701	Assert(cRanges > 0);
702
703	unsigned cMaxDepth = ASMBitLastSetU32(cRanges);
704	if ((1U << cMaxDepth) < cRanges)
705	cMaxDepth++;
706
707	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
708	{
709	unsigned cDepth = 0;
710	PPGMRAMRANGE pRam2 = pVM->pgm.s.pRamRangeTreeR3;
711	for (;;)
712	{
713	if (pRam == pRam2)
714	break;
715	Assert(pRam2);
716	if (pRam->GCPhys < pRam2->GCPhys)
717	pRam2 = pRam2->pLeftR3;
718	else
719	pRam2 = pRam2->pRightR3;
720	}
721	AssertMsg(cDepth <= cMaxDepth, ("cDepth=%d cMaxDepth=%d\n", cDepth, cMaxDepth));
722	}
723	#endif /* VBOX_STRICT */
724	}
725
726	#undef MAKE_LEAF
727	#undef INSERT_LEFT
728	#undef INSERT_RIGHT
729
730	/**
731	* Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
732	*
733	* Called when anything was relocated.
734	*
735	* @param pVM Pointer to the shared VM structure.
736	*/
737	void pgmR3PhysRelinkRamRanges(PVM pVM)
738	{
739	PPGMRAMRANGE pCur;
740
741	#ifdef VBOX_STRICT
742	for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
743	{
744	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
745	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfRC == MMHyperCCToRC(pVM, pCur));
746	Assert((pCur->GCPhys & PAGE_OFFSET_MASK) == 0);
747	Assert((pCur->GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
748	Assert((pCur->cb & PAGE_OFFSET_MASK) == 0);
749	Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
750	for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesXR3; pCur2; pCur2 = pCur2->pNextR3)
751	Assert( pCur2 == pCur
752	\|\| strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
753	}
754	#endif
755
756	pCur = pVM->pgm.s.pRamRangesXR3;
757	if (pCur)
758	{
759	pVM->pgm.s.pRamRangesXR0 = pCur->pSelfR0;
760	pVM->pgm.s.pRamRangesXRC = pCur->pSelfRC;
761
762	for (; pCur->pNextR3; pCur = pCur->pNextR3)
763	{
764	pCur->pNextR0 = pCur->pNextR3->pSelfR0;
765	pCur->pNextRC = pCur->pNextR3->pSelfRC;
766	}
767
768	Assert(pCur->pNextR0 == NIL_RTR0PTR);
769	Assert(pCur->pNextRC == NIL_RTRCPTR);
770	}
771	else
772	{
773	Assert(pVM->pgm.s.pRamRangesXR0 == NIL_RTR0PTR);
774	Assert(pVM->pgm.s.pRamRangesXRC == NIL_RTRCPTR);
775	}
776	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
777
778	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
779	}
780
781
782	/**
783	* Links a new RAM range into the list.
784	*
785	* @param pVM Pointer to the shared VM structure.
786	* @param pNew Pointer to the new list entry.
787	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
788	*/
789	static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
790	{
791	AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
792	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfR0 == MMHyperCCToR0(pVM, pNew));
793	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfRC == MMHyperCCToRC(pVM, pNew));
794
795	pgmLock(pVM);
796
797	PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesXR3;
798	pNew->pNextR3 = pRam;
799	pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
800	pNew->pNextRC = pRam ? pRam->pSelfRC : NIL_RTRCPTR;
801
802	if (pPrev)
803	{
804	pPrev->pNextR3 = pNew;
805	pPrev->pNextR0 = pNew->pSelfR0;
806	pPrev->pNextRC = pNew->pSelfRC;
807	}
808	else
809	{
810	pVM->pgm.s.pRamRangesXR3 = pNew;
811	pVM->pgm.s.pRamRangesXR0 = pNew->pSelfR0;
812	pVM->pgm.s.pRamRangesXRC = pNew->pSelfRC;
813	}
814	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
815
816	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
817	pgmUnlock(pVM);
818	}
819
820
821	/**
822	* Unlink an existing RAM range from the list.
823	*
824	* @param pVM Pointer to the shared VM structure.
825	* @param pRam Pointer to the new list entry.
826	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
827	*/
828	static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
829	{
830	Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesXR3 == pRam);
831	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
832	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfRC == MMHyperCCToRC(pVM, pRam));
833
834	pgmLock(pVM);
835
836	PPGMRAMRANGE pNext = pRam->pNextR3;
837	if (pPrev)
838	{
839	pPrev->pNextR3 = pNext;
840	pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
841	pPrev->pNextRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
842	}
843	else
844	{
845	Assert(pVM->pgm.s.pRamRangesXR3 == pRam);
846	pVM->pgm.s.pRamRangesXR3 = pNext;
847	pVM->pgm.s.pRamRangesXR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
848	pVM->pgm.s.pRamRangesXRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
849	}
850	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
851
852	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
853	pgmUnlock(pVM);
854	}
855
856
857	/**
858	* Unlink an existing RAM range from the list.
859	*
860	* @param pVM Pointer to the shared VM structure.
861	* @param pRam Pointer to the new list entry.
862	*/
863	static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
864	{
865	pgmLock(pVM);
866
867	/* find prev. */
868	PPGMRAMRANGE pPrev = NULL;
869	PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesXR3;
870	while (pCur != pRam)
871	{
872	pPrev = pCur;
873	pCur = pCur->pNextR3;
874	}
875	AssertFatal(pCur);
876
877	pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
878	pgmUnlock(pVM);
879	}
880
881
882	/**
883	* Frees a range of pages, replacing them with ZERO pages of the specified type.
884	*
885	* @returns VBox status code.
886	* @param pVM The VM handle.
887	* @param pRam The RAM range in which the pages resides.
888	* @param GCPhys The address of the first page.
889	* @param GCPhysLast The address of the last page.
890	* @param uType The page type to replace then with.
891	*/
892	static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint8_t uType)
893	{
894	PGM_LOCK_ASSERT_OWNER(pVM);
895	uint32_t cPendingPages = 0;
896	PGMMFREEPAGESREQ pReq;
897	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
898	AssertLogRelRCReturn(rc, rc);
899
900	/* Iterate the pages. */
901	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
902	uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
903	while (cPagesLeft-- > 0)
904	{
905	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
906	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
907
908	PGM_PAGE_SET_TYPE(pVM, pPageDst, uType);
909
910	GCPhys += PAGE_SIZE;
911	pPageDst++;
912	}
913
914	if (cPendingPages)
915	{
916	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
917	AssertLogRelRCReturn(rc, rc);
918	}
919	GMMR3FreePagesCleanup(pReq);
920
921	return rc;
922	}
923
924	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
925	/**
926	* Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
927	*
928	* This is only called on one of the EMTs while the other ones are waiting for
929	* it to complete this function.
930	*
931	* @returns VINF_SUCCESS (VBox strict status code).
932	* @param pVM The VM handle.
933	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
934	* @param pvUser User parameter
935	*/
936	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
937	{
938	uintptr_t paUser = (uintptr_t )pvUser;
939	bool fInflate = !!paUser[0];
940	unsigned cPages = paUser[1];
941	RTGCPHYS paPhysPage = (RTGCPHYS )paUser[2];
942	uint32_t cPendingPages = 0;
943	PGMMFREEPAGESREQ pReq;
944	int rc;
945
946	Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
947	pgmLock(pVM);
948
949	if (fInflate)
950	{
951	/* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
952	pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
953
954	/* Replace pages with ZERO pages. */
955	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
956	if (RT_FAILURE(rc))
957	{
958	pgmUnlock(pVM);
959	AssertLogRelRC(rc);
960	return rc;
961	}
962
963	/* Iterate the pages. */
964	for (unsigned i = 0; i < cPages; i++)
965	{
966	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
967	if ( pPage == NULL
968	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM)
969	{
970	Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], pPage ? PGM_PAGE_GET_TYPE(pPage) : 0));
971	break;
972	}
973
974	LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
975
976	/* Flush the shadow PT if this page was previously used as a guest page table. */
977	pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
978
979	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i]);
980	if (RT_FAILURE(rc))
981	{
982	pgmUnlock(pVM);
983	AssertLogRelRC(rc);
984	return rc;
985	}
986	Assert(PGM_PAGE_IS_ZERO(pPage));
987	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_BALLOONED);
988	}
989
990	if (cPendingPages)
991	{
992	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
993	if (RT_FAILURE(rc))
994	{
995	pgmUnlock(pVM);
996	AssertLogRelRC(rc);
997	return rc;
998	}
999	}
1000	GMMR3FreePagesCleanup(pReq);
1001	}
1002	else
1003	{
1004	/* Iterate the pages. */
1005	for (unsigned i = 0; i < cPages; i++)
1006	{
1007	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
1008	AssertBreak(pPage && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
1009
1010	LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
1011
1012	Assert(PGM_PAGE_IS_BALLOONED(pPage));
1013
1014	/* Change back to zero page. */
1015	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1016	}
1017
1018	/* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
1019	}
1020
1021	/* Notify GMM about the balloon change. */
1022	rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
1023	if (RT_SUCCESS(rc))
1024	{
1025	if (!fInflate)
1026	{
1027	Assert(pVM->pgm.s.cBalloonedPages >= cPages);
1028	pVM->pgm.s.cBalloonedPages -= cPages;
1029	}
1030	else
1031	pVM->pgm.s.cBalloonedPages += cPages;
1032	}
1033
1034	pgmUnlock(pVM);
1035
1036	/* Flush the recompiler's TLB as well. */
1037	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1038	CPUMSetChangedFlags(&pVM->aCpus[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1039
1040	AssertLogRelRC(rc);
1041	return rc;
1042	}
1043
1044	/**
1045	* Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
1046	*
1047	* @returns VBox status code.
1048	* @param pVM The VM handle.
1049	* @param fInflate Inflate or deflate memory balloon
1050	* @param cPages Number of pages to free
1051	* @param paPhysPage Array of guest physical addresses
1052	*/
1053	static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1054	{
1055	uintptr_t paUser[3];
1056
1057	paUser[0] = fInflate;
1058	paUser[1] = cPages;
1059	paUser[2] = (uintptr_t)paPhysPage;
1060	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1061	AssertRC(rc);
1062
1063	/* Made a copy in PGMR3PhysFreeRamPages; free it here. */
1064	RTMemFree(paPhysPage);
1065	}
1066	#endif
1067
1068	/**
1069	* Inflate or deflate a memory balloon
1070	*
1071	* @returns VBox status code.
1072	* @param pVM The VM handle.
1073	* @param fInflate Inflate or deflate memory balloon
1074	* @param cPages Number of pages to free
1075	* @param paPhysPage Array of guest physical addresses
1076	*/
1077	VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1078	{
1079	/* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
1080	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
1081	int rc;
1082
1083	/* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
1084	AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
1085
1086	/* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
1087	* In the SMP case we post a request packet to postpone the job.
1088	*/
1089	if (pVM->cCpus > 1)
1090	{
1091	unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
1092	RTGCPHYS paPhysPageCopy = (RTGCPHYS )RTMemAlloc(cbPhysPage);
1093	AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
1094
1095	memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
1096
1097	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
1098	AssertRC(rc);
1099	}
1100	else
1101	{
1102	uintptr_t paUser[3];
1103
1104	paUser[0] = fInflate;
1105	paUser[1] = cPages;
1106	paUser[2] = (uintptr_t)paPhysPage;
1107	rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1108	AssertRC(rc);
1109	}
1110	return rc;
1111	#else
1112	return VERR_NOT_IMPLEMENTED;
1113	#endif
1114	}
1115
1116	/**
1117	* Rendezvous callback used by PGMR3WriteProtectRAM that write protects all
1118	* physical RAM.
1119	*
1120	* This is only called on one of the EMTs while the other ones are waiting for
1121	* it to complete this function.
1122	*
1123	* @returns VINF_SUCCESS (VBox strict status code).
1124	* @param pVM The VM handle.
1125	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
1126	* @param pvUser User parameter, unused.
1127	*/
1128	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1129	{
1130	int rc = VINF_SUCCESS;
1131	NOREF(pvUser);
1132
1133	pgmLock(pVM);
1134	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1135	pgmPoolResetDirtyPages(pVM);
1136	#endif
1137
1138	/** @todo pointless to write protect the physical page pointed to by RSP. */
1139
1140	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1141	pRam;
1142	pRam = pRam->CTX_SUFF(pNext))
1143	{
1144	uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1145	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1146	{
1147	PPGMPAGE pPage = &pRam->aPages[iPage];
1148	PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1149
1150	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1151	\|\| enmPageType == PGMPAGETYPE_MMIO2)
1152	{
1153	/*
1154	* A RAM page.
1155	*/
1156	switch (PGM_PAGE_GET_STATE(pPage))
1157	{
1158	case PGM_PAGE_STATE_ALLOCATED:
1159	/** @todo Optimize this: Don't always re-enable write
1160	* monitoring if the page is known to be very busy. */
1161	if (PGM_PAGE_IS_WRITTEN_TO(pPage))
1162	{
1163	PGM_PAGE_CLEAR_WRITTEN_TO(pVM, pPage);
1164	/* Remember this dirty page for the next (memory) sync. */
1165	PGM_PAGE_SET_FT_DIRTY(pPage);
1166	}
1167
1168	pgmPhysPageWriteMonitor(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1169	break;
1170
1171	case PGM_PAGE_STATE_SHARED:
1172	AssertFailed();
1173	break;
1174
1175	case PGM_PAGE_STATE_WRITE_MONITORED: /* nothing to change. */
1176	default:
1177	break;
1178	}
1179	}
1180	}
1181	}
1182	pgmR3PoolWriteProtectPages(pVM);
1183	PGM_INVL_ALL_VCPU_TLBS(pVM);
1184	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1185	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1186
1187	pgmUnlock(pVM);
1188	return rc;
1189	}
1190
1191	/**
1192	* Protect all physical RAM to monitor writes
1193	*
1194	* @returns VBox status code.
1195	* @param pVM The VM handle.
1196	*/
1197	VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
1198	{
1199	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1200
1201	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
1202	AssertRC(rc);
1203	return rc;
1204	}
1205
1206	/**
1207	* Enumerate all dirty FT pages.
1208	*
1209	* @returns VBox status code.
1210	* @param pVM The VM handle.
1211	* @param pfnEnum Enumerate callback handler.
1212	* @param pvUser Enumerate callback handler parameter.
1213	*/
1214	VMMR3DECL(int) PGMR3PhysEnumDirtyFTPages(PVM pVM, PFNPGMENUMDIRTYFTPAGES pfnEnum, void *pvUser)
1215	{
1216	int rc = VINF_SUCCESS;
1217
1218	pgmLock(pVM);
1219	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1220	pRam;
1221	pRam = pRam->CTX_SUFF(pNext))
1222	{
1223	uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1224	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1225	{
1226	PPGMPAGE pPage = &pRam->aPages[iPage];
1227	PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1228
1229	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1230	\|\| enmPageType == PGMPAGETYPE_MMIO2)
1231	{
1232	/*
1233	* A RAM page.
1234	*/
1235	switch (PGM_PAGE_GET_STATE(pPage))
1236	{
1237	case PGM_PAGE_STATE_ALLOCATED:
1238	case PGM_PAGE_STATE_WRITE_MONITORED:
1239	if ( !PGM_PAGE_IS_WRITTEN_TO(pPage) /* not very recently updated? */
1240	&& PGM_PAGE_IS_FT_DIRTY(pPage))
1241	{
1242	unsigned cbPageRange = PAGE_SIZE;
1243	unsigned iPageClean = iPage + 1;
1244	RTGCPHYS GCPhysPage = pRam->GCPhys + iPage * PAGE_SIZE;
1245	uint8_t *pu8Page = NULL;
1246	PGMPAGEMAPLOCK Lock;
1247
1248	/* Find the next clean page, so we can merge adjacent dirty pages. */
1249	for (; iPageClean < cPages; iPageClean++)
1250	{
1251	PPGMPAGE pPageNext = &pRam->aPages[iPageClean];
1252	if ( RT_UNLIKELY(PGM_PAGE_GET_TYPE(pPageNext) != PGMPAGETYPE_RAM)
1253	\|\| PGM_PAGE_GET_STATE(pPageNext) != PGM_PAGE_STATE_ALLOCATED
1254	\|\| PGM_PAGE_IS_WRITTEN_TO(pPageNext)
1255	\|\| !PGM_PAGE_IS_FT_DIRTY(pPageNext)
1256	/* Crossing a chunk boundary? */
1257	\|\| (GCPhysPage & GMM_PAGEID_IDX_MASK) != ((GCPhysPage + cbPageRange) & GMM_PAGEID_IDX_MASK)
1258	)
1259	break;
1260
1261	cbPageRange += PAGE_SIZE;
1262	}
1263
1264	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysPage, (const void **)&pu8Page, &Lock);
1265	if (RT_SUCCESS(rc))
1266	{
1267	/** @todo this is risky; the range might be changed, but little choice as the sync
1268	* costs a lot of time. */
1269	pgmUnlock(pVM);
1270	pfnEnum(pVM, GCPhysPage, pu8Page, cbPageRange, pvUser);
1271	pgmLock(pVM);
1272	PGMPhysReleasePageMappingLock(pVM, &Lock);
1273	}
1274
1275	for (iPage; iPage < iPageClean; iPage++)
1276	PGM_PAGE_CLEAR_FT_DIRTY(&pRam->aPages[iPage]);
1277
1278	iPage = iPageClean - 1;
1279	}
1280	break;
1281	}
1282	}
1283	}
1284	}
1285	pgmUnlock(pVM);
1286	return rc;
1287	}
1288
1289
1290	/**
1291	* Gets the number of ram ranges.
1292	*
1293	* @returns Number of ram ranges. Returns UINT32_MAX if @a pVM is invalid.
1294	* @param pVM The VM handle.
1295	*/
1296	VMMR3DECL(uint32_t) PGMR3PhysGetRamRangeCount(PVM pVM)
1297	{
1298	VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
1299
1300	pgmLock(pVM);
1301	uint32_t cRamRanges = 0;
1302	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext))
1303	cRamRanges++;
1304	pgmUnlock(pVM);
1305	return cRamRanges;
1306	}
1307
1308
1309	/**
1310	* Get information about a range.
1311	*
1312	* @returns VINF_SUCCESS or VERR_OUT_OF_RANGE.
1313	* @param pVM The VM handle
1314	* @param iRange The ordinal of the range.
1315	* @param pGCPhysStart Where to return the start of the range. Optional.
1316	* @param pGCPhysLast Where to return the address of the last byte in the
1317	* range. Optional.
1318	* @param pfIsMmio Where to indicate that this is a pure MMIO range.
1319	* Optional.
1320	*/
1321	VMMR3DECL(int) PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
1322	const char *ppszDesc, bool pfIsMmio)
1323	{
1324	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1325
1326	pgmLock(pVM);
1327	uint32_t iCurRange = 0;
1328	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext), iCurRange++)
1329	if (iCurRange == iRange)
1330	{
1331	if (pGCPhysStart)
1332	*pGCPhysStart = pCur->GCPhys;
1333	if (pGCPhysLast)
1334	*pGCPhysLast = pCur->GCPhysLast;
1335	if (pfIsMmio)
1336	*pfIsMmio = !!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO);
1337
1338	pgmUnlock(pVM);
1339	return VINF_SUCCESS;
1340	}
1341	pgmUnlock(pVM);
1342	return VERR_OUT_OF_RANGE;
1343	}
1344
1345
1346	/**
1347	* Query the amount of free memory inside VMMR0
1348	*
1349	* @returns VBox status code.
1350	* @param pVM The VM handle.
1351	* @param pcbAllocMem Where to return the amount of memory allocated
1352	* by VMs.
1353	* @param pcbFreeMem Where to return the amount of memory that is
1354	* allocated from the host but not currently used
1355	* by any VMs.
1356	* @param pcbBallonedMem Where to return the sum of memory that is
1357	* currently ballooned by the VMs.
1358	* @param pcbSharedMem Where to return the amount of memory that is
1359	* currently shared.
1360	*/
1361	VMMR3DECL(int) PGMR3QueryGlobalMemoryStats(PVM pVM, uint64_t pcbAllocMem, uint64_t pcbFreeMem,
1362	uint64_t pcbBallonedMem, uint64_t pcbSharedMem)
1363	{
1364	uint64_t cAllocPages = 0;
1365	uint64_t cFreePages = 0;
1366	uint64_t cBalloonPages = 0;
1367	uint64_t cSharedPages = 0;
1368	int rc = GMMR3QueryHypervisorMemoryStats(pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
1369	AssertRCReturn(rc, rc);
1370
1371	if (pcbAllocMem)
1372	pcbAllocMem = cAllocPages _4K;
1373
1374	if (pcbFreeMem)
1375	pcbFreeMem = cFreePages _4K;
1376
1377	if (pcbBallonedMem)
1378	pcbBallonedMem = cBalloonPages _4K;
1379
1380	if (pcbSharedMem)
1381	pcbSharedMem = cSharedPages _4K;
1382
1383	Log(("PGMR3QueryVMMMemoryStats: all=%llx free=%llx ballooned=%llx shared=%llx\n",
1384	cAllocPages, cFreePages, cBalloonPages, cSharedPages));
1385	return VINF_SUCCESS;
1386	}
1387
1388
1389	/**
1390	* Query memory stats for the VM.
1391	*
1392	* @returns VBox status code.
1393	* @param pVM The VM handle.
1394	* @param pcbTotalMem Where to return total amount memory the VM may
1395	* possibly use.
1396	* @param pcbPrivateMem Where to return the amount of private memory
1397	* currently allocated.
1398	* @param pcbSharedMem Where to return the amount of actually shared
1399	* memory currently used by the VM.
1400	* @param pcbZeroMem Where to return the amount of memory backed by
1401	* zero pages.
1402	*
1403	* @remarks The total mem is normally larger than the sum of the three
1404	* components. There are two reasons for this, first the amount of
1405	* shared memory is what we're sure is shared instead of what could
1406	* possibly be shared with someone. Secondly, because the total may
1407	* include some pure MMIO pages that doesn't go into any of the three
1408	* sub-counts.
1409	*
1410	* @todo Why do we return reused shared pages instead of anything that could
1411	* potentially be shared? Doesn't this mean the first VM gets a much
1412	* lower number of shared pages?
1413	*/
1414	VMMR3DECL(int) PGMR3QueryMemoryStats(PVM pVM, uint64_t pcbTotalMem, uint64_t pcbPrivateMem,
1415	uint64_t pcbSharedMem, uint64_t pcbZeroMem)
1416	{
1417	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1418
1419	if (pcbTotalMem)
1420	pcbTotalMem = (uint64_t)pVM->pgm.s.cAllPages PAGE_SIZE;
1421
1422	if (pcbPrivateMem)
1423	pcbPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages PAGE_SIZE;
1424
1425	if (pcbSharedMem)
1426	pcbSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages PAGE_SIZE;
1427
1428	if (pcbZeroMem)
1429	pcbZeroMem = (uint64_t)pVM->pgm.s.cZeroPages PAGE_SIZE;
1430
1431	Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
1432	return VINF_SUCCESS;
1433	}
1434
1435
1436	/**
1437	* PGMR3PhysRegisterRam worker that initializes and links a RAM range.
1438	*
1439	* @param pVM The VM handle.
1440	* @param pNew The new RAM range.
1441	* @param GCPhys The address of the RAM range.
1442	* @param GCPhysLast The last address of the RAM range.
1443	* @param RCPtrNew The RC address if the range is floating. NIL_RTRCPTR
1444	* if in HMA.
1445	* @param R0PtrNew Ditto for R0.
1446	* @param pszDesc The description.
1447	* @param pPrev The previous RAM range (for linking).
1448	*/
1449	static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
1450	RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
1451	{
1452	/*
1453	* Initialize the range.
1454	*/
1455	pNew->pSelfR0 = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
1456	pNew->pSelfRC = RCPtrNew != NIL_RTRCPTR ? RCPtrNew : MMHyperCCToRC(pVM, pNew);
1457	pNew->GCPhys = GCPhys;
1458	pNew->GCPhysLast = GCPhysLast;
1459	pNew->cb = GCPhysLast - GCPhys + 1;
1460	pNew->pszDesc = pszDesc;
1461	pNew->fFlags = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
1462	pNew->pvR3 = NULL;
1463	pNew->paLSPages = NULL;
1464
1465	uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
1466	RTGCPHYS iPage = cPages;
1467	while (iPage-- > 0)
1468	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
1469
1470	/* Update the page count stats. */
1471	pVM->pgm.s.cZeroPages += cPages;
1472	pVM->pgm.s.cAllPages += cPages;
1473
1474	/*
1475	* Link it.
1476	*/
1477	pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
1478	}
1479
1480
1481	/**
1482	* Relocate a floating RAM range.
1483	*
1484	* @copydoc FNPGMRELOCATE.
1485	*/
1486	static DECLCALLBACK(bool) pgmR3PhysRamRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser)
1487	{
1488	PPGMRAMRANGE pRam = (PPGMRAMRANGE)pvUser;
1489	Assert(pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
1490	Assert(pRam->pSelfRC == GCPtrOld + PAGE_SIZE);
1491
1492	switch (enmMode)
1493	{
1494	case PGMRELOCATECALL_SUGGEST:
1495	return true;
1496
1497	case PGMRELOCATECALL_RELOCATE:
1498	{
1499	/*
1500	* Update myself, then relink all the ranges and flush the RC TLB.
1501	*/
1502	pgmLock(pVM);
1503
1504	pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
1505
1506	pgmR3PhysRelinkRamRanges(pVM);
1507	for (unsigned i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
1508	pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
1509
1510	pgmUnlock(pVM);
1511	return true;
1512	}
1513
1514	default:
1515	AssertFailedReturn(false);
1516	}
1517	}
1518
1519
1520	/**
1521	* PGMR3PhysRegisterRam worker that registers a high chunk.
1522	*
1523	* @returns VBox status code.
1524	* @param pVM The VM handle.
1525	* @param GCPhys The address of the RAM.
1526	* @param cRamPages The number of RAM pages to register.
1527	* @param cbChunk The size of the PGMRAMRANGE guest mapping.
1528	* @param iChunk The chunk number.
1529	* @param pszDesc The RAM range description.
1530	* @param ppPrev Previous RAM range pointer. In/Out.
1531	*/
1532	static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages,
1533	uint32_t cbChunk, uint32_t iChunk, const char *pszDesc,
1534	PPGMRAMRANGE *ppPrev)
1535	{
1536	const char *pszDescChunk = iChunk == 0
1537	? pszDesc
1538	: MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
1539	AssertReturn(pszDescChunk, VERR_NO_MEMORY);
1540
1541	/*
1542	* Allocate memory for the new chunk.
1543	*/
1544	size_t const cChunkPages = RT_ALIGN_Z(RT_UOFFSETOF(PGMRAMRANGE, aPages[cRamPages]), PAGE_SIZE) >> PAGE_SHIFT;
1545	PSUPPAGE paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
1546	AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
1547	RTR0PTR R0PtrChunk = NIL_RTR0PTR;
1548	void *pvChunk = NULL;
1549	int rc = SUPR3PageAllocEx(cChunkPages, 0 /fFlags/, &pvChunk,
1550	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1551	VMMIsHwVirtExtForced(pVM) ? &R0PtrChunk : NULL,
1552	#else
1553	NULL,
1554	#endif
1555	paChunkPages);
1556	if (RT_SUCCESS(rc))
1557	{
1558	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1559	if (!VMMIsHwVirtExtForced(pVM))
1560	R0PtrChunk = NIL_RTR0PTR;
1561	#else
1562	R0PtrChunk = (uintptr_t)pvChunk;
1563	#endif
1564	memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
1565
1566	PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
1567
1568	/*
1569	* Create a mapping and map the pages into it.
1570	* We push these in below the HMA.
1571	*/
1572	RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
1573	rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /fFlags/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
1574	if (RT_SUCCESS(rc))
1575	{
1576	pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
1577
1578	RTGCPTR const GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
1579	RTGCPTR GCPtrPage = GCPtrChunk;
1580	for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
1581	rc = PGMMap(pVM, GCPtrPage, paChunkPages[iPage].Phys, PAGE_SIZE, 0);
1582	if (RT_SUCCESS(rc))
1583	{
1584	/*
1585	* Ok, init and link the range.
1586	*/
1587	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
1588	(RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
1589	*ppPrev = pNew;
1590	}
1591	}
1592
1593	if (RT_FAILURE(rc))
1594	SUPR3PageFreeEx(pvChunk, cChunkPages);
1595	}
1596
1597	RTMemTmpFree(paChunkPages);
1598	return rc;
1599	}
1600
1601
1602	/**
1603	* Sets up a range RAM.
1604	*
1605	* This will check for conflicting registrations, make a resource
1606	* reservation for the memory (with GMM), and setup the per-page
1607	* tracking structures (PGMPAGE).
1608	*
1609	* @returns VBox status code.
1610	* @param pVM Pointer to the shared VM structure.
1611	* @param GCPhys The physical address of the RAM.
1612	* @param cb The size of the RAM.
1613	* @param pszDesc The description - not copied, so, don't free or change it.
1614	*/
1615	VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
1616	{
1617	/*
1618	* Validate input.
1619	*/
1620	Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
1621	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
1622	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
1623	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
1624	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1625	AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
1626	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1627	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1628
1629	pgmLock(pVM);
1630
1631	/*
1632	* Find range location and check for conflicts.
1633	* (We don't lock here because the locking by EMT is only required on update.)
1634	*/
1635	PPGMRAMRANGE pPrev = NULL;
1636	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
1637	while (pRam && GCPhysLast >= pRam->GCPhys)
1638	{
1639	if ( GCPhysLast >= pRam->GCPhys
1640	&& GCPhys <= pRam->GCPhysLast)
1641	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
1642	GCPhys, GCPhysLast, pszDesc,
1643	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1644	VERR_PGM_RAM_CONFLICT);
1645
1646	/* next */
1647	pPrev = pRam;
1648	pRam = pRam->pNextR3;
1649	}
1650
1651	/*
1652	* Register it with GMM (the API bitches).
1653	*/
1654	const RTGCPHYS cPages = cb >> PAGE_SHIFT;
1655	int rc = MMR3IncreaseBaseReservation(pVM, cPages);
1656	if (RT_FAILURE(rc))
1657	{
1658	pgmUnlock(pVM);
1659	return rc;
1660	}
1661
1662	if ( GCPhys >= _4G
1663	&& cPages > 256)
1664	{
1665	/*
1666	* The PGMRAMRANGE structures for the high memory can get very big.
1667	* In order to avoid SUPR3PageAllocEx allocation failures due to the
1668	* allocation size limit there and also to avoid being unable to find
1669	* guest mapping space for them, we split this memory up into 4MB in
1670	* (potential) raw-mode configs and 16MB chunks in forced AMD-V/VT-x
1671	* mode.
1672	*
1673	* The first and last page of each mapping are guard pages and marked
1674	* not-present. So, we've got 4186112 and 16769024 bytes available for
1675	* the PGMRAMRANGE structure.
1676	*
1677	* Note! The sizes used here will influence the saved state.
1678	*/
1679	uint32_t cbChunk;
1680	uint32_t cPagesPerChunk;
1681	if (VMMIsHwVirtExtForced(pVM))
1682	{
1683	cbChunk = 16U*_1M;
1684	cPagesPerChunk = 1048048; /* max ~1048059 */
1685	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 1048048 < 16U_1M - PAGE_SIZE 2);
1686	}
1687	else
1688	{
1689	cbChunk = 4U*_1M;
1690	cPagesPerChunk = 261616; /* max ~261627 */
1691	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 261616 < 4U_1M - PAGE_SIZE 2);
1692	}
1693	AssertRelease(RT_UOFFSETOF(PGMRAMRANGE, aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
1694
1695	RTGCPHYS cPagesLeft = cPages;
1696	RTGCPHYS GCPhysChunk = GCPhys;
1697	uint32_t iChunk = 0;
1698	while (cPagesLeft > 0)
1699	{
1700	uint32_t cPagesInChunk = cPagesLeft;
1701	if (cPagesInChunk > cPagesPerChunk)
1702	cPagesInChunk = cPagesPerChunk;
1703
1704	rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, cbChunk, iChunk, pszDesc, &pPrev);
1705	AssertRCReturn(rc, rc);
1706
1707	/* advance */
1708	GCPhysChunk += (RTGCPHYS)cPagesInChunk << PAGE_SHIFT;
1709	cPagesLeft -= cPagesInChunk;
1710	iChunk++;
1711	}
1712	}
1713	else
1714	{
1715	/*
1716	* Allocate, initialize and link the new RAM range.
1717	*/
1718	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
1719	PPGMRAMRANGE pNew;
1720	rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
1721	AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
1722
1723	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
1724	}
1725	pgmPhysInvalidatePageMapTLB(pVM);
1726	pgmUnlock(pVM);
1727
1728	/*
1729	* Notify REM.
1730	*/
1731	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_RAM);
1732
1733	return VINF_SUCCESS;
1734	}
1735
1736
1737	/**
1738	* Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
1739	*
1740	* We do this late in the init process so that all the ROM and MMIO ranges have
1741	* been registered already and we don't go wasting memory on them.
1742	*
1743	* @returns VBox status code.
1744	*
1745	* @param pVM Pointer to the shared VM structure.
1746	*/
1747	int pgmR3PhysRamPreAllocate(PVM pVM)
1748	{
1749	Assert(pVM->pgm.s.fRamPreAlloc);
1750	Log(("pgmR3PhysRamPreAllocate: enter\n"));
1751
1752	/*
1753	* Walk the RAM ranges and allocate all RAM pages, halt at
1754	* the first allocation error.
1755	*/
1756	uint64_t cPages = 0;
1757	uint64_t NanoTS = RTTimeNanoTS();
1758	pgmLock(pVM);
1759	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1760	{
1761	PPGMPAGE pPage = &pRam->aPages[0];
1762	RTGCPHYS GCPhys = pRam->GCPhys;
1763	uint32_t cLeft = pRam->cb >> PAGE_SHIFT;
1764	while (cLeft-- > 0)
1765	{
1766	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1767	{
1768	switch (PGM_PAGE_GET_STATE(pPage))
1769	{
1770	case PGM_PAGE_STATE_ZERO:
1771	{
1772	int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
1773	if (RT_FAILURE(rc))
1774	{
1775	LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
1776	pgmUnlock(pVM);
1777	return rc;
1778	}
1779	cPages++;
1780	break;
1781	}
1782
1783	case PGM_PAGE_STATE_BALLOONED:
1784	case PGM_PAGE_STATE_ALLOCATED:
1785	case PGM_PAGE_STATE_WRITE_MONITORED:
1786	case PGM_PAGE_STATE_SHARED:
1787	/* nothing to do here. */
1788	break;
1789	}
1790	}
1791
1792	/* next */
1793	pPage++;
1794	GCPhys += PAGE_SIZE;
1795	}
1796	}
1797	pgmUnlock(pVM);
1798	NanoTS = RTTimeNanoTS() - NanoTS;
1799
1800	LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
1801	Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
1802	return VINF_SUCCESS;
1803	}
1804
1805
1806	/**
1807	* Resets (zeros) the RAM.
1808	*
1809	* ASSUMES that the caller owns the PGM lock.
1810	*
1811	* @returns VBox status code.
1812	* @param pVM Pointer to the shared VM structure.
1813	*/
1814	int pgmR3PhysRamReset(PVM pVM)
1815	{
1816	PGM_LOCK_ASSERT_OWNER(pVM);
1817
1818	/* Reset the memory balloon. */
1819	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1820	AssertRC(rc);
1821
1822	#ifdef VBOX_WITH_PAGE_SHARING
1823	/* Clear all registered shared modules. */
1824	rc = GMMR3ResetSharedModules(pVM);
1825	AssertRC(rc);
1826	#endif
1827	/* Reset counters. */
1828	pVM->pgm.s.cReusedSharedPages = 0;
1829	pVM->pgm.s.cBalloonedPages = 0;
1830
1831	/*
1832	* We batch up pages that should be freed instead of calling GMM for
1833	* each and every one of them.
1834	*/
1835	uint32_t cPendingPages = 0;
1836	PGMMFREEPAGESREQ pReq;
1837	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1838	AssertLogRelRCReturn(rc, rc);
1839
1840	/*
1841	* Walk the ram ranges.
1842	*/
1843	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1844	{
1845	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1846	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1847
1848	if (!pVM->pgm.s.fRamPreAlloc)
1849	{
1850	/* Replace all RAM pages by ZERO pages. */
1851	while (iPage-- > 0)
1852	{
1853	PPGMPAGE pPage = &pRam->aPages[iPage];
1854	switch (PGM_PAGE_GET_TYPE(pPage))
1855	{
1856	case PGMPAGETYPE_RAM:
1857	/* Do not replace pages part of a 2 MB continuous range
1858	with zero pages, but zero them instead. */
1859	if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE
1860	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
1861	{
1862	void *pvPage;
1863	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1864	AssertLogRelRCReturn(rc, rc);
1865	ASMMemZeroPage(pvPage);
1866	}
1867	else if (PGM_PAGE_IS_BALLOONED(pPage))
1868	{
1869	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
1870	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1871	}
1872	else if (!PGM_PAGE_IS_ZERO(pPage))
1873	{
1874	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1875	AssertLogRelRCReturn(rc, rc);
1876	}
1877	break;
1878
1879	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1880	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
1881	true /fDoAccounting/);
1882	break;
1883
1884	case PGMPAGETYPE_MMIO2:
1885	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
1886	case PGMPAGETYPE_ROM:
1887	case PGMPAGETYPE_MMIO:
1888	break;
1889	default:
1890	AssertFailed();
1891	}
1892	} /* for each page */
1893	}
1894	else
1895	{
1896	/* Zero the memory. */
1897	while (iPage-- > 0)
1898	{
1899	PPGMPAGE pPage = &pRam->aPages[iPage];
1900	switch (PGM_PAGE_GET_TYPE(pPage))
1901	{
1902	case PGMPAGETYPE_RAM:
1903	switch (PGM_PAGE_GET_STATE(pPage))
1904	{
1905	case PGM_PAGE_STATE_ZERO:
1906	break;
1907
1908	case PGM_PAGE_STATE_BALLOONED:
1909	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
1910	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1911	break;
1912
1913	case PGM_PAGE_STATE_SHARED:
1914	case PGM_PAGE_STATE_WRITE_MONITORED:
1915	rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1916	AssertLogRelRCReturn(rc, rc);
1917	/* no break */
1918
1919	case PGM_PAGE_STATE_ALLOCATED:
1920	{
1921	void *pvPage;
1922	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1923	AssertLogRelRCReturn(rc, rc);
1924	ASMMemZeroPage(pvPage);
1925	break;
1926	}
1927	}
1928	break;
1929
1930	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1931	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
1932	true /fDoAccounting/);
1933	break;
1934
1935	case PGMPAGETYPE_MMIO2:
1936	case PGMPAGETYPE_ROM_SHADOW:
1937	case PGMPAGETYPE_ROM:
1938	case PGMPAGETYPE_MMIO:
1939	break;
1940	default:
1941	AssertFailed();
1942
1943	}
1944	} /* for each page */
1945	}
1946
1947	}
1948
1949	/*
1950	* Finish off any pages pending freeing.
1951	*/
1952	if (cPendingPages)
1953	{
1954	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1955	AssertLogRelRCReturn(rc, rc);
1956	}
1957	GMMR3FreePagesCleanup(pReq);
1958
1959	return VINF_SUCCESS;
1960	}
1961
1962	/**
1963	* Frees all RAM during VM termination
1964	*
1965	* ASSUMES that the caller owns the PGM lock.
1966	*
1967	* @returns VBox status code.
1968	* @param pVM Pointer to the shared VM structure.
1969	*/
1970	int pgmR3PhysRamTerm(PVM pVM)
1971	{
1972	PGM_LOCK_ASSERT_OWNER(pVM);
1973
1974	/* Reset the memory balloon. */
1975	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1976	AssertRC(rc);
1977
1978	#ifdef VBOX_WITH_PAGE_SHARING
1979	/* Clear all registered shared modules. */
1980	rc = GMMR3ResetSharedModules(pVM);
1981	AssertRC(rc);
1982	#endif
1983
1984	/*
1985	* We batch up pages that should be freed instead of calling GMM for
1986	* each and every one of them.
1987	*/
1988	uint32_t cPendingPages = 0;
1989	PGMMFREEPAGESREQ pReq;
1990	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1991	AssertLogRelRCReturn(rc, rc);
1992
1993	/*
1994	* Walk the ram ranges.
1995	*/
1996	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1997	{
1998	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1999	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
2000
2001	/* Replace all RAM pages by ZERO pages. */
2002	while (iPage-- > 0)
2003	{
2004	PPGMPAGE pPage = &pRam->aPages[iPage];
2005	switch (PGM_PAGE_GET_TYPE(pPage))
2006	{
2007	case PGMPAGETYPE_RAM:
2008	/* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
2009	if (PGM_PAGE_IS_SHARED(pPage))
2010	{
2011	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
2012	AssertLogRelRCReturn(rc, rc);
2013	}
2014	break;
2015
2016	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2017	case PGMPAGETYPE_MMIO2:
2018	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2019	case PGMPAGETYPE_ROM:
2020	case PGMPAGETYPE_MMIO:
2021	break;
2022	default:
2023	AssertFailed();
2024	}
2025	} /* for each page */
2026	}
2027
2028	/*
2029	* Finish off any pages pending freeing.
2030	*/
2031	if (cPendingPages)
2032	{
2033	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2034	AssertLogRelRCReturn(rc, rc);
2035	}
2036	GMMR3FreePagesCleanup(pReq);
2037	return VINF_SUCCESS;
2038	}
2039
2040	/**
2041	* This is the interface IOM is using to register an MMIO region.
2042	*
2043	* It will check for conflicts and ensure that a RAM range structure
2044	* is present before calling the PGMR3HandlerPhysicalRegister API to
2045	* register the callbacks.
2046	*
2047	* @returns VBox status code.
2048	*
2049	* @param pVM Pointer to the shared VM structure.
2050	* @param GCPhys The start of the MMIO region.
2051	* @param cb The size of the MMIO region.
2052	* @param pfnHandlerR3 The address of the ring-3 handler. (IOMR3MMIOHandler)
2053	* @param pvUserR3 The user argument for R3.
2054	* @param pfnHandlerR0 The address of the ring-0 handler. (IOMMMIOHandler)
2055	* @param pvUserR0 The user argument for R0.
2056	* @param pfnHandlerRC The address of the RC handler. (IOMMMIOHandler)
2057	* @param pvUserRC The user argument for RC.
2058	* @param pszDesc The description of the MMIO region.
2059	*/
2060	VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb,
2061	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3, RTR3PTR pvUserR3,
2062	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0, RTR0PTR pvUserR0,
2063	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC, RTRCPTR pvUserRC,
2064	R3PTRTYPE(const char *) pszDesc)
2065	{
2066	/*
2067	* Assert on some assumption.
2068	*/
2069	VM_ASSERT_EMT(pVM);
2070	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2071	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2072	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2073	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2074
2075	int rc = pgmLock(pVM);
2076	AssertRCReturn(rc, rc);
2077
2078	/*
2079	* Make sure there's a RAM range structure for the region.
2080	*/
2081	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2082	bool fRamExists = false;
2083	PPGMRAMRANGE pRamPrev = NULL;
2084	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2085	while (pRam && GCPhysLast >= pRam->GCPhys)
2086	{
2087	if ( GCPhysLast >= pRam->GCPhys
2088	&& GCPhys <= pRam->GCPhysLast)
2089	{
2090	/* Simplification: all within the same range. */
2091	AssertLogRelMsgReturnStmt( GCPhys >= pRam->GCPhys
2092	&& GCPhysLast <= pRam->GCPhysLast,
2093	("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
2094	GCPhys, GCPhysLast, pszDesc,
2095	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2096	pgmUnlock(pVM),
2097	VERR_PGM_RAM_CONFLICT);
2098
2099	/* Check that it's all RAM or MMIO pages. */
2100	PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2101	uint32_t cLeft = cb >> PAGE_SHIFT;
2102	while (cLeft-- > 0)
2103	{
2104	AssertLogRelMsgReturnStmt( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2105	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
2106	("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
2107	GCPhys, GCPhysLast, pszDesc, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
2108	pgmUnlock(pVM),
2109	VERR_PGM_RAM_CONFLICT);
2110	pPage++;
2111	}
2112
2113	/* Looks good. */
2114	fRamExists = true;
2115	break;
2116	}
2117
2118	/* next */
2119	pRamPrev = pRam;
2120	pRam = pRam->pNextR3;
2121	}
2122	PPGMRAMRANGE pNew;
2123	if (fRamExists)
2124	{
2125	pNew = NULL;
2126
2127	/*
2128	* Make all the pages in the range MMIO/ZERO pages, freeing any
2129	* RAM pages currently mapped here. This might not be 100% correct
2130	* for PCI memory, but we're doing the same thing for MMIO2 pages.
2131	*/
2132	rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
2133	AssertRCReturnStmt(rc, pgmUnlock(pVM), rc);
2134
2135	/* Force a PGM pool flush as guest ram references have been changed. */
2136	/** @todo not entirely SMP safe; assuming for now the guest takes
2137	* care of this internally (not touch mapped mmio while changing the
2138	* mapping). */
2139	PVMCPU pVCpu = VMMGetCpu(pVM);
2140	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2141	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2142	}
2143	else
2144	{
2145
2146	/*
2147	* No RAM range, insert an ad hoc one.
2148	*
2149	* Note that we don't have to tell REM about this range because
2150	* PGMHandlerPhysicalRegisterEx will do that for us.
2151	*/
2152	Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
2153
2154	const uint32_t cPages = cb >> PAGE_SHIFT;
2155	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
2156	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
2157	AssertLogRelMsgRCReturnStmt(rc, ("cbRamRange=%zu\n", cbRamRange), pgmUnlock(pVM), rc);
2158
2159	/* Initialize the range. */
2160	pNew->pSelfR0 = MMHyperCCToR0(pVM, pNew);
2161	pNew->pSelfRC = MMHyperCCToRC(pVM, pNew);
2162	pNew->GCPhys = GCPhys;
2163	pNew->GCPhysLast = GCPhysLast;
2164	pNew->cb = cb;
2165	pNew->pszDesc = pszDesc;
2166	pNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
2167	pNew->pvR3 = NULL;
2168	pNew->paLSPages = NULL;
2169
2170	uint32_t iPage = cPages;
2171	while (iPage-- > 0)
2172	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
2173	Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
2174
2175	/* update the page count stats. */
2176	pVM->pgm.s.cPureMmioPages += cPages;
2177	pVM->pgm.s.cAllPages += cPages;
2178
2179	/* link it */
2180	pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
2181	}
2182
2183	/*
2184	* Register the access handler.
2185	*/
2186	rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_MMIO, GCPhys, GCPhysLast,
2187	pfnHandlerR3, pvUserR3,
2188	pfnHandlerR0, pvUserR0,
2189	pfnHandlerRC, pvUserRC, pszDesc);
2190	if ( RT_FAILURE(rc)
2191	&& !fRamExists)
2192	{
2193	pVM->pgm.s.cPureMmioPages -= cb >> PAGE_SHIFT;
2194	pVM->pgm.s.cAllPages -= cb >> PAGE_SHIFT;
2195
2196	/* remove the ad hoc range. */
2197	pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
2198	pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
2199	MMHyperFree(pVM, pRam);
2200	}
2201	pgmPhysInvalidatePageMapTLB(pVM);
2202
2203	pgmUnlock(pVM);
2204	return rc;
2205	}
2206
2207
2208	/**
2209	* This is the interface IOM is using to register an MMIO region.
2210	*
2211	* It will take care of calling PGMHandlerPhysicalDeregister and clean up
2212	* any ad hoc PGMRAMRANGE left behind.
2213	*
2214	* @returns VBox status code.
2215	* @param pVM Pointer to the shared VM structure.
2216	* @param GCPhys The start of the MMIO region.
2217	* @param cb The size of the MMIO region.
2218	*/
2219	VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
2220	{
2221	VM_ASSERT_EMT(pVM);
2222
2223	int rc = pgmLock(pVM);
2224	AssertRCReturn(rc, rc);
2225
2226	/*
2227	* First deregister the handler, then check if we should remove the ram range.
2228	*/
2229	rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
2230	if (RT_SUCCESS(rc))
2231	{
2232	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2233	PPGMRAMRANGE pRamPrev = NULL;
2234	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2235	while (pRam && GCPhysLast >= pRam->GCPhys)
2236	{
2237	/** @todo We're being a bit too careful here. rewrite. */
2238	if ( GCPhysLast == pRam->GCPhysLast
2239	&& GCPhys == pRam->GCPhys)
2240	{
2241	Assert(pRam->cb == cb);
2242
2243	/*
2244	* See if all the pages are dead MMIO pages.
2245	*/
2246	uint32_t const cPages = cb >> PAGE_SHIFT;
2247	bool fAllMMIO = true;
2248	uint32_t iPage = 0;
2249	uint32_t cLeft = cPages;
2250	while (cLeft-- > 0)
2251	{
2252	PPGMPAGE pPage = &pRam->aPages[iPage];
2253	if ( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO
2254	/\|\| not-out-of-action later /)
2255	{
2256	fAllMMIO = false;
2257	Assert(PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO2_ALIAS_MMIO);
2258	AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2259	break;
2260	}
2261	Assert(PGM_PAGE_IS_ZERO(pPage));
2262	pPage++;
2263	}
2264	if (fAllMMIO)
2265	{
2266	/*
2267	* Ad-hoc range, unlink and free it.
2268	*/
2269	Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
2270	GCPhys, GCPhysLast, pRam->pszDesc));
2271
2272	pVM->pgm.s.cAllPages -= cPages;
2273	pVM->pgm.s.cPureMmioPages -= cPages;
2274
2275	pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
2276	pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
2277	MMHyperFree(pVM, pRam);
2278	break;
2279	}
2280	}
2281
2282	/*
2283	* Range match? It will all be within one range (see PGMAllHandler.cpp).
2284	*/
2285	if ( GCPhysLast >= pRam->GCPhys
2286	&& GCPhys <= pRam->GCPhysLast)
2287	{
2288	Assert(GCPhys >= pRam->GCPhys);
2289	Assert(GCPhysLast <= pRam->GCPhysLast);
2290
2291	/*
2292	* Turn the pages back into RAM pages.
2293	*/
2294	uint32_t iPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2295	uint32_t cLeft = cb >> PAGE_SHIFT;
2296	while (cLeft--)
2297	{
2298	PPGMPAGE pPage = &pRam->aPages[iPage];
2299	AssertMsg(PGM_PAGE_IS_MMIO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2300	AssertMsg(PGM_PAGE_IS_ZERO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2301	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO)
2302	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_RAM);
2303	}
2304	break;
2305	}
2306
2307	/* next */
2308	pRamPrev = pRam;
2309	pRam = pRam->pNextR3;
2310	}
2311	}
2312
2313	/* Force a PGM pool flush as guest ram references have been changed. */
2314	/** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
2315	PVMCPU pVCpu = VMMGetCpu(pVM);
2316	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2317	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2318
2319	pgmPhysInvalidatePageMapTLB(pVM);
2320	pgmPhysInvalidRamRangeTlbs(pVM);
2321	pgmUnlock(pVM);
2322	return rc;
2323	}
2324
2325
2326	/**
2327	* Locate a MMIO2 range.
2328	*
2329	* @returns Pointer to the MMIO2 range.
2330	* @param pVM Pointer to the shared VM structure.
2331	* @param pDevIns The device instance owning the region.
2332	* @param iRegion The region.
2333	*/
2334	DECLINLINE(PPGMMMIO2RANGE) pgmR3PhysMMIO2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2335	{
2336	/*
2337	* Search the list.
2338	*/
2339	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2340	if ( pCur->pDevInsR3 == pDevIns
2341	&& pCur->iRegion == iRegion)
2342	return pCur;
2343	return NULL;
2344	}
2345
2346
2347	/**
2348	* Allocate and register an MMIO2 region.
2349	*
2350	* As mentioned elsewhere, MMIO2 is just RAM spelled differently. It's RAM
2351	* associated with a device. It is also non-shared memory with a permanent
2352	* ring-3 mapping and page backing (presently).
2353	*
2354	* A MMIO2 range may overlap with base memory if a lot of RAM is configured for
2355	* the VM, in which case we'll drop the base memory pages. Presently we will
2356	* make no attempt to preserve anything that happens to be present in the base
2357	* memory that is replaced, this is of course incorrectly but it's too much
2358	* effort.
2359	*
2360	* @returns VBox status code.
2361	* @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
2362	* memory.
2363	* @retval VERR_ALREADY_EXISTS if the region already exists.
2364	*
2365	* @param pVM Pointer to the shared VM structure.
2366	* @param pDevIns The device instance owning the region.
2367	* @param iRegion The region number. If the MMIO2 memory is a PCI
2368	* I/O region this number has to be the number of that
2369	* region. Otherwise it can be any number safe
2370	* UINT8_MAX.
2371	* @param cb The size of the region. Must be page aligned.
2372	* @param fFlags Reserved for future use, must be zero.
2373	* @param ppv Where to store the pointer to the ring-3 mapping of
2374	* the memory.
2375	* @param pszDesc The description.
2376	*/
2377	VMMR3DECL(int) PGMR3PhysMMIO2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags, void *ppv, const char pszDesc)
2378	{
2379	/*
2380	* Validate input.
2381	*/
2382	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2383	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2384	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2385	AssertPtrReturn(ppv, VERR_INVALID_POINTER);
2386	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2387	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2388	AssertReturn(pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion) == NULL, VERR_ALREADY_EXISTS);
2389	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2390	AssertReturn(cb, VERR_INVALID_PARAMETER);
2391	AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
2392
2393	const uint32_t cPages = cb >> PAGE_SHIFT;
2394	AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
2395	AssertLogRelReturn(cPages <= INT32_MAX / 2, VERR_NO_MEMORY);
2396
2397	/*
2398	* For the 2nd+ instance, mangle the description string so it's unique.
2399	*/
2400	if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
2401	{
2402	pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
2403	if (!pszDesc)
2404	return VERR_NO_MEMORY;
2405	}
2406
2407	/*
2408	* Try reserve and allocate the backing memory first as this is what is
2409	* most likely to fail.
2410	*/
2411	int rc = MMR3AdjustFixedReservation(pVM, cPages, pszDesc);
2412	if (RT_SUCCESS(rc))
2413	{
2414	void *pvPages;
2415	PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(SUPPAGE));
2416	if (RT_SUCCESS(rc))
2417	rc = SUPR3PageAllocEx(cPages, 0 /fFlags/, &pvPages, NULL /pR0Ptr/, paPages);
2418	if (RT_SUCCESS(rc))
2419	{
2420	memset(pvPages, 0, cPages * PAGE_SIZE);
2421
2422	/*
2423	* Create the MMIO2 range record for it.
2424	*/
2425	const size_t cbRange = RT_OFFSETOF(PGMMMIO2RANGE, RamRange.aPages[cPages]);
2426	PPGMMMIO2RANGE pNew;
2427	rc = MMR3HyperAllocOnceNoRel(pVM, cbRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
2428	AssertLogRelMsgRC(rc, ("cbRamRange=%zu\n", cbRange));
2429	if (RT_SUCCESS(rc))
2430	{
2431	pNew->pDevInsR3 = pDevIns;
2432	pNew->pvR3 = pvPages;
2433	//pNew->pNext = NULL;
2434	//pNew->fMapped = false;
2435	//pNew->fOverlapping = false;
2436	pNew->iRegion = iRegion;
2437	pNew->idSavedState = UINT8_MAX;
2438	pNew->RamRange.pSelfR0 = MMHyperCCToR0(pVM, &pNew->RamRange);
2439	pNew->RamRange.pSelfRC = MMHyperCCToRC(pVM, &pNew->RamRange);
2440	pNew->RamRange.GCPhys = NIL_RTGCPHYS;
2441	pNew->RamRange.GCPhysLast = NIL_RTGCPHYS;
2442	pNew->RamRange.pszDesc = pszDesc;
2443	pNew->RamRange.cb = cb;
2444	pNew->RamRange.fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2;
2445	pNew->RamRange.pvR3 = pvPages;
2446	//pNew->RamRange.paLSPages = NULL;
2447
2448	uint32_t iPage = cPages;
2449	while (iPage-- > 0)
2450	{
2451	PGM_PAGE_INIT(&pNew->RamRange.aPages[iPage],
2452	paPages[iPage].Phys, NIL_GMM_PAGEID,
2453	PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
2454	}
2455
2456	/* update page count stats */
2457	pVM->pgm.s.cAllPages += cPages;
2458	pVM->pgm.s.cPrivatePages += cPages;
2459
2460	/*
2461	* Link it into the list.
2462	* Since there is no particular order, just push it.
2463	*/
2464	pgmLock(pVM);
2465	pNew->pNextR3 = pVM->pgm.s.pMmio2RangesR3;
2466	pVM->pgm.s.pMmio2RangesR3 = pNew;
2467	pgmUnlock(pVM);
2468
2469	*ppv = pvPages;
2470	RTMemTmpFree(paPages);
2471	pgmPhysInvalidatePageMapTLB(pVM);
2472	return VINF_SUCCESS;
2473	}
2474
2475	SUPR3PageFreeEx(pvPages, cPages);
2476	}
2477	RTMemTmpFree(paPages);
2478	MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pszDesc);
2479	}
2480	if (pDevIns->iInstance > 0)
2481	MMR3HeapFree((void *)pszDesc);
2482	return rc;
2483	}
2484
2485
2486	/**
2487	* Deregisters and frees an MMIO2 region.
2488	*
2489	* Any physical (and virtual) access handlers registered for the region must
2490	* be deregistered before calling this function.
2491	*
2492	* @returns VBox status code.
2493	* @param pVM Pointer to the shared VM structure.
2494	* @param pDevIns The device instance owning the region.
2495	* @param iRegion The region. If it's UINT32_MAX it'll be a wildcard match.
2496	*/
2497	VMMR3DECL(int) PGMR3PhysMMIO2Deregister(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2498	{
2499	/*
2500	* Validate input.
2501	*/
2502	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2503	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2504	AssertReturn(iRegion <= UINT8_MAX \|\| iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
2505
2506	pgmLock(pVM);
2507	int rc = VINF_SUCCESS;
2508	unsigned cFound = 0;
2509	PPGMMMIO2RANGE pPrev = NULL;
2510	PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3;
2511	while (pCur)
2512	{
2513	if ( pCur->pDevInsR3 == pDevIns
2514	&& ( iRegion == UINT32_MAX
2515	\|\| pCur->iRegion == iRegion))
2516	{
2517	cFound++;
2518
2519	/*
2520	* Unmap it if it's mapped.
2521	*/
2522	if (pCur->fMapped)
2523	{
2524	int rc2 = PGMR3PhysMMIO2Unmap(pVM, pCur->pDevInsR3, pCur->iRegion, pCur->RamRange.GCPhys);
2525	AssertRC(rc2);
2526	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2527	rc = rc2;
2528	}
2529
2530	/*
2531	* Unlink it
2532	*/
2533	PPGMMMIO2RANGE pNext = pCur->pNextR3;
2534	if (pPrev)
2535	pPrev->pNextR3 = pNext;
2536	else
2537	pVM->pgm.s.pMmio2RangesR3 = pNext;
2538	pCur->pNextR3 = NULL;
2539
2540	/*
2541	* Free the memory.
2542	*/
2543	int rc2 = SUPR3PageFreeEx(pCur->pvR3, pCur->RamRange.cb >> PAGE_SHIFT);
2544	AssertRC(rc2);
2545	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2546	rc = rc2;
2547
2548	uint32_t const cPages = pCur->RamRange.cb >> PAGE_SHIFT;
2549	rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pCur->RamRange.pszDesc);
2550	AssertRC(rc2);
2551	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2552	rc = rc2;
2553
2554	/* we're leaking hyper memory here if done at runtime. */
2555	#ifdef VBOX_STRICT
2556	VMSTATE const enmState = VMR3GetState(pVM);
2557	AssertMsg( enmState == VMSTATE_POWERING_OFF
2558	\|\| enmState == VMSTATE_POWERING_OFF_LS
2559	\|\| enmState == VMSTATE_OFF
2560	\|\| enmState == VMSTATE_OFF_LS
2561	\|\| enmState == VMSTATE_DESTROYING
2562	\|\| enmState == VMSTATE_TERMINATED
2563	\|\| enmState == VMSTATE_CREATING
2564	, ("%s\n", VMR3GetStateName(enmState)));
2565	#endif
2566	/*rc = MMHyperFree(pVM, pCur);
2567	AssertRCReturn(rc, rc); - not safe, see the alloc call. */
2568
2569
2570	/* update page count stats */
2571	pVM->pgm.s.cAllPages -= cPages;
2572	pVM->pgm.s.cPrivatePages -= cPages;
2573
2574	/* next */
2575	pCur = pNext;
2576	}
2577	else
2578	{
2579	pPrev = pCur;
2580	pCur = pCur->pNextR3;
2581	}
2582	}
2583	pgmPhysInvalidatePageMapTLB(pVM);
2584	pgmUnlock(pVM);
2585	return !cFound && iRegion != UINT32_MAX ? VERR_NOT_FOUND : rc;
2586	}
2587
2588
2589	/**
2590	* Maps a MMIO2 region.
2591	*
2592	* This is done when a guest / the bios / state loading changes the
2593	* PCI config. The replacing of base memory has the same restrictions
2594	* as during registration, of course.
2595	*
2596	* @returns VBox status code.
2597	*
2598	* @param pVM Pointer to the shared VM structure.
2599	* @param pDevIns The
2600	*/
2601	VMMR3DECL(int) PGMR3PhysMMIO2Map(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2602	{
2603	/*
2604	* Validate input
2605	*/
2606	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2607	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2608	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2609	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2610	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2611	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2612
2613	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2614	AssertReturn(pCur, VERR_NOT_FOUND);
2615	AssertReturn(!pCur->fMapped, VERR_WRONG_ORDER);
2616	Assert(pCur->RamRange.GCPhys == NIL_RTGCPHYS);
2617	Assert(pCur->RamRange.GCPhysLast == NIL_RTGCPHYS);
2618
2619	const RTGCPHYS GCPhysLast = GCPhys + pCur->RamRange.cb - 1;
2620	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2621
2622	/*
2623	* Find our location in the ram range list, checking for
2624	* restriction we don't bother implementing yet (partially overlapping).
2625	*/
2626	bool fRamExists = false;
2627	PPGMRAMRANGE pRamPrev = NULL;
2628	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2629	while (pRam && GCPhysLast >= pRam->GCPhys)
2630	{
2631	if ( GCPhys <= pRam->GCPhysLast
2632	&& GCPhysLast >= pRam->GCPhys)
2633	{
2634	/* completely within? */
2635	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
2636	&& GCPhysLast <= pRam->GCPhysLast,
2637	("%RGp-%RGp (MMIO2/%s) falls partly outside %RGp-%RGp (%s)\n",
2638	GCPhys, GCPhysLast, pCur->RamRange.pszDesc,
2639	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2640	VERR_PGM_RAM_CONFLICT);
2641	fRamExists = true;
2642	break;
2643	}
2644
2645	/* next */
2646	pRamPrev = pRam;
2647	pRam = pRam->pNextR3;
2648	}
2649	if (fRamExists)
2650	{
2651	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2652	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2653	while (cPagesLeft-- > 0)
2654	{
2655	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
2656	("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
2657	GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pCur->RamRange.pszDesc),
2658	VERR_PGM_RAM_CONFLICT);
2659	pPage++;
2660	}
2661	}
2662	Log(("PGMR3PhysMMIO2Map: %RGp-%RGp fRamExists=%RTbool %s\n",
2663	GCPhys, GCPhysLast, fRamExists, pCur->RamRange.pszDesc));
2664
2665	/*
2666	* Make the changes.
2667	*/
2668	pgmLock(pVM);
2669
2670	pCur->RamRange.GCPhys = GCPhys;
2671	pCur->RamRange.GCPhysLast = GCPhysLast;
2672	pCur->fMapped = true;
2673	pCur->fOverlapping = fRamExists;
2674
2675	if (fRamExists)
2676	{
2677	/** @todo use pgmR3PhysFreePageRange here. */
2678	uint32_t cPendingPages = 0;
2679	PGMMFREEPAGESREQ pReq;
2680	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2681	AssertLogRelRCReturn(rc, rc);
2682
2683	/* replace the pages, freeing all present RAM pages. */
2684	PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2685	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2686	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2687	while (cPagesLeft-- > 0)
2688	{
2689	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
2690	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
2691
2692	RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
2693	PGM_PAGE_SET_HCPHYS(pVM, pPageDst, HCPhys);
2694	PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO2);
2695	PGM_PAGE_SET_STATE(pVM, pPageDst, PGM_PAGE_STATE_ALLOCATED);
2696	PGM_PAGE_SET_PDE_TYPE(pVM, pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
2697	PGM_PAGE_SET_PTE_INDEX(pVM, pPageDst, 0);
2698	PGM_PAGE_SET_TRACKING(pVM, pPageDst, 0);
2699
2700	pVM->pgm.s.cZeroPages--;
2701	GCPhys += PAGE_SIZE;
2702	pPageSrc++;
2703	pPageDst++;
2704	}
2705
2706	/* Flush physical page map TLB. */
2707	pgmPhysInvalidatePageMapTLB(pVM);
2708
2709	if (cPendingPages)
2710	{
2711	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2712	AssertLogRelRCReturn(rc, rc);
2713	}
2714	GMMR3FreePagesCleanup(pReq);
2715
2716	/* Force a PGM pool flush as guest ram references have been changed. */
2717	/** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
2718	PVMCPU pVCpu = VMMGetCpu(pVM);
2719	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2720	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2721
2722	pgmUnlock(pVM);
2723	}
2724	else
2725	{
2726	RTGCPHYS cb = pCur->RamRange.cb;
2727
2728	/* Clear the tracking data of pages we're going to reactivate. */
2729	PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2730	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2731	while (cPagesLeft-- > 0)
2732	{
2733	PGM_PAGE_SET_TRACKING(pVM, pPageSrc, 0);
2734	PGM_PAGE_SET_PTE_INDEX(pVM, pPageSrc, 0);
2735	pPageSrc++;
2736	}
2737
2738	/* link in the ram range */
2739	pgmR3PhysLinkRamRange(pVM, &pCur->RamRange, pRamPrev);
2740	pgmUnlock(pVM);
2741
2742	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_MMIO2);
2743	}
2744
2745	pgmPhysInvalidatePageMapTLB(pVM);
2746	return VINF_SUCCESS;
2747	}
2748
2749
2750	/**
2751	* Unmaps a MMIO2 region.
2752	*
2753	* This is done when a guest / the bios / state loading changes the
2754	* PCI config. The replacing of base memory has the same restrictions
2755	* as during registration, of course.
2756	*/
2757	VMMR3DECL(int) PGMR3PhysMMIO2Unmap(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2758	{
2759	/*
2760	* Validate input
2761	*/
2762	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2763	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2764	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2765	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2766	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2767	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2768
2769	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2770	AssertReturn(pCur, VERR_NOT_FOUND);
2771	AssertReturn(pCur->fMapped, VERR_WRONG_ORDER);
2772	AssertReturn(pCur->RamRange.GCPhys == GCPhys, VERR_INVALID_PARAMETER);
2773	Assert(pCur->RamRange.GCPhysLast != NIL_RTGCPHYS);
2774
2775	Log(("PGMR3PhysMMIO2Unmap: %RGp-%RGp %s\n",
2776	pCur->RamRange.GCPhys, pCur->RamRange.GCPhysLast, pCur->RamRange.pszDesc));
2777
2778	/*
2779	* Unmap it.
2780	*/
2781	pgmLock(pVM);
2782
2783	RTGCPHYS GCPhysRangeREM;
2784	RTGCPHYS cbRangeREM;
2785	bool fInformREM;
2786	if (pCur->fOverlapping)
2787	{
2788	/* Restore the RAM pages we've replaced. */
2789	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2790	while (pRam->GCPhys > pCur->RamRange.GCPhysLast)
2791	pRam = pRam->pNextR3;
2792
2793	PPGMPAGE pPageDst = &pRam->aPages[(pCur->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2794	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2795	while (cPagesLeft-- > 0)
2796	{
2797	PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
2798	pVM->pgm.s.cZeroPages++;
2799	pPageDst++;
2800	}
2801
2802	/* Flush physical page map TLB. */
2803	pgmPhysInvalidatePageMapTLB(pVM);
2804
2805	GCPhysRangeREM = NIL_RTGCPHYS; /* shuts up gcc */
2806	cbRangeREM = RTGCPHYS_MAX; /* ditto */
2807	fInformREM = false;
2808	}
2809	else
2810	{
2811	GCPhysRangeREM = pCur->RamRange.GCPhys;
2812	cbRangeREM = pCur->RamRange.cb;
2813	fInformREM = true;
2814
2815	pgmR3PhysUnlinkRamRange(pVM, &pCur->RamRange);
2816	}
2817
2818	pCur->RamRange.GCPhys = NIL_RTGCPHYS;
2819	pCur->RamRange.GCPhysLast = NIL_RTGCPHYS;
2820	pCur->fOverlapping = false;
2821	pCur->fMapped = false;
2822
2823	/* Force a PGM pool flush as guest ram references have been changed. */
2824	/** @todo not entirely SMP safe; assuming for now the guest takes care
2825	* of this internally (not touch mapped mmio while changing the
2826	* mapping). */
2827	PVMCPU pVCpu = VMMGetCpu(pVM);
2828	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2829	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2830
2831	pgmPhysInvalidatePageMapTLB(pVM);
2832	pgmPhysInvalidRamRangeTlbs(pVM);
2833	pgmUnlock(pVM);
2834
2835	if (fInformREM)
2836	REMR3NotifyPhysRamDeregister(pVM, GCPhysRangeREM, cbRangeREM);
2837
2838	return VINF_SUCCESS;
2839	}
2840
2841
2842	/**
2843	* Checks if the given address is an MMIO2 base address or not.
2844	*
2845	* @returns true/false accordingly.
2846	* @param pVM Pointer to the shared VM structure.
2847	* @param pDevIns The owner of the memory, optional.
2848	* @param GCPhys The address to check.
2849	*/
2850	VMMR3DECL(bool) PGMR3PhysMMIO2IsBase(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
2851	{
2852	/*
2853	* Validate input
2854	*/
2855	VM_ASSERT_EMT_RETURN(pVM, false);
2856	AssertPtrReturn(pDevIns, false);
2857	AssertReturn(GCPhys != NIL_RTGCPHYS, false);
2858	AssertReturn(GCPhys != 0, false);
2859	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), false);
2860
2861	/*
2862	* Search the list.
2863	*/
2864	pgmLock(pVM);
2865	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2866	if (pCur->RamRange.GCPhys == GCPhys)
2867	{
2868	Assert(pCur->fMapped);
2869	pgmUnlock(pVM);
2870	return true;
2871	}
2872	pgmUnlock(pVM);
2873	return false;
2874	}
2875
2876
2877	/**
2878	* Gets the HC physical address of a page in the MMIO2 region.
2879	*
2880	* This is API is intended for MMHyper and shouldn't be called
2881	* by anyone else...
2882	*
2883	* @returns VBox status code.
2884	* @param pVM Pointer to the shared VM structure.
2885	* @param pDevIns The owner of the memory, optional.
2886	* @param iRegion The region.
2887	* @param off The page expressed an offset into the MMIO2 region.
2888	* @param pHCPhys Where to store the result.
2889	*/
2890	VMMR3DECL(int) PGMR3PhysMMIO2GetHCPhys(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, PRTHCPHYS pHCPhys)
2891	{
2892	/*
2893	* Validate input
2894	*/
2895	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2896	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2897	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2898
2899	pgmLock(pVM);
2900	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2901	AssertReturn(pCur, VERR_NOT_FOUND);
2902	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2903
2904	PCPGMPAGE pPage = &pCur->RamRange.aPages[off >> PAGE_SHIFT];
2905	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2906	pgmUnlock(pVM);
2907	return VINF_SUCCESS;
2908	}
2909
2910
2911	/**
2912	* Maps a portion of an MMIO2 region into kernel space (host).
2913	*
2914	* The kernel mapping will become invalid when the MMIO2 memory is deregistered
2915	* or the VM is terminated.
2916	*
2917	* @return VBox status code.
2918	*
2919	* @param pVM Pointer to the shared VM structure.
2920	* @param pDevIns The device owning the MMIO2 memory.
2921	* @param iRegion The region.
2922	* @param off The offset into the region. Must be page aligned.
2923	* @param cb The number of bytes to map. Must be page aligned.
2924	* @param pszDesc Mapping description.
2925	* @param pR0Ptr Where to store the R0 address.
2926	*/
2927	VMMR3DECL(int) PGMR3PhysMMIO2MapKernel(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
2928	const char *pszDesc, PRTR0PTR pR0Ptr)
2929	{
2930	/*
2931	* Validate input.
2932	*/
2933	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2934	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2935	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2936
2937	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2938	AssertReturn(pCur, VERR_NOT_FOUND);
2939	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2940	AssertReturn(cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2941	AssertReturn(off + cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2942
2943	/*
2944	* Pass the request on to the support library/driver.
2945	*/
2946	int rc = SUPR3PageMapKernel(pCur->pvR3, off, cb, 0, pR0Ptr);
2947
2948	return rc;
2949	}
2950
2951
2952	/**
2953	* Worker for PGMR3PhysRomRegister.
2954	*
2955	* This is here to simplify lock management, i.e. the caller does all the
2956	* locking and we can simply return without needing to remember to unlock
2957	* anything first.
2958	*
2959	* @returns VBox status.
2960	* @param pVM VM Handle.
2961	* @param pDevIns The device instance owning the ROM.
2962	* @param GCPhys First physical address in the range.
2963	* Must be page aligned!
2964	* @param cb The size of the range (in bytes).
2965	* Must be page aligned!
2966	* @param pvBinary Pointer to the binary data backing the ROM image.
2967	* @param cbBinary The size of the binary data pvBinary points to.
2968	* This must be less or equal to @a cb.
2969	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
2970	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
2971	* @param pszDesc Pointer to description string. This must not be freed.
2972	*/
2973	static int pgmR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
2974	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
2975	{
2976	/*
2977	* Validate input.
2978	*/
2979	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2980	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
2981	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
2982	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2983	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2984	AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
2985	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2986	AssertReturn(!(fFlags & ~(PGMPHYS_ROM_FLAGS_SHADOWED \| PGMPHYS_ROM_FLAGS_PERMANENT_BINARY)), VERR_INVALID_PARAMETER);
2987	VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
2988
2989	const uint32_t cPages = cb >> PAGE_SHIFT;
2990
2991	/*
2992	* Find the ROM location in the ROM list first.
2993	*/
2994	PPGMROMRANGE pRomPrev = NULL;
2995	PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3;
2996	while (pRom && GCPhysLast >= pRom->GCPhys)
2997	{
2998	if ( GCPhys <= pRom->GCPhysLast
2999	&& GCPhysLast >= pRom->GCPhys)
3000	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
3001	GCPhys, GCPhysLast, pszDesc,
3002	pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
3003	VERR_PGM_RAM_CONFLICT);
3004	/* next */
3005	pRomPrev = pRom;
3006	pRom = pRom->pNextR3;
3007	}
3008
3009	/*
3010	* Find the RAM location and check for conflicts.
3011	*
3012	* Conflict detection is a bit different than for RAM
3013	* registration since a ROM can be located within a RAM
3014	* range. So, what we have to check for is other memory
3015	* types (other than RAM that is) and that we don't span
3016	* more than one RAM range (layz).
3017	*/
3018	bool fRamExists = false;
3019	PPGMRAMRANGE pRamPrev = NULL;
3020	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
3021	while (pRam && GCPhysLast >= pRam->GCPhys)
3022	{
3023	if ( GCPhys <= pRam->GCPhysLast
3024	&& GCPhysLast >= pRam->GCPhys)
3025	{
3026	/* completely within? */
3027	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
3028	&& GCPhysLast <= pRam->GCPhysLast,
3029	("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
3030	GCPhys, GCPhysLast, pszDesc,
3031	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
3032	VERR_PGM_RAM_CONFLICT);
3033	fRamExists = true;
3034	break;
3035	}
3036
3037	/* next */
3038	pRamPrev = pRam;
3039	pRam = pRam->pNextR3;
3040	}
3041	if (fRamExists)
3042	{
3043	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3044	uint32_t cPagesLeft = cPages;
3045	while (cPagesLeft-- > 0)
3046	{
3047	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
3048	("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
3049	pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
3050	pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
3051	Assert(PGM_PAGE_IS_ZERO(pPage));
3052	pPage++;
3053	}
3054	}
3055
3056	/*
3057	* Update the base memory reservation if necessary.
3058	*/
3059	uint32_t cExtraBaseCost = fRamExists ? 0 : cPages;
3060	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3061	cExtraBaseCost += cPages;
3062	if (cExtraBaseCost)
3063	{
3064	int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
3065	if (RT_FAILURE(rc))
3066	return rc;
3067	}
3068
3069	/*
3070	* Allocate memory for the virgin copy of the RAM.
3071	*/
3072	PGMMALLOCATEPAGESREQ pReq;
3073	int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
3074	AssertRCReturn(rc, rc);
3075
3076	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3077	{
3078	pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
3079	pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
3080	pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
3081	}
3082
3083	rc = GMMR3AllocatePagesPerform(pVM, pReq);
3084	if (RT_FAILURE(rc))
3085	{
3086	GMMR3AllocatePagesCleanup(pReq);
3087	return rc;
3088	}
3089
3090	/*
3091	* Allocate the new ROM range and RAM range (if necessary).
3092	*/
3093	PPGMROMRANGE pRomNew;
3094	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
3095	if (RT_SUCCESS(rc))
3096	{
3097	PPGMRAMRANGE pRamNew = NULL;
3098	if (!fRamExists)
3099	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), sizeof(PGMPAGE), MM_TAG_PGM_PHYS, (void **)&pRamNew);
3100	if (RT_SUCCESS(rc))
3101	{
3102	/*
3103	* Initialize and insert the RAM range (if required).
3104	*/
3105	PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
3106	if (!fRamExists)
3107	{
3108	pRamNew->pSelfR0 = MMHyperCCToR0(pVM, pRamNew);
3109	pRamNew->pSelfRC = MMHyperCCToRC(pVM, pRamNew);
3110	pRamNew->GCPhys = GCPhys;
3111	pRamNew->GCPhysLast = GCPhysLast;
3112	pRamNew->cb = cb;
3113	pRamNew->pszDesc = pszDesc;
3114	pRamNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
3115	pRamNew->pvR3 = NULL;
3116	pRamNew->paLSPages = NULL;
3117
3118	PPGMPAGE pPage = &pRamNew->aPages[0];
3119	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3120	{
3121	PGM_PAGE_INIT(pPage,
3122	pReq->aPages[iPage].HCPhysGCPhys,
3123	pReq->aPages[iPage].idPage,
3124	PGMPAGETYPE_ROM,
3125	PGM_PAGE_STATE_ALLOCATED);
3126
3127	pRomPage->Virgin = *pPage;
3128	}
3129
3130	pVM->pgm.s.cAllPages += cPages;
3131	pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
3132	}
3133	else
3134	{
3135	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3136	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3137	{
3138	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_ROM);
3139	PGM_PAGE_SET_HCPHYS(pVM, pPage, pReq->aPages[iPage].HCPhysGCPhys);
3140	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
3141	PGM_PAGE_SET_PAGEID(pVM, pPage, pReq->aPages[iPage].idPage);
3142	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
3143	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
3144	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
3145
3146	pRomPage->Virgin = *pPage;
3147	}
3148
3149	pRamNew = pRam;
3150
3151	pVM->pgm.s.cZeroPages -= cPages;
3152	}
3153	pVM->pgm.s.cPrivatePages += cPages;
3154
3155	/* Flush physical page map TLB. */
3156	pgmPhysInvalidatePageMapTLB(pVM);
3157
3158
3159	/*
3160	* !HACK ALERT! REM + (Shadowed) ROM ==> mess.
3161	*
3162	* If it's shadowed we'll register the handler after the ROM notification
3163	* so we get the access handler callbacks that we should. If it isn't
3164	* shadowed we'll do it the other way around to make REM use the built-in
3165	* ROM behavior and not the handler behavior (which is to route all access
3166	* to PGM atm).
3167	*/
3168	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3169	{
3170	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, true /* fShadowed */);
3171	rc = PGMR3HandlerPhysicalRegister(pVM,
3172	fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
3173	? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
3174	: PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
3175	GCPhys, GCPhysLast,
3176	pgmR3PhysRomWriteHandler, pRomNew,
3177	NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
3178	NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
3179	}
3180	else
3181	{
3182	rc = PGMR3HandlerPhysicalRegister(pVM,
3183	fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
3184	? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
3185	: PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
3186	GCPhys, GCPhysLast,
3187	pgmR3PhysRomWriteHandler, pRomNew,
3188	NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
3189	NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
3190	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, false /* fShadowed */);
3191	}
3192	if (RT_SUCCESS(rc))
3193	{
3194	/*
3195	* Copy the image over to the virgin pages.
3196	* This must be done after linking in the RAM range.
3197	*/
3198	size_t cbBinaryLeft = cbBinary;
3199	PPGMPAGE pRamPage = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
3200	for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
3201	{
3202	void *pvDstPage;
3203	rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << PAGE_SHIFT), &pvDstPage);
3204	if (RT_FAILURE(rc))
3205	{
3206	VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
3207	break;
3208	}
3209	if (cbBinaryLeft >= PAGE_SIZE)
3210	{
3211	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), PAGE_SIZE);
3212	cbBinaryLeft -= PAGE_SIZE;
3213	}
3214	else
3215	{
3216	ASMMemZeroPage(pvDstPage); /* (shouldn't be necessary, but can't hurt either) */
3217	if (cbBinaryLeft > 0)
3218	{
3219	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), cbBinaryLeft);
3220	cbBinaryLeft = 0;
3221	}
3222	}
3223	}
3224	if (RT_SUCCESS(rc))
3225	{
3226	/*
3227	* Initialize the ROM range.
3228	* Note that the Virgin member of the pages has already been initialized above.
3229	*/
3230	pRomNew->GCPhys = GCPhys;
3231	pRomNew->GCPhysLast = GCPhysLast;
3232	pRomNew->cb = cb;
3233	pRomNew->fFlags = fFlags;
3234	pRomNew->idSavedState = UINT8_MAX;
3235	pRomNew->cbOriginal = cbBinary;
3236	#ifdef VBOX_STRICT
3237	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
3238	? pvBinary : RTMemDup(pvBinary, cbBinary);
3239	#else
3240	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY ? pvBinary : NULL;
3241	#endif
3242	pRomNew->pszDesc = pszDesc;
3243
3244	for (unsigned iPage = 0; iPage < cPages; iPage++)
3245	{
3246	PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
3247	pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
3248	PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
3249	}
3250
3251	/* update the page count stats for the shadow pages. */
3252	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3253	{
3254	pVM->pgm.s.cZeroPages += cPages;
3255	pVM->pgm.s.cAllPages += cPages;
3256	}
3257
3258	/*
3259	* Insert the ROM range, tell REM and return successfully.
3260	*/
3261	pRomNew->pNextR3 = pRom;
3262	pRomNew->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
3263	pRomNew->pNextRC = pRom ? MMHyperCCToRC(pVM, pRom) : NIL_RTRCPTR;
3264
3265	if (pRomPrev)
3266	{
3267	pRomPrev->pNextR3 = pRomNew;
3268	pRomPrev->pNextR0 = MMHyperCCToR0(pVM, pRomNew);
3269	pRomPrev->pNextRC = MMHyperCCToRC(pVM, pRomNew);
3270	}
3271	else
3272	{
3273	pVM->pgm.s.pRomRangesR3 = pRomNew;
3274	pVM->pgm.s.pRomRangesR0 = MMHyperCCToR0(pVM, pRomNew);
3275	pVM->pgm.s.pRomRangesRC = MMHyperCCToRC(pVM, pRomNew);
3276	}
3277
3278	pgmPhysInvalidatePageMapTLB(pVM);
3279	GMMR3AllocatePagesCleanup(pReq);
3280	return VINF_SUCCESS;
3281	}
3282
3283	/* bail out */
3284
3285	int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
3286	AssertRC(rc2);
3287	}
3288
3289	if (!fRamExists)
3290	{
3291	pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
3292	MMHyperFree(pVM, pRamNew);
3293	}
3294	}
3295	MMHyperFree(pVM, pRomNew);
3296	}
3297
3298	/** @todo Purge the mapping cache or something... */
3299	GMMR3FreeAllocatedPages(pVM, pReq);
3300	GMMR3AllocatePagesCleanup(pReq);
3301	return rc;
3302	}
3303
3304
3305	/**
3306	* Registers a ROM image.
3307	*
3308	* Shadowed ROM images requires double the amount of backing memory, so,
3309	* don't use that unless you have to. Shadowing of ROM images is process
3310	* where we can select where the reads go and where the writes go. On real
3311	* hardware the chipset provides means to configure this. We provide
3312	* PGMR3PhysProtectROM() for this purpose.
3313	*
3314	* A read-only copy of the ROM image will always be kept around while we
3315	* will allocate RAM pages for the changes on demand (unless all memory
3316	* is configured to be preallocated).
3317	*
3318	* @returns VBox status.
3319	* @param pVM VM Handle.
3320	* @param pDevIns The device instance owning the ROM.
3321	* @param GCPhys First physical address in the range.
3322	* Must be page aligned!
3323	* @param cb The size of the range (in bytes).
3324	* Must be page aligned!
3325	* @param pvBinary Pointer to the binary data backing the ROM image.
3326	* @param cbBinary The size of the binary data pvBinary points to.
3327	* This must be less or equal to @a cb.
3328	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
3329	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
3330	* @param pszDesc Pointer to description string. This must not be freed.
3331	*
3332	* @remark There is no way to remove the rom, automatically on device cleanup or
3333	* manually from the device yet. This isn't difficult in any way, it's
3334	* just not something we expect to be necessary for a while.
3335	*/
3336	VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
3337	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
3338	{
3339	Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
3340	pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
3341	pgmLock(pVM);
3342	int rc = pgmR3PhysRomRegister(pVM, pDevIns, GCPhys, cb, pvBinary, cbBinary, fFlags, pszDesc);
3343	pgmUnlock(pVM);
3344	return rc;
3345	}
3346
3347
3348	/**
3349	* \#PF Handler callback for ROM write accesses.
3350	*
3351	* @returns VINF_SUCCESS if the handler have carried out the operation.
3352	* @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
3353	* @param pVM VM Handle.
3354	* @param GCPhys The physical address the guest is writing to.
3355	* @param pvPhys The HC mapping of that address.
3356	* @param pvBuf What the guest is reading/writing.
3357	* @param cbBuf How much it's reading/writing.
3358	* @param enmAccessType The access type.
3359	* @param pvUser User argument.
3360	*/
3361	static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
3362	PGMACCESSTYPE enmAccessType, void *pvUser)
3363	{
3364	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
3365	const uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3366	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
3367	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3368	Log5(("pgmR3PhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
3369
3370	if (enmAccessType == PGMACCESSTYPE_READ)
3371	{
3372	switch (pRomPage->enmProt)
3373	{
3374	/*
3375	* Take the default action.
3376	*/
3377	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3378	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3379	case PGMROMPROT_READ_ROM_WRITE_RAM:
3380	case PGMROMPROT_READ_RAM_WRITE_RAM:
3381	return VINF_PGM_HANDLER_DO_DEFAULT;
3382
3383	default:
3384	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3385	pRom->aPages[iPage].enmProt, iPage, GCPhys),
3386	VERR_INTERNAL_ERROR);
3387	}
3388	}
3389	else
3390	{
3391	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
3392	switch (pRomPage->enmProt)
3393	{
3394	/*
3395	* Ignore writes.
3396	*/
3397	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3398	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3399	return VINF_SUCCESS;
3400
3401	/*
3402	* Write to the RAM page.
3403	*/
3404	case PGMROMPROT_READ_ROM_WRITE_RAM:
3405	case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's way simpler that way. */
3406	{
3407	/* This should be impossible now, pvPhys doesn't work cross page anylonger. */
3408	Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
3409
3410	/*
3411	* Take the lock, do lazy allocation, map the page and copy the data.
3412	*
3413	* Note that we have to bypass the mapping TLB since it works on
3414	* guest physical addresses and entering the shadow page would
3415	* kind of screw things up...
3416	*/
3417	int rc = pgmLock(pVM);
3418	AssertRC(rc);
3419
3420	PPGMPAGE pShadowPage = &pRomPage->Shadow;
3421	if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
3422	{
3423	pShadowPage = pgmPhysGetPage(pVM, GCPhys);
3424	AssertLogRelReturn(pShadowPage, VERR_INTERNAL_ERROR);
3425	}
3426
3427	void *pvDstPage;
3428	rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
3429	if (RT_SUCCESS(rc))
3430	{
3431	memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
3432	pRomPage->LiveSave.fWrittenTo = true;
3433	}
3434
3435	pgmUnlock(pVM);
3436	return rc;
3437	}
3438
3439	default:
3440	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3441	pRom->aPages[iPage].enmProt, iPage, GCPhys),
3442	VERR_INTERNAL_ERROR);
3443	}
3444	}
3445	}
3446
3447
3448	/**
3449	* Called by PGMR3Reset to reset the shadow, switch to the virgin,
3450	* and verify that the virgin part is untouched.
3451	*
3452	* This is done after the normal memory has been cleared.
3453	*
3454	* ASSUMES that the caller owns the PGM lock.
3455	*
3456	* @param pVM The VM handle.
3457	*/
3458	int pgmR3PhysRomReset(PVM pVM)
3459	{
3460	PGM_LOCK_ASSERT_OWNER(pVM);
3461	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3462	{
3463	const uint32_t cPages = pRom->cb >> PAGE_SHIFT;
3464
3465	if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3466	{
3467	/*
3468	* Reset the physical handler.
3469	*/
3470	int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
3471	AssertRCReturn(rc, rc);
3472
3473	/*
3474	* What we do with the shadow pages depends on the memory
3475	* preallocation option. If not enabled, we'll just throw
3476	* out all the dirty pages and replace them by the zero page.
3477	*/
3478	if (!pVM->pgm.s.fRamPreAlloc)
3479	{
3480	/* Free the dirty pages. */
3481	uint32_t cPendingPages = 0;
3482	PGMMFREEPAGESREQ pReq;
3483	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
3484	AssertRCReturn(rc, rc);
3485
3486	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3487	if ( !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
3488	&& !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
3489	{
3490	Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
3491	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
3492	pRom->GCPhys + (iPage << PAGE_SHIFT));
3493	AssertLogRelRCReturn(rc, rc);
3494	}
3495
3496	if (cPendingPages)
3497	{
3498	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
3499	AssertLogRelRCReturn(rc, rc);
3500	}
3501	GMMR3FreePagesCleanup(pReq);
3502	}
3503	else
3504	{
3505	/* clear all the shadow pages. */
3506	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3507	{
3508	if (PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow))
3509	continue;
3510	Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
3511	void *pvDstPage;
3512	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3513	rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
3514	if (RT_FAILURE(rc))
3515	break;
3516	ASMMemZeroPage(pvDstPage);
3517	}
3518	AssertRCReturn(rc, rc);
3519	}
3520	}
3521
3522	#ifdef VBOX_STRICT
3523	/*
3524	* Verify that the virgin page is unchanged if possible.
3525	*/
3526	if (pRom->pvOriginal)
3527	{
3528	size_t cbSrcLeft = pRom->cbOriginal;
3529	uint8_t const pbSrcPage = (uint8_t const )pRom->pvOriginal;
3530	for (uint32_t iPage = 0; iPage < cPages && cbSrcLeft > 0; iPage++, pbSrcPage += PAGE_SIZE)
3531	{
3532	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3533	void const *pvDstPage;
3534	int rc = pgmPhysPageMapReadOnly(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPage);
3535	if (RT_FAILURE(rc))
3536	break;
3537
3538	if (memcmp(pvDstPage, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE)))
3539	LogRel(("pgmR3PhysRomReset: %RGp rom page changed (%s) - loaded saved state?\n",
3540	GCPhys, pRom->pszDesc));
3541	cbSrcLeft -= RT_MIN(cbSrcLeft, PAGE_SIZE);
3542	}
3543	}
3544	#endif
3545	}
3546
3547	return VINF_SUCCESS;
3548	}
3549
3550
3551	/**
3552	* Called by PGMR3Term to free resources.
3553	*
3554	* ASSUMES that the caller owns the PGM lock.
3555	*
3556	* @param pVM The VM handle.
3557	*/
3558	void pgmR3PhysRomTerm(PVM pVM)
3559	{
3560	#ifdef RT_STRICT
3561	/*
3562	* Free the heap copy of the original bits.
3563	*/
3564	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3565	{
3566	if ( pRom->pvOriginal
3567	&& !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
3568	{
3569	RTMemFree((void *)pRom->pvOriginal);
3570	pRom->pvOriginal = NULL;
3571	}
3572	}
3573	#endif
3574	}
3575
3576
3577	/**
3578	* Change the shadowing of a range of ROM pages.
3579	*
3580	* This is intended for implementing chipset specific memory registers
3581	* and will not be very strict about the input. It will silently ignore
3582	* any pages that are not the part of a shadowed ROM.
3583	*
3584	* @returns VBox status code.
3585	* @retval VINF_PGM_SYNC_CR3
3586	*
3587	* @param pVM Pointer to the shared VM structure.
3588	* @param GCPhys Where to start. Page aligned.
3589	* @param cb How much to change. Page aligned.
3590	* @param enmProt The new ROM protection.
3591	*/
3592	VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
3593	{
3594	/*
3595	* Check input
3596	*/
3597	if (!cb)
3598	return VINF_SUCCESS;
3599	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3600	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3601	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
3602	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3603	AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
3604
3605	/*
3606	* Process the request.
3607	*/
3608	pgmLock(pVM);
3609	int rc = VINF_SUCCESS;
3610	bool fFlushTLB = false;
3611	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3612	{
3613	if ( GCPhys <= pRom->GCPhysLast
3614	&& GCPhysLast >= pRom->GCPhys
3615	&& (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
3616	{
3617	/*
3618	* Iterate the relevant pages and make necessary the changes.
3619	*/
3620	bool fChanges = false;
3621	uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
3622	? pRom->cb >> PAGE_SHIFT
3623	: (GCPhysLast - pRom->GCPhys + 1) >> PAGE_SHIFT;
3624	for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3625	iPage < cPages;
3626	iPage++)
3627	{
3628	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3629	if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
3630	{
3631	fChanges = true;
3632
3633	/* flush references to the page. */
3634	PPGMPAGE pRamPage = pgmPhysGetPage(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT));
3635	int rc2 = pgmPoolTrackUpdateGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage,
3636	true /fFlushPTEs/, &fFlushTLB);
3637	if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS \|\| RT_FAILURE(rc2)))
3638	rc = rc2;
3639
3640	PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
3641	PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
3642
3643	pOld = pRamPage;
3644	pRamPage = pNew;
3645	/** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
3646	}
3647	pRomPage->enmProt = enmProt;
3648	}
3649
3650	/*
3651	* Reset the access handler if we made changes, no need
3652	* to optimize this.
3653	*/
3654	if (fChanges)
3655	{
3656	int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
3657	if (RT_FAILURE(rc2))
3658	{
3659	pgmUnlock(pVM);
3660	AssertRC(rc);
3661	return rc2;
3662	}
3663	}
3664
3665	/* Advance - cb isn't updated. */
3666	GCPhys = pRom->GCPhys + (cPages << PAGE_SHIFT);
3667	}
3668	}
3669	pgmUnlock(pVM);
3670	if (fFlushTLB)
3671	PGM_INVL_ALL_VCPU_TLBS(pVM);
3672
3673	return rc;
3674	}
3675
3676
3677	/**
3678	* Sets the Address Gate 20 state.
3679	*
3680	* @param pVCpu The VCPU to operate on.
3681	* @param fEnable True if the gate should be enabled.
3682	* False if the gate should be disabled.
3683	*/
3684	VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
3685	{
3686	LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
3687	if (pVCpu->pgm.s.fA20Enabled != fEnable)
3688	{
3689	pVCpu->pgm.s.fA20Enabled = fEnable;
3690	pVCpu->pgm.s.GCPhysA20Mask = ~(RTGCPHYS)(!fEnable << 20);
3691	REMR3A20Set(pVCpu->pVMR3, pVCpu, fEnable);
3692	/** @todo we're not handling this correctly for VT-x / AMD-V. See #2911 */
3693	}
3694	}
3695
3696
3697	/**
3698	* Tree enumeration callback for dealing with age rollover.
3699	* It will perform a simple compression of the current age.
3700	*/
3701	static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
3702	{
3703	/* Age compression - ASSUMES iNow == 4. */
3704	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3705	if (pChunk->iAge >= UINT32_C(0xffffff00))
3706	pChunk->iAge = 3;
3707	else if (pChunk->iAge >= UINT32_C(0xfffff000))
3708	pChunk->iAge = 2;
3709	else if (pChunk->iAge)
3710	pChunk->iAge = 1;
3711	else /* iAge = 0 */
3712	pChunk->iAge = 4;
3713	return 0;
3714	}
3715
3716
3717	/**
3718	* Tree enumeration callback that updates the chunks that have
3719	* been used since the last
3720	*/
3721	static DECLCALLBACK(int) pgmR3PhysChunkAgeingCallback(PAVLU32NODECORE pNode, void *pvUser)
3722	{
3723	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3724	if (!pChunk->iAge)
3725	{
3726	PVM pVM = (PVM)pvUser;
3727	pChunk->iAge = pVM->pgm.s.ChunkR3Map.iNow;
3728	}
3729	return 0;
3730	}
3731
3732
3733	/**
3734	* Performs ageing of the ring-3 chunk mappings.
3735	*
3736	* @param pVM The VM handle.
3737	*/
3738	VMMR3DECL(void) PGMR3PhysChunkAgeing(PVM pVM)
3739	{
3740	pgmLock(pVM);
3741	pVM->pgm.s.ChunkR3Map.AgeingCountdown = RT_MIN(pVM->pgm.s.ChunkR3Map.cMax / 4, 1024);
3742	pVM->pgm.s.ChunkR3Map.iNow++;
3743	if (pVM->pgm.s.ChunkR3Map.iNow == 0)
3744	{
3745	pVM->pgm.s.ChunkR3Map.iNow = 4;
3746	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkAgeingRolloverCallback, pVM);
3747	}
3748	else
3749	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkAgeingCallback, pVM);
3750	pgmUnlock(pVM);
3751	}
3752
3753
3754	/**
3755	* The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
3756	*/
3757	typedef struct PGMR3PHYSCHUNKUNMAPCB
3758	{
3759	PVM pVM; /*< The VM handle. /
3760	PPGMCHUNKR3MAP pChunk; /*< The chunk to unmap. /
3761	uint32_t iLastAge; /*< Highest age found so far. /
3762	} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
3763
3764
3765	/**
3766	* Callback used to find the mapping that's been unused for
3767	* the longest time.
3768	*/
3769	static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
3770	{
3771	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3772	PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
3773
3774	/*
3775	* Check for locks and age.
3776	*/
3777	if (pChunk->cRefs)
3778	return 0;
3779	if (!pChunk->iAge)
3780	return 0;
3781	if (pArg->iLastAge >= pChunk->iAge)
3782	return 0;
3783
3784	/*
3785	* Check that it's not in any of the TLBs.
3786	*/
3787	PVM pVM = pArg->pVM;
3788	if ( pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(pChunk->Core.Key)].idChunk
3789	== pChunk->Core.Key)
3790	{
3791	pChunk = NULL;
3792	return 0;
3793	}
3794	#ifdef VBOX_STRICT
3795	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
3796	{
3797	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk != pChunk);
3798	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk != pChunk->Core.Key);
3799	}
3800	#endif
3801
3802	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
3803	if (pVM->pgm.s.PhysTlbHC.aEntries[i].pMap == pChunk)
3804	return 0;
3805
3806	pArg->pChunk = pChunk;
3807	pArg->iLastAge = pChunk->iAge;
3808	return 0;
3809	}
3810
3811
3812	/**
3813	* Finds a good candidate for unmapping when the ring-3 mapping cache is full.
3814	*
3815	* The candidate will not be part of any TLBs, so no need to flush
3816	* anything afterwards.
3817	*
3818	* @returns Chunk id.
3819	* @param pVM The VM handle.
3820	*/
3821	static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
3822	{
3823	PGM_LOCK_ASSERT_OWNER(pVM);
3824
3825	/*
3826	* Do tree ageing first?
3827	*/
3828	if (pVM->pgm.s.ChunkR3Map.AgeingCountdown-- == 0)
3829	{
3830	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkAging, a);
3831	PGMR3PhysChunkAgeing(pVM);
3832	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkAging, a);
3833	}
3834
3835	/*
3836	* Enumerate the age tree starting with the left most node.
3837	*/
3838	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3839	PGMR3PHYSCHUNKUNMAPCB Args;
3840	Args.pVM = pVM;
3841	Args.pChunk = NULL;
3842	Args.iLastAge = 0;
3843	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
3844	Assert(Args.pChunk);
3845	if (Args.pChunk)
3846	{
3847	Assert(Args.pChunk->cRefs == 0);
3848	Assert(Args.pChunk->cPermRefs == 0);
3849	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3850	return Args.pChunk->Core.Key;
3851	}
3852
3853	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3854	return INT32_MAX;
3855	}
3856
3857
3858	/**
3859	* Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
3860	*
3861	* This is only called on one of the EMTs while the other ones are waiting for
3862	* it to complete this function.
3863	*
3864	* @returns VINF_SUCCESS (VBox strict status code).
3865	* @param pVM The VM handle.
3866	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
3867	* @param pvUser User pointer. Unused
3868	*
3869	*/
3870	DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
3871	{
3872	int rc = VINF_SUCCESS;
3873	pgmLock(pVM);
3874
3875	if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
3876	{
3877	/* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
3878	/** @todo also not really efficient to unmap a chunk that contains PD
3879	* or PT pages. */
3880	pgmR3PoolClearAllRendezvous(pVM, &pVM->aCpus[0], NULL /* no need to flush the REM TLB as we already did that above */);
3881
3882	/*
3883	* Request the ring-0 part to unmap a chunk to make space in the mapping cache.
3884	*/
3885	GMMMAPUNMAPCHUNKREQ Req;
3886	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3887	Req.Hdr.cbReq = sizeof(Req);
3888	Req.pvR3 = NULL;
3889	Req.idChunkMap = NIL_GMM_CHUNKID;
3890	Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
3891	if (Req.idChunkUnmap != INT32_MAX)
3892	{
3893	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3894	rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
3895	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3896	if (RT_SUCCESS(rc))
3897	{
3898	/*
3899	* Remove the unmapped one.
3900	*/
3901	PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
3902	AssertRelease(pUnmappedChunk);
3903	AssertRelease(!pUnmappedChunk->cRefs);
3904	AssertRelease(!pUnmappedChunk->cPermRefs);
3905	pUnmappedChunk->pv = NULL;
3906	pUnmappedChunk->Core.Key = UINT32_MAX;
3907	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3908	MMR3HeapFree(pUnmappedChunk);
3909	#else
3910	MMR3UkHeapFree(pVM, pUnmappedChunk, MM_TAG_PGM_CHUNK_MAPPING);
3911	#endif
3912	pVM->pgm.s.ChunkR3Map.c--;
3913	pVM->pgm.s.cUnmappedChunks++;
3914
3915	/*
3916	* Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses).
3917	*/
3918	/** todo: we should not flush chunks which include cr3 mappings. */
3919	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3920	{
3921	PPGMCPU pPGM = &pVM->aCpus[idCpu].pgm.s;
3922
3923	pPGM->pGst32BitPdR3 = NULL;
3924	pPGM->pGstPaePdptR3 = NULL;
3925	pPGM->pGstAmd64Pml4R3 = NULL;
3926	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3927	pPGM->pGst32BitPdR0 = NIL_RTR0PTR;
3928	pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
3929	pPGM->pGstAmd64Pml4R0 = NIL_RTR0PTR;
3930	#endif
3931	for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
3932	{
3933	pPGM->apGstPaePDsR3[i] = NULL;
3934	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3935	pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
3936	#endif
3937	}
3938
3939	/* Flush REM TLBs. */
3940	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3941	}
3942
3943	/* Flush REM translation blocks. */
3944	REMFlushTBs(pVM);
3945	}
3946	}
3947	}
3948	pgmUnlock(pVM);
3949	return rc;
3950	}
3951
3952	/**
3953	* Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
3954	*
3955	* @returns VBox status code.
3956	* @param pVM The VM to operate on.
3957	*/
3958	void pgmR3PhysUnmapChunk(PVM pVM)
3959	{
3960	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
3961	AssertRC(rc);
3962	}
3963
3964
3965	/**
3966	* Maps the given chunk into the ring-3 mapping cache.
3967	*
3968	* This will call ring-0.
3969	*
3970	* @returns VBox status code.
3971	* @param pVM The VM handle.
3972	* @param idChunk The chunk in question.
3973	* @param ppChunk Where to store the chunk tracking structure.
3974	*
3975	* @remarks Called from within the PGM critical section.
3976	* @remarks Can be called from any thread!
3977	*/
3978	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
3979	{
3980	int rc;
3981
3982	PGM_LOCK_ASSERT_OWNER(pVM);
3983
3984	/*
3985	* Allocate a new tracking structure first.
3986	*/
3987	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3988	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
3989	#else
3990	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3UkHeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk), NULL);
3991	#endif
3992	AssertReturn(pChunk, VERR_NO_MEMORY);
3993	pChunk->Core.Key = idChunk;
3994
3995	/*
3996	* Request the ring-0 part to map the chunk in question.
3997	*/
3998	GMMMAPUNMAPCHUNKREQ Req;
3999	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
4000	Req.Hdr.cbReq = sizeof(Req);
4001	Req.pvR3 = NULL;
4002	Req.idChunkMap = idChunk;
4003	Req.idChunkUnmap = NIL_GMM_CHUNKID;
4004
4005	/* Must be callable from any thread, so can't use VMMR3CallR0. */
4006	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
4007	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
4008	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
4009	if (RT_SUCCESS(rc))
4010	{
4011	/*
4012	* If we're running out of virtual address space, then we should
4013	* unmap another chunk.
4014	*
4015	* Currently, an unmap operation requires that all other virtual CPUs
4016	* are idling and not by chance making use of the memory we're
4017	* unmapping. So, we create an async unmap operation here.
4018	*
4019	* Now, when creating or restoring a saved state this wont work very
4020	* well since we may want to restore all guest RAM + a little something.
4021	* So, we have to do the unmap synchronously. Fortunately for us
4022	* though, during these operations the other virtual CPUs are inactive
4023	* and it should be safe to do this.
4024	*/
4025	/** @todo Eventually we should lock all memory when used and do
4026	* map+unmap as one kernel call without any rendezvous or
4027	* other precautions. */
4028	if (pVM->pgm.s.ChunkR3Map.c + 1 >= pVM->pgm.s.ChunkR3Map.cMax)
4029	{
4030	switch (VMR3GetState(pVM))
4031	{
4032	case VMSTATE_LOADING:
4033	case VMSTATE_SAVING:
4034	{
4035	PVMCPU pVCpu = VMMGetCpu(pVM);
4036	if ( pVCpu
4037	&& pVM->pgm.s.cDeprecatedPageLocks == 0)
4038	{
4039	pgmR3PhysUnmapChunkRendezvous(pVM, pVCpu, NULL);
4040	break;
4041	}
4042	/* fall thru */
4043	}
4044	default:
4045	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
4046	AssertRC(rc);
4047	break;
4048	}
4049	}
4050
4051	/*
4052	* Update the tree. We must do this after any unmapping to make sure
4053	* the chunk we're going to return isn't unmapped by accident.
4054	*/
4055	/* insert the new one. */
4056	AssertPtr(Req.pvR3);
4057	pChunk->pv = Req.pvR3;
4058	bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
4059	AssertRelease(fRc);
4060	pVM->pgm.s.ChunkR3Map.c++;
4061	pVM->pgm.s.cMappedChunks++;
4062	}
4063	else
4064	{
4065	/** @todo this may fail because of /proc/sys/vm/max_map_count, so we
4066	* should probably restrict ourselves on linux. */
4067	AssertRC(rc);
4068	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4069	MMR3HeapFree(pChunk);
4070	#else
4071	MMR3UkHeapFree(pVM, pChunk, MM_TAG_PGM_CHUNK_MAPPING);
4072	#endif
4073	pChunk = NULL;
4074	}
4075
4076	*ppChunk = pChunk;
4077	return rc;
4078	}
4079
4080
4081	/**
4082	* For VMMCALLRING3_PGM_MAP_CHUNK, considered internal.
4083	*
4084	* @returns see pgmR3PhysChunkMap.
4085	* @param pVM The VM handle.
4086	* @param idChunk The chunk to map.
4087	*/
4088	VMMR3DECL(int) PGMR3PhysChunkMap(PVM pVM, uint32_t idChunk)
4089	{
4090	PPGMCHUNKR3MAP pChunk;
4091	int rc;
4092
4093	pgmLock(pVM);
4094	rc = pgmR3PhysChunkMap(pVM, idChunk, &pChunk);
4095	pgmUnlock(pVM);
4096	return rc;
4097	}
4098
4099
4100	/**
4101	* Invalidates the TLB for the ring-3 mapping cache.
4102	*
4103	* @param pVM The VM handle.
4104	*/
4105	VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
4106	{
4107	pgmLock(pVM);
4108	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
4109	{
4110	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
4111	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
4112	}
4113	/* The page map TLB references chunks, so invalidate that one too. */
4114	pgmPhysInvalidatePageMapTLB(pVM);
4115	pgmUnlock(pVM);
4116	}
4117
4118
4119	/**
4120	* Response to VMMCALLRING3_PGM_ALLOCATE_LARGE_PAGE to allocate a large (2MB) page
4121	* for use with a nested paging PDE.
4122	*
4123	* @returns The following VBox status codes.
4124	* @retval VINF_SUCCESS on success.
4125	* @retval VINF_EM_NO_MEMORY if we're out of memory.
4126	*
4127	* @param pVM The VM handle.
4128	* @param GCPhys GC physical start address of the 2 MB range
4129	*/
4130	VMMR3DECL(int) PGMR3PhysAllocateLargeHandyPage(PVM pVM, RTGCPHYS GCPhys)
4131	{
4132	#ifdef PGM_WITH_LARGE_PAGES
4133	uint64_t u64TimeStamp1, u64TimeStamp2;
4134
4135	pgmLock(pVM);
4136
4137	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4138	u64TimeStamp1 = RTTimeMilliTS();
4139	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE, 0, NULL);
4140	u64TimeStamp2 = RTTimeMilliTS();
4141	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4142	if (RT_SUCCESS(rc))
4143	{
4144	Assert(pVM->pgm.s.cLargeHandyPages == 1);
4145
4146	uint32_t idPage = pVM->pgm.s.aLargeHandyPage[0].idPage;
4147	RTHCPHYS HCPhys = pVM->pgm.s.aLargeHandyPage[0].HCPhysGCPhys;
4148
4149	void *pv;
4150
4151	/* Map the large page into our address space.
4152	*
4153	* Note: assuming that within the 2 MB range:
4154	* - GCPhys + PAGE_SIZE = HCPhys + PAGE_SIZE (whole point of this exercise)
4155	* - user space mapping is continuous as well
4156	* - page id (GCPhys) + 1 = page id (GCPhys + PAGE_SIZE)
4157	*/
4158	rc = pgmPhysPageMapByPageID(pVM, idPage, HCPhys, &pv);
4159	AssertLogRelMsg(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", idPage, HCPhys, rc));
4160
4161	if (RT_SUCCESS(rc))
4162	{
4163	/*
4164	* Clear the pages.
4165	*/
4166	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4167	for (unsigned i = 0; i < _2M/PAGE_SIZE; i++)
4168	{
4169	ASMMemZeroPage(pv);
4170
4171	PPGMPAGE pPage;
4172	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
4173	AssertRC(rc);
4174
4175	Assert(PGM_PAGE_IS_ZERO(pPage));
4176	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
4177	pVM->pgm.s.cZeroPages--;
4178
4179	/*
4180	* Do the PGMPAGE modifications.
4181	*/
4182	pVM->pgm.s.cPrivatePages++;
4183	PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
4184	PGM_PAGE_SET_PAGEID(pVM, pPage, idPage);
4185	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
4186	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE);
4187	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4188	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4189
4190	/* Somewhat dirty assumption that page ids are increasing. */
4191	idPage++;
4192
4193	HCPhys += PAGE_SIZE;
4194	GCPhys += PAGE_SIZE;
4195
4196	pv = (void *)((uintptr_t)pv + PAGE_SIZE);
4197
4198	Log3(("PGMR3PhysAllocateLargePage: idPage=%#x HCPhys=%RGp\n", idPage, HCPhys));
4199	}
4200	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4201
4202	/* Flush all TLBs. */
4203	PGM_INVL_ALL_VCPU_TLBS(pVM);
4204	pgmPhysInvalidatePageMapTLB(pVM);
4205	}
4206	pVM->pgm.s.cLargeHandyPages = 0;
4207	}
4208
4209	if (RT_SUCCESS(rc))
4210	{
4211	static uint32_t cTimeOut = 0;
4212	uint64_t u64TimeStampDelta = u64TimeStamp2 - u64TimeStamp1;
4213
4214	if (u64TimeStampDelta > 100)
4215	{
4216	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatLargePageOverflow);
4217	if ( ++cTimeOut > 10
4218	\|\| u64TimeStampDelta > 1000 /* more than one second forces an early retirement from allocating large pages. */)
4219	{
4220	/* If repeated attempts to allocate a large page takes more than 100 ms, then we fall back to normal 4k pages.
4221	* E.g. Vista 64 tries to move memory around, which takes a huge amount of time.
4222	*/
4223	LogRel(("PGMR3PhysAllocateLargePage: allocating large pages takes too long (last attempt %d ms; nr of timeouts %d); DISABLE\n", u64TimeStampDelta, cTimeOut));
4224	PGMSetLargePageUsage(pVM, false);
4225	}
4226	}
4227	else
4228	if (cTimeOut > 0)
4229	cTimeOut--;
4230	}
4231
4232	pgmUnlock(pVM);
4233	return rc;
4234	#else
4235	return VERR_NOT_IMPLEMENTED;
4236	#endif /* PGM_WITH_LARGE_PAGES */
4237	}
4238
4239
4240	/**
4241	* Response to VM_FF_PGM_NEED_HANDY_PAGES and VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES.
4242	*
4243	* This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
4244	* signal and clear the out of memory condition. When contracted, this API is
4245	* used to try clear the condition when the user wants to resume.
4246	*
4247	* @returns The following VBox status codes.
4248	* @retval VINF_SUCCESS on success. FFs cleared.
4249	* @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
4250	* this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
4251	*
4252	* @param pVM The VM handle.
4253	*
4254	* @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
4255	* in EM.cpp and shouldn't be propagated outside TRPM, HWACCM, EM and
4256	* pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
4257	* handler.
4258	*/
4259	VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
4260	{
4261	pgmLock(pVM);
4262
4263	/*
4264	* Allocate more pages, noting down the index of the first new page.
4265	*/
4266	uint32_t iClear = pVM->pgm.s.cHandyPages;
4267	AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_INTERNAL_ERROR);
4268	Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
4269	int rcAlloc = VINF_SUCCESS;
4270	int rcSeed = VINF_SUCCESS;
4271	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4272	while (rc == VERR_GMM_SEED_ME)
4273	{
4274	void *pvChunk;
4275	rcAlloc = rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
4276	if (RT_SUCCESS(rc))
4277	{
4278	rcSeed = rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
4279	if (RT_FAILURE(rc))
4280	SUPR3PageFree(pvChunk, GMM_CHUNK_SIZE >> PAGE_SHIFT);
4281	}
4282	if (RT_SUCCESS(rc))
4283	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4284	}
4285
4286	/* todo: we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
4287	if ( rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
4288	&& pVM->pgm.s.cHandyPages > 0)
4289	{
4290	/* Still handy pages left, so don't panic. */
4291	rc = VINF_SUCCESS;
4292	}
4293
4294	if (RT_SUCCESS(rc))
4295	{
4296	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
4297	Assert(pVM->pgm.s.cHandyPages > 0);
4298	VM_FF_CLEAR(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4299	VM_FF_CLEAR(pVM, VM_FF_PGM_NO_MEMORY);
4300
4301	#ifdef VBOX_STRICT
4302	bool fOk = true;
4303	uint32_t i;
4304	for (i = iClear; i < pVM->pgm.s.cHandyPages; i++)
4305	if ( pVM->pgm.s.aHandyPages[i].idPage == NIL_GMM_PAGEID
4306	\|\| pVM->pgm.s.aHandyPages[i].idSharedPage != NIL_GMM_PAGEID
4307	\|\| (pVM->pgm.s.aHandyPages[i].HCPhysGCPhys & PAGE_OFFSET_MASK))
4308	break;
4309	if (i != pVM->pgm.s.cHandyPages)
4310	{
4311	RTAssertMsg1Weak(NULL, __LINE__, __FILE__, __FUNCTION__);
4312	RTAssertMsg2Weak("i=%d iClear=%d cHandyPages=%d\n", i, iClear, pVM->pgm.s.cHandyPages);
4313	for (uint32_t j = iClear; j < pVM->pgm.s.cHandyPages; j++)
4314	RTAssertMsg2Add(("%03d: idPage=%d HCPhysGCPhys=%RHp idSharedPage=%d%\n", j,
4315	pVM->pgm.s.aHandyPages[j].idPage,
4316	pVM->pgm.s.aHandyPages[j].HCPhysGCPhys,
4317	pVM->pgm.s.aHandyPages[j].idSharedPage,
4318	j == i ? " <---" : ""));
4319	RTAssertPanic();
4320	}
4321	#endif
4322	/*
4323	* Clear the pages.
4324	*/
4325	while (iClear < pVM->pgm.s.cHandyPages)
4326	{
4327	PGMMPAGEDESC pPage = &pVM->pgm.s.aHandyPages[iClear];
4328	void *pv;
4329	rc = pgmPhysPageMapByPageID(pVM, pPage->idPage, pPage->HCPhysGCPhys, &pv);
4330	AssertLogRelMsgBreak(RT_SUCCESS(rc),
4331	("%u/%u: idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n",
4332	iClear, pVM->pgm.s.cHandyPages, pPage->idPage, pPage->HCPhysGCPhys, rc));
4333	ASMMemZeroPage(pv);
4334	iClear++;
4335	Log3(("PGMR3PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
4336	}
4337	}
4338	else
4339	{
4340	uint64_t cAllocPages, cMaxPages, cBalloonPages;
4341
4342	/*
4343	* We should never get here unless there is a genuine shortage of
4344	* memory (or some internal error). Flag the error so the VM can be
4345	* suspended ASAP and the user informed. If we're totally out of
4346	* handy pages we will return failure.
4347	*/
4348	/* Report the failure. */
4349	LogRel(("PGM: Failed to procure handy pages; rc=%Rrc rcAlloc=%Rrc rcSeed=%Rrc cHandyPages=%#x\n"
4350	" cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
4351	rc, rcAlloc, rcSeed,
4352	pVM->pgm.s.cHandyPages,
4353	pVM->pgm.s.cAllPages,
4354	pVM->pgm.s.cPrivatePages,
4355	pVM->pgm.s.cSharedPages,
4356	pVM->pgm.s.cZeroPages));
4357
4358	if (GMMR3QueryMemoryStats(pVM, &cAllocPages, &cMaxPages, &cBalloonPages) == VINF_SUCCESS)
4359	{
4360	LogRel(("GMM: Statistics:\n"
4361	" Allocated pages: %RX64\n"
4362	" Maximum pages: %RX64\n"
4363	" Ballooned pages: %RX64\n", cAllocPages, cMaxPages, cBalloonPages));
4364	}
4365
4366	if ( rc != VERR_NO_MEMORY
4367	&& rc != VERR_LOCK_FAILED)
4368	{
4369	for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4370	{
4371	LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
4372	i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
4373	pVM->pgm.s.aHandyPages[i].idSharedPage));
4374	uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
4375	if (idPage != NIL_GMM_PAGEID)
4376	{
4377	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
4378	pRam;
4379	pRam = pRam->pNextR3)
4380	{
4381	uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
4382	for (uint32_t iPage = 0; iPage < cPages; iPage++)
4383	if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
4384	LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
4385	pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
4386	}
4387	}
4388	}
4389	}
4390
4391	/* Set the FFs and adjust rc. */
4392	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4393	VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
4394	if ( rc == VERR_NO_MEMORY
4395	\|\| rc == VERR_LOCK_FAILED)
4396	rc = VINF_EM_NO_MEMORY;
4397	}
4398
4399	pgmUnlock(pVM);
4400	return rc;
4401	}
4402
4403
4404	/**
4405	* Frees the specified RAM page and replaces it with the ZERO page.
4406	*
4407	* This is used by ballooning, remapping MMIO2, RAM reset and state loading.
4408	*
4409	* @param pVM Pointer to the shared VM structure.
4410	* @param pReq Pointer to the request.
4411	* @param pcPendingPages Where the number of pages waiting to be freed are
4412	* kept. This will normally be incremented.
4413	* @param pPage Pointer to the page structure.
4414	* @param GCPhys The guest physical address of the page, if applicable.
4415	*
4416	* @remarks The caller must own the PGM lock.
4417	*/
4418	int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys)
4419	{
4420	/*
4421	* Assert sanity.
4422	*/
4423	PGM_LOCK_ASSERT_OWNER(pVM);
4424	if (RT_UNLIKELY( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
4425	&& PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
4426	{
4427	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4428	return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
4429	}
4430
4431	/** @todo What about ballooning of large pages??! */
4432	Assert( PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
4433	&& PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED);
4434
4435	if ( PGM_PAGE_IS_ZERO(pPage)
4436	\|\| PGM_PAGE_IS_BALLOONED(pPage))
4437	return VINF_SUCCESS;
4438
4439	const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
4440	Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
4441	if (RT_UNLIKELY( idPage == NIL_GMM_PAGEID
4442	\|\| idPage > GMM_PAGEID_LAST
4443	\|\| PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
4444	{
4445	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4446	return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
4447	}
4448
4449	/* update page count stats. */
4450	if (PGM_PAGE_IS_SHARED(pPage))
4451	pVM->pgm.s.cSharedPages--;
4452	else
4453	pVM->pgm.s.cPrivatePages--;
4454	pVM->pgm.s.cZeroPages++;
4455
4456	/* Deal with write monitored pages. */
4457	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
4458	{
4459	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
4460	pVM->pgm.s.cWrittenToPages++;
4461	}
4462
4463	/*
4464	* pPage = ZERO page.
4465	*/
4466	PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg);
4467	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
4468	PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
4469	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4470	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4471	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4472
4473	/* Flush physical page map TLB entry. */
4474	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
4475
4476	/*
4477	* Make sure it's not in the handy page array.
4478	*/
4479	for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4480	{
4481	if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
4482	{
4483	pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
4484	break;
4485	}
4486	if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
4487	{
4488	pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
4489	break;
4490	}
4491	}
4492
4493	/*
4494	* Push it onto the page array.
4495	*/
4496	uint32_t iPage = *pcPendingPages;
4497	Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
4498	*pcPendingPages += 1;
4499
4500	pReq->aPages[iPage].idPage = idPage;
4501
4502	if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
4503	return VINF_SUCCESS;
4504
4505	/*
4506	* Flush the pages.
4507	*/
4508	int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
4509	if (RT_SUCCESS(rc))
4510	{
4511	GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
4512	*pcPendingPages = 0;
4513	}
4514	return rc;
4515	}
4516
4517
4518	/**
4519	* Converts a GC physical address to a HC ring-3 pointer, with some
4520	* additional checks.
4521	*
4522	* @returns VBox status code.
4523	* @retval VINF_SUCCESS on success.
4524	* @retval VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4525	* access handler of some kind.
4526	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4527	* accesses or is odd in any way.
4528	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4529	*
4530	* @param pVM The VM handle.
4531	* @param GCPhys The GC physical address to convert.
4532	* @param fWritable Whether write access is required.
4533	* @param ppv Where to store the pointer corresponding to GCPhys on
4534	* success.
4535	*/
4536	VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
4537	{
4538	pgmLock(pVM);
4539
4540	PPGMRAMRANGE pRam;
4541	PPGMPAGE pPage;
4542	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4543	if (RT_SUCCESS(rc))
4544	{
4545	if (PGM_PAGE_IS_BALLOONED(pPage))
4546	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4547	else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
4548	rc = VINF_SUCCESS;
4549	else
4550	{
4551	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4552	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4553	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
4554	{
4555	/** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
4556	* in -norawr0 mode. */
4557	if (fWritable)
4558	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4559	}
4560	else
4561	{
4562	/* Temporarily disabled physical handler(s), since the recompiler
4563	doesn't get notified when it's reset we'll have to pretend it's
4564	operating normally. */
4565	if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
4566	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4567	else
4568	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4569	}
4570	}
4571	if (RT_SUCCESS(rc))
4572	{
4573	int rc2;
4574
4575	/* Make sure what we return is writable. */
4576	if (fWritable)
4577	switch (PGM_PAGE_GET_STATE(pPage))
4578	{
4579	case PGM_PAGE_STATE_ALLOCATED:
4580	break;
4581	case PGM_PAGE_STATE_BALLOONED:
4582	AssertFailed();
4583	break;
4584	case PGM_PAGE_STATE_ZERO:
4585	case PGM_PAGE_STATE_SHARED:
4586	if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
4587	break;
4588	case PGM_PAGE_STATE_WRITE_MONITORED:
4589	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
4590	AssertLogRelRCReturn(rc2, rc2);
4591	break;
4592	}
4593
4594	/* Get a ring-3 mapping of the address. */
4595	PPGMPAGER3MAPTLBE pTlbe;
4596	rc2 = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
4597	AssertLogRelRCReturn(rc2, rc2);
4598	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4599	/** @todo mapping/locking hell; this isn't horribly efficient since
4600	* pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
4601
4602	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
4603	}
4604	else
4605	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
4606
4607	/* else: handler catching all access, no pointer returned. */
4608	}
4609	else
4610	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
4611
4612	pgmUnlock(pVM);
4613	return rc;
4614	}
4615

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp@ 38955

Download in other formats: