PGMPhys.cpp@ 39427

Last change on this file since 39427 was 39427, checked in by vboxsync, 13 years ago
r=bird: pre-review comments.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 164.1 KB

Line
1	/* $Id: PGMPhys.cpp 39427 2011-11-25 14:29:43Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM_PHYS
23	#include <VBox/vmm/pgm.h>
24	#include <VBox/vmm/iom.h>
25	#include <VBox/vmm/mm.h>
26	#include <VBox/vmm/stam.h>
27	#include <VBox/vmm/rem.h>
28	#include <VBox/vmm/pdmdev.h>
29	#include "PGMInternal.h"
30	#include <VBox/vmm/vm.h>
31	#include "PGMInline.h"
32	#include <VBox/sup.h>
33	#include <VBox/param.h>
34	#include <VBox/err.h>
35	#include <VBox/log.h>
36	#include <iprt/assert.h>
37	#include <iprt/alloc.h>
38	#include <iprt/asm.h>
39	#include <iprt/thread.h>
40	#include <iprt/string.h>
41	#include <iprt/system.h>
42
43
44	/*******************************************************************************
45	* Defined Constants And Macros *
46	*******************************************************************************/
47	/** The number of pages to free in one batch. */
48	#define PGMPHYS_FREE_PAGE_BATCH_SIZE 128
49
50
51	/*******************************************************************************
52	* Internal Functions *
53	*******************************************************************************/
54	static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
55
56
57	/*
58	* PGMR3PhysReadU8-64
59	* PGMR3PhysWriteU8-64
60	*/
61	#define PGMPHYSFN_READNAME PGMR3PhysReadU8
62	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
63	#define PGMPHYS_DATASIZE 1
64	#define PGMPHYS_DATATYPE uint8_t
65	#include "PGMPhysRWTmpl.h"
66
67	#define PGMPHYSFN_READNAME PGMR3PhysReadU16
68	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
69	#define PGMPHYS_DATASIZE 2
70	#define PGMPHYS_DATATYPE uint16_t
71	#include "PGMPhysRWTmpl.h"
72
73	#define PGMPHYSFN_READNAME PGMR3PhysReadU32
74	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
75	#define PGMPHYS_DATASIZE 4
76	#define PGMPHYS_DATATYPE uint32_t
77	#include "PGMPhysRWTmpl.h"
78
79	#define PGMPHYSFN_READNAME PGMR3PhysReadU64
80	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
81	#define PGMPHYS_DATASIZE 8
82	#define PGMPHYS_DATATYPE uint64_t
83	#include "PGMPhysRWTmpl.h"
84
85
86	/**
87	* EMT worker for PGMR3PhysReadExternal.
88	*/
89	static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead)
90	{
91	PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead);
92	return VINF_SUCCESS;
93	}
94
95
96	/**
97	* Read from physical memory, external users.
98	*
99	* @returns VBox status code.
100	* @retval VINF_SUCCESS.
101	*
102	* @param pVM VM Handle.
103	* @param GCPhys Physical address to read from.
104	* @param pvBuf Where to read into.
105	* @param cbRead How many bytes to read.
106	*
107	* @thread Any but EMTs.
108	*/
109	VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
110	{
111	VM_ASSERT_OTHER_THREAD(pVM);
112
113	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
114	LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
115
116	pgmLock(pVM);
117
118	/*
119	* Copy loop on ram ranges.
120	*/
121	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
122	for (;;)
123	{
124	/* Inside range or not? */
125	if (pRam && GCPhys >= pRam->GCPhys)
126	{
127	/*
128	* Must work our way thru this page by page.
129	*/
130	RTGCPHYS off = GCPhys - pRam->GCPhys;
131	while (off < pRam->cb)
132	{
133	unsigned iPage = off >> PAGE_SHIFT;
134	PPGMPAGE pPage = &pRam->aPages[iPage];
135
136	/*
137	* If the page has an ALL access handler, we'll have to
138	* delegate the job to EMT.
139	*/
140	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
141	{
142	pgmUnlock(pVM);
143
144	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 4,
145	pVM, &GCPhys, pvBuf, cbRead);
146	}
147	Assert(!PGM_PAGE_IS_MMIO(pPage));
148
149	/*
150	* Simple stuff, go ahead.
151	*/
152	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
153	if (cb > cbRead)
154	cb = cbRead;
155	PGMPAGEMAPLOCK PgMpLck;
156	const void *pvSrc;
157	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
158	if (RT_SUCCESS(rc))
159	{
160	memcpy(pvBuf, pvSrc, cb);
161	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
162	}
163	else
164	{
165	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
166	pRam->GCPhys + off, pPage, rc));
167	memset(pvBuf, 0xff, cb);
168	}
169
170	/* next page */
171	if (cb >= cbRead)
172	{
173	pgmUnlock(pVM);
174	return VINF_SUCCESS;
175	}
176	cbRead -= cb;
177	off += cb;
178	GCPhys += cb;
179	pvBuf = (char *)pvBuf + cb;
180	} /* walk pages in ram range. */
181	}
182	else
183	{
184	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
185
186	/*
187	* Unassigned address space.
188	*/
189	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
190	if (cb >= cbRead)
191	{
192	memset(pvBuf, 0xff, cbRead);
193	break;
194	}
195	memset(pvBuf, 0xff, cb);
196
197	cbRead -= cb;
198	pvBuf = (char *)pvBuf + cb;
199	GCPhys += cb;
200	}
201
202	/* Advance range if necessary. */
203	while (pRam && GCPhys > pRam->GCPhysLast)
204	pRam = pRam->CTX_SUFF(pNext);
205	} /* Ram range walk */
206
207	pgmUnlock(pVM);
208
209	return VINF_SUCCESS;
210	}
211
212
213	/**
214	* EMT worker for PGMR3PhysWriteExternal.
215	*/
216	static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite)
217	{
218	/** @todo VERR_EM_NO_MEMORY */
219	PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite);
220	return VINF_SUCCESS;
221	}
222
223
224	/**
225	* Write to physical memory, external users.
226	*
227	* @returns VBox status code.
228	* @retval VINF_SUCCESS.
229	* @retval VERR_EM_NO_MEMORY.
230	*
231	* @param pVM VM Handle.
232	* @param GCPhys Physical address to write to.
233	* @param pvBuf What to write.
234	* @param cbWrite How many bytes to write.
235	* @param pszWho Who is writing. For tracking down who is writing
236	* after we've saved the state.
237	*
238	* @thread Any but EMTs.
239	*/
240	VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void pvBuf, size_t cbWrite, const char pszWho)
241	{
242	VM_ASSERT_OTHER_THREAD(pVM);
243
244	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
245	("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x pszWho=%s\n",
246	GCPhys, cbWrite, pszWho));
247	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
248	LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
249
250	pgmLock(pVM);
251
252	/*
253	* Copy loop on ram ranges, stop when we hit something difficult.
254	*/
255	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
256	for (;;)
257	{
258	/* Inside range or not? */
259	if (pRam && GCPhys >= pRam->GCPhys)
260	{
261	/*
262	* Must work our way thru this page by page.
263	*/
264	RTGCPTR off = GCPhys - pRam->GCPhys;
265	while (off < pRam->cb)
266	{
267	RTGCPTR iPage = off >> PAGE_SHIFT;
268	PPGMPAGE pPage = &pRam->aPages[iPage];
269
270	/*
271	* Is the page problematic, we have to do the work on the EMT.
272	*
273	* Allocating writable pages and access handlers are
274	* problematic, write monitored pages are simple and can be
275	* dealt with here.
276	*/
277	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
278	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
279	{
280	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
281	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
282	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
283	else
284	{
285	pgmUnlock(pVM);
286
287	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 4,
288	pVM, &GCPhys, pvBuf, cbWrite);
289	}
290	}
291	Assert(!PGM_PAGE_IS_MMIO(pPage));
292
293	/*
294	* Simple stuff, go ahead.
295	*/
296	size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
297	if (cb > cbWrite)
298	cb = cbWrite;
299	PGMPAGEMAPLOCK PgMpLck;
300	void *pvDst;
301	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
302	if (RT_SUCCESS(rc))
303	{
304	memcpy(pvDst, pvBuf, cb);
305	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
306	}
307	else
308	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
309	pRam->GCPhys + off, pPage, rc));
310
311	/* next page */
312	if (cb >= cbWrite)
313	{
314	pgmUnlock(pVM);
315	return VINF_SUCCESS;
316	}
317
318	cbWrite -= cb;
319	off += cb;
320	GCPhys += cb;
321	pvBuf = (const char *)pvBuf + cb;
322	} /* walk pages in ram range */
323	}
324	else
325	{
326	/*
327	* Unassigned address space, skip it.
328	*/
329	if (!pRam)
330	break;
331	size_t cb = pRam->GCPhys - GCPhys;
332	if (cb >= cbWrite)
333	break;
334	cbWrite -= cb;
335	pvBuf = (const char *)pvBuf + cb;
336	GCPhys += cb;
337	}
338
339	/* Advance range if necessary. */
340	while (pRam && GCPhys > pRam->GCPhysLast)
341	pRam = pRam->CTX_SUFF(pNext);
342	} /* Ram range walk */
343
344	pgmUnlock(pVM);
345	return VINF_SUCCESS;
346	}
347
348
349	/**
350	* VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
351	*
352	* @returns see PGMR3PhysGCPhys2CCPtrExternal
353	* @param pVM The VM handle.
354	* @param pGCPhys Pointer to the guest physical address.
355	* @param ppv Where to store the mapping address.
356	* @param pLock Where to store the lock.
357	*/
358	static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
359	{
360	/*
361	* Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
362	* an access handler after it succeeds.
363	*/
364	int rc = pgmLock(pVM);
365	AssertRCReturn(rc, rc);
366
367	rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
368	if (RT_SUCCESS(rc))
369	{
370	PPGMPAGEMAPTLBE pTlbe;
371	int rc2 = pgmPhysPageQueryTlbe(pVM, *pGCPhys, &pTlbe);
372	AssertFatalRC(rc2);
373	PPGMPAGE pPage = pTlbe->pPage;
374	if (PGM_PAGE_IS_MMIO(pPage))
375	{
376	PGMPhysReleasePageMappingLock(pVM, pLock);
377	rc = VERR_PGM_PHYS_PAGE_RESERVED;
378	}
379	else if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
380	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
381	\|\| pgmPoolIsDirtyPage(pVM, *pGCPhys)
382	#endif
383	)
384	{
385	/* We must flush any corresponding pgm pool page here, otherwise we'll
386	* not be informed about writes and keep bogus gst->shw mappings around.
387	*/
388	pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
389	Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
390	/** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
391	* active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
392	}
393	}
394
395	pgmUnlock(pVM);
396	return rc;
397	}
398
399
400	/**
401	* Requests the mapping of a guest page into ring-3, external threads.
402	*
403	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
404	* release it.
405	*
406	* This API will assume your intention is to write to the page, and will
407	* therefore replace shared and zero pages. If you do not intend to modify the
408	* page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
409	*
410	* @returns VBox status code.
411	* @retval VINF_SUCCESS on success.
412	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
413	* backing or if the page has any active access handlers. The caller
414	* must fall back on using PGMR3PhysWriteExternal.
415	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
416	*
417	* @param pVM The VM handle.
418	* @param GCPhys The guest physical address of the page that should be mapped.
419	* @param ppv Where to store the address corresponding to GCPhys.
420	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
421	*
422	* @remark Avoid calling this API from within critical sections (other than the
423	* PGM one) because of the deadlock risk when we have to delegating the
424	* task to an EMT.
425	* @thread Any.
426	*/
427	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
428	{
429	AssertPtr(ppv);
430	AssertPtr(pLock);
431
432	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
433
434	int rc = pgmLock(pVM);
435	AssertRCReturn(rc, rc);
436
437	/*
438	* Query the Physical TLB entry for the page (may fail).
439	*/
440	PPGMPAGEMAPTLBE pTlbe;
441	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
442	if (RT_SUCCESS(rc))
443	{
444	PPGMPAGE pPage = pTlbe->pPage;
445	if (PGM_PAGE_IS_MMIO(pPage))
446	rc = VERR_PGM_PHYS_PAGE_RESERVED;
447	else
448	{
449	/*
450	* If the page is shared, the zero page, or being write monitored
451	* it must be converted to an page that's writable if possible.
452	* We can only deal with write monitored pages here, the rest have
453	* to be on an EMT.
454	*/
455	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
456	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
457	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
458	\|\| pgmPoolIsDirtyPage(pVM, GCPhys)
459	#endif
460	)
461	{
462	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
463	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
464	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
465	&& !pgmPoolIsDirtyPage(pVM, GCPhys)
466	#endif
467	)
468	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
469	else
470	{
471	pgmUnlock(pVM);
472
473	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
474	pVM, &GCPhys, ppv, pLock);
475	}
476	}
477
478	/*
479	* Now, just perform the locking and calculate the return address.
480	*/
481	PPGMPAGEMAP pMap = pTlbe->pMap;
482	if (pMap)
483	pMap->cRefs++;
484
485	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
486	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
487	{
488	if (cLocks == 0)
489	pVM->pgm.s.cWriteLockedPages++;
490	PGM_PAGE_INC_WRITE_LOCKS(pPage);
491	}
492	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
493	{
494	PGM_PAGE_INC_WRITE_LOCKS(pPage);
495	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
496	if (pMap)
497	pMap->cRefs++; /* Extra ref to prevent it from going away. */
498	}
499
500	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
501	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
502	pLock->pvMap = pMap;
503	}
504	}
505
506	pgmUnlock(pVM);
507	return rc;
508	}
509
510
511	/**
512	* Requests the mapping of a guest page into ring-3, external threads.
513	*
514	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
515	* release it.
516	*
517	* @returns VBox status code.
518	* @retval VINF_SUCCESS on success.
519	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
520	* backing or if the page as an active ALL access handler. The caller
521	* must fall back on using PGMPhysRead.
522	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
523	*
524	* @param pVM The VM handle.
525	* @param GCPhys The guest physical address of the page that should be mapped.
526	* @param ppv Where to store the address corresponding to GCPhys.
527	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
528	*
529	* @remark Avoid calling this API from within critical sections (other than
530	* the PGM one) because of the deadlock risk.
531	* @thread Any.
532	*/
533	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
534	{
535	int rc = pgmLock(pVM);
536	AssertRCReturn(rc, rc);
537
538	/*
539	* Query the Physical TLB entry for the page (may fail).
540	*/
541	PPGMPAGEMAPTLBE pTlbe;
542	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
543	if (RT_SUCCESS(rc))
544	{
545	PPGMPAGE pPage = pTlbe->pPage;
546	#if 1
547	/* MMIO pages doesn't have any readable backing. */
548	if (PGM_PAGE_IS_MMIO(pPage))
549	rc = VERR_PGM_PHYS_PAGE_RESERVED;
550	#else
551	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
552	rc = VERR_PGM_PHYS_PAGE_RESERVED;
553	#endif
554	else
555	{
556	/*
557	* Now, just perform the locking and calculate the return address.
558	*/
559	PPGMPAGEMAP pMap = pTlbe->pMap;
560	if (pMap)
561	pMap->cRefs++;
562
563	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
564	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
565	{
566	if (cLocks == 0)
567	pVM->pgm.s.cReadLockedPages++;
568	PGM_PAGE_INC_READ_LOCKS(pPage);
569	}
570	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
571	{
572	PGM_PAGE_INC_READ_LOCKS(pPage);
573	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
574	if (pMap)
575	pMap->cRefs++; /* Extra ref to prevent it from going away. */
576	}
577
578	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
579	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
580	pLock->pvMap = pMap;
581	}
582	}
583
584	pgmUnlock(pVM);
585	return rc;
586	}
587
588
589	#define MAKE_LEAF(a_pNode) \
590	do { \
591	(a_pNode)->pLeftR3 = NIL_RTR3PTR; \
592	(a_pNode)->pRightR3 = NIL_RTR3PTR; \
593	(a_pNode)->pLeftR0 = NIL_RTR0PTR; \
594	(a_pNode)->pRightR0 = NIL_RTR0PTR; \
595	(a_pNode)->pLeftRC = NIL_RTRCPTR; \
596	(a_pNode)->pRightRC = NIL_RTRCPTR; \
597	} while (0)
598
599	#define INSERT_LEFT(a_pParent, a_pNode) \
600	do { \
601	(a_pParent)->pLeftR3 = (a_pNode); \
602	(a_pParent)->pLeftR0 = (a_pNode)->pSelfR0; \
603	(a_pParent)->pLeftRC = (a_pNode)->pSelfRC; \
604	} while (0)
605	#define INSERT_RIGHT(a_pParent, a_pNode) \
606	do { \
607	(a_pParent)->pRightR3 = (a_pNode); \
608	(a_pParent)->pRightR0 = (a_pNode)->pSelfR0; \
609	(a_pParent)->pRightRC = (a_pNode)->pSelfRC; \
610	} while (0)
611
612
613	/**
614	* Recursive tree builder.
615	*
616	* @param ppRam Pointer to the iterator variable.
617	* @param iHeight The hight about normal leaf nodes. Inserts a leaf
618	* node if 0.
619	*/
620	static PPGMRAMRANGE pgmR3PhysRebuildRamRangeSearchTreesRecursively(PPGMRAMRANGE *ppRam, int iDepth)
621	{
622	PPGMRAMRANGE pRam;
623	if (iDepth <= 0)
624	{
625	/*
626	* Leaf node.
627	*/
628	pRam = *ppRam;
629	if (pRam)
630	{
631	*ppRam = pRam->pNextR3;
632	MAKE_LEAF(pRam);
633	}
634	}
635	else
636	{
637
638	/*
639	* Intermediate node.
640	*/
641	PPGMRAMRANGE pLeft = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
642
643	pRam = *ppRam;
644	if (!pRam)
645	return pLeft;
646	*ppRam = pRam->pNextR3;
647	MAKE_LEAF(pRam);
648	INSERT_LEFT(pRam, pLeft);
649
650	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
651	if (pRight)
652	INSERT_RIGHT(pRam, pRight);
653	}
654	return pRam;
655	}
656
657
658	/**
659	* Rebuilds the RAM range search trees.
660	*
661	* @param pVM The VM handle.
662	*/
663	static void pgmR3PhysRebuildRamRangeSearchTrees(PVM pVM)
664	{
665
666	/*
667	* Create the reasonably balanced tree in a sequential fashion.
668	* For simplicity (laziness) we use standard recursion here.
669	*/
670	int iDepth = 0;
671	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
672	PPGMRAMRANGE pRoot = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, 0);
673	while (pRam)
674	{
675	PPGMRAMRANGE pLeft = pRoot;
676
677	pRoot = pRam;
678	pRam = pRam->pNextR3;
679	MAKE_LEAF(pRoot);
680	INSERT_LEFT(pRoot, pLeft);
681
682	PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, iDepth);
683	if (pRight)
684	INSERT_RIGHT(pRoot, pRight);
685	/** @todo else: rotate the tree. */
686
687	iDepth++;
688	}
689
690	pVM->pgm.s.pRamRangeTreeR3 = pRoot;
691	pVM->pgm.s.pRamRangeTreeR0 = pRoot ? pRoot->pSelfR0 : NIL_RTR0PTR;
692	pVM->pgm.s.pRamRangeTreeRC = pRoot ? pRoot->pSelfRC : NIL_RTRCPTR;
693
694	#ifdef VBOX_STRICT
695	/*
696	* Verify that the above code works.
697	*/
698	unsigned cRanges = 0;
699	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
700	cRanges++;
701	Assert(cRanges > 0);
702
703	unsigned cMaxDepth = ASMBitLastSetU32(cRanges);
704	if ((1U << cMaxDepth) < cRanges)
705	cMaxDepth++;
706
707	for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
708	{
709	unsigned cDepth = 0;
710	PPGMRAMRANGE pRam2 = pVM->pgm.s.pRamRangeTreeR3;
711	for (;;)
712	{
713	if (pRam == pRam2)
714	break;
715	Assert(pRam2);
716	if (pRam->GCPhys < pRam2->GCPhys)
717	pRam2 = pRam2->pLeftR3;
718	else
719	pRam2 = pRam2->pRightR3;
720	}
721	AssertMsg(cDepth <= cMaxDepth, ("cDepth=%d cMaxDepth=%d\n", cDepth, cMaxDepth));
722	}
723	#endif /* VBOX_STRICT */
724	}
725
726	#undef MAKE_LEAF
727	#undef INSERT_LEFT
728	#undef INSERT_RIGHT
729
730	/**
731	* Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
732	*
733	* Called when anything was relocated.
734	*
735	* @param pVM Pointer to the shared VM structure.
736	*/
737	void pgmR3PhysRelinkRamRanges(PVM pVM)
738	{
739	PPGMRAMRANGE pCur;
740
741	#ifdef VBOX_STRICT
742	for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
743	{
744	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
745	Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pCur->pSelfRC == MMHyperCCToRC(pVM, pCur));
746	Assert((pCur->GCPhys & PAGE_OFFSET_MASK) == 0);
747	Assert((pCur->GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
748	Assert((pCur->cb & PAGE_OFFSET_MASK) == 0);
749	Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
750	for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesXR3; pCur2; pCur2 = pCur2->pNextR3)
751	Assert( pCur2 == pCur
752	\|\| strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
753	}
754	#endif
755
756	pCur = pVM->pgm.s.pRamRangesXR3;
757	if (pCur)
758	{
759	pVM->pgm.s.pRamRangesXR0 = pCur->pSelfR0;
760	pVM->pgm.s.pRamRangesXRC = pCur->pSelfRC;
761
762	for (; pCur->pNextR3; pCur = pCur->pNextR3)
763	{
764	pCur->pNextR0 = pCur->pNextR3->pSelfR0;
765	pCur->pNextRC = pCur->pNextR3->pSelfRC;
766	}
767
768	Assert(pCur->pNextR0 == NIL_RTR0PTR);
769	Assert(pCur->pNextRC == NIL_RTRCPTR);
770	}
771	else
772	{
773	Assert(pVM->pgm.s.pRamRangesXR0 == NIL_RTR0PTR);
774	Assert(pVM->pgm.s.pRamRangesXRC == NIL_RTRCPTR);
775	}
776	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
777
778	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
779	}
780
781
782	/**
783	* Links a new RAM range into the list.
784	*
785	* @param pVM Pointer to the shared VM structure.
786	* @param pNew Pointer to the new list entry.
787	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
788	*/
789	static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
790	{
791	AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
792	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfR0 == MMHyperCCToR0(pVM, pNew));
793	Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pNew->pSelfRC == MMHyperCCToRC(pVM, pNew));
794
795	pgmLock(pVM);
796
797	PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesXR3;
798	pNew->pNextR3 = pRam;
799	pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
800	pNew->pNextRC = pRam ? pRam->pSelfRC : NIL_RTRCPTR;
801
802	if (pPrev)
803	{
804	pPrev->pNextR3 = pNew;
805	pPrev->pNextR0 = pNew->pSelfR0;
806	pPrev->pNextRC = pNew->pSelfRC;
807	}
808	else
809	{
810	pVM->pgm.s.pRamRangesXR3 = pNew;
811	pVM->pgm.s.pRamRangesXR0 = pNew->pSelfR0;
812	pVM->pgm.s.pRamRangesXRC = pNew->pSelfRC;
813	}
814	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
815
816	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
817	pgmUnlock(pVM);
818	}
819
820
821	/**
822	* Unlink an existing RAM range from the list.
823	*
824	* @param pVM Pointer to the shared VM structure.
825	* @param pRam Pointer to the new list entry.
826	* @param pPrev Pointer to the previous list entry. If NULL, insert as head.
827	*/
828	static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
829	{
830	Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesXR3 == pRam);
831	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
832	Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) \|\| pRam->pSelfRC == MMHyperCCToRC(pVM, pRam));
833
834	pgmLock(pVM);
835
836	PPGMRAMRANGE pNext = pRam->pNextR3;
837	if (pPrev)
838	{
839	pPrev->pNextR3 = pNext;
840	pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
841	pPrev->pNextRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
842	}
843	else
844	{
845	Assert(pVM->pgm.s.pRamRangesXR3 == pRam);
846	pVM->pgm.s.pRamRangesXR3 = pNext;
847	pVM->pgm.s.pRamRangesXR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
848	pVM->pgm.s.pRamRangesXRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
849	}
850	ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
851
852	pgmR3PhysRebuildRamRangeSearchTrees(pVM);
853	pgmUnlock(pVM);
854	}
855
856
857	/**
858	* Unlink an existing RAM range from the list.
859	*
860	* @param pVM Pointer to the shared VM structure.
861	* @param pRam Pointer to the new list entry.
862	*/
863	static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
864	{
865	pgmLock(pVM);
866
867	/* find prev. */
868	PPGMRAMRANGE pPrev = NULL;
869	PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesXR3;
870	while (pCur != pRam)
871	{
872	pPrev = pCur;
873	pCur = pCur->pNextR3;
874	}
875	AssertFatal(pCur);
876
877	pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
878	pgmUnlock(pVM);
879	}
880
881
882	/**
883	* Frees a range of pages, replacing them with ZERO pages of the specified type.
884	*
885	* @returns VBox status code.
886	* @param pVM The VM handle.
887	* @param pRam The RAM range in which the pages resides.
888	* @param GCPhys The address of the first page.
889	* @param GCPhysLast The address of the last page.
890	* @param uType The page type to replace then with.
891	*/
892	static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint8_t uType)
893	{
894	PGM_LOCK_ASSERT_OWNER(pVM);
895	uint32_t cPendingPages = 0;
896	PGMMFREEPAGESREQ pReq;
897	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
898	AssertLogRelRCReturn(rc, rc);
899
900	/* Iterate the pages. */
901	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
902	uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
903	while (cPagesLeft-- > 0)
904	{
905	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
906	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
907
908	PGM_PAGE_SET_TYPE(pVM, pPageDst, uType);
909
910	GCPhys += PAGE_SIZE;
911	pPageDst++;
912	}
913
914	if (cPendingPages)
915	{
916	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
917	AssertLogRelRCReturn(rc, rc);
918	}
919	GMMR3FreePagesCleanup(pReq);
920
921	return rc;
922	}
923
924	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
925
926	/**
927	* Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
928	*
929	* This is only called on one of the EMTs while the other ones are waiting for
930	* it to complete this function.
931	*
932	* @returns VINF_SUCCESS (VBox strict status code).
933	* @param pVM The VM handle.
934	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
935	* @param pvUser User parameter
936	*/
937	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
938	{
939	uintptr_t paUser = (uintptr_t )pvUser;
940	bool fInflate = !!paUser[0];
941	unsigned cPages = paUser[1];
942	RTGCPHYS paPhysPage = (RTGCPHYS )paUser[2];
943	uint32_t cPendingPages = 0;
944	PGMMFREEPAGESREQ pReq;
945	int rc;
946
947	Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
948	pgmLock(pVM);
949
950	if (fInflate)
951	{
952	/* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
953	pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
954
955	/* Replace pages with ZERO pages. */
956	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
957	if (RT_FAILURE(rc))
958	{
959	pgmUnlock(pVM);
960	AssertLogRelRC(rc);
961	return rc;
962	}
963
964	/* Iterate the pages. */
965	for (unsigned i = 0; i < cPages; i++)
966	{
967	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
968	if ( pPage == NULL
969	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM)
970	{
971	Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], pPage ? PGM_PAGE_GET_TYPE(pPage) : 0));
972	break;
973	}
974
975	LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
976
977	/* Flush the shadow PT if this page was previously used as a guest page table. */
978	pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
979
980	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i]);
981	if (RT_FAILURE(rc))
982	{
983	pgmUnlock(pVM);
984	AssertLogRelRC(rc);
985	return rc;
986	}
987	Assert(PGM_PAGE_IS_ZERO(pPage));
988	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_BALLOONED);
989	}
990
991	if (cPendingPages)
992	{
993	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
994	if (RT_FAILURE(rc))
995	{
996	pgmUnlock(pVM);
997	AssertLogRelRC(rc);
998	return rc;
999	}
1000	}
1001	GMMR3FreePagesCleanup(pReq);
1002	}
1003	else
1004	{
1005	/* Iterate the pages. */
1006	for (unsigned i = 0; i < cPages; i++)
1007	{
1008	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
1009	AssertBreak(pPage && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
1010
1011	LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
1012
1013	Assert(PGM_PAGE_IS_BALLOONED(pPage));
1014
1015	/* Change back to zero page. */
1016	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1017	}
1018
1019	/* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
1020	}
1021
1022	/* Notify GMM about the balloon change. */
1023	rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
1024	if (RT_SUCCESS(rc))
1025	{
1026	if (!fInflate)
1027	{
1028	Assert(pVM->pgm.s.cBalloonedPages >= cPages);
1029	pVM->pgm.s.cBalloonedPages -= cPages;
1030	}
1031	else
1032	pVM->pgm.s.cBalloonedPages += cPages;
1033	}
1034
1035	pgmUnlock(pVM);
1036
1037	/* Flush the recompiler's TLB as well. */
1038	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1039	CPUMSetChangedFlags(&pVM->aCpus[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1040
1041	AssertLogRelRC(rc);
1042	return rc;
1043	}
1044
1045
1046	/**
1047	* Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
1048	*
1049	* @returns VBox status code.
1050	* @param pVM The VM handle.
1051	* @param fInflate Inflate or deflate memory balloon
1052	* @param cPages Number of pages to free
1053	* @param paPhysPage Array of guest physical addresses
1054	*/
1055	static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1056	{
1057	uintptr_t paUser[3];
1058
1059	paUser[0] = fInflate;
1060	paUser[1] = cPages;
1061	paUser[2] = (uintptr_t)paPhysPage;
1062	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1063	AssertRC(rc);
1064
1065	/* Made a copy in PGMR3PhysFreeRamPages; free it here. */
1066	RTMemFree(paPhysPage);
1067	}
1068
1069	#endif /* 64-bit host && (Windows \|\| Solaris \|\| Linux \|\| FreeBSD) */
1070
1071	/**
1072	* Inflate or deflate a memory balloon
1073	*
1074	* @returns VBox status code.
1075	* @param pVM The VM handle.
1076	* @param fInflate Inflate or deflate memory balloon
1077	* @param cPages Number of pages to free
1078	* @param paPhysPage Array of guest physical addresses
1079	*/
1080	VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1081	{
1082	/* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
1083	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
1084	int rc;
1085
1086	/* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
1087	AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
1088
1089	/* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
1090	* In the SMP case we post a request packet to postpone the job.
1091	*/
1092	if (pVM->cCpus > 1)
1093	{
1094	unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
1095	RTGCPHYS paPhysPageCopy = (RTGCPHYS )RTMemAlloc(cbPhysPage);
1096	AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
1097
1098	memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
1099
1100	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
1101	AssertRC(rc);
1102	}
1103	else
1104	{
1105	uintptr_t paUser[3];
1106
1107	paUser[0] = fInflate;
1108	paUser[1] = cPages;
1109	paUser[2] = (uintptr_t)paPhysPage;
1110	rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1111	AssertRC(rc);
1112	}
1113	return rc;
1114
1115	#else
1116	NOREF(pVM); NOREF(fInflate); NOREF(cPages); NOREF(paPhysPage);
1117	return VERR_NOT_IMPLEMENTED;
1118	#endif
1119	}
1120
1121
1122	/**
1123	* Rendezvous callback used by PGMR3WriteProtectRAM that write protects all
1124	* physical RAM.
1125	*
1126	* This is only called on one of the EMTs while the other ones are waiting for
1127	* it to complete this function.
1128	*
1129	* @returns VINF_SUCCESS (VBox strict status code).
1130	* @param pVM The VM handle.
1131	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
1132	* @param pvUser User parameter, unused.
1133	*/
1134	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1135	{
1136	int rc = VINF_SUCCESS;
1137	NOREF(pvUser); NOREF(pVCpu);
1138
1139	pgmLock(pVM);
1140	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1141	pgmPoolResetDirtyPages(pVM);
1142	#endif
1143
1144	/** @todo pointless to write protect the physical page pointed to by RSP. */
1145
1146	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1147	pRam;
1148	pRam = pRam->CTX_SUFF(pNext))
1149	{
1150	uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1151	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1152	{
1153	PPGMPAGE pPage = &pRam->aPages[iPage];
1154	PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1155
1156	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1157	\|\| enmPageType == PGMPAGETYPE_MMIO2)
1158	{
1159	/*
1160	* A RAM page.
1161	*/
1162	switch (PGM_PAGE_GET_STATE(pPage))
1163	{
1164	case PGM_PAGE_STATE_ALLOCATED:
1165	/** @todo Optimize this: Don't always re-enable write
1166	* monitoring if the page is known to be very busy. */
1167	if (PGM_PAGE_IS_WRITTEN_TO(pPage))
1168	{
1169	PGM_PAGE_CLEAR_WRITTEN_TO(pVM, pPage);
1170	/* Remember this dirty page for the next (memory) sync. */
1171	PGM_PAGE_SET_FT_DIRTY(pPage);
1172	}
1173
1174	pgmPhysPageWriteMonitor(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1175	break;
1176
1177	case PGM_PAGE_STATE_SHARED:
1178	AssertFailed();
1179	break;
1180
1181	case PGM_PAGE_STATE_WRITE_MONITORED: /* nothing to change. */
1182	default:
1183	break;
1184	}
1185	}
1186	}
1187	}
1188	pgmR3PoolWriteProtectPages(pVM);
1189	PGM_INVL_ALL_VCPU_TLBS(pVM);
1190	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1191	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1192
1193	pgmUnlock(pVM);
1194	return rc;
1195	}
1196
1197	/**
1198	* Protect all physical RAM to monitor writes
1199	*
1200	* @returns VBox status code.
1201	* @param pVM The VM handle.
1202	*/
1203	VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
1204	{
1205	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1206
1207	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
1208	AssertRC(rc);
1209	return rc;
1210	}
1211
1212	/**
1213	* Enumerate all dirty FT pages.
1214	*
1215	* @returns VBox status code.
1216	* @param pVM The VM handle.
1217	* @param pfnEnum Enumerate callback handler.
1218	* @param pvUser Enumerate callback handler parameter.
1219	*/
1220	VMMR3DECL(int) PGMR3PhysEnumDirtyFTPages(PVM pVM, PFNPGMENUMDIRTYFTPAGES pfnEnum, void *pvUser)
1221	{
1222	int rc = VINF_SUCCESS;
1223
1224	pgmLock(pVM);
1225	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1226	pRam;
1227	pRam = pRam->CTX_SUFF(pNext))
1228	{
1229	uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1230	for (uint32_t iPage = 0; iPage < cPages; iPage++)
1231	{
1232	PPGMPAGE pPage = &pRam->aPages[iPage];
1233	PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1234
1235	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1236	\|\| enmPageType == PGMPAGETYPE_MMIO2)
1237	{
1238	/*
1239	* A RAM page.
1240	*/
1241	switch (PGM_PAGE_GET_STATE(pPage))
1242	{
1243	case PGM_PAGE_STATE_ALLOCATED:
1244	case PGM_PAGE_STATE_WRITE_MONITORED:
1245	if ( !PGM_PAGE_IS_WRITTEN_TO(pPage) /* not very recently updated? */
1246	&& PGM_PAGE_IS_FT_DIRTY(pPage))
1247	{
1248	unsigned cbPageRange = PAGE_SIZE;
1249	unsigned iPageClean = iPage + 1;
1250	RTGCPHYS GCPhysPage = pRam->GCPhys + iPage * PAGE_SIZE;
1251	uint8_t *pu8Page = NULL;
1252	PGMPAGEMAPLOCK Lock;
1253
1254	/* Find the next clean page, so we can merge adjacent dirty pages. */
1255	for (; iPageClean < cPages; iPageClean++)
1256	{
1257	PPGMPAGE pPageNext = &pRam->aPages[iPageClean];
1258	if ( RT_UNLIKELY(PGM_PAGE_GET_TYPE(pPageNext) != PGMPAGETYPE_RAM)
1259	\|\| PGM_PAGE_GET_STATE(pPageNext) != PGM_PAGE_STATE_ALLOCATED
1260	\|\| PGM_PAGE_IS_WRITTEN_TO(pPageNext)
1261	\|\| !PGM_PAGE_IS_FT_DIRTY(pPageNext)
1262	/* Crossing a chunk boundary? */
1263	\|\| (GCPhysPage & GMM_PAGEID_IDX_MASK) != ((GCPhysPage + cbPageRange) & GMM_PAGEID_IDX_MASK)
1264	)
1265	break;
1266
1267	cbPageRange += PAGE_SIZE;
1268	}
1269
1270	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysPage, (const void **)&pu8Page, &Lock);
1271	if (RT_SUCCESS(rc))
1272	{
1273	/** @todo this is risky; the range might be changed, but little choice as the sync
1274	* costs a lot of time. */
1275	pgmUnlock(pVM);
1276	pfnEnum(pVM, GCPhysPage, pu8Page, cbPageRange, pvUser);
1277	pgmLock(pVM);
1278	PGMPhysReleasePageMappingLock(pVM, &Lock);
1279	}
1280
1281	for (iPage; iPage < iPageClean; iPage++)
1282	PGM_PAGE_CLEAR_FT_DIRTY(&pRam->aPages[iPage]);
1283
1284	iPage = iPageClean - 1;
1285	}
1286	break;
1287	}
1288	}
1289	}
1290	}
1291	pgmUnlock(pVM);
1292	return rc;
1293	}
1294
1295
1296	/**
1297	* Gets the number of ram ranges.
1298	*
1299	* @returns Number of ram ranges. Returns UINT32_MAX if @a pVM is invalid.
1300	* @param pVM The VM handle.
1301	*/
1302	VMMR3DECL(uint32_t) PGMR3PhysGetRamRangeCount(PVM pVM)
1303	{
1304	VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
1305
1306	pgmLock(pVM);
1307	uint32_t cRamRanges = 0;
1308	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext))
1309	cRamRanges++;
1310	pgmUnlock(pVM);
1311	return cRamRanges;
1312	}
1313
1314
1315	/**
1316	* Get information about a range.
1317	*
1318	* @returns VINF_SUCCESS or VERR_OUT_OF_RANGE.
1319	* @param pVM The VM handle
1320	* @param iRange The ordinal of the range.
1321	* @param pGCPhysStart Where to return the start of the range. Optional.
1322	* @param pGCPhysLast Where to return the address of the last byte in the
1323	* range. Optional.
1324	* @param pfIsMmio Where to indicate that this is a pure MMIO range.
1325	* Optional.
1326	*/
1327	VMMR3DECL(int) PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
1328	const char *ppszDesc, bool pfIsMmio)
1329	{
1330	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1331
1332	pgmLock(pVM);
1333	uint32_t iCurRange = 0;
1334	for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext), iCurRange++)
1335	if (iCurRange == iRange)
1336	{
1337	if (pGCPhysStart)
1338	*pGCPhysStart = pCur->GCPhys;
1339	if (pGCPhysLast)
1340	*pGCPhysLast = pCur->GCPhysLast;
1341	if (ppszDesc)
1342	*ppszDesc = pCur->pszDesc;
1343	if (pfIsMmio)
1344	*pfIsMmio = !!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO);
1345
1346	pgmUnlock(pVM);
1347	return VINF_SUCCESS;
1348	}
1349	pgmUnlock(pVM);
1350	return VERR_OUT_OF_RANGE;
1351	}
1352
1353
1354	/**
1355	* Query the amount of free memory inside VMMR0
1356	*
1357	* @returns VBox status code.
1358	* @param pVM The VM handle.
1359	* @param pcbAllocMem Where to return the amount of memory allocated
1360	* by VMs.
1361	* @param pcbFreeMem Where to return the amount of memory that is
1362	* allocated from the host but not currently used
1363	* by any VMs.
1364	* @param pcbBallonedMem Where to return the sum of memory that is
1365	* currently ballooned by the VMs.
1366	* @param pcbSharedMem Where to return the amount of memory that is
1367	* currently shared.
1368	*/
1369	VMMR3DECL(int) PGMR3QueryGlobalMemoryStats(PVM pVM, uint64_t pcbAllocMem, uint64_t pcbFreeMem,
1370	uint64_t pcbBallonedMem, uint64_t pcbSharedMem)
1371	{
1372	uint64_t cAllocPages = 0;
1373	uint64_t cFreePages = 0;
1374	uint64_t cBalloonPages = 0;
1375	uint64_t cSharedPages = 0;
1376	int rc = GMMR3QueryHypervisorMemoryStats(pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
1377	AssertRCReturn(rc, rc);
1378
1379	if (pcbAllocMem)
1380	pcbAllocMem = cAllocPages _4K;
1381
1382	if (pcbFreeMem)
1383	pcbFreeMem = cFreePages _4K;
1384
1385	if (pcbBallonedMem)
1386	pcbBallonedMem = cBalloonPages _4K;
1387
1388	if (pcbSharedMem)
1389	pcbSharedMem = cSharedPages _4K;
1390
1391	Log(("PGMR3QueryVMMMemoryStats: all=%llx free=%llx ballooned=%llx shared=%llx\n",
1392	cAllocPages, cFreePages, cBalloonPages, cSharedPages));
1393	return VINF_SUCCESS;
1394	}
1395
1396
1397	/**
1398	* Query memory stats for the VM.
1399	*
1400	* @returns VBox status code.
1401	* @param pVM The VM handle.
1402	* @param pcbTotalMem Where to return total amount memory the VM may
1403	* possibly use.
1404	* @param pcbPrivateMem Where to return the amount of private memory
1405	* currently allocated.
1406	* @param pcbSharedMem Where to return the amount of actually shared
1407	* memory currently used by the VM.
1408	* @param pcbZeroMem Where to return the amount of memory backed by
1409	* zero pages.
1410	*
1411	* @remarks The total mem is normally larger than the sum of the three
1412	* components. There are two reasons for this, first the amount of
1413	* shared memory is what we're sure is shared instead of what could
1414	* possibly be shared with someone. Secondly, because the total may
1415	* include some pure MMIO pages that doesn't go into any of the three
1416	* sub-counts.
1417	*
1418	* @todo Why do we return reused shared pages instead of anything that could
1419	* potentially be shared? Doesn't this mean the first VM gets a much
1420	* lower number of shared pages?
1421	*/
1422	VMMR3DECL(int) PGMR3QueryMemoryStats(PVM pVM, uint64_t pcbTotalMem, uint64_t pcbPrivateMem,
1423	uint64_t pcbSharedMem, uint64_t pcbZeroMem)
1424	{
1425	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1426
1427	if (pcbTotalMem)
1428	pcbTotalMem = (uint64_t)pVM->pgm.s.cAllPages PAGE_SIZE;
1429
1430	if (pcbPrivateMem)
1431	pcbPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages PAGE_SIZE;
1432
1433	if (pcbSharedMem)
1434	pcbSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages PAGE_SIZE;
1435
1436	if (pcbZeroMem)
1437	pcbZeroMem = (uint64_t)pVM->pgm.s.cZeroPages PAGE_SIZE;
1438
1439	Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
1440	return VINF_SUCCESS;
1441	}
1442
1443
1444	/**
1445	* PGMR3PhysRegisterRam worker that initializes and links a RAM range.
1446	*
1447	* @param pVM The VM handle.
1448	* @param pNew The new RAM range.
1449	* @param GCPhys The address of the RAM range.
1450	* @param GCPhysLast The last address of the RAM range.
1451	* @param RCPtrNew The RC address if the range is floating. NIL_RTRCPTR
1452	* if in HMA.
1453	* @param R0PtrNew Ditto for R0.
1454	* @param pszDesc The description.
1455	* @param pPrev The previous RAM range (for linking).
1456	*/
1457	static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
1458	RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
1459	{
1460	/*
1461	* Initialize the range.
1462	*/
1463	pNew->pSelfR0 = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
1464	pNew->pSelfRC = RCPtrNew != NIL_RTRCPTR ? RCPtrNew : MMHyperCCToRC(pVM, pNew);
1465	pNew->GCPhys = GCPhys;
1466	pNew->GCPhysLast = GCPhysLast;
1467	pNew->cb = GCPhysLast - GCPhys + 1;
1468	pNew->pszDesc = pszDesc;
1469	pNew->fFlags = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
1470	pNew->pvR3 = NULL;
1471	pNew->paLSPages = NULL;
1472
1473	uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
1474	RTGCPHYS iPage = cPages;
1475	while (iPage-- > 0)
1476	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
1477
1478	/* Update the page count stats. */
1479	pVM->pgm.s.cZeroPages += cPages;
1480	pVM->pgm.s.cAllPages += cPages;
1481
1482	/*
1483	* Link it.
1484	*/
1485	pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
1486	}
1487
1488
1489	/**
1490	* Relocate a floating RAM range.
1491	*
1492	* @copydoc FNPGMRELOCATE.
1493	*/
1494	static DECLCALLBACK(bool) pgmR3PhysRamRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser)
1495	{
1496	PPGMRAMRANGE pRam = (PPGMRAMRANGE)pvUser;
1497	Assert(pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
1498	Assert(pRam->pSelfRC == GCPtrOld + PAGE_SIZE);
1499
1500	switch (enmMode)
1501	{
1502	case PGMRELOCATECALL_SUGGEST:
1503	return true;
1504
1505	case PGMRELOCATECALL_RELOCATE:
1506	{
1507	/*
1508	* Update myself, then relink all the ranges and flush the RC TLB.
1509	*/
1510	pgmLock(pVM);
1511
1512	pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
1513
1514	pgmR3PhysRelinkRamRanges(pVM);
1515	for (unsigned i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
1516	pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
1517
1518	pgmUnlock(pVM);
1519	return true;
1520	}
1521
1522	default:
1523	AssertFailedReturn(false);
1524	}
1525	}
1526
1527
1528	/**
1529	* PGMR3PhysRegisterRam worker that registers a high chunk.
1530	*
1531	* @returns VBox status code.
1532	* @param pVM The VM handle.
1533	* @param GCPhys The address of the RAM.
1534	* @param cRamPages The number of RAM pages to register.
1535	* @param cbChunk The size of the PGMRAMRANGE guest mapping.
1536	* @param iChunk The chunk number.
1537	* @param pszDesc The RAM range description.
1538	* @param ppPrev Previous RAM range pointer. In/Out.
1539	*/
1540	static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages,
1541	uint32_t cbChunk, uint32_t iChunk, const char *pszDesc,
1542	PPGMRAMRANGE *ppPrev)
1543	{
1544	const char *pszDescChunk = iChunk == 0
1545	? pszDesc
1546	: MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
1547	AssertReturn(pszDescChunk, VERR_NO_MEMORY);
1548
1549	/*
1550	* Allocate memory for the new chunk.
1551	*/
1552	size_t const cChunkPages = RT_ALIGN_Z(RT_UOFFSETOF(PGMRAMRANGE, aPages[cRamPages]), PAGE_SIZE) >> PAGE_SHIFT;
1553	PSUPPAGE paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
1554	AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
1555	RTR0PTR R0PtrChunk = NIL_RTR0PTR;
1556	void *pvChunk = NULL;
1557	int rc = SUPR3PageAllocEx(cChunkPages, 0 /fFlags/, &pvChunk,
1558	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1559	VMMIsHwVirtExtForced(pVM) ? &R0PtrChunk : NULL,
1560	#else
1561	NULL,
1562	#endif
1563	paChunkPages);
1564	if (RT_SUCCESS(rc))
1565	{
1566	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1567	if (!VMMIsHwVirtExtForced(pVM))
1568	R0PtrChunk = NIL_RTR0PTR;
1569	#else
1570	R0PtrChunk = (uintptr_t)pvChunk;
1571	#endif
1572	memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
1573
1574	PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
1575
1576	/*
1577	* Create a mapping and map the pages into it.
1578	* We push these in below the HMA.
1579	*/
1580	RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
1581	rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /fFlags/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
1582	if (RT_SUCCESS(rc))
1583	{
1584	pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
1585
1586	RTGCPTR const GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
1587	RTGCPTR GCPtrPage = GCPtrChunk;
1588	for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
1589	rc = PGMMap(pVM, GCPtrPage, paChunkPages[iPage].Phys, PAGE_SIZE, 0);
1590	if (RT_SUCCESS(rc))
1591	{
1592	/*
1593	* Ok, init and link the range.
1594	*/
1595	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
1596	(RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
1597	*ppPrev = pNew;
1598	}
1599	}
1600
1601	if (RT_FAILURE(rc))
1602	SUPR3PageFreeEx(pvChunk, cChunkPages);
1603	}
1604
1605	RTMemTmpFree(paChunkPages);
1606	return rc;
1607	}
1608
1609
1610	/**
1611	* Sets up a range RAM.
1612	*
1613	* This will check for conflicting registrations, make a resource
1614	* reservation for the memory (with GMM), and setup the per-page
1615	* tracking structures (PGMPAGE).
1616	*
1617	* @returns VBox status code.
1618	* @param pVM Pointer to the shared VM structure.
1619	* @param GCPhys The physical address of the RAM.
1620	* @param cb The size of the RAM.
1621	* @param pszDesc The description - not copied, so, don't free or change it.
1622	*/
1623	VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
1624	{
1625	/*
1626	* Validate input.
1627	*/
1628	Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
1629	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
1630	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
1631	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
1632	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1633	AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
1634	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1635	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1636
1637	pgmLock(pVM);
1638
1639	/*
1640	* Find range location and check for conflicts.
1641	* (We don't lock here because the locking by EMT is only required on update.)
1642	*/
1643	PPGMRAMRANGE pPrev = NULL;
1644	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
1645	while (pRam && GCPhysLast >= pRam->GCPhys)
1646	{
1647	if ( GCPhysLast >= pRam->GCPhys
1648	&& GCPhys <= pRam->GCPhysLast)
1649	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
1650	GCPhys, GCPhysLast, pszDesc,
1651	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1652	VERR_PGM_RAM_CONFLICT);
1653
1654	/* next */
1655	pPrev = pRam;
1656	pRam = pRam->pNextR3;
1657	}
1658
1659	/*
1660	* Register it with GMM (the API bitches).
1661	*/
1662	const RTGCPHYS cPages = cb >> PAGE_SHIFT;
1663	int rc = MMR3IncreaseBaseReservation(pVM, cPages);
1664	if (RT_FAILURE(rc))
1665	{
1666	pgmUnlock(pVM);
1667	return rc;
1668	}
1669
1670	if ( GCPhys >= _4G
1671	&& cPages > 256)
1672	{
1673	/*
1674	* The PGMRAMRANGE structures for the high memory can get very big.
1675	* In order to avoid SUPR3PageAllocEx allocation failures due to the
1676	* allocation size limit there and also to avoid being unable to find
1677	* guest mapping space for them, we split this memory up into 4MB in
1678	* (potential) raw-mode configs and 16MB chunks in forced AMD-V/VT-x
1679	* mode.
1680	*
1681	* The first and last page of each mapping are guard pages and marked
1682	* not-present. So, we've got 4186112 and 16769024 bytes available for
1683	* the PGMRAMRANGE structure.
1684	*
1685	* Note! The sizes used here will influence the saved state.
1686	*/
1687	uint32_t cbChunk;
1688	uint32_t cPagesPerChunk;
1689	if (VMMIsHwVirtExtForced(pVM))
1690	{
1691	cbChunk = 16U*_1M;
1692	cPagesPerChunk = 1048048; /* max ~1048059 */
1693	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 1048048 < 16U_1M - PAGE_SIZE 2);
1694	}
1695	else
1696	{
1697	cbChunk = 4U*_1M;
1698	cPagesPerChunk = 261616; /* max ~261627 */
1699	AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 261616 < 4U_1M - PAGE_SIZE 2);
1700	}
1701	AssertRelease(RT_UOFFSETOF(PGMRAMRANGE, aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
1702
1703	RTGCPHYS cPagesLeft = cPages;
1704	RTGCPHYS GCPhysChunk = GCPhys;
1705	uint32_t iChunk = 0;
1706	while (cPagesLeft > 0)
1707	{
1708	uint32_t cPagesInChunk = cPagesLeft;
1709	if (cPagesInChunk > cPagesPerChunk)
1710	cPagesInChunk = cPagesPerChunk;
1711
1712	rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, cbChunk, iChunk, pszDesc, &pPrev);
1713	AssertRCReturn(rc, rc);
1714
1715	/* advance */
1716	GCPhysChunk += (RTGCPHYS)cPagesInChunk << PAGE_SHIFT;
1717	cPagesLeft -= cPagesInChunk;
1718	iChunk++;
1719	}
1720	}
1721	else
1722	{
1723	/*
1724	* Allocate, initialize and link the new RAM range.
1725	*/
1726	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
1727	PPGMRAMRANGE pNew;
1728	rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
1729	AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
1730
1731	pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
1732	}
1733	pgmPhysInvalidatePageMapTLB(pVM);
1734	pgmUnlock(pVM);
1735
1736	/*
1737	* Notify REM.
1738	*/
1739	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_RAM);
1740
1741	return VINF_SUCCESS;
1742	}
1743
1744
1745	/**
1746	* Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
1747	*
1748	* We do this late in the init process so that all the ROM and MMIO ranges have
1749	* been registered already and we don't go wasting memory on them.
1750	*
1751	* @returns VBox status code.
1752	*
1753	* @param pVM Pointer to the shared VM structure.
1754	*/
1755	int pgmR3PhysRamPreAllocate(PVM pVM)
1756	{
1757	Assert(pVM->pgm.s.fRamPreAlloc);
1758	Log(("pgmR3PhysRamPreAllocate: enter\n"));
1759
1760	/*
1761	* Walk the RAM ranges and allocate all RAM pages, halt at
1762	* the first allocation error.
1763	*/
1764	uint64_t cPages = 0;
1765	uint64_t NanoTS = RTTimeNanoTS();
1766	pgmLock(pVM);
1767	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1768	{
1769	PPGMPAGE pPage = &pRam->aPages[0];
1770	RTGCPHYS GCPhys = pRam->GCPhys;
1771	uint32_t cLeft = pRam->cb >> PAGE_SHIFT;
1772	while (cLeft-- > 0)
1773	{
1774	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1775	{
1776	switch (PGM_PAGE_GET_STATE(pPage))
1777	{
1778	case PGM_PAGE_STATE_ZERO:
1779	{
1780	int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
1781	if (RT_FAILURE(rc))
1782	{
1783	LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
1784	pgmUnlock(pVM);
1785	return rc;
1786	}
1787	cPages++;
1788	break;
1789	}
1790
1791	case PGM_PAGE_STATE_BALLOONED:
1792	case PGM_PAGE_STATE_ALLOCATED:
1793	case PGM_PAGE_STATE_WRITE_MONITORED:
1794	case PGM_PAGE_STATE_SHARED:
1795	/* nothing to do here. */
1796	break;
1797	}
1798	}
1799
1800	/* next */
1801	pPage++;
1802	GCPhys += PAGE_SIZE;
1803	}
1804	}
1805	pgmUnlock(pVM);
1806	NanoTS = RTTimeNanoTS() - NanoTS;
1807
1808	LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
1809	Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
1810	return VINF_SUCCESS;
1811	}
1812
1813
1814	/**
1815	* Resets (zeros) the RAM.
1816	*
1817	* ASSUMES that the caller owns the PGM lock.
1818	*
1819	* @returns VBox status code.
1820	* @param pVM Pointer to the shared VM structure.
1821	*/
1822	int pgmR3PhysRamReset(PVM pVM)
1823	{
1824	PGM_LOCK_ASSERT_OWNER(pVM);
1825
1826	/* Reset the memory balloon. */
1827	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1828	AssertRC(rc);
1829
1830	#ifdef VBOX_WITH_PAGE_SHARING
1831	/* Clear all registered shared modules. */
1832	rc = GMMR3ResetSharedModules(pVM);
1833	AssertRC(rc);
1834	#endif
1835	/* Reset counters. */
1836	pVM->pgm.s.cReusedSharedPages = 0;
1837	pVM->pgm.s.cBalloonedPages = 0;
1838
1839	/*
1840	* We batch up pages that should be freed instead of calling GMM for
1841	* each and every one of them.
1842	*/
1843	uint32_t cPendingPages = 0;
1844	PGMMFREEPAGESREQ pReq;
1845	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1846	AssertLogRelRCReturn(rc, rc);
1847
1848	/*
1849	* Walk the ram ranges.
1850	*/
1851	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1852	{
1853	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1854	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1855
1856	#ifndef NO_RAM_RESET
1857	if (!pVM->pgm.s.fRamPreAlloc)
1858	#else
1859	if (0)
1860	#endif
1861	{
1862	/* Replace all RAM pages by ZERO pages. */
1863	while (iPage-- > 0)
1864	{
1865	PPGMPAGE pPage = &pRam->aPages[iPage];
1866	switch (PGM_PAGE_GET_TYPE(pPage))
1867	{
1868	case PGMPAGETYPE_RAM:
1869	/* Do not replace pages part of a 2 MB continuous range
1870	with zero pages, but zero them instead. */
1871	if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE
1872	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
1873	{
1874	void *pvPage;
1875	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1876	AssertLogRelRCReturn(rc, rc);
1877	ASMMemZeroPage(pvPage);
1878	}
1879	else if (PGM_PAGE_IS_BALLOONED(pPage))
1880	{
1881	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
1882	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1883	}
1884	else if (!PGM_PAGE_IS_ZERO(pPage))
1885	{
1886	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1887	AssertLogRelRCReturn(rc, rc);
1888	}
1889	break;
1890
1891	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1892	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
1893	true /fDoAccounting/);
1894	break;
1895
1896	case PGMPAGETYPE_MMIO2:
1897	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
1898	case PGMPAGETYPE_ROM:
1899	case PGMPAGETYPE_MMIO:
1900	break;
1901	default:
1902	AssertFailed();
1903	}
1904	} /* for each page */
1905	}
1906	else
1907	{
1908	/* Zero the memory. */
1909	while (iPage-- > 0)
1910	{
1911	PPGMPAGE pPage = &pRam->aPages[iPage];
1912	switch (PGM_PAGE_GET_TYPE(pPage))
1913	{
1914	case PGMPAGETYPE_RAM:
1915	switch (PGM_PAGE_GET_STATE(pPage))
1916	{
1917	case PGM_PAGE_STATE_ZERO:
1918	break;
1919
1920	case PGM_PAGE_STATE_BALLOONED:
1921	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
1922	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1923	break;
1924
1925	case PGM_PAGE_STATE_SHARED:
1926	case PGM_PAGE_STATE_WRITE_MONITORED:
1927	rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1928	AssertLogRelRCReturn(rc, rc);
1929	/* no break */
1930
1931	case PGM_PAGE_STATE_ALLOCATED:
1932	{
1933	void *pvPage;
1934	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1935	AssertLogRelRCReturn(rc, rc);
1936	#ifndef NO_RAM_RESET
1937	ASMMemZeroPage(pvPage);
1938	#endif
1939	break;
1940	}
1941	}
1942	break;
1943
1944	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1945	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
1946	true /fDoAccounting/);
1947	break;
1948
1949	case PGMPAGETYPE_MMIO2:
1950	case PGMPAGETYPE_ROM_SHADOW:
1951	case PGMPAGETYPE_ROM:
1952	case PGMPAGETYPE_MMIO:
1953	break;
1954	default:
1955	AssertFailed();
1956
1957	}
1958	} /* for each page */
1959	}
1960
1961	}
1962
1963	/*
1964	* Finish off any pages pending freeing.
1965	*/
1966	if (cPendingPages)
1967	{
1968	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1969	AssertLogRelRCReturn(rc, rc);
1970	}
1971	GMMR3FreePagesCleanup(pReq);
1972
1973	return VINF_SUCCESS;
1974	}
1975
1976	/**
1977	* Frees all RAM during VM termination
1978	*
1979	* ASSUMES that the caller owns the PGM lock.
1980	*
1981	* @returns VBox status code.
1982	* @param pVM Pointer to the shared VM structure.
1983	*/
1984	int pgmR3PhysRamTerm(PVM pVM)
1985	{
1986	PGM_LOCK_ASSERT_OWNER(pVM);
1987
1988	/* Reset the memory balloon. */
1989	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1990	AssertRC(rc);
1991
1992	#ifdef VBOX_WITH_PAGE_SHARING
1993	/* Clear all registered shared modules. */
1994	rc = GMMR3ResetSharedModules(pVM);
1995	AssertRC(rc);
1996	#endif
1997
1998	/*
1999	* We batch up pages that should be freed instead of calling GMM for
2000	* each and every one of them.
2001	*/
2002	uint32_t cPendingPages = 0;
2003	PGMMFREEPAGESREQ pReq;
2004	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2005	AssertLogRelRCReturn(rc, rc);
2006
2007	/*
2008	* Walk the ram ranges.
2009	*/
2010	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
2011	{
2012	uint32_t iPage = pRam->cb >> PAGE_SHIFT;
2013	AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
2014
2015	/* Replace all RAM pages by ZERO pages. */
2016	while (iPage-- > 0)
2017	{
2018	PPGMPAGE pPage = &pRam->aPages[iPage];
2019	switch (PGM_PAGE_GET_TYPE(pPage))
2020	{
2021	case PGMPAGETYPE_RAM:
2022	/* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
2023	if (PGM_PAGE_IS_SHARED(pPage))
2024	{
2025	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
2026	AssertLogRelRCReturn(rc, rc);
2027	}
2028	break;
2029
2030	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2031	case PGMPAGETYPE_MMIO2:
2032	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2033	case PGMPAGETYPE_ROM:
2034	case PGMPAGETYPE_MMIO:
2035	break;
2036	default:
2037	AssertFailed();
2038	}
2039	} /* for each page */
2040	}
2041
2042	/*
2043	* Finish off any pages pending freeing.
2044	*/
2045	if (cPendingPages)
2046	{
2047	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2048	AssertLogRelRCReturn(rc, rc);
2049	}
2050	GMMR3FreePagesCleanup(pReq);
2051	return VINF_SUCCESS;
2052	}
2053
2054	/**
2055	* This is the interface IOM is using to register an MMIO region.
2056	*
2057	* It will check for conflicts and ensure that a RAM range structure
2058	* is present before calling the PGMR3HandlerPhysicalRegister API to
2059	* register the callbacks.
2060	*
2061	* @returns VBox status code.
2062	*
2063	* @param pVM Pointer to the shared VM structure.
2064	* @param GCPhys The start of the MMIO region.
2065	* @param cb The size of the MMIO region.
2066	* @param pfnHandlerR3 The address of the ring-3 handler. (IOMR3MMIOHandler)
2067	* @param pvUserR3 The user argument for R3.
2068	* @param pfnHandlerR0 The address of the ring-0 handler. (IOMMMIOHandler)
2069	* @param pvUserR0 The user argument for R0.
2070	* @param pfnHandlerRC The address of the RC handler. (IOMMMIOHandler)
2071	* @param pvUserRC The user argument for RC.
2072	* @param pszDesc The description of the MMIO region.
2073	*/
2074	VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb,
2075	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3, RTR3PTR pvUserR3,
2076	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0, RTR0PTR pvUserR0,
2077	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC, RTRCPTR pvUserRC,
2078	R3PTRTYPE(const char *) pszDesc)
2079	{
2080	/*
2081	* Assert on some assumption.
2082	*/
2083	VM_ASSERT_EMT(pVM);
2084	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2085	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2086	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2087	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2088
2089	int rc = pgmLock(pVM);
2090	AssertRCReturn(rc, rc);
2091
2092	/*
2093	* Make sure there's a RAM range structure for the region.
2094	*/
2095	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2096	bool fRamExists = false;
2097	PPGMRAMRANGE pRamPrev = NULL;
2098	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2099	while (pRam && GCPhysLast >= pRam->GCPhys)
2100	{
2101	if ( GCPhysLast >= pRam->GCPhys
2102	&& GCPhys <= pRam->GCPhysLast)
2103	{
2104	/* Simplification: all within the same range. */
2105	AssertLogRelMsgReturnStmt( GCPhys >= pRam->GCPhys
2106	&& GCPhysLast <= pRam->GCPhysLast,
2107	("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
2108	GCPhys, GCPhysLast, pszDesc,
2109	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2110	pgmUnlock(pVM),
2111	VERR_PGM_RAM_CONFLICT);
2112
2113	/* Check that it's all RAM or MMIO pages. */
2114	PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2115	uint32_t cLeft = cb >> PAGE_SHIFT;
2116	while (cLeft-- > 0)
2117	{
2118	AssertLogRelMsgReturnStmt( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2119	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
2120	("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
2121	GCPhys, GCPhysLast, pszDesc, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
2122	pgmUnlock(pVM),
2123	VERR_PGM_RAM_CONFLICT);
2124	pPage++;
2125	}
2126
2127	/* Looks good. */
2128	fRamExists = true;
2129	break;
2130	}
2131
2132	/* next */
2133	pRamPrev = pRam;
2134	pRam = pRam->pNextR3;
2135	}
2136	PPGMRAMRANGE pNew;
2137	if (fRamExists)
2138	{
2139	pNew = NULL;
2140
2141	/*
2142	* Make all the pages in the range MMIO/ZERO pages, freeing any
2143	* RAM pages currently mapped here. This might not be 100% correct
2144	* for PCI memory, but we're doing the same thing for MMIO2 pages.
2145	*/
2146	rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
2147	AssertRCReturnStmt(rc, pgmUnlock(pVM), rc);
2148
2149	/* Force a PGM pool flush as guest ram references have been changed. */
2150	/** @todo not entirely SMP safe; assuming for now the guest takes
2151	* care of this internally (not touch mapped mmio while changing the
2152	* mapping). */
2153	PVMCPU pVCpu = VMMGetCpu(pVM);
2154	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2155	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2156	}
2157	else
2158	{
2159
2160	/*
2161	* No RAM range, insert an ad hoc one.
2162	*
2163	* Note that we don't have to tell REM about this range because
2164	* PGMHandlerPhysicalRegisterEx will do that for us.
2165	*/
2166	Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
2167
2168	const uint32_t cPages = cb >> PAGE_SHIFT;
2169	const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
2170	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
2171	AssertLogRelMsgRCReturnStmt(rc, ("cbRamRange=%zu\n", cbRamRange), pgmUnlock(pVM), rc);
2172
2173	/* Initialize the range. */
2174	pNew->pSelfR0 = MMHyperCCToR0(pVM, pNew);
2175	pNew->pSelfRC = MMHyperCCToRC(pVM, pNew);
2176	pNew->GCPhys = GCPhys;
2177	pNew->GCPhysLast = GCPhysLast;
2178	pNew->cb = cb;
2179	pNew->pszDesc = pszDesc;
2180	pNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
2181	pNew->pvR3 = NULL;
2182	pNew->paLSPages = NULL;
2183
2184	uint32_t iPage = cPages;
2185	while (iPage-- > 0)
2186	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
2187	Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
2188
2189	/* update the page count stats. */
2190	pVM->pgm.s.cPureMmioPages += cPages;
2191	pVM->pgm.s.cAllPages += cPages;
2192
2193	/* link it */
2194	pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
2195	}
2196
2197	/*
2198	* Register the access handler.
2199	*/
2200	rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_MMIO, GCPhys, GCPhysLast,
2201	pfnHandlerR3, pvUserR3,
2202	pfnHandlerR0, pvUserR0,
2203	pfnHandlerRC, pvUserRC, pszDesc);
2204	if ( RT_FAILURE(rc)
2205	&& !fRamExists)
2206	{
2207	pVM->pgm.s.cPureMmioPages -= cb >> PAGE_SHIFT;
2208	pVM->pgm.s.cAllPages -= cb >> PAGE_SHIFT;
2209
2210	/* remove the ad hoc range. */
2211	pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
2212	pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
2213	MMHyperFree(pVM, pRam);
2214	}
2215	pgmPhysInvalidatePageMapTLB(pVM);
2216
2217	pgmUnlock(pVM);
2218	return rc;
2219	}
2220
2221
2222	/**
2223	* This is the interface IOM is using to register an MMIO region.
2224	*
2225	* It will take care of calling PGMHandlerPhysicalDeregister and clean up
2226	* any ad hoc PGMRAMRANGE left behind.
2227	*
2228	* @returns VBox status code.
2229	* @param pVM Pointer to the shared VM structure.
2230	* @param GCPhys The start of the MMIO region.
2231	* @param cb The size of the MMIO region.
2232	*/
2233	VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
2234	{
2235	VM_ASSERT_EMT(pVM);
2236
2237	int rc = pgmLock(pVM);
2238	AssertRCReturn(rc, rc);
2239
2240	/*
2241	* First deregister the handler, then check if we should remove the ram range.
2242	*/
2243	rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
2244	if (RT_SUCCESS(rc))
2245	{
2246	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2247	PPGMRAMRANGE pRamPrev = NULL;
2248	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2249	while (pRam && GCPhysLast >= pRam->GCPhys)
2250	{
2251	/** @todo We're being a bit too careful here. rewrite. */
2252	if ( GCPhysLast == pRam->GCPhysLast
2253	&& GCPhys == pRam->GCPhys)
2254	{
2255	Assert(pRam->cb == cb);
2256
2257	/*
2258	* See if all the pages are dead MMIO pages.
2259	*/
2260	uint32_t const cPages = cb >> PAGE_SHIFT;
2261	bool fAllMMIO = true;
2262	uint32_t iPage = 0;
2263	uint32_t cLeft = cPages;
2264	while (cLeft-- > 0)
2265	{
2266	PPGMPAGE pPage = &pRam->aPages[iPage];
2267	if ( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO
2268	/\|\| not-out-of-action later /)
2269	{
2270	fAllMMIO = false;
2271	Assert(PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO2_ALIAS_MMIO);
2272	AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2273	break;
2274	}
2275	Assert(PGM_PAGE_IS_ZERO(pPage));
2276	pPage++;
2277	}
2278	if (fAllMMIO)
2279	{
2280	/*
2281	* Ad-hoc range, unlink and free it.
2282	*/
2283	Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
2284	GCPhys, GCPhysLast, pRam->pszDesc));
2285
2286	pVM->pgm.s.cAllPages -= cPages;
2287	pVM->pgm.s.cPureMmioPages -= cPages;
2288
2289	pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
2290	pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
2291	MMHyperFree(pVM, pRam);
2292	break;
2293	}
2294	}
2295
2296	/*
2297	* Range match? It will all be within one range (see PGMAllHandler.cpp).
2298	*/
2299	if ( GCPhysLast >= pRam->GCPhys
2300	&& GCPhys <= pRam->GCPhysLast)
2301	{
2302	Assert(GCPhys >= pRam->GCPhys);
2303	Assert(GCPhysLast <= pRam->GCPhysLast);
2304
2305	/*
2306	* Turn the pages back into RAM pages.
2307	*/
2308	uint32_t iPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2309	uint32_t cLeft = cb >> PAGE_SHIFT;
2310	while (cLeft--)
2311	{
2312	PPGMPAGE pPage = &pRam->aPages[iPage];
2313	AssertMsg(PGM_PAGE_IS_MMIO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2314	AssertMsg(PGM_PAGE_IS_ZERO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2315	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO)
2316	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_RAM);
2317	}
2318	break;
2319	}
2320
2321	/* next */
2322	pRamPrev = pRam;
2323	pRam = pRam->pNextR3;
2324	}
2325	}
2326
2327	/* Force a PGM pool flush as guest ram references have been changed. */
2328	/** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
2329	PVMCPU pVCpu = VMMGetCpu(pVM);
2330	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2331	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2332
2333	pgmPhysInvalidatePageMapTLB(pVM);
2334	pgmPhysInvalidRamRangeTlbs(pVM);
2335	pgmUnlock(pVM);
2336	return rc;
2337	}
2338
2339
2340	/**
2341	* Locate a MMIO2 range.
2342	*
2343	* @returns Pointer to the MMIO2 range.
2344	* @param pVM Pointer to the shared VM structure.
2345	* @param pDevIns The device instance owning the region.
2346	* @param iRegion The region.
2347	*/
2348	DECLINLINE(PPGMMMIO2RANGE) pgmR3PhysMMIO2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2349	{
2350	/*
2351	* Search the list.
2352	*/
2353	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2354	if ( pCur->pDevInsR3 == pDevIns
2355	&& pCur->iRegion == iRegion)
2356	return pCur;
2357	return NULL;
2358	}
2359
2360
2361	/**
2362	* Allocate and register an MMIO2 region.
2363	*
2364	* As mentioned elsewhere, MMIO2 is just RAM spelled differently. It's RAM
2365	* associated with a device. It is also non-shared memory with a permanent
2366	* ring-3 mapping and page backing (presently).
2367	*
2368	* A MMIO2 range may overlap with base memory if a lot of RAM is configured for
2369	* the VM, in which case we'll drop the base memory pages. Presently we will
2370	* make no attempt to preserve anything that happens to be present in the base
2371	* memory that is replaced, this is of course incorrectly but it's too much
2372	* effort.
2373	*
2374	* @returns VBox status code.
2375	* @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
2376	* memory.
2377	* @retval VERR_ALREADY_EXISTS if the region already exists.
2378	*
2379	* @param pVM Pointer to the shared VM structure.
2380	* @param pDevIns The device instance owning the region.
2381	* @param iRegion The region number. If the MMIO2 memory is a PCI
2382	* I/O region this number has to be the number of that
2383	* region. Otherwise it can be any number safe
2384	* UINT8_MAX.
2385	* @param cb The size of the region. Must be page aligned.
2386	* @param fFlags Reserved for future use, must be zero.
2387	* @param ppv Where to store the pointer to the ring-3 mapping of
2388	* the memory.
2389	* @param pszDesc The description.
2390	*/
2391	VMMR3DECL(int) PGMR3PhysMMIO2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags, void *ppv, const char pszDesc)
2392	{
2393	/*
2394	* Validate input.
2395	*/
2396	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2397	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2398	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2399	AssertPtrReturn(ppv, VERR_INVALID_POINTER);
2400	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2401	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2402	AssertReturn(pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion) == NULL, VERR_ALREADY_EXISTS);
2403	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2404	AssertReturn(cb, VERR_INVALID_PARAMETER);
2405	AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
2406
2407	const uint32_t cPages = cb >> PAGE_SHIFT;
2408	AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
2409	AssertLogRelReturn(cPages <= INT32_MAX / 2, VERR_NO_MEMORY);
2410
2411	/*
2412	* For the 2nd+ instance, mangle the description string so it's unique.
2413	*/
2414	if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
2415	{
2416	pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
2417	if (!pszDesc)
2418	return VERR_NO_MEMORY;
2419	}
2420
2421	/*
2422	* Try reserve and allocate the backing memory first as this is what is
2423	* most likely to fail.
2424	*/
2425	int rc = MMR3AdjustFixedReservation(pVM, cPages, pszDesc);
2426	if (RT_SUCCESS(rc))
2427	{
2428	void *pvPages;
2429	PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(SUPPAGE));
2430	if (RT_SUCCESS(rc))
2431	rc = SUPR3PageAllocEx(cPages, 0 /fFlags/, &pvPages, NULL /pR0Ptr/, paPages);
2432	if (RT_SUCCESS(rc))
2433	{
2434	memset(pvPages, 0, cPages * PAGE_SIZE);
2435
2436	/*
2437	* Create the MMIO2 range record for it.
2438	*/
2439	const size_t cbRange = RT_OFFSETOF(PGMMMIO2RANGE, RamRange.aPages[cPages]);
2440	PPGMMMIO2RANGE pNew;
2441	rc = MMR3HyperAllocOnceNoRel(pVM, cbRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
2442	AssertLogRelMsgRC(rc, ("cbRamRange=%zu\n", cbRange));
2443	if (RT_SUCCESS(rc))
2444	{
2445	pNew->pDevInsR3 = pDevIns;
2446	pNew->pvR3 = pvPages;
2447	//pNew->pNext = NULL;
2448	//pNew->fMapped = false;
2449	//pNew->fOverlapping = false;
2450	pNew->iRegion = iRegion;
2451	pNew->idSavedState = UINT8_MAX;
2452	pNew->RamRange.pSelfR0 = MMHyperCCToR0(pVM, &pNew->RamRange);
2453	pNew->RamRange.pSelfRC = MMHyperCCToRC(pVM, &pNew->RamRange);
2454	pNew->RamRange.GCPhys = NIL_RTGCPHYS;
2455	pNew->RamRange.GCPhysLast = NIL_RTGCPHYS;
2456	pNew->RamRange.pszDesc = pszDesc;
2457	pNew->RamRange.cb = cb;
2458	pNew->RamRange.fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2;
2459	pNew->RamRange.pvR3 = pvPages;
2460	//pNew->RamRange.paLSPages = NULL;
2461
2462	uint32_t iPage = cPages;
2463	while (iPage-- > 0)
2464	{
2465	PGM_PAGE_INIT(&pNew->RamRange.aPages[iPage],
2466	paPages[iPage].Phys, NIL_GMM_PAGEID,
2467	PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
2468	}
2469
2470	/* update page count stats */
2471	pVM->pgm.s.cAllPages += cPages;
2472	pVM->pgm.s.cPrivatePages += cPages;
2473
2474	/*
2475	* Link it into the list.
2476	* Since there is no particular order, just push it.
2477	*/
2478	pgmLock(pVM);
2479	pNew->pNextR3 = pVM->pgm.s.pMmio2RangesR3;
2480	pVM->pgm.s.pMmio2RangesR3 = pNew;
2481	pgmUnlock(pVM);
2482
2483	*ppv = pvPages;
2484	RTMemTmpFree(paPages);
2485	pgmPhysInvalidatePageMapTLB(pVM);
2486	return VINF_SUCCESS;
2487	}
2488
2489	SUPR3PageFreeEx(pvPages, cPages);
2490	}
2491	RTMemTmpFree(paPages);
2492	MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pszDesc);
2493	}
2494	if (pDevIns->iInstance > 0)
2495	MMR3HeapFree((void *)pszDesc);
2496	return rc;
2497	}
2498
2499
2500	/**
2501	* Deregisters and frees an MMIO2 region.
2502	*
2503	* Any physical (and virtual) access handlers registered for the region must
2504	* be deregistered before calling this function.
2505	*
2506	* @returns VBox status code.
2507	* @param pVM Pointer to the shared VM structure.
2508	* @param pDevIns The device instance owning the region.
2509	* @param iRegion The region. If it's UINT32_MAX it'll be a wildcard match.
2510	*/
2511	VMMR3DECL(int) PGMR3PhysMMIO2Deregister(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2512	{
2513	/*
2514	* Validate input.
2515	*/
2516	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2517	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2518	AssertReturn(iRegion <= UINT8_MAX \|\| iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
2519
2520	pgmLock(pVM);
2521	int rc = VINF_SUCCESS;
2522	unsigned cFound = 0;
2523	PPGMMMIO2RANGE pPrev = NULL;
2524	PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3;
2525	while (pCur)
2526	{
2527	if ( pCur->pDevInsR3 == pDevIns
2528	&& ( iRegion == UINT32_MAX
2529	\|\| pCur->iRegion == iRegion))
2530	{
2531	cFound++;
2532
2533	/*
2534	* Unmap it if it's mapped.
2535	*/
2536	if (pCur->fMapped)
2537	{
2538	int rc2 = PGMR3PhysMMIO2Unmap(pVM, pCur->pDevInsR3, pCur->iRegion, pCur->RamRange.GCPhys);
2539	AssertRC(rc2);
2540	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2541	rc = rc2;
2542	}
2543
2544	/*
2545	* Unlink it
2546	*/
2547	PPGMMMIO2RANGE pNext = pCur->pNextR3;
2548	if (pPrev)
2549	pPrev->pNextR3 = pNext;
2550	else
2551	pVM->pgm.s.pMmio2RangesR3 = pNext;
2552	pCur->pNextR3 = NULL;
2553
2554	/*
2555	* Free the memory.
2556	*/
2557	int rc2 = SUPR3PageFreeEx(pCur->pvR3, pCur->RamRange.cb >> PAGE_SHIFT);
2558	AssertRC(rc2);
2559	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2560	rc = rc2;
2561
2562	uint32_t const cPages = pCur->RamRange.cb >> PAGE_SHIFT;
2563	rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pCur->RamRange.pszDesc);
2564	AssertRC(rc2);
2565	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2566	rc = rc2;
2567
2568	/* we're leaking hyper memory here if done at runtime. */
2569	#ifdef VBOX_STRICT
2570	VMSTATE const enmState = VMR3GetState(pVM);
2571	AssertMsg( enmState == VMSTATE_POWERING_OFF
2572	\|\| enmState == VMSTATE_POWERING_OFF_LS
2573	\|\| enmState == VMSTATE_OFF
2574	\|\| enmState == VMSTATE_OFF_LS
2575	\|\| enmState == VMSTATE_DESTROYING
2576	\|\| enmState == VMSTATE_TERMINATED
2577	\|\| enmState == VMSTATE_CREATING
2578	, ("%s\n", VMR3GetStateName(enmState)));
2579	#endif
2580	/*rc = MMHyperFree(pVM, pCur);
2581	AssertRCReturn(rc, rc); - not safe, see the alloc call. */
2582
2583
2584	/* update page count stats */
2585	pVM->pgm.s.cAllPages -= cPages;
2586	pVM->pgm.s.cPrivatePages -= cPages;
2587
2588	/* next */
2589	pCur = pNext;
2590	}
2591	else
2592	{
2593	pPrev = pCur;
2594	pCur = pCur->pNextR3;
2595	}
2596	}
2597	pgmPhysInvalidatePageMapTLB(pVM);
2598	pgmUnlock(pVM);
2599	return !cFound && iRegion != UINT32_MAX ? VERR_NOT_FOUND : rc;
2600	}
2601
2602
2603	/**
2604	* Maps a MMIO2 region.
2605	*
2606	* This is done when a guest / the bios / state loading changes the
2607	* PCI config. The replacing of base memory has the same restrictions
2608	* as during registration, of course.
2609	*
2610	* @returns VBox status code.
2611	*
2612	* @param pVM Pointer to the shared VM structure.
2613	* @param pDevIns The
2614	*/
2615	VMMR3DECL(int) PGMR3PhysMMIO2Map(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2616	{
2617	/*
2618	* Validate input
2619	*/
2620	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2621	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2622	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2623	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2624	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2625	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2626
2627	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2628	AssertReturn(pCur, VERR_NOT_FOUND);
2629	AssertReturn(!pCur->fMapped, VERR_WRONG_ORDER);
2630	Assert(pCur->RamRange.GCPhys == NIL_RTGCPHYS);
2631	Assert(pCur->RamRange.GCPhysLast == NIL_RTGCPHYS);
2632
2633	const RTGCPHYS GCPhysLast = GCPhys + pCur->RamRange.cb - 1;
2634	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2635
2636	/*
2637	* Find our location in the ram range list, checking for
2638	* restriction we don't bother implementing yet (partially overlapping).
2639	*/
2640	bool fRamExists = false;
2641	PPGMRAMRANGE pRamPrev = NULL;
2642	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2643	while (pRam && GCPhysLast >= pRam->GCPhys)
2644	{
2645	if ( GCPhys <= pRam->GCPhysLast
2646	&& GCPhysLast >= pRam->GCPhys)
2647	{
2648	/* completely within? */
2649	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
2650	&& GCPhysLast <= pRam->GCPhysLast,
2651	("%RGp-%RGp (MMIO2/%s) falls partly outside %RGp-%RGp (%s)\n",
2652	GCPhys, GCPhysLast, pCur->RamRange.pszDesc,
2653	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2654	VERR_PGM_RAM_CONFLICT);
2655	fRamExists = true;
2656	break;
2657	}
2658
2659	/* next */
2660	pRamPrev = pRam;
2661	pRam = pRam->pNextR3;
2662	}
2663	if (fRamExists)
2664	{
2665	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2666	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2667	while (cPagesLeft-- > 0)
2668	{
2669	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
2670	("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
2671	GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pCur->RamRange.pszDesc),
2672	VERR_PGM_RAM_CONFLICT);
2673	pPage++;
2674	}
2675	}
2676	Log(("PGMR3PhysMMIO2Map: %RGp-%RGp fRamExists=%RTbool %s\n",
2677	GCPhys, GCPhysLast, fRamExists, pCur->RamRange.pszDesc));
2678
2679	/*
2680	* Make the changes.
2681	*/
2682	pgmLock(pVM);
2683
2684	pCur->RamRange.GCPhys = GCPhys;
2685	pCur->RamRange.GCPhysLast = GCPhysLast;
2686	pCur->fMapped = true;
2687	pCur->fOverlapping = fRamExists;
2688
2689	if (fRamExists)
2690	{
2691	/** @todo use pgmR3PhysFreePageRange here. */
2692	uint32_t cPendingPages = 0;
2693	PGMMFREEPAGESREQ pReq;
2694	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2695	AssertLogRelRCReturn(rc, rc);
2696
2697	/* replace the pages, freeing all present RAM pages. */
2698	PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2699	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2700	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2701	while (cPagesLeft-- > 0)
2702	{
2703	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
2704	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
2705
2706	RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
2707	PGM_PAGE_SET_HCPHYS(pVM, pPageDst, HCPhys);
2708	PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO2);
2709	PGM_PAGE_SET_STATE(pVM, pPageDst, PGM_PAGE_STATE_ALLOCATED);
2710	PGM_PAGE_SET_PDE_TYPE(pVM, pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
2711	PGM_PAGE_SET_PTE_INDEX(pVM, pPageDst, 0);
2712	PGM_PAGE_SET_TRACKING(pVM, pPageDst, 0);
2713
2714	pVM->pgm.s.cZeroPages--;
2715	GCPhys += PAGE_SIZE;
2716	pPageSrc++;
2717	pPageDst++;
2718	}
2719
2720	/* Flush physical page map TLB. */
2721	pgmPhysInvalidatePageMapTLB(pVM);
2722
2723	if (cPendingPages)
2724	{
2725	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2726	AssertLogRelRCReturn(rc, rc);
2727	}
2728	GMMR3FreePagesCleanup(pReq);
2729
2730	/* Force a PGM pool flush as guest ram references have been changed. */
2731	/** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
2732	PVMCPU pVCpu = VMMGetCpu(pVM);
2733	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2734	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2735
2736	pgmUnlock(pVM);
2737	}
2738	else
2739	{
2740	RTGCPHYS cb = pCur->RamRange.cb;
2741
2742	/* Clear the tracking data of pages we're going to reactivate. */
2743	PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2744	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2745	while (cPagesLeft-- > 0)
2746	{
2747	PGM_PAGE_SET_TRACKING(pVM, pPageSrc, 0);
2748	PGM_PAGE_SET_PTE_INDEX(pVM, pPageSrc, 0);
2749	pPageSrc++;
2750	}
2751
2752	/* link in the ram range */
2753	pgmR3PhysLinkRamRange(pVM, &pCur->RamRange, pRamPrev);
2754	pgmUnlock(pVM);
2755
2756	REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_MMIO2);
2757	}
2758
2759	pgmPhysInvalidatePageMapTLB(pVM);
2760	return VINF_SUCCESS;
2761	}
2762
2763
2764	/**
2765	* Unmaps a MMIO2 region.
2766	*
2767	* This is done when a guest / the bios / state loading changes the
2768	* PCI config. The replacing of base memory has the same restrictions
2769	* as during registration, of course.
2770	*/
2771	VMMR3DECL(int) PGMR3PhysMMIO2Unmap(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2772	{
2773	/*
2774	* Validate input
2775	*/
2776	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2777	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2778	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2779	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2780	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2781	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2782
2783	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2784	AssertReturn(pCur, VERR_NOT_FOUND);
2785	AssertReturn(pCur->fMapped, VERR_WRONG_ORDER);
2786	AssertReturn(pCur->RamRange.GCPhys == GCPhys, VERR_INVALID_PARAMETER);
2787	Assert(pCur->RamRange.GCPhysLast != NIL_RTGCPHYS);
2788
2789	Log(("PGMR3PhysMMIO2Unmap: %RGp-%RGp %s\n",
2790	pCur->RamRange.GCPhys, pCur->RamRange.GCPhysLast, pCur->RamRange.pszDesc));
2791
2792	/*
2793	* Unmap it.
2794	*/
2795	pgmLock(pVM);
2796
2797	RTGCPHYS GCPhysRangeREM;
2798	RTGCPHYS cbRangeREM;
2799	bool fInformREM;
2800	if (pCur->fOverlapping)
2801	{
2802	/* Restore the RAM pages we've replaced. */
2803	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2804	while (pRam->GCPhys > pCur->RamRange.GCPhysLast)
2805	pRam = pRam->pNextR3;
2806
2807	PPGMPAGE pPageDst = &pRam->aPages[(pCur->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2808	uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2809	while (cPagesLeft-- > 0)
2810	{
2811	PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
2812	pVM->pgm.s.cZeroPages++;
2813	pPageDst++;
2814	}
2815
2816	/* Flush physical page map TLB. */
2817	pgmPhysInvalidatePageMapTLB(pVM);
2818
2819	GCPhysRangeREM = NIL_RTGCPHYS; /* shuts up gcc */
2820	cbRangeREM = RTGCPHYS_MAX; /* ditto */
2821	fInformREM = false;
2822	}
2823	else
2824	{
2825	GCPhysRangeREM = pCur->RamRange.GCPhys;
2826	cbRangeREM = pCur->RamRange.cb;
2827	fInformREM = true;
2828
2829	pgmR3PhysUnlinkRamRange(pVM, &pCur->RamRange);
2830	}
2831
2832	pCur->RamRange.GCPhys = NIL_RTGCPHYS;
2833	pCur->RamRange.GCPhysLast = NIL_RTGCPHYS;
2834	pCur->fOverlapping = false;
2835	pCur->fMapped = false;
2836
2837	/* Force a PGM pool flush as guest ram references have been changed. */
2838	/** @todo not entirely SMP safe; assuming for now the guest takes care
2839	* of this internally (not touch mapped mmio while changing the
2840	* mapping). */
2841	PVMCPU pVCpu = VMMGetCpu(pVM);
2842	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2843	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2844
2845	pgmPhysInvalidatePageMapTLB(pVM);
2846	pgmPhysInvalidRamRangeTlbs(pVM);
2847	pgmUnlock(pVM);
2848
2849	if (fInformREM)
2850	REMR3NotifyPhysRamDeregister(pVM, GCPhysRangeREM, cbRangeREM);
2851
2852	return VINF_SUCCESS;
2853	}
2854
2855
2856	/**
2857	* Checks if the given address is an MMIO2 base address or not.
2858	*
2859	* @returns true/false accordingly.
2860	* @param pVM Pointer to the shared VM structure.
2861	* @param pDevIns The owner of the memory, optional.
2862	* @param GCPhys The address to check.
2863	*/
2864	VMMR3DECL(bool) PGMR3PhysMMIO2IsBase(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
2865	{
2866	/*
2867	* Validate input
2868	*/
2869	VM_ASSERT_EMT_RETURN(pVM, false);
2870	AssertPtrReturn(pDevIns, false);
2871	AssertReturn(GCPhys != NIL_RTGCPHYS, false);
2872	AssertReturn(GCPhys != 0, false);
2873	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), false);
2874
2875	/*
2876	* Search the list.
2877	*/
2878	pgmLock(pVM);
2879	for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2880	if (pCur->RamRange.GCPhys == GCPhys)
2881	{
2882	Assert(pCur->fMapped);
2883	pgmUnlock(pVM);
2884	return true;
2885	}
2886	pgmUnlock(pVM);
2887	return false;
2888	}
2889
2890
2891	/**
2892	* Gets the HC physical address of a page in the MMIO2 region.
2893	*
2894	* This is API is intended for MMHyper and shouldn't be called
2895	* by anyone else...
2896	*
2897	* @returns VBox status code.
2898	* @param pVM Pointer to the shared VM structure.
2899	* @param pDevIns The owner of the memory, optional.
2900	* @param iRegion The region.
2901	* @param off The page expressed an offset into the MMIO2 region.
2902	* @param pHCPhys Where to store the result.
2903	*/
2904	VMMR3DECL(int) PGMR3PhysMMIO2GetHCPhys(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, PRTHCPHYS pHCPhys)
2905	{
2906	/*
2907	* Validate input
2908	*/
2909	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2910	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2911	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2912
2913	pgmLock(pVM);
2914	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2915	AssertReturn(pCur, VERR_NOT_FOUND);
2916	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2917
2918	PCPGMPAGE pPage = &pCur->RamRange.aPages[off >> PAGE_SHIFT];
2919	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage);
2920	pgmUnlock(pVM);
2921	return VINF_SUCCESS;
2922	}
2923
2924
2925	/**
2926	* Maps a portion of an MMIO2 region into kernel space (host).
2927	*
2928	* The kernel mapping will become invalid when the MMIO2 memory is deregistered
2929	* or the VM is terminated.
2930	*
2931	* @return VBox status code.
2932	*
2933	* @param pVM Pointer to the shared VM structure.
2934	* @param pDevIns The device owning the MMIO2 memory.
2935	* @param iRegion The region.
2936	* @param off The offset into the region. Must be page aligned.
2937	* @param cb The number of bytes to map. Must be page aligned.
2938	* @param pszDesc Mapping description.
2939	* @param pR0Ptr Where to store the R0 address.
2940	*/
2941	VMMR3DECL(int) PGMR3PhysMMIO2MapKernel(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
2942	const char *pszDesc, PRTR0PTR pR0Ptr)
2943	{
2944	/*
2945	* Validate input.
2946	*/
2947	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2948	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2949	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2950
2951	PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2952	AssertReturn(pCur, VERR_NOT_FOUND);
2953	AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2954	AssertReturn(cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2955	AssertReturn(off + cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
2956	NOREF(pszDesc);
2957
2958	/*
2959	* Pass the request on to the support library/driver.
2960	*/
2961	int rc = SUPR3PageMapKernel(pCur->pvR3, off, cb, 0, pR0Ptr);
2962
2963	return rc;
2964	}
2965
2966
2967	/**
2968	* Worker for PGMR3PhysRomRegister.
2969	*
2970	* This is here to simplify lock management, i.e. the caller does all the
2971	* locking and we can simply return without needing to remember to unlock
2972	* anything first.
2973	*
2974	* @returns VBox status.
2975	* @param pVM VM Handle.
2976	* @param pDevIns The device instance owning the ROM.
2977	* @param GCPhys First physical address in the range.
2978	* Must be page aligned!
2979	* @param cb The size of the range (in bytes).
2980	* Must be page aligned!
2981	* @param pvBinary Pointer to the binary data backing the ROM image.
2982	* @param cbBinary The size of the binary data pvBinary points to.
2983	* This must be less or equal to @a cb.
2984	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
2985	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
2986	* @param pszDesc Pointer to description string. This must not be freed.
2987	*/
2988	static int pgmR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
2989	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
2990	{
2991	/*
2992	* Validate input.
2993	*/
2994	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2995	AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
2996	AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
2997	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2998	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2999	AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
3000	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
3001	AssertReturn(!(fFlags & ~(PGMPHYS_ROM_FLAGS_SHADOWED \| PGMPHYS_ROM_FLAGS_PERMANENT_BINARY)), VERR_INVALID_PARAMETER);
3002	VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
3003
3004	const uint32_t cPages = cb >> PAGE_SHIFT;
3005
3006	/*
3007	* Find the ROM location in the ROM list first.
3008	*/
3009	PPGMROMRANGE pRomPrev = NULL;
3010	PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3;
3011	while (pRom && GCPhysLast >= pRom->GCPhys)
3012	{
3013	if ( GCPhys <= pRom->GCPhysLast
3014	&& GCPhysLast >= pRom->GCPhys)
3015	AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
3016	GCPhys, GCPhysLast, pszDesc,
3017	pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
3018	VERR_PGM_RAM_CONFLICT);
3019	/* next */
3020	pRomPrev = pRom;
3021	pRom = pRom->pNextR3;
3022	}
3023
3024	/*
3025	* Find the RAM location and check for conflicts.
3026	*
3027	* Conflict detection is a bit different than for RAM
3028	* registration since a ROM can be located within a RAM
3029	* range. So, what we have to check for is other memory
3030	* types (other than RAM that is) and that we don't span
3031	* more than one RAM range (layz).
3032	*/
3033	bool fRamExists = false;
3034	PPGMRAMRANGE pRamPrev = NULL;
3035	PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
3036	while (pRam && GCPhysLast >= pRam->GCPhys)
3037	{
3038	if ( GCPhys <= pRam->GCPhysLast
3039	&& GCPhysLast >= pRam->GCPhys)
3040	{
3041	/* completely within? */
3042	AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
3043	&& GCPhysLast <= pRam->GCPhysLast,
3044	("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
3045	GCPhys, GCPhysLast, pszDesc,
3046	pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
3047	VERR_PGM_RAM_CONFLICT);
3048	fRamExists = true;
3049	break;
3050	}
3051
3052	/* next */
3053	pRamPrev = pRam;
3054	pRam = pRam->pNextR3;
3055	}
3056	if (fRamExists)
3057	{
3058	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3059	uint32_t cPagesLeft = cPages;
3060	while (cPagesLeft-- > 0)
3061	{
3062	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
3063	("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
3064	pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
3065	pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
3066	Assert(PGM_PAGE_IS_ZERO(pPage));
3067	pPage++;
3068	}
3069	}
3070
3071	/*
3072	* Update the base memory reservation if necessary.
3073	*/
3074	uint32_t cExtraBaseCost = fRamExists ? 0 : cPages;
3075	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3076	cExtraBaseCost += cPages;
3077	if (cExtraBaseCost)
3078	{
3079	int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
3080	if (RT_FAILURE(rc))
3081	return rc;
3082	}
3083
3084	/*
3085	* Allocate memory for the virgin copy of the RAM.
3086	*/
3087	PGMMALLOCATEPAGESREQ pReq;
3088	int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
3089	AssertRCReturn(rc, rc);
3090
3091	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3092	{
3093	pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
3094	pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
3095	pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
3096	}
3097
3098	rc = GMMR3AllocatePagesPerform(pVM, pReq);
3099	if (RT_FAILURE(rc))
3100	{
3101	GMMR3AllocatePagesCleanup(pReq);
3102	return rc;
3103	}
3104
3105	/*
3106	* Allocate the new ROM range and RAM range (if necessary).
3107	*/
3108	PPGMROMRANGE pRomNew;
3109	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
3110	if (RT_SUCCESS(rc))
3111	{
3112	PPGMRAMRANGE pRamNew = NULL;
3113	if (!fRamExists)
3114	rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), sizeof(PGMPAGE), MM_TAG_PGM_PHYS, (void **)&pRamNew);
3115	if (RT_SUCCESS(rc))
3116	{
3117	/*
3118	* Initialize and insert the RAM range (if required).
3119	*/
3120	PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
3121	if (!fRamExists)
3122	{
3123	pRamNew->pSelfR0 = MMHyperCCToR0(pVM, pRamNew);
3124	pRamNew->pSelfRC = MMHyperCCToRC(pVM, pRamNew);
3125	pRamNew->GCPhys = GCPhys;
3126	pRamNew->GCPhysLast = GCPhysLast;
3127	pRamNew->cb = cb;
3128	pRamNew->pszDesc = pszDesc;
3129	pRamNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
3130	pRamNew->pvR3 = NULL;
3131	pRamNew->paLSPages = NULL;
3132
3133	PPGMPAGE pPage = &pRamNew->aPages[0];
3134	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3135	{
3136	PGM_PAGE_INIT(pPage,
3137	pReq->aPages[iPage].HCPhysGCPhys,
3138	pReq->aPages[iPage].idPage,
3139	PGMPAGETYPE_ROM,
3140	PGM_PAGE_STATE_ALLOCATED);
3141
3142	pRomPage->Virgin = *pPage;
3143	}
3144
3145	pVM->pgm.s.cAllPages += cPages;
3146	pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
3147	}
3148	else
3149	{
3150	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3151	for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3152	{
3153	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_ROM);
3154	PGM_PAGE_SET_HCPHYS(pVM, pPage, pReq->aPages[iPage].HCPhysGCPhys);
3155	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
3156	PGM_PAGE_SET_PAGEID(pVM, pPage, pReq->aPages[iPage].idPage);
3157	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
3158	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
3159	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
3160
3161	pRomPage->Virgin = *pPage;
3162	}
3163
3164	pRamNew = pRam;
3165
3166	pVM->pgm.s.cZeroPages -= cPages;
3167	}
3168	pVM->pgm.s.cPrivatePages += cPages;
3169
3170	/* Flush physical page map TLB. */
3171	pgmPhysInvalidatePageMapTLB(pVM);
3172
3173
3174	/*
3175	* !HACK ALERT! REM + (Shadowed) ROM ==> mess.
3176	*
3177	* If it's shadowed we'll register the handler after the ROM notification
3178	* so we get the access handler callbacks that we should. If it isn't
3179	* shadowed we'll do it the other way around to make REM use the built-in
3180	* ROM behavior and not the handler behavior (which is to route all access
3181	* to PGM atm).
3182	*/
3183	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3184	{
3185	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, true /* fShadowed */);
3186	rc = PGMR3HandlerPhysicalRegister(pVM,
3187	fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
3188	? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
3189	: PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
3190	GCPhys, GCPhysLast,
3191	pgmR3PhysRomWriteHandler, pRomNew,
3192	NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
3193	NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
3194	}
3195	else
3196	{
3197	rc = PGMR3HandlerPhysicalRegister(pVM,
3198	fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
3199	? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
3200	: PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
3201	GCPhys, GCPhysLast,
3202	pgmR3PhysRomWriteHandler, pRomNew,
3203	NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
3204	NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
3205	REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, false /* fShadowed */);
3206	}
3207	if (RT_SUCCESS(rc))
3208	{
3209	/*
3210	* Copy the image over to the virgin pages.
3211	* This must be done after linking in the RAM range.
3212	*/
3213	size_t cbBinaryLeft = cbBinary;
3214	PPGMPAGE pRamPage = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
3215	for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
3216	{
3217	void *pvDstPage;
3218	rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << PAGE_SHIFT), &pvDstPage);
3219	if (RT_FAILURE(rc))
3220	{
3221	VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
3222	break;
3223	}
3224	if (cbBinaryLeft >= PAGE_SIZE)
3225	{
3226	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), PAGE_SIZE);
3227	cbBinaryLeft -= PAGE_SIZE;
3228	}
3229	else
3230	{
3231	ASMMemZeroPage(pvDstPage); /* (shouldn't be necessary, but can't hurt either) */
3232	if (cbBinaryLeft > 0)
3233	{
3234	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), cbBinaryLeft);
3235	cbBinaryLeft = 0;
3236	}
3237	}
3238	}
3239	if (RT_SUCCESS(rc))
3240	{
3241	/*
3242	* Initialize the ROM range.
3243	* Note that the Virgin member of the pages has already been initialized above.
3244	*/
3245	pRomNew->GCPhys = GCPhys;
3246	pRomNew->GCPhysLast = GCPhysLast;
3247	pRomNew->cb = cb;
3248	pRomNew->fFlags = fFlags;
3249	pRomNew->idSavedState = UINT8_MAX;
3250	pRomNew->cbOriginal = cbBinary;
3251	#ifdef VBOX_STRICT
3252	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
3253	? pvBinary : RTMemDup(pvBinary, cbBinary);
3254	#else
3255	pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY ? pvBinary : NULL;
3256	#endif
3257	pRomNew->pszDesc = pszDesc;
3258
3259	for (unsigned iPage = 0; iPage < cPages; iPage++)
3260	{
3261	PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
3262	pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
3263	PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
3264	}
3265
3266	/* update the page count stats for the shadow pages. */
3267	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3268	{
3269	pVM->pgm.s.cZeroPages += cPages;
3270	pVM->pgm.s.cAllPages += cPages;
3271	}
3272
3273	/*
3274	* Insert the ROM range, tell REM and return successfully.
3275	*/
3276	pRomNew->pNextR3 = pRom;
3277	pRomNew->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
3278	pRomNew->pNextRC = pRom ? MMHyperCCToRC(pVM, pRom) : NIL_RTRCPTR;
3279
3280	if (pRomPrev)
3281	{
3282	pRomPrev->pNextR3 = pRomNew;
3283	pRomPrev->pNextR0 = MMHyperCCToR0(pVM, pRomNew);
3284	pRomPrev->pNextRC = MMHyperCCToRC(pVM, pRomNew);
3285	}
3286	else
3287	{
3288	pVM->pgm.s.pRomRangesR3 = pRomNew;
3289	pVM->pgm.s.pRomRangesR0 = MMHyperCCToR0(pVM, pRomNew);
3290	pVM->pgm.s.pRomRangesRC = MMHyperCCToRC(pVM, pRomNew);
3291	}
3292
3293	pgmPhysInvalidatePageMapTLB(pVM);
3294	GMMR3AllocatePagesCleanup(pReq);
3295	return VINF_SUCCESS;
3296	}
3297
3298	/* bail out */
3299
3300	int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
3301	AssertRC(rc2);
3302	}
3303
3304	if (!fRamExists)
3305	{
3306	pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
3307	MMHyperFree(pVM, pRamNew);
3308	}
3309	}
3310	MMHyperFree(pVM, pRomNew);
3311	}
3312
3313	/** @todo Purge the mapping cache or something... */
3314	GMMR3FreeAllocatedPages(pVM, pReq);
3315	GMMR3AllocatePagesCleanup(pReq);
3316	return rc;
3317	}
3318
3319
3320	/**
3321	* Registers a ROM image.
3322	*
3323	* Shadowed ROM images requires double the amount of backing memory, so,
3324	* don't use that unless you have to. Shadowing of ROM images is process
3325	* where we can select where the reads go and where the writes go. On real
3326	* hardware the chipset provides means to configure this. We provide
3327	* PGMR3PhysProtectROM() for this purpose.
3328	*
3329	* A read-only copy of the ROM image will always be kept around while we
3330	* will allocate RAM pages for the changes on demand (unless all memory
3331	* is configured to be preallocated).
3332	*
3333	* @returns VBox status.
3334	* @param pVM VM Handle.
3335	* @param pDevIns The device instance owning the ROM.
3336	* @param GCPhys First physical address in the range.
3337	* Must be page aligned!
3338	* @param cb The size of the range (in bytes).
3339	* Must be page aligned!
3340	* @param pvBinary Pointer to the binary data backing the ROM image.
3341	* @param cbBinary The size of the binary data pvBinary points to.
3342	* This must be less or equal to @a cb.
3343	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
3344	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
3345	* @param pszDesc Pointer to description string. This must not be freed.
3346	*
3347	* @remark There is no way to remove the rom, automatically on device cleanup or
3348	* manually from the device yet. This isn't difficult in any way, it's
3349	* just not something we expect to be necessary for a while.
3350	*/
3351	VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
3352	const void pvBinary, uint32_t cbBinary, uint32_t fFlags, const char pszDesc)
3353	{
3354	Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
3355	pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
3356	pgmLock(pVM);
3357	int rc = pgmR3PhysRomRegister(pVM, pDevIns, GCPhys, cb, pvBinary, cbBinary, fFlags, pszDesc);
3358	pgmUnlock(pVM);
3359	return rc;
3360	}
3361
3362
3363	/**
3364	* \#PF Handler callback for ROM write accesses.
3365	*
3366	* @returns VINF_SUCCESS if the handler have carried out the operation.
3367	* @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
3368	* @param pVM VM Handle.
3369	* @param GCPhys The physical address the guest is writing to.
3370	* @param pvPhys The HC mapping of that address.
3371	* @param pvBuf What the guest is reading/writing.
3372	* @param cbBuf How much it's reading/writing.
3373	* @param enmAccessType The access type.
3374	* @param pvUser User argument.
3375	*/
3376	static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
3377	PGMACCESSTYPE enmAccessType, void *pvUser)
3378	{
3379	PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
3380	const uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3381	Assert(iPage < (pRom->cb >> PAGE_SHIFT));
3382	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3383	Log5(("pgmR3PhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
3384	NOREF(pvPhys);
3385
3386	if (enmAccessType == PGMACCESSTYPE_READ)
3387	{
3388	switch (pRomPage->enmProt)
3389	{
3390	/*
3391	* Take the default action.
3392	*/
3393	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3394	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3395	case PGMROMPROT_READ_ROM_WRITE_RAM:
3396	case PGMROMPROT_READ_RAM_WRITE_RAM:
3397	return VINF_PGM_HANDLER_DO_DEFAULT;
3398
3399	default:
3400	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3401	pRom->aPages[iPage].enmProt, iPage, GCPhys),
3402	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
3403	}
3404	}
3405	else
3406	{
3407	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
3408	switch (pRomPage->enmProt)
3409	{
3410	/*
3411	* Ignore writes.
3412	*/
3413	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3414	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3415	return VINF_SUCCESS;
3416
3417	/*
3418	* Write to the RAM page.
3419	*/
3420	case PGMROMPROT_READ_ROM_WRITE_RAM:
3421	case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's way simpler that way. */
3422	{
3423	/* This should be impossible now, pvPhys doesn't work cross page anylonger. */
3424	Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
3425
3426	/*
3427	* Take the lock, do lazy allocation, map the page and copy the data.
3428	*
3429	* Note that we have to bypass the mapping TLB since it works on
3430	* guest physical addresses and entering the shadow page would
3431	* kind of screw things up...
3432	*/
3433	int rc = pgmLock(pVM);
3434	AssertRC(rc);
3435
3436	PPGMPAGE pShadowPage = &pRomPage->Shadow;
3437	if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
3438	{
3439	pShadowPage = pgmPhysGetPage(pVM, GCPhys);
3440	AssertLogRelReturn(pShadowPage, VERR_PGM_PHYS_PAGE_GET_IPE);
3441	}
3442
3443	void *pvDstPage;
3444	rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
3445	if (RT_SUCCESS(rc))
3446	{
3447	memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
3448	pRomPage->LiveSave.fWrittenTo = true;
3449	}
3450
3451	pgmUnlock(pVM);
3452	return rc;
3453	}
3454
3455	default:
3456	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3457	pRom->aPages[iPage].enmProt, iPage, GCPhys),
3458	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
3459	}
3460	}
3461	}
3462
3463
3464	/**
3465	* Called by PGMR3Reset to reset the shadow, switch to the virgin,
3466	* and verify that the virgin part is untouched.
3467	*
3468	* This is done after the normal memory has been cleared.
3469	*
3470	* ASSUMES that the caller owns the PGM lock.
3471	*
3472	* @param pVM The VM handle.
3473	*/
3474	int pgmR3PhysRomReset(PVM pVM)
3475	{
3476	PGM_LOCK_ASSERT_OWNER(pVM);
3477	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3478	{
3479	const uint32_t cPages = pRom->cb >> PAGE_SHIFT;
3480
3481	if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3482	{
3483	/*
3484	* Reset the physical handler.
3485	*/
3486	int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
3487	AssertRCReturn(rc, rc);
3488
3489	/*
3490	* What we do with the shadow pages depends on the memory
3491	* preallocation option. If not enabled, we'll just throw
3492	* out all the dirty pages and replace them by the zero page.
3493	*/
3494	if (!pVM->pgm.s.fRamPreAlloc)
3495	{
3496	/* Free the dirty pages. */
3497	uint32_t cPendingPages = 0;
3498	PGMMFREEPAGESREQ pReq;
3499	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
3500	AssertRCReturn(rc, rc);
3501
3502	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3503	if ( !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
3504	&& !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
3505	{
3506	Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
3507	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
3508	pRom->GCPhys + (iPage << PAGE_SHIFT));
3509	AssertLogRelRCReturn(rc, rc);
3510	}
3511
3512	if (cPendingPages)
3513	{
3514	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
3515	AssertLogRelRCReturn(rc, rc);
3516	}
3517	GMMR3FreePagesCleanup(pReq);
3518	}
3519	else
3520	{
3521	/* clear all the shadow pages. */
3522	for (uint32_t iPage = 0; iPage < cPages; iPage++)
3523	{
3524	if (PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow))
3525	continue;
3526	Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
3527	void *pvDstPage;
3528	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3529	rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
3530	if (RT_FAILURE(rc))
3531	break;
3532	ASMMemZeroPage(pvDstPage);
3533	}
3534	AssertRCReturn(rc, rc);
3535	}
3536	}
3537
3538	#ifdef VBOX_STRICT
3539	/*
3540	* Verify that the virgin page is unchanged if possible.
3541	*/
3542	if (pRom->pvOriginal)
3543	{
3544	size_t cbSrcLeft = pRom->cbOriginal;
3545	uint8_t const pbSrcPage = (uint8_t const )pRom->pvOriginal;
3546	for (uint32_t iPage = 0; iPage < cPages && cbSrcLeft > 0; iPage++, pbSrcPage += PAGE_SIZE)
3547	{
3548	const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3549	void const *pvDstPage;
3550	int rc = pgmPhysPageMapReadOnly(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPage);
3551	if (RT_FAILURE(rc))
3552	break;
3553
3554	if (memcmp(pvDstPage, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE)))
3555	LogRel(("pgmR3PhysRomReset: %RGp rom page changed (%s) - loaded saved state?\n",
3556	GCPhys, pRom->pszDesc));
3557	cbSrcLeft -= RT_MIN(cbSrcLeft, PAGE_SIZE);
3558	}
3559	}
3560	#endif
3561	}
3562
3563	return VINF_SUCCESS;
3564	}
3565
3566
3567	/**
3568	* Called by PGMR3Term to free resources.
3569	*
3570	* ASSUMES that the caller owns the PGM lock.
3571	*
3572	* @param pVM The VM handle.
3573	*/
3574	void pgmR3PhysRomTerm(PVM pVM)
3575	{
3576	#ifdef RT_STRICT
3577	/*
3578	* Free the heap copy of the original bits.
3579	*/
3580	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3581	{
3582	if ( pRom->pvOriginal
3583	&& !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
3584	{
3585	RTMemFree((void *)pRom->pvOriginal);
3586	pRom->pvOriginal = NULL;
3587	}
3588	}
3589	#endif
3590	}
3591
3592
3593	/**
3594	* Change the shadowing of a range of ROM pages.
3595	*
3596	* This is intended for implementing chipset specific memory registers
3597	* and will not be very strict about the input. It will silently ignore
3598	* any pages that are not the part of a shadowed ROM.
3599	*
3600	* @returns VBox status code.
3601	* @retval VINF_PGM_SYNC_CR3
3602	*
3603	* @param pVM Pointer to the shared VM structure.
3604	* @param GCPhys Where to start. Page aligned.
3605	* @param cb How much to change. Page aligned.
3606	* @param enmProt The new ROM protection.
3607	*/
3608	VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
3609	{
3610	/*
3611	* Check input
3612	*/
3613	if (!cb)
3614	return VINF_SUCCESS;
3615	AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3616	AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3617	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
3618	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3619	AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
3620
3621	/*
3622	* Process the request.
3623	*/
3624	pgmLock(pVM);
3625	int rc = VINF_SUCCESS;
3626	bool fFlushTLB = false;
3627	for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3628	{
3629	if ( GCPhys <= pRom->GCPhysLast
3630	&& GCPhysLast >= pRom->GCPhys
3631	&& (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
3632	{
3633	/*
3634	* Iterate the relevant pages and make necessary the changes.
3635	*/
3636	bool fChanges = false;
3637	uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
3638	? pRom->cb >> PAGE_SHIFT
3639	: (GCPhysLast - pRom->GCPhys + 1) >> PAGE_SHIFT;
3640	for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3641	iPage < cPages;
3642	iPage++)
3643	{
3644	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3645	if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
3646	{
3647	fChanges = true;
3648
3649	/* flush references to the page. */
3650	PPGMPAGE pRamPage = pgmPhysGetPage(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT));
3651	int rc2 = pgmPoolTrackUpdateGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage,
3652	true /fFlushPTEs/, &fFlushTLB);
3653	if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS \|\| RT_FAILURE(rc2)))
3654	rc = rc2;
3655
3656	PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
3657	PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
3658
3659	pOld = pRamPage;
3660	pRamPage = pNew;
3661	/** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
3662	}
3663	pRomPage->enmProt = enmProt;
3664	}
3665
3666	/*
3667	* Reset the access handler if we made changes, no need
3668	* to optimize this.
3669	*/
3670	if (fChanges)
3671	{
3672	int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
3673	if (RT_FAILURE(rc2))
3674	{
3675	pgmUnlock(pVM);
3676	AssertRC(rc);
3677	return rc2;
3678	}
3679	}
3680
3681	/* Advance - cb isn't updated. */
3682	GCPhys = pRom->GCPhys + (cPages << PAGE_SHIFT);
3683	}
3684	}
3685	pgmUnlock(pVM);
3686	if (fFlushTLB)
3687	PGM_INVL_ALL_VCPU_TLBS(pVM);
3688
3689	return rc;
3690	}
3691
3692
3693	/**
3694	* Sets the Address Gate 20 state.
3695	*
3696	* @param pVCpu The VCPU to operate on.
3697	* @param fEnable True if the gate should be enabled.
3698	* False if the gate should be disabled.
3699	*/
3700	VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
3701	{
3702	LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
3703	if (pVCpu->pgm.s.fA20Enabled != fEnable)
3704	{
3705	pVCpu->pgm.s.fA20Enabled = fEnable;
3706	pVCpu->pgm.s.GCPhysA20Mask = ~(RTGCPHYS)(!fEnable << 20);
3707	REMR3A20Set(pVCpu->pVMR3, pVCpu, fEnable);
3708	/** @todo we're not handling this correctly for VT-x / AMD-V. See #2911 */
3709	}
3710	}
3711
3712
3713	/**
3714	* Tree enumeration callback for dealing with age rollover.
3715	* It will perform a simple compression of the current age.
3716	*/
3717	static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
3718	{
3719	/* Age compression - ASSUMES iNow == 4. */
3720	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3721	if (pChunk->iLastUsed >= UINT32_C(0xffffff00))
3722	pChunk->iLastUsed = 3;
3723	else if (pChunk->iLastUsed >= UINT32_C(0xfffff000))
3724	pChunk->iLastUsed = 2;
3725	else if (pChunk->iLastUsed)
3726	pChunk->iLastUsed = 1;
3727	else /* iLastUsed = 0 */
3728	pChunk->iLastUsed = 4;
3729
3730	NOREF(pvUser);
3731	return 0;
3732	}
3733
3734
3735	/**
3736	* The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
3737	*/
3738	typedef struct PGMR3PHYSCHUNKUNMAPCB
3739	{
3740	PVM pVM; /*< The VM handle. /
3741	PPGMCHUNKR3MAP pChunk; /*< The chunk to unmap. /
3742	} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
3743
3744
3745	/**
3746	* Callback used to find the mapping that's been unused for
3747	* the longest time.
3748	*/
3749	static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
3750	{
3751	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3752	PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
3753
3754	/*
3755	* Check for locks and compare when last used.
3756	*/
3757	if (pChunk->cRefs)
3758	return 0;
3759	if (pChunk->cPermRefs)
3760	return 0;
3761	if ( pArg->pChunk
3762	&& pChunk->iLastUsed >= pArg->pChunk->iLastUsed)
3763	return 0;
3764
3765	/*
3766	* Check that it's not in any of the TLBs.
3767	*/
3768	PVM pVM = pArg->pVM;
3769	if ( pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(pChunk->Core.Key)].idChunk
3770	== pChunk->Core.Key)
3771	{
3772	pChunk = NULL;
3773	return 0;
3774	}
3775	#ifdef VBOX_STRICT
3776	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
3777	{
3778	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk != pChunk);
3779	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk != pChunk->Core.Key);
3780	}
3781	#endif
3782
3783	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
3784	if (pVM->pgm.s.PhysTlbHC.aEntries[i].pMap == pChunk)
3785	return 0;
3786
3787	pArg->pChunk = pChunk;
3788	return 0;
3789	}
3790
3791
3792	/**
3793	* Finds a good candidate for unmapping when the ring-3 mapping cache is full.
3794	*
3795	* The candidate will not be part of any TLBs, so no need to flush
3796	* anything afterwards.
3797	*
3798	* @returns Chunk id.
3799	* @param pVM The VM handle.
3800	*/
3801	static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
3802	{
3803	PGM_LOCK_ASSERT_OWNER(pVM);
3804
3805	/*
3806	* Enumerate the age tree starting with the left most node.
3807	*/
3808	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3809	PGMR3PHYSCHUNKUNMAPCB Args;
3810	Args.pVM = pVM;
3811	Args.pChunk = NULL;
3812	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
3813	Assert(Args.pChunk);
3814	if (Args.pChunk)
3815	{
3816	Assert(Args.pChunk->cRefs == 0);
3817	Assert(Args.pChunk->cPermRefs == 0);
3818	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3819	return Args.pChunk->Core.Key;
3820	}
3821
3822	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3823	return INT32_MAX;
3824	}
3825
3826
3827	/**
3828	* Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
3829	*
3830	* This is only called on one of the EMTs while the other ones are waiting for
3831	* it to complete this function.
3832	*
3833	* @returns VINF_SUCCESS (VBox strict status code).
3834	* @param pVM The VM handle.
3835	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
3836	* @param pvUser User pointer. Unused
3837	*
3838	*/
3839	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
3840	{
3841	int rc = VINF_SUCCESS;
3842	pgmLock(pVM);
3843	NOREF(pVCpu); NOREF(pvUser);
3844
3845	if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
3846	{
3847	/* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
3848	/** @todo also not really efficient to unmap a chunk that contains PD
3849	* or PT pages. */
3850	pgmR3PoolClearAllRendezvous(pVM, &pVM->aCpus[0], NULL /* no need to flush the REM TLB as we already did that above */);
3851
3852	/*
3853	* Request the ring-0 part to unmap a chunk to make space in the mapping cache.
3854	*/
3855	GMMMAPUNMAPCHUNKREQ Req;
3856	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3857	Req.Hdr.cbReq = sizeof(Req);
3858	Req.pvR3 = NULL;
3859	Req.idChunkMap = NIL_GMM_CHUNKID;
3860	Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
3861	if (Req.idChunkUnmap != INT32_MAX)
3862	{
3863	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3864	rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
3865	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
3866	if (RT_SUCCESS(rc))
3867	{
3868	/*
3869	* Remove the unmapped one.
3870	*/
3871	PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
3872	AssertRelease(pUnmappedChunk);
3873	AssertRelease(!pUnmappedChunk->cRefs);
3874	AssertRelease(!pUnmappedChunk->cPermRefs);
3875	pUnmappedChunk->pv = NULL;
3876	pUnmappedChunk->Core.Key = UINT32_MAX;
3877	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3878	MMR3HeapFree(pUnmappedChunk);
3879	#else
3880	MMR3UkHeapFree(pVM, pUnmappedChunk, MM_TAG_PGM_CHUNK_MAPPING);
3881	#endif
3882	pVM->pgm.s.ChunkR3Map.c--;
3883	pVM->pgm.s.cUnmappedChunks++;
3884
3885	/*
3886	* Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses).
3887	*/
3888	/** todo: we should not flush chunks which include cr3 mappings. */
3889	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3890	{
3891	PPGMCPU pPGM = &pVM->aCpus[idCpu].pgm.s;
3892
3893	pPGM->pGst32BitPdR3 = NULL;
3894	pPGM->pGstPaePdptR3 = NULL;
3895	pPGM->pGstAmd64Pml4R3 = NULL;
3896	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3897	pPGM->pGst32BitPdR0 = NIL_RTR0PTR;
3898	pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
3899	pPGM->pGstAmd64Pml4R0 = NIL_RTR0PTR;
3900	#endif
3901	for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
3902	{
3903	pPGM->apGstPaePDsR3[i] = NULL;
3904	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3905	pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
3906	#endif
3907	}
3908
3909	/* Flush REM TLBs. */
3910	CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3911	}
3912
3913	/* Flush REM translation blocks. */
3914	REMFlushTBs(pVM);
3915	}
3916	}
3917	}
3918	pgmUnlock(pVM);
3919	return rc;
3920	}
3921
3922	/**
3923	* Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
3924	*
3925	* @returns VBox status code.
3926	* @param pVM The VM to operate on.
3927	*/
3928	void pgmR3PhysUnmapChunk(PVM pVM)
3929	{
3930	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
3931	AssertRC(rc);
3932	}
3933
3934
3935	/**
3936	* Maps the given chunk into the ring-3 mapping cache.
3937	*
3938	* This will call ring-0.
3939	*
3940	* @returns VBox status code.
3941	* @param pVM The VM handle.
3942	* @param idChunk The chunk in question.
3943	* @param ppChunk Where to store the chunk tracking structure.
3944	*
3945	* @remarks Called from within the PGM critical section.
3946	* @remarks Can be called from any thread!
3947	*/
3948	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
3949	{
3950	int rc;
3951
3952	PGM_LOCK_ASSERT_OWNER(pVM);
3953
3954	/*
3955	* Move the chunk time forward.
3956	*/
3957	pVM->pgm.s.ChunkR3Map.iNow++;
3958	if (pVM->pgm.s.ChunkR3Map.iNow == 0)
3959	{
3960	pVM->pgm.s.ChunkR3Map.iNow = 4;
3961	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkAgeingRolloverCallback, NULL);
3962	}
3963
3964	/*
3965	* Allocate a new tracking structure first.
3966	*/
3967	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3968	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
3969	#else
3970	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3UkHeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk), NULL);
3971	#endif
3972	AssertReturn(pChunk, VERR_NO_MEMORY);
3973	pChunk->Core.Key = idChunk;
3974	pChunk->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
3975
3976	/*
3977	* Request the ring-0 part to map the chunk in question.
3978	*/
3979	GMMMAPUNMAPCHUNKREQ Req;
3980	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3981	Req.Hdr.cbReq = sizeof(Req);
3982	Req.pvR3 = NULL;
3983	Req.idChunkMap = idChunk;
3984	Req.idChunkUnmap = NIL_GMM_CHUNKID;
3985
3986	/* Must be callable from any thread, so can't use VMMR3CallR0. */
3987	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
3988	rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
3989	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
3990	if (RT_SUCCESS(rc))
3991	{
3992	pChunk->pv = Req.pvR3;
3993
3994	/*
3995	* If we're running out of virtual address space, then we should
3996	* unmap another chunk.
3997	*
3998	* Currently, an unmap operation requires that all other virtual CPUs
3999	* are idling and not by chance making use of the memory we're
4000	* unmapping. So, we create an async unmap operation here.
4001	*
4002	* Now, when creating or restoring a saved state this wont work very
4003	* well since we may want to restore all guest RAM + a little something.
4004	* So, we have to do the unmap synchronously. Fortunately for us
4005	* though, during these operations the other virtual CPUs are inactive
4006	* and it should be safe to do this.
4007	*/
4008	/** @todo Eventually we should lock all memory when used and do
4009	* map+unmap as one kernel call without any rendezvous or
4010	* other precautions. */
4011	if (pVM->pgm.s.ChunkR3Map.c + 1 >= pVM->pgm.s.ChunkR3Map.cMax)
4012	{
4013	switch (VMR3GetState(pVM))
4014	{
4015	case VMSTATE_LOADING:
4016	case VMSTATE_SAVING:
4017	{
4018	PVMCPU pVCpu = VMMGetCpu(pVM);
4019	if ( pVCpu
4020	&& pVM->pgm.s.cDeprecatedPageLocks == 0)
4021	{
4022	pgmR3PhysUnmapChunkRendezvous(pVM, pVCpu, NULL);
4023	break;
4024	}
4025	/* fall thru */
4026	}
4027	default:
4028	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
4029	AssertRC(rc);
4030	break;
4031	}
4032	}
4033
4034	/*
4035	* Update the tree. We must do this after any unmapping to make sure
4036	* the chunk we're going to return isn't unmapped by accident.
4037	*/
4038	AssertPtr(Req.pvR3);
4039	bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
4040	AssertRelease(fRc);
4041	pVM->pgm.s.ChunkR3Map.c++;
4042	pVM->pgm.s.cMappedChunks++;
4043	}
4044	else
4045	{
4046	/** @todo this may fail because of /proc/sys/vm/max_map_count, so we
4047	* should probably restrict ourselves on linux. */
4048	AssertRC(rc);
4049	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4050	MMR3HeapFree(pChunk);
4051	#else
4052	MMR3UkHeapFree(pVM, pChunk, MM_TAG_PGM_CHUNK_MAPPING);
4053	#endif
4054	pChunk = NULL;
4055	}
4056
4057	*ppChunk = pChunk;
4058	return rc;
4059	}
4060
4061
4062	/**
4063	* For VMMCALLRING3_PGM_MAP_CHUNK, considered internal.
4064	*
4065	* @returns see pgmR3PhysChunkMap.
4066	* @param pVM The VM handle.
4067	* @param idChunk The chunk to map.
4068	*/
4069	VMMR3DECL(int) PGMR3PhysChunkMap(PVM pVM, uint32_t idChunk)
4070	{
4071	PPGMCHUNKR3MAP pChunk;
4072	int rc;
4073
4074	pgmLock(pVM);
4075	rc = pgmR3PhysChunkMap(pVM, idChunk, &pChunk);
4076	pgmUnlock(pVM);
4077	return rc;
4078	}
4079
4080
4081	/**
4082	* Invalidates the TLB for the ring-3 mapping cache.
4083	*
4084	* @param pVM The VM handle.
4085	*/
4086	VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
4087	{
4088	pgmLock(pVM);
4089	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
4090	{
4091	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
4092	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
4093	}
4094	/* The page map TLB references chunks, so invalidate that one too. */
4095	pgmPhysInvalidatePageMapTLB(pVM);
4096	pgmUnlock(pVM);
4097	}
4098
4099
4100	/**
4101	* Response to VMMCALLRING3_PGM_ALLOCATE_LARGE_PAGE to allocate a large (2MB) page
4102	* for use with a nested paging PDE.
4103	*
4104	* @returns The following VBox status codes.
4105	* @retval VINF_SUCCESS on success.
4106	* @retval VINF_EM_NO_MEMORY if we're out of memory.
4107	*
4108	* @param pVM The VM handle.
4109	* @param GCPhys GC physical start address of the 2 MB range
4110	*/
4111	VMMR3DECL(int) PGMR3PhysAllocateLargeHandyPage(PVM pVM, RTGCPHYS GCPhys)
4112	{
4113	#ifdef PGM_WITH_LARGE_PAGES
4114	uint64_t u64TimeStamp1, u64TimeStamp2;
4115
4116	pgmLock(pVM);
4117
4118	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4119	u64TimeStamp1 = RTTimeMilliTS();
4120	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE, 0, NULL);
4121	u64TimeStamp2 = RTTimeMilliTS();
4122	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4123	if (RT_SUCCESS(rc))
4124	{
4125	Assert(pVM->pgm.s.cLargeHandyPages == 1);
4126
4127	uint32_t idPage = pVM->pgm.s.aLargeHandyPage[0].idPage;
4128	RTHCPHYS HCPhys = pVM->pgm.s.aLargeHandyPage[0].HCPhysGCPhys;
4129
4130	void *pv;
4131
4132	/* Map the large page into our address space.
4133	*
4134	* Note: assuming that within the 2 MB range:
4135	* - GCPhys + PAGE_SIZE = HCPhys + PAGE_SIZE (whole point of this exercise)
4136	* - user space mapping is continuous as well
4137	* - page id (GCPhys) + 1 = page id (GCPhys + PAGE_SIZE)
4138	*/
4139	rc = pgmPhysPageMapByPageID(pVM, idPage, HCPhys, &pv);
4140	AssertLogRelMsg(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", idPage, HCPhys, rc));
4141
4142	if (RT_SUCCESS(rc))
4143	{
4144	/*
4145	* Clear the pages.
4146	*/
4147	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4148	for (unsigned i = 0; i < _2M/PAGE_SIZE; i++)
4149	{
4150	ASMMemZeroPage(pv);
4151
4152	PPGMPAGE pPage;
4153	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
4154	AssertRC(rc);
4155
4156	Assert(PGM_PAGE_IS_ZERO(pPage));
4157	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
4158	pVM->pgm.s.cZeroPages--;
4159
4160	/*
4161	* Do the PGMPAGE modifications.
4162	*/
4163	pVM->pgm.s.cPrivatePages++;
4164	PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
4165	PGM_PAGE_SET_PAGEID(pVM, pPage, idPage);
4166	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
4167	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE);
4168	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4169	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4170
4171	/* Somewhat dirty assumption that page ids are increasing. */
4172	idPage++;
4173
4174	HCPhys += PAGE_SIZE;
4175	GCPhys += PAGE_SIZE;
4176
4177	pv = (void *)((uintptr_t)pv + PAGE_SIZE);
4178
4179	Log3(("PGMR3PhysAllocateLargePage: idPage=%#x HCPhys=%RGp\n", idPage, HCPhys));
4180	}
4181	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4182
4183	/* Flush all TLBs. */
4184	PGM_INVL_ALL_VCPU_TLBS(pVM);
4185	pgmPhysInvalidatePageMapTLB(pVM);
4186	}
4187	pVM->pgm.s.cLargeHandyPages = 0;
4188	}
4189
4190	if (RT_SUCCESS(rc))
4191	{
4192	static uint32_t cTimeOut = 0;
4193	uint64_t u64TimeStampDelta = u64TimeStamp2 - u64TimeStamp1;
4194
4195	if (u64TimeStampDelta > 100)
4196	{
4197	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatLargePageOverflow);
4198	if ( ++cTimeOut > 10
4199	\|\| u64TimeStampDelta > 1000 /* more than one second forces an early retirement from allocating large pages. */)
4200	{
4201	/* If repeated attempts to allocate a large page takes more than 100 ms, then we fall back to normal 4k pages.
4202	* E.g. Vista 64 tries to move memory around, which takes a huge amount of time.
4203	*/
4204	LogRel(("PGMR3PhysAllocateLargePage: allocating large pages takes too long (last attempt %d ms; nr of timeouts %d); DISABLE\n", u64TimeStampDelta, cTimeOut));
4205	PGMSetLargePageUsage(pVM, false);
4206	}
4207	}
4208	else
4209	if (cTimeOut > 0)
4210	cTimeOut--;
4211	}
4212
4213	pgmUnlock(pVM);
4214	return rc;
4215	#else
4216	return VERR_NOT_IMPLEMENTED;
4217	#endif /* PGM_WITH_LARGE_PAGES */
4218	}
4219
4220
4221	/**
4222	* Response to VM_FF_PGM_NEED_HANDY_PAGES and VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES.
4223	*
4224	* This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
4225	* signal and clear the out of memory condition. When contracted, this API is
4226	* used to try clear the condition when the user wants to resume.
4227	*
4228	* @returns The following VBox status codes.
4229	* @retval VINF_SUCCESS on success. FFs cleared.
4230	* @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
4231	* this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
4232	*
4233	* @param pVM The VM handle.
4234	*
4235	* @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
4236	* in EM.cpp and shouldn't be propagated outside TRPM, HWACCM, EM and
4237	* pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
4238	* handler.
4239	*/
4240	VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
4241	{
4242	pgmLock(pVM);
4243
4244	/*
4245	* Allocate more pages, noting down the index of the first new page.
4246	*/
4247	uint32_t iClear = pVM->pgm.s.cHandyPages;
4248	AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_PGM_HANDY_PAGE_IPE);
4249	Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
4250	int rcAlloc = VINF_SUCCESS;
4251	int rcSeed = VINF_SUCCESS;
4252	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4253	while (rc == VERR_GMM_SEED_ME)
4254	{
4255	void *pvChunk;
4256	rcAlloc = rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
4257	if (RT_SUCCESS(rc))
4258	{
4259	rcSeed = rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
4260	if (RT_FAILURE(rc))
4261	SUPR3PageFree(pvChunk, GMM_CHUNK_SIZE >> PAGE_SHIFT);
4262	}
4263	if (RT_SUCCESS(rc))
4264	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4265	}
4266
4267	/* todo: we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
4268	if ( rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
4269	&& pVM->pgm.s.cHandyPages > 0)
4270	{
4271	/* Still handy pages left, so don't panic. */
4272	rc = VINF_SUCCESS;
4273	}
4274
4275	if (RT_SUCCESS(rc))
4276	{
4277	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
4278	Assert(pVM->pgm.s.cHandyPages > 0);
4279	VM_FF_CLEAR(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4280	VM_FF_CLEAR(pVM, VM_FF_PGM_NO_MEMORY);
4281
4282	#ifdef VBOX_STRICT
4283	uint32_t i;
4284	for (i = iClear; i < pVM->pgm.s.cHandyPages; i++)
4285	if ( pVM->pgm.s.aHandyPages[i].idPage == NIL_GMM_PAGEID
4286	\|\| pVM->pgm.s.aHandyPages[i].idSharedPage != NIL_GMM_PAGEID
4287	\|\| (pVM->pgm.s.aHandyPages[i].HCPhysGCPhys & PAGE_OFFSET_MASK))
4288	break;
4289	if (i != pVM->pgm.s.cHandyPages)
4290	{
4291	RTAssertMsg1Weak(NULL, __LINE__, __FILE__, __FUNCTION__);
4292	RTAssertMsg2Weak("i=%d iClear=%d cHandyPages=%d\n", i, iClear, pVM->pgm.s.cHandyPages);
4293	for (uint32_t j = iClear; j < pVM->pgm.s.cHandyPages; j++)
4294	RTAssertMsg2Add(("%03d: idPage=%d HCPhysGCPhys=%RHp idSharedPage=%d%\n", j,
4295	pVM->pgm.s.aHandyPages[j].idPage,
4296	pVM->pgm.s.aHandyPages[j].HCPhysGCPhys,
4297	pVM->pgm.s.aHandyPages[j].idSharedPage,
4298	j == i ? " <---" : ""));
4299	RTAssertPanic();
4300	}
4301	#endif
4302	/*
4303	* Clear the pages.
4304	*/
4305	while (iClear < pVM->pgm.s.cHandyPages)
4306	{
4307	PGMMPAGEDESC pPage = &pVM->pgm.s.aHandyPages[iClear];
4308	void *pv;
4309	rc = pgmPhysPageMapByPageID(pVM, pPage->idPage, pPage->HCPhysGCPhys, &pv);
4310	AssertLogRelMsgBreak(RT_SUCCESS(rc),
4311	("%u/%u: idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n",
4312	iClear, pVM->pgm.s.cHandyPages, pPage->idPage, pPage->HCPhysGCPhys, rc));
4313	ASMMemZeroPage(pv);
4314	iClear++;
4315	Log3(("PGMR3PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
4316	}
4317	}
4318	else
4319	{
4320	uint64_t cAllocPages, cMaxPages, cBalloonPages;
4321
4322	/*
4323	* We should never get here unless there is a genuine shortage of
4324	* memory (or some internal error). Flag the error so the VM can be
4325	* suspended ASAP and the user informed. If we're totally out of
4326	* handy pages we will return failure.
4327	*/
4328	/* Report the failure. */
4329	LogRel(("PGM: Failed to procure handy pages; rc=%Rrc rcAlloc=%Rrc rcSeed=%Rrc cHandyPages=%#x\n"
4330	" cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
4331	rc, rcAlloc, rcSeed,
4332	pVM->pgm.s.cHandyPages,
4333	pVM->pgm.s.cAllPages,
4334	pVM->pgm.s.cPrivatePages,
4335	pVM->pgm.s.cSharedPages,
4336	pVM->pgm.s.cZeroPages));
4337
4338	if (GMMR3QueryMemoryStats(pVM, &cAllocPages, &cMaxPages, &cBalloonPages) == VINF_SUCCESS)
4339	{
4340	LogRel(("GMM: Statistics:\n"
4341	" Allocated pages: %RX64\n"
4342	" Maximum pages: %RX64\n"
4343	" Ballooned pages: %RX64\n", cAllocPages, cMaxPages, cBalloonPages));
4344	}
4345
4346	if ( rc != VERR_NO_MEMORY
4347	&& rc != VERR_LOCK_FAILED)
4348	{
4349	for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4350	{
4351	LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
4352	i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
4353	pVM->pgm.s.aHandyPages[i].idSharedPage));
4354	uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
4355	if (idPage != NIL_GMM_PAGEID)
4356	{
4357	for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
4358	pRam;
4359	pRam = pRam->pNextR3)
4360	{
4361	uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
4362	for (uint32_t iPage = 0; iPage < cPages; iPage++)
4363	if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
4364	LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
4365	pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
4366	}
4367	}
4368	}
4369	}
4370
4371	/* Set the FFs and adjust rc. */
4372	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4373	VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
4374	if ( rc == VERR_NO_MEMORY
4375	\|\| rc == VERR_LOCK_FAILED)
4376	rc = VINF_EM_NO_MEMORY;
4377	}
4378
4379	pgmUnlock(pVM);
4380	return rc;
4381	}
4382
4383
4384	/**
4385	* Frees the specified RAM page and replaces it with the ZERO page.
4386	*
4387	* This is used by ballooning, remapping MMIO2, RAM reset and state loading.
4388	*
4389	* @param pVM Pointer to the shared VM structure.
4390	* @param pReq Pointer to the request.
4391	* @param pcPendingPages Where the number of pages waiting to be freed are
4392	* kept. This will normally be incremented.
4393	* @param pPage Pointer to the page structure.
4394	* @param GCPhys The guest physical address of the page, if applicable.
4395	*
4396	* @remarks The caller must own the PGM lock.
4397	*/
4398	int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys)
4399	{
4400	/*
4401	* Assert sanity.
4402	*/
4403	PGM_LOCK_ASSERT_OWNER(pVM);
4404	if (RT_UNLIKELY( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
4405	&& PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
4406	{
4407	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4408	return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
4409	}
4410
4411	/** @todo What about ballooning of large pages??! */
4412	Assert( PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
4413	&& PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED);
4414
4415	if ( PGM_PAGE_IS_ZERO(pPage)
4416	\|\| PGM_PAGE_IS_BALLOONED(pPage))
4417	return VINF_SUCCESS;
4418
4419	const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
4420	Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
4421	if (RT_UNLIKELY( idPage == NIL_GMM_PAGEID
4422	\|\| idPage > GMM_PAGEID_LAST
4423	\|\| PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
4424	{
4425	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4426	return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
4427	}
4428
4429	/* update page count stats. */
4430	if (PGM_PAGE_IS_SHARED(pPage))
4431	pVM->pgm.s.cSharedPages--;
4432	else
4433	pVM->pgm.s.cPrivatePages--;
4434	pVM->pgm.s.cZeroPages++;
4435
4436	/* Deal with write monitored pages. */
4437	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
4438	{
4439	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
4440	pVM->pgm.s.cWrittenToPages++;
4441	}
4442
4443	/*
4444	* pPage = ZERO page.
4445	*/
4446	PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg);
4447	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
4448	PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
4449	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4450	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4451	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4452
4453	/* Flush physical page map TLB entry. */
4454	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
4455
4456	/*
4457	* Make sure it's not in the handy page array.
4458	*/
4459	for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4460	{
4461	if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
4462	{
4463	pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
4464	break;
4465	}
4466	if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
4467	{
4468	pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
4469	break;
4470	}
4471	}
4472
4473	/*
4474	* Push it onto the page array.
4475	*/
4476	uint32_t iPage = *pcPendingPages;
4477	Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
4478	*pcPendingPages += 1;
4479
4480	pReq->aPages[iPage].idPage = idPage;
4481
4482	if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
4483	return VINF_SUCCESS;
4484
4485	/*
4486	* Flush the pages.
4487	*/
4488	int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
4489	if (RT_SUCCESS(rc))
4490	{
4491	GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
4492	*pcPendingPages = 0;
4493	}
4494	return rc;
4495	}
4496
4497
4498	/**
4499	* Converts a GC physical address to a HC ring-3 pointer, with some
4500	* additional checks.
4501	*
4502	* @returns VBox status code.
4503	* @retval VINF_SUCCESS on success.
4504	* @retval VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4505	* access handler of some kind.
4506	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4507	* accesses or is odd in any way.
4508	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4509	*
4510	* @param pVM The VM handle.
4511	* @param GCPhys The GC physical address to convert.
4512	* @param fWritable Whether write access is required.
4513	* @param ppv Where to store the pointer corresponding to GCPhys on
4514	* success.
4515	*/
4516	VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
4517	{
4518	pgmLock(pVM);
4519
4520	PPGMRAMRANGE pRam;
4521	PPGMPAGE pPage;
4522	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4523	if (RT_SUCCESS(rc))
4524	{
4525	if (PGM_PAGE_IS_BALLOONED(pPage))
4526	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4527	else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
4528	rc = VINF_SUCCESS;
4529	else
4530	{
4531	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4532	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4533	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
4534	{
4535	/** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
4536	* in -norawr0 mode. */
4537	if (fWritable)
4538	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4539	}
4540	else
4541	{
4542	/* Temporarily disabled physical handler(s), since the recompiler
4543	doesn't get notified when it's reset we'll have to pretend it's
4544	operating normally. */
4545	if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
4546	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4547	else
4548	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4549	}
4550	}
4551	if (RT_SUCCESS(rc))
4552	{
4553	int rc2;
4554
4555	/* Make sure what we return is writable. */
4556	if (fWritable)
4557	switch (PGM_PAGE_GET_STATE(pPage))
4558	{
4559	case PGM_PAGE_STATE_ALLOCATED:
4560	break;
4561	case PGM_PAGE_STATE_BALLOONED:
4562	AssertFailed();
4563	break;
4564	case PGM_PAGE_STATE_ZERO:
4565	case PGM_PAGE_STATE_SHARED:
4566	if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
4567	break;
4568	case PGM_PAGE_STATE_WRITE_MONITORED:
4569	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
4570	AssertLogRelRCReturn(rc2, rc2);
4571	break;
4572	}
4573
4574	/* Get a ring-3 mapping of the address. */
4575	PPGMPAGER3MAPTLBE pTlbe;
4576	rc2 = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
4577	AssertLogRelRCReturn(rc2, rc2);
4578	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4579	/** @todo mapping/locking hell; this isn't horribly efficient since
4580	* pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
4581
4582	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
4583	}
4584	else
4585	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
4586
4587	/* else: handler catching all access, no pointer returned. */
4588	}
4589	else
4590	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
4591
4592	pgmUnlock(pVM);
4593	return rc;
4594	}
4595

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp@ 39427

Download in other formats: