VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp@ 53414

Last change on this file since 53414 was 52692, checked in by vboxsync, 10 years ago

VMM: properly restore A20 from the saved state (fixes potential misbehavior after restoring a VM from a saved state with A20 disabled)

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 169.8 KB
Line 
1/* $Id: PGMPhys.cpp 52692 2014-09-11 11:46:59Z vboxsync $ */
2/** @file
3 * PGM - Page Manager and Monitor, Physical Memory Addressing.
4 */
5
6/*
7 * Copyright (C) 2006-2014 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_PGM_PHYS
23#include <VBox/vmm/pgm.h>
24#include <VBox/vmm/iom.h>
25#include <VBox/vmm/mm.h>
26#include <VBox/vmm/stam.h>
27#ifdef VBOX_WITH_REM
28# include <VBox/vmm/rem.h>
29#endif
30#include <VBox/vmm/pdmdev.h>
31#include "PGMInternal.h"
32#include <VBox/vmm/vm.h>
33#include <VBox/vmm/uvm.h>
34#include "PGMInline.h"
35#include <VBox/sup.h>
36#include <VBox/param.h>
37#include <VBox/err.h>
38#include <VBox/log.h>
39#include <iprt/assert.h>
40#include <iprt/alloc.h>
41#include <iprt/asm.h>
42#ifdef VBOX_STRICT
43# include <iprt/crc.h>
44#endif
45#include <iprt/thread.h>
46#include <iprt/string.h>
47#include <iprt/system.h>
48
49
50/*******************************************************************************
51* Defined Constants And Macros *
52*******************************************************************************/
53/** The number of pages to free in one batch. */
54#define PGMPHYS_FREE_PAGE_BATCH_SIZE 128
55
56
57/*******************************************************************************
58* Internal Functions *
59*******************************************************************************/
60static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
61
62
63/*
64 * PGMR3PhysReadU8-64
65 * PGMR3PhysWriteU8-64
66 */
67#define PGMPHYSFN_READNAME PGMR3PhysReadU8
68#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
69#define PGMPHYS_DATASIZE 1
70#define PGMPHYS_DATATYPE uint8_t
71#include "PGMPhysRWTmpl.h"
72
73#define PGMPHYSFN_READNAME PGMR3PhysReadU16
74#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
75#define PGMPHYS_DATASIZE 2
76#define PGMPHYS_DATATYPE uint16_t
77#include "PGMPhysRWTmpl.h"
78
79#define PGMPHYSFN_READNAME PGMR3PhysReadU32
80#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
81#define PGMPHYS_DATASIZE 4
82#define PGMPHYS_DATATYPE uint32_t
83#include "PGMPhysRWTmpl.h"
84
85#define PGMPHYSFN_READNAME PGMR3PhysReadU64
86#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
87#define PGMPHYS_DATASIZE 8
88#define PGMPHYS_DATATYPE uint64_t
89#include "PGMPhysRWTmpl.h"
90
91
92/**
93 * EMT worker for PGMR3PhysReadExternal.
94 */
95static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead)
96{
97 PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead);
98 return VINF_SUCCESS;
99}
100
101
102/**
103 * Read from physical memory, external users.
104 *
105 * @returns VBox status code.
106 * @retval VINF_SUCCESS.
107 *
108 * @param pVM Pointer to the VM.
109 * @param GCPhys Physical address to read from.
110 * @param pvBuf Where to read into.
111 * @param cbRead How many bytes to read.
112 *
113 * @thread Any but EMTs.
114 */
115VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
116{
117 VM_ASSERT_OTHER_THREAD(pVM);
118
119 AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
120 LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
121
122 pgmLock(pVM);
123
124 /*
125 * Copy loop on ram ranges.
126 */
127 PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
128 for (;;)
129 {
130 /* Inside range or not? */
131 if (pRam && GCPhys >= pRam->GCPhys)
132 {
133 /*
134 * Must work our way thru this page by page.
135 */
136 RTGCPHYS off = GCPhys - pRam->GCPhys;
137 while (off < pRam->cb)
138 {
139 unsigned iPage = off >> PAGE_SHIFT;
140 PPGMPAGE pPage = &pRam->aPages[iPage];
141
142 /*
143 * If the page has an ALL access handler, we'll have to
144 * delegate the job to EMT.
145 */
146 if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
147 || PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
148 {
149 pgmUnlock(pVM);
150
151 return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 4,
152 pVM, &GCPhys, pvBuf, cbRead);
153 }
154 Assert(!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage));
155
156 /*
157 * Simple stuff, go ahead.
158 */
159 size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
160 if (cb > cbRead)
161 cb = cbRead;
162 PGMPAGEMAPLOCK PgMpLck;
163 const void *pvSrc;
164 int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
165 if (RT_SUCCESS(rc))
166 {
167 memcpy(pvBuf, pvSrc, cb);
168 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
169 }
170 else
171 {
172 AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
173 pRam->GCPhys + off, pPage, rc));
174 memset(pvBuf, 0xff, cb);
175 }
176
177 /* next page */
178 if (cb >= cbRead)
179 {
180 pgmUnlock(pVM);
181 return VINF_SUCCESS;
182 }
183 cbRead -= cb;
184 off += cb;
185 GCPhys += cb;
186 pvBuf = (char *)pvBuf + cb;
187 } /* walk pages in ram range. */
188 }
189 else
190 {
191 LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
192
193 /*
194 * Unassigned address space.
195 */
196 size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
197 if (cb >= cbRead)
198 {
199 memset(pvBuf, 0xff, cbRead);
200 break;
201 }
202 memset(pvBuf, 0xff, cb);
203
204 cbRead -= cb;
205 pvBuf = (char *)pvBuf + cb;
206 GCPhys += cb;
207 }
208
209 /* Advance range if necessary. */
210 while (pRam && GCPhys > pRam->GCPhysLast)
211 pRam = pRam->CTX_SUFF(pNext);
212 } /* Ram range walk */
213
214 pgmUnlock(pVM);
215
216 return VINF_SUCCESS;
217}
218
219
220/**
221 * EMT worker for PGMR3PhysWriteExternal.
222 */
223static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite)
224{
225 /** @todo VERR_EM_NO_MEMORY */
226 PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite);
227 return VINF_SUCCESS;
228}
229
230
231/**
232 * Write to physical memory, external users.
233 *
234 * @returns VBox status code.
235 * @retval VINF_SUCCESS.
236 * @retval VERR_EM_NO_MEMORY.
237 *
238 * @param pVM Pointer to the VM.
239 * @param GCPhys Physical address to write to.
240 * @param pvBuf What to write.
241 * @param cbWrite How many bytes to write.
242 * @param pszWho Who is writing. For tracking down who is writing
243 * after we've saved the state.
244 *
245 * @thread Any but EMTs.
246 */
247VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, const char *pszWho)
248{
249 VM_ASSERT_OTHER_THREAD(pVM);
250
251 AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
252 ("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x pszWho=%s\n",
253 GCPhys, cbWrite, pszWho));
254 AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
255 LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
256
257 pgmLock(pVM);
258
259 /*
260 * Copy loop on ram ranges, stop when we hit something difficult.
261 */
262 PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
263 for (;;)
264 {
265 /* Inside range or not? */
266 if (pRam && GCPhys >= pRam->GCPhys)
267 {
268 /*
269 * Must work our way thru this page by page.
270 */
271 RTGCPTR off = GCPhys - pRam->GCPhys;
272 while (off < pRam->cb)
273 {
274 RTGCPTR iPage = off >> PAGE_SHIFT;
275 PPGMPAGE pPage = &pRam->aPages[iPage];
276
277 /*
278 * Is the page problematic, we have to do the work on the EMT.
279 *
280 * Allocating writable pages and access handlers are
281 * problematic, write monitored pages are simple and can be
282 * dealt with here.
283 */
284 if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
285 || PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
286 || PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
287 {
288 if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
289 && !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
290 pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
291 else
292 {
293 pgmUnlock(pVM);
294
295 return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 4,
296 pVM, &GCPhys, pvBuf, cbWrite);
297 }
298 }
299 Assert(!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage));
300
301 /*
302 * Simple stuff, go ahead.
303 */
304 size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
305 if (cb > cbWrite)
306 cb = cbWrite;
307 PGMPAGEMAPLOCK PgMpLck;
308 void *pvDst;
309 int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
310 if (RT_SUCCESS(rc))
311 {
312 memcpy(pvDst, pvBuf, cb);
313 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
314 }
315 else
316 AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
317 pRam->GCPhys + off, pPage, rc));
318
319 /* next page */
320 if (cb >= cbWrite)
321 {
322 pgmUnlock(pVM);
323 return VINF_SUCCESS;
324 }
325
326 cbWrite -= cb;
327 off += cb;
328 GCPhys += cb;
329 pvBuf = (const char *)pvBuf + cb;
330 } /* walk pages in ram range */
331 }
332 else
333 {
334 /*
335 * Unassigned address space, skip it.
336 */
337 if (!pRam)
338 break;
339 size_t cb = pRam->GCPhys - GCPhys;
340 if (cb >= cbWrite)
341 break;
342 cbWrite -= cb;
343 pvBuf = (const char *)pvBuf + cb;
344 GCPhys += cb;
345 }
346
347 /* Advance range if necessary. */
348 while (pRam && GCPhys > pRam->GCPhysLast)
349 pRam = pRam->CTX_SUFF(pNext);
350 } /* Ram range walk */
351
352 pgmUnlock(pVM);
353 return VINF_SUCCESS;
354}
355
356
357/**
358 * VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
359 *
360 * @returns see PGMR3PhysGCPhys2CCPtrExternal
361 * @param pVM Pointer to the VM.
362 * @param pGCPhys Pointer to the guest physical address.
363 * @param ppv Where to store the mapping address.
364 * @param pLock Where to store the lock.
365 */
366static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
367{
368 /*
369 * Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
370 * an access handler after it succeeds.
371 */
372 int rc = pgmLock(pVM);
373 AssertRCReturn(rc, rc);
374
375 rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
376 if (RT_SUCCESS(rc))
377 {
378 PPGMPAGEMAPTLBE pTlbe;
379 int rc2 = pgmPhysPageQueryTlbe(pVM, *pGCPhys, &pTlbe);
380 AssertFatalRC(rc2);
381 PPGMPAGE pPage = pTlbe->pPage;
382 if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
383 {
384 PGMPhysReleasePageMappingLock(pVM, pLock);
385 rc = VERR_PGM_PHYS_PAGE_RESERVED;
386 }
387 else if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
388#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
389 || pgmPoolIsDirtyPage(pVM, *pGCPhys)
390#endif
391 )
392 {
393 /* We *must* flush any corresponding pgm pool page here, otherwise we'll
394 * not be informed about writes and keep bogus gst->shw mappings around.
395 */
396 pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
397 Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
398 /** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
399 * active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
400 }
401 }
402
403 pgmUnlock(pVM);
404 return rc;
405}
406
407
408/**
409 * Requests the mapping of a guest page into ring-3, external threads.
410 *
411 * When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
412 * release it.
413 *
414 * This API will assume your intention is to write to the page, and will
415 * therefore replace shared and zero pages. If you do not intend to modify the
416 * page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
417 *
418 * @returns VBox status code.
419 * @retval VINF_SUCCESS on success.
420 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
421 * backing or if the page has any active access handlers. The caller
422 * must fall back on using PGMR3PhysWriteExternal.
423 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
424 *
425 * @param pVM Pointer to the VM.
426 * @param GCPhys The guest physical address of the page that should be mapped.
427 * @param ppv Where to store the address corresponding to GCPhys.
428 * @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
429 *
430 * @remark Avoid calling this API from within critical sections (other than the
431 * PGM one) because of the deadlock risk when we have to delegating the
432 * task to an EMT.
433 * @thread Any.
434 */
435VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
436{
437 AssertPtr(ppv);
438 AssertPtr(pLock);
439
440 Assert(VM_IS_EMT(pVM) || !PGMIsLockOwner(pVM));
441
442 int rc = pgmLock(pVM);
443 AssertRCReturn(rc, rc);
444
445 /*
446 * Query the Physical TLB entry for the page (may fail).
447 */
448 PPGMPAGEMAPTLBE pTlbe;
449 rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
450 if (RT_SUCCESS(rc))
451 {
452 PPGMPAGE pPage = pTlbe->pPage;
453 if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
454 rc = VERR_PGM_PHYS_PAGE_RESERVED;
455 else
456 {
457 /*
458 * If the page is shared, the zero page, or being write monitored
459 * it must be converted to an page that's writable if possible.
460 * We can only deal with write monitored pages here, the rest have
461 * to be on an EMT.
462 */
463 if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
464 || PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
465#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
466 || pgmPoolIsDirtyPage(pVM, GCPhys)
467#endif
468 )
469 {
470 if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
471 && !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
472#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
473 && !pgmPoolIsDirtyPage(pVM, GCPhys)
474#endif
475 )
476 pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
477 else
478 {
479 pgmUnlock(pVM);
480
481 return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
482 pVM, &GCPhys, ppv, pLock);
483 }
484 }
485
486 /*
487 * Now, just perform the locking and calculate the return address.
488 */
489 PPGMPAGEMAP pMap = pTlbe->pMap;
490 if (pMap)
491 pMap->cRefs++;
492
493 unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
494 if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
495 {
496 if (cLocks == 0)
497 pVM->pgm.s.cWriteLockedPages++;
498 PGM_PAGE_INC_WRITE_LOCKS(pPage);
499 }
500 else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
501 {
502 PGM_PAGE_INC_WRITE_LOCKS(pPage);
503 AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
504 if (pMap)
505 pMap->cRefs++; /* Extra ref to prevent it from going away. */
506 }
507
508 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
509 pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_WRITE;
510 pLock->pvMap = pMap;
511 }
512 }
513
514 pgmUnlock(pVM);
515 return rc;
516}
517
518
519/**
520 * Requests the mapping of a guest page into ring-3, external threads.
521 *
522 * When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
523 * release it.
524 *
525 * @returns VBox status code.
526 * @retval VINF_SUCCESS on success.
527 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
528 * backing or if the page as an active ALL access handler. The caller
529 * must fall back on using PGMPhysRead.
530 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
531 *
532 * @param pVM Pointer to the VM.
533 * @param GCPhys The guest physical address of the page that should be mapped.
534 * @param ppv Where to store the address corresponding to GCPhys.
535 * @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
536 *
537 * @remark Avoid calling this API from within critical sections (other than
538 * the PGM one) because of the deadlock risk.
539 * @thread Any.
540 */
541VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
542{
543 int rc = pgmLock(pVM);
544 AssertRCReturn(rc, rc);
545
546 /*
547 * Query the Physical TLB entry for the page (may fail).
548 */
549 PPGMPAGEMAPTLBE pTlbe;
550 rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
551 if (RT_SUCCESS(rc))
552 {
553 PPGMPAGE pPage = pTlbe->pPage;
554#if 1
555 /* MMIO pages doesn't have any readable backing. */
556 if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
557 rc = VERR_PGM_PHYS_PAGE_RESERVED;
558#else
559 if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
560 rc = VERR_PGM_PHYS_PAGE_RESERVED;
561#endif
562 else
563 {
564 /*
565 * Now, just perform the locking and calculate the return address.
566 */
567 PPGMPAGEMAP pMap = pTlbe->pMap;
568 if (pMap)
569 pMap->cRefs++;
570
571 unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
572 if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
573 {
574 if (cLocks == 0)
575 pVM->pgm.s.cReadLockedPages++;
576 PGM_PAGE_INC_READ_LOCKS(pPage);
577 }
578 else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
579 {
580 PGM_PAGE_INC_READ_LOCKS(pPage);
581 AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
582 if (pMap)
583 pMap->cRefs++; /* Extra ref to prevent it from going away. */
584 }
585
586 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
587 pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_READ;
588 pLock->pvMap = pMap;
589 }
590 }
591
592 pgmUnlock(pVM);
593 return rc;
594}
595
596
597#define MAKE_LEAF(a_pNode) \
598 do { \
599 (a_pNode)->pLeftR3 = NIL_RTR3PTR; \
600 (a_pNode)->pRightR3 = NIL_RTR3PTR; \
601 (a_pNode)->pLeftR0 = NIL_RTR0PTR; \
602 (a_pNode)->pRightR0 = NIL_RTR0PTR; \
603 (a_pNode)->pLeftRC = NIL_RTRCPTR; \
604 (a_pNode)->pRightRC = NIL_RTRCPTR; \
605 } while (0)
606
607#define INSERT_LEFT(a_pParent, a_pNode) \
608 do { \
609 (a_pParent)->pLeftR3 = (a_pNode); \
610 (a_pParent)->pLeftR0 = (a_pNode)->pSelfR0; \
611 (a_pParent)->pLeftRC = (a_pNode)->pSelfRC; \
612 } while (0)
613#define INSERT_RIGHT(a_pParent, a_pNode) \
614 do { \
615 (a_pParent)->pRightR3 = (a_pNode); \
616 (a_pParent)->pRightR0 = (a_pNode)->pSelfR0; \
617 (a_pParent)->pRightRC = (a_pNode)->pSelfRC; \
618 } while (0)
619
620
621/**
622 * Recursive tree builder.
623 *
624 * @param ppRam Pointer to the iterator variable.
625 * @param iHeight The hight about normal leaf nodes. Inserts a leaf
626 * node if 0.
627 */
628static PPGMRAMRANGE pgmR3PhysRebuildRamRangeSearchTreesRecursively(PPGMRAMRANGE *ppRam, int iDepth)
629{
630 PPGMRAMRANGE pRam;
631 if (iDepth <= 0)
632 {
633 /*
634 * Leaf node.
635 */
636 pRam = *ppRam;
637 if (pRam)
638 {
639 *ppRam = pRam->pNextR3;
640 MAKE_LEAF(pRam);
641 }
642 }
643 else
644 {
645
646 /*
647 * Intermediate node.
648 */
649 PPGMRAMRANGE pLeft = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
650
651 pRam = *ppRam;
652 if (!pRam)
653 return pLeft;
654 *ppRam = pRam->pNextR3;
655 MAKE_LEAF(pRam);
656 INSERT_LEFT(pRam, pLeft);
657
658 PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(ppRam, iDepth - 1);
659 if (pRight)
660 INSERT_RIGHT(pRam, pRight);
661 }
662 return pRam;
663}
664
665
666/**
667 * Rebuilds the RAM range search trees.
668 *
669 * @param pVM Pointer to the VM.
670 */
671static void pgmR3PhysRebuildRamRangeSearchTrees(PVM pVM)
672{
673
674 /*
675 * Create the reasonably balanced tree in a sequential fashion.
676 * For simplicity (laziness) we use standard recursion here.
677 */
678 int iDepth = 0;
679 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
680 PPGMRAMRANGE pRoot = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, 0);
681 while (pRam)
682 {
683 PPGMRAMRANGE pLeft = pRoot;
684
685 pRoot = pRam;
686 pRam = pRam->pNextR3;
687 MAKE_LEAF(pRoot);
688 INSERT_LEFT(pRoot, pLeft);
689
690 PPGMRAMRANGE pRight = pgmR3PhysRebuildRamRangeSearchTreesRecursively(&pRam, iDepth);
691 if (pRight)
692 INSERT_RIGHT(pRoot, pRight);
693 /** @todo else: rotate the tree. */
694
695 iDepth++;
696 }
697
698 pVM->pgm.s.pRamRangeTreeR3 = pRoot;
699 pVM->pgm.s.pRamRangeTreeR0 = pRoot ? pRoot->pSelfR0 : NIL_RTR0PTR;
700 pVM->pgm.s.pRamRangeTreeRC = pRoot ? pRoot->pSelfRC : NIL_RTRCPTR;
701
702#ifdef VBOX_STRICT
703 /*
704 * Verify that the above code works.
705 */
706 unsigned cRanges = 0;
707 for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
708 cRanges++;
709 Assert(cRanges > 0);
710
711 unsigned cMaxDepth = ASMBitLastSetU32(cRanges);
712 if ((1U << cMaxDepth) < cRanges)
713 cMaxDepth++;
714
715 for (pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
716 {
717 unsigned cDepth = 0;
718 PPGMRAMRANGE pRam2 = pVM->pgm.s.pRamRangeTreeR3;
719 for (;;)
720 {
721 if (pRam == pRam2)
722 break;
723 Assert(pRam2);
724 if (pRam->GCPhys < pRam2->GCPhys)
725 pRam2 = pRam2->pLeftR3;
726 else
727 pRam2 = pRam2->pRightR3;
728 }
729 AssertMsg(cDepth <= cMaxDepth, ("cDepth=%d cMaxDepth=%d\n", cDepth, cMaxDepth));
730 }
731#endif /* VBOX_STRICT */
732}
733
734#undef MAKE_LEAF
735#undef INSERT_LEFT
736#undef INSERT_RIGHT
737
738/**
739 * Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
740 *
741 * Called when anything was relocated.
742 *
743 * @param pVM Pointer to the VM.
744 */
745void pgmR3PhysRelinkRamRanges(PVM pVM)
746{
747 PPGMRAMRANGE pCur;
748
749#ifdef VBOX_STRICT
750 for (pCur = pVM->pgm.s.pRamRangesXR3; pCur; pCur = pCur->pNextR3)
751 {
752 Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
753 Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pCur->pSelfRC == MMHyperCCToRC(pVM, pCur));
754 Assert((pCur->GCPhys & PAGE_OFFSET_MASK) == 0);
755 Assert((pCur->GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
756 Assert((pCur->cb & PAGE_OFFSET_MASK) == 0);
757 Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
758 for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesXR3; pCur2; pCur2 = pCur2->pNextR3)
759 Assert( pCur2 == pCur
760 || strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
761 }
762#endif
763
764 pCur = pVM->pgm.s.pRamRangesXR3;
765 if (pCur)
766 {
767 pVM->pgm.s.pRamRangesXR0 = pCur->pSelfR0;
768 pVM->pgm.s.pRamRangesXRC = pCur->pSelfRC;
769
770 for (; pCur->pNextR3; pCur = pCur->pNextR3)
771 {
772 pCur->pNextR0 = pCur->pNextR3->pSelfR0;
773 pCur->pNextRC = pCur->pNextR3->pSelfRC;
774 }
775
776 Assert(pCur->pNextR0 == NIL_RTR0PTR);
777 Assert(pCur->pNextRC == NIL_RTRCPTR);
778 }
779 else
780 {
781 Assert(pVM->pgm.s.pRamRangesXR0 == NIL_RTR0PTR);
782 Assert(pVM->pgm.s.pRamRangesXRC == NIL_RTRCPTR);
783 }
784 ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
785
786 pgmR3PhysRebuildRamRangeSearchTrees(pVM);
787}
788
789
790/**
791 * Links a new RAM range into the list.
792 *
793 * @param pVM Pointer to the VM.
794 * @param pNew Pointer to the new list entry.
795 * @param pPrev Pointer to the previous list entry. If NULL, insert as head.
796 */
797static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
798{
799 AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
800 Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pNew->pSelfR0 == MMHyperCCToR0(pVM, pNew));
801 Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pNew->pSelfRC == MMHyperCCToRC(pVM, pNew));
802
803 pgmLock(pVM);
804
805 PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesXR3;
806 pNew->pNextR3 = pRam;
807 pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
808 pNew->pNextRC = pRam ? pRam->pSelfRC : NIL_RTRCPTR;
809
810 if (pPrev)
811 {
812 pPrev->pNextR3 = pNew;
813 pPrev->pNextR0 = pNew->pSelfR0;
814 pPrev->pNextRC = pNew->pSelfRC;
815 }
816 else
817 {
818 pVM->pgm.s.pRamRangesXR3 = pNew;
819 pVM->pgm.s.pRamRangesXR0 = pNew->pSelfR0;
820 pVM->pgm.s.pRamRangesXRC = pNew->pSelfRC;
821 }
822 ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
823
824 pgmR3PhysRebuildRamRangeSearchTrees(pVM);
825 pgmUnlock(pVM);
826}
827
828
829/**
830 * Unlink an existing RAM range from the list.
831 *
832 * @param pVM Pointer to the VM.
833 * @param pRam Pointer to the new list entry.
834 * @param pPrev Pointer to the previous list entry. If NULL, insert as head.
835 */
836static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
837{
838 Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesXR3 == pRam);
839 Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
840 Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pRam->pSelfRC == MMHyperCCToRC(pVM, pRam));
841
842 pgmLock(pVM);
843
844 PPGMRAMRANGE pNext = pRam->pNextR3;
845 if (pPrev)
846 {
847 pPrev->pNextR3 = pNext;
848 pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
849 pPrev->pNextRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
850 }
851 else
852 {
853 Assert(pVM->pgm.s.pRamRangesXR3 == pRam);
854 pVM->pgm.s.pRamRangesXR3 = pNext;
855 pVM->pgm.s.pRamRangesXR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
856 pVM->pgm.s.pRamRangesXRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
857 }
858 ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
859
860 pgmR3PhysRebuildRamRangeSearchTrees(pVM);
861 pgmUnlock(pVM);
862}
863
864
865/**
866 * Unlink an existing RAM range from the list.
867 *
868 * @param pVM Pointer to the VM.
869 * @param pRam Pointer to the new list entry.
870 */
871static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
872{
873 pgmLock(pVM);
874
875 /* find prev. */
876 PPGMRAMRANGE pPrev = NULL;
877 PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesXR3;
878 while (pCur != pRam)
879 {
880 pPrev = pCur;
881 pCur = pCur->pNextR3;
882 }
883 AssertFatal(pCur);
884
885 pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
886 pgmUnlock(pVM);
887}
888
889
890/**
891 * Frees a range of pages, replacing them with ZERO pages of the specified type.
892 *
893 * @returns VBox status code.
894 * @param pVM Pointer to the VM.
895 * @param pRam The RAM range in which the pages resides.
896 * @param GCPhys The address of the first page.
897 * @param GCPhysLast The address of the last page.
898 * @param uType The page type to replace then with.
899 */
900static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint8_t uType)
901{
902 PGM_LOCK_ASSERT_OWNER(pVM);
903 uint32_t cPendingPages = 0;
904 PGMMFREEPAGESREQ pReq;
905 int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
906 AssertLogRelRCReturn(rc, rc);
907
908 /* Iterate the pages. */
909 PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
910 uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
911 while (cPagesLeft-- > 0)
912 {
913 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
914 AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
915
916 PGM_PAGE_SET_TYPE(pVM, pPageDst, uType);
917
918 GCPhys += PAGE_SIZE;
919 pPageDst++;
920 }
921
922 if (cPendingPages)
923 {
924 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
925 AssertLogRelRCReturn(rc, rc);
926 }
927 GMMR3FreePagesCleanup(pReq);
928
929 return rc;
930}
931
932#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) || defined(RT_OS_SOLARIS) || defined(RT_OS_LINUX) || defined(RT_OS_FREEBSD))
933
934/**
935 * Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
936 *
937 * This is only called on one of the EMTs while the other ones are waiting for
938 * it to complete this function.
939 *
940 * @returns VINF_SUCCESS (VBox strict status code).
941 * @param pVM Pointer to the VM.
942 * @param pVCpu The VMCPU for the EMT we're being called on. Unused.
943 * @param pvUser User parameter
944 */
945static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
946{
947 uintptr_t *paUser = (uintptr_t *)pvUser;
948 bool fInflate = !!paUser[0];
949 unsigned cPages = paUser[1];
950 RTGCPHYS *paPhysPage = (RTGCPHYS *)paUser[2];
951 uint32_t cPendingPages = 0;
952 PGMMFREEPAGESREQ pReq;
953 int rc;
954
955 Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
956 pgmLock(pVM);
957
958 if (fInflate)
959 {
960 /* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
961 pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
962
963 /* Replace pages with ZERO pages. */
964 rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
965 if (RT_FAILURE(rc))
966 {
967 pgmUnlock(pVM);
968 AssertLogRelRC(rc);
969 return rc;
970 }
971
972 /* Iterate the pages. */
973 for (unsigned i = 0; i < cPages; i++)
974 {
975 PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
976 if ( pPage == NULL
977 || PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM)
978 {
979 Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], pPage ? PGM_PAGE_GET_TYPE(pPage) : 0));
980 break;
981 }
982
983 LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
984
985 /* Flush the shadow PT if this page was previously used as a guest page table. */
986 pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
987
988 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i]);
989 if (RT_FAILURE(rc))
990 {
991 pgmUnlock(pVM);
992 AssertLogRelRC(rc);
993 return rc;
994 }
995 Assert(PGM_PAGE_IS_ZERO(pPage));
996 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_BALLOONED);
997 }
998
999 if (cPendingPages)
1000 {
1001 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1002 if (RT_FAILURE(rc))
1003 {
1004 pgmUnlock(pVM);
1005 AssertLogRelRC(rc);
1006 return rc;
1007 }
1008 }
1009 GMMR3FreePagesCleanup(pReq);
1010 }
1011 else
1012 {
1013 /* Iterate the pages. */
1014 for (unsigned i = 0; i < cPages; i++)
1015 {
1016 PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
1017 AssertBreak(pPage && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
1018
1019 LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
1020
1021 Assert(PGM_PAGE_IS_BALLOONED(pPage));
1022
1023 /* Change back to zero page. */
1024 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1025 }
1026
1027 /* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
1028 }
1029
1030 /* Notify GMM about the balloon change. */
1031 rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
1032 if (RT_SUCCESS(rc))
1033 {
1034 if (!fInflate)
1035 {
1036 Assert(pVM->pgm.s.cBalloonedPages >= cPages);
1037 pVM->pgm.s.cBalloonedPages -= cPages;
1038 }
1039 else
1040 pVM->pgm.s.cBalloonedPages += cPages;
1041 }
1042
1043 pgmUnlock(pVM);
1044
1045 /* Flush the recompiler's TLB as well. */
1046 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1047 CPUMSetChangedFlags(&pVM->aCpus[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1048
1049 AssertLogRelRC(rc);
1050 return rc;
1051}
1052
1053
1054/**
1055 * Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
1056 *
1057 * @returns VBox status code.
1058 * @param pVM Pointer to the VM.
1059 * @param fInflate Inflate or deflate memory balloon
1060 * @param cPages Number of pages to free
1061 * @param paPhysPage Array of guest physical addresses
1062 */
1063static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1064{
1065 uintptr_t paUser[3];
1066
1067 paUser[0] = fInflate;
1068 paUser[1] = cPages;
1069 paUser[2] = (uintptr_t)paPhysPage;
1070 int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1071 AssertRC(rc);
1072
1073 /* Made a copy in PGMR3PhysFreeRamPages; free it here. */
1074 RTMemFree(paPhysPage);
1075}
1076
1077#endif /* 64-bit host && (Windows || Solaris || Linux || FreeBSD) */
1078
1079/**
1080 * Inflate or deflate a memory balloon
1081 *
1082 * @returns VBox status code.
1083 * @param pVM Pointer to the VM.
1084 * @param fInflate Inflate or deflate memory balloon
1085 * @param cPages Number of pages to free
1086 * @param paPhysPage Array of guest physical addresses
1087 */
1088VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
1089{
1090 /* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
1091#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) || defined(RT_OS_SOLARIS) || defined(RT_OS_LINUX) || defined(RT_OS_FREEBSD))
1092 int rc;
1093
1094 /* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
1095 AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
1096
1097 /* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
1098 * In the SMP case we post a request packet to postpone the job.
1099 */
1100 if (pVM->cCpus > 1)
1101 {
1102 unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
1103 RTGCPHYS *paPhysPageCopy = (RTGCPHYS *)RTMemAlloc(cbPhysPage);
1104 AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
1105
1106 memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
1107
1108 rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
1109 AssertRC(rc);
1110 }
1111 else
1112 {
1113 uintptr_t paUser[3];
1114
1115 paUser[0] = fInflate;
1116 paUser[1] = cPages;
1117 paUser[2] = (uintptr_t)paPhysPage;
1118 rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
1119 AssertRC(rc);
1120 }
1121 return rc;
1122
1123#else
1124 NOREF(pVM); NOREF(fInflate); NOREF(cPages); NOREF(paPhysPage);
1125 return VERR_NOT_IMPLEMENTED;
1126#endif
1127}
1128
1129
1130/**
1131 * Rendezvous callback used by PGMR3WriteProtectRAM that write protects all
1132 * physical RAM.
1133 *
1134 * This is only called on one of the EMTs while the other ones are waiting for
1135 * it to complete this function.
1136 *
1137 * @returns VINF_SUCCESS (VBox strict status code).
1138 * @param pVM Pointer to the VM.
1139 * @param pVCpu The VMCPU for the EMT we're being called on. Unused.
1140 * @param pvUser User parameter, unused.
1141 */
1142static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1143{
1144 int rc = VINF_SUCCESS;
1145 NOREF(pvUser); NOREF(pVCpu);
1146
1147 pgmLock(pVM);
1148#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1149 pgmPoolResetDirtyPages(pVM);
1150#endif
1151
1152 /** @todo pointless to write protect the physical page pointed to by RSP. */
1153
1154 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1155 pRam;
1156 pRam = pRam->CTX_SUFF(pNext))
1157 {
1158 uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1159 for (uint32_t iPage = 0; iPage < cPages; iPage++)
1160 {
1161 PPGMPAGE pPage = &pRam->aPages[iPage];
1162 PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1163
1164 if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1165 || enmPageType == PGMPAGETYPE_MMIO2)
1166 {
1167 /*
1168 * A RAM page.
1169 */
1170 switch (PGM_PAGE_GET_STATE(pPage))
1171 {
1172 case PGM_PAGE_STATE_ALLOCATED:
1173 /** @todo Optimize this: Don't always re-enable write
1174 * monitoring if the page is known to be very busy. */
1175 if (PGM_PAGE_IS_WRITTEN_TO(pPage))
1176 {
1177 PGM_PAGE_CLEAR_WRITTEN_TO(pVM, pPage);
1178 /* Remember this dirty page for the next (memory) sync. */
1179 PGM_PAGE_SET_FT_DIRTY(pPage);
1180 }
1181
1182 pgmPhysPageWriteMonitor(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1183 break;
1184
1185 case PGM_PAGE_STATE_SHARED:
1186 AssertFailed();
1187 break;
1188
1189 case PGM_PAGE_STATE_WRITE_MONITORED: /* nothing to change. */
1190 default:
1191 break;
1192 }
1193 }
1194 }
1195 }
1196 pgmR3PoolWriteProtectPages(pVM);
1197 PGM_INVL_ALL_VCPU_TLBS(pVM);
1198 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1199 CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
1200
1201 pgmUnlock(pVM);
1202 return rc;
1203}
1204
1205/**
1206 * Protect all physical RAM to monitor writes
1207 *
1208 * @returns VBox status code.
1209 * @param pVM Pointer to the VM.
1210 */
1211VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
1212{
1213 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1214
1215 int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
1216 AssertRC(rc);
1217 return rc;
1218}
1219
1220/**
1221 * Enumerate all dirty FT pages.
1222 *
1223 * @returns VBox status code.
1224 * @param pVM Pointer to the VM.
1225 * @param pfnEnum Enumerate callback handler.
1226 * @param pvUser Enumerate callback handler parameter.
1227 */
1228VMMR3DECL(int) PGMR3PhysEnumDirtyFTPages(PVM pVM, PFNPGMENUMDIRTYFTPAGES pfnEnum, void *pvUser)
1229{
1230 int rc = VINF_SUCCESS;
1231
1232 pgmLock(pVM);
1233 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1234 pRam;
1235 pRam = pRam->CTX_SUFF(pNext))
1236 {
1237 uint32_t cPages = pRam->cb >> PAGE_SHIFT;
1238 for (uint32_t iPage = 0; iPage < cPages; iPage++)
1239 {
1240 PPGMPAGE pPage = &pRam->aPages[iPage];
1241 PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1242
1243 if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
1244 || enmPageType == PGMPAGETYPE_MMIO2)
1245 {
1246 /*
1247 * A RAM page.
1248 */
1249 switch (PGM_PAGE_GET_STATE(pPage))
1250 {
1251 case PGM_PAGE_STATE_ALLOCATED:
1252 case PGM_PAGE_STATE_WRITE_MONITORED:
1253 if ( !PGM_PAGE_IS_WRITTEN_TO(pPage) /* not very recently updated? */
1254 && PGM_PAGE_IS_FT_DIRTY(pPage))
1255 {
1256 unsigned cbPageRange = PAGE_SIZE;
1257 unsigned iPageClean = iPage + 1;
1258 RTGCPHYS GCPhysPage = pRam->GCPhys + iPage * PAGE_SIZE;
1259 uint8_t *pu8Page = NULL;
1260 PGMPAGEMAPLOCK Lock;
1261
1262 /* Find the next clean page, so we can merge adjacent dirty pages. */
1263 for (; iPageClean < cPages; iPageClean++)
1264 {
1265 PPGMPAGE pPageNext = &pRam->aPages[iPageClean];
1266 if ( RT_UNLIKELY(PGM_PAGE_GET_TYPE(pPageNext) != PGMPAGETYPE_RAM)
1267 || PGM_PAGE_GET_STATE(pPageNext) != PGM_PAGE_STATE_ALLOCATED
1268 || PGM_PAGE_IS_WRITTEN_TO(pPageNext)
1269 || !PGM_PAGE_IS_FT_DIRTY(pPageNext)
1270 /* Crossing a chunk boundary? */
1271 || (GCPhysPage & GMM_PAGEID_IDX_MASK) != ((GCPhysPage + cbPageRange) & GMM_PAGEID_IDX_MASK)
1272 )
1273 break;
1274
1275 cbPageRange += PAGE_SIZE;
1276 }
1277
1278 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysPage, (const void **)&pu8Page, &Lock);
1279 if (RT_SUCCESS(rc))
1280 {
1281 /** @todo this is risky; the range might be changed, but little choice as the sync
1282 * costs a lot of time. */
1283 pgmUnlock(pVM);
1284 pfnEnum(pVM, GCPhysPage, pu8Page, cbPageRange, pvUser);
1285 pgmLock(pVM);
1286 PGMPhysReleasePageMappingLock(pVM, &Lock);
1287 }
1288
1289 for (iPage; iPage < iPageClean; iPage++)
1290 PGM_PAGE_CLEAR_FT_DIRTY(&pRam->aPages[iPage]);
1291
1292 iPage = iPageClean - 1;
1293 }
1294 break;
1295 }
1296 }
1297 }
1298 }
1299 pgmUnlock(pVM);
1300 return rc;
1301}
1302
1303
1304/**
1305 * Gets the number of ram ranges.
1306 *
1307 * @returns Number of ram ranges. Returns UINT32_MAX if @a pVM is invalid.
1308 * @param pVM Pointer to the VM.
1309 */
1310VMMR3DECL(uint32_t) PGMR3PhysGetRamRangeCount(PVM pVM)
1311{
1312 VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
1313
1314 pgmLock(pVM);
1315 uint32_t cRamRanges = 0;
1316 for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext))
1317 cRamRanges++;
1318 pgmUnlock(pVM);
1319 return cRamRanges;
1320}
1321
1322
1323/**
1324 * Get information about a range.
1325 *
1326 * @returns VINF_SUCCESS or VERR_OUT_OF_RANGE.
1327 * @param pVM Pointer to the VM.
1328 * @param iRange The ordinal of the range.
1329 * @param pGCPhysStart Where to return the start of the range. Optional.
1330 * @param pGCPhysLast Where to return the address of the last byte in the
1331 * range. Optional.
1332 * @param pfIsMmio Where to indicate that this is a pure MMIO range.
1333 * Optional.
1334 */
1335VMMR3DECL(int) PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
1336 const char **ppszDesc, bool *pfIsMmio)
1337{
1338 VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1339
1340 pgmLock(pVM);
1341 uint32_t iCurRange = 0;
1342 for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRangesX); pCur; pCur = pCur->CTX_SUFF(pNext), iCurRange++)
1343 if (iCurRange == iRange)
1344 {
1345 if (pGCPhysStart)
1346 *pGCPhysStart = pCur->GCPhys;
1347 if (pGCPhysLast)
1348 *pGCPhysLast = pCur->GCPhysLast;
1349 if (ppszDesc)
1350 *ppszDesc = pCur->pszDesc;
1351 if (pfIsMmio)
1352 *pfIsMmio = !!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO);
1353
1354 pgmUnlock(pVM);
1355 return VINF_SUCCESS;
1356 }
1357 pgmUnlock(pVM);
1358 return VERR_OUT_OF_RANGE;
1359}
1360
1361
1362/**
1363 * Query the amount of free memory inside VMMR0
1364 *
1365 * @returns VBox status code.
1366 * @param pUVM The user mode VM handle.
1367 * @param pcbAllocMem Where to return the amount of memory allocated
1368 * by VMs.
1369 * @param pcbFreeMem Where to return the amount of memory that is
1370 * allocated from the host but not currently used
1371 * by any VMs.
1372 * @param pcbBallonedMem Where to return the sum of memory that is
1373 * currently ballooned by the VMs.
1374 * @param pcbSharedMem Where to return the amount of memory that is
1375 * currently shared.
1376 */
1377VMMR3DECL(int) PGMR3QueryGlobalMemoryStats(PUVM pUVM, uint64_t *pcbAllocMem, uint64_t *pcbFreeMem,
1378 uint64_t *pcbBallonedMem, uint64_t *pcbSharedMem)
1379{
1380 UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
1381 VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
1382
1383 uint64_t cAllocPages = 0;
1384 uint64_t cFreePages = 0;
1385 uint64_t cBalloonPages = 0;
1386 uint64_t cSharedPages = 0;
1387 int rc = GMMR3QueryHypervisorMemoryStats(pUVM->pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
1388 AssertRCReturn(rc, rc);
1389
1390 if (pcbAllocMem)
1391 *pcbAllocMem = cAllocPages * _4K;
1392
1393 if (pcbFreeMem)
1394 *pcbFreeMem = cFreePages * _4K;
1395
1396 if (pcbBallonedMem)
1397 *pcbBallonedMem = cBalloonPages * _4K;
1398
1399 if (pcbSharedMem)
1400 *pcbSharedMem = cSharedPages * _4K;
1401
1402 Log(("PGMR3QueryVMMMemoryStats: all=%llx free=%llx ballooned=%llx shared=%llx\n",
1403 cAllocPages, cFreePages, cBalloonPages, cSharedPages));
1404 return VINF_SUCCESS;
1405}
1406
1407
1408/**
1409 * Query memory stats for the VM.
1410 *
1411 * @returns VBox status code.
1412 * @param pUVM The user mode VM handle.
1413 * @param pcbTotalMem Where to return total amount memory the VM may
1414 * possibly use.
1415 * @param pcbPrivateMem Where to return the amount of private memory
1416 * currently allocated.
1417 * @param pcbSharedMem Where to return the amount of actually shared
1418 * memory currently used by the VM.
1419 * @param pcbZeroMem Where to return the amount of memory backed by
1420 * zero pages.
1421 *
1422 * @remarks The total mem is normally larger than the sum of the three
1423 * components. There are two reasons for this, first the amount of
1424 * shared memory is what we're sure is shared instead of what could
1425 * possibly be shared with someone. Secondly, because the total may
1426 * include some pure MMIO pages that doesn't go into any of the three
1427 * sub-counts.
1428 *
1429 * @todo Why do we return reused shared pages instead of anything that could
1430 * potentially be shared? Doesn't this mean the first VM gets a much
1431 * lower number of shared pages?
1432 */
1433VMMR3DECL(int) PGMR3QueryMemoryStats(PUVM pUVM, uint64_t *pcbTotalMem, uint64_t *pcbPrivateMem,
1434 uint64_t *pcbSharedMem, uint64_t *pcbZeroMem)
1435{
1436 UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
1437 PVM pVM = pUVM->pVM;
1438 VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1439
1440 if (pcbTotalMem)
1441 *pcbTotalMem = (uint64_t)pVM->pgm.s.cAllPages * PAGE_SIZE;
1442
1443 if (pcbPrivateMem)
1444 *pcbPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages * PAGE_SIZE;
1445
1446 if (pcbSharedMem)
1447 *pcbSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages * PAGE_SIZE;
1448
1449 if (pcbZeroMem)
1450 *pcbZeroMem = (uint64_t)pVM->pgm.s.cZeroPages * PAGE_SIZE;
1451
1452 Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
1453 return VINF_SUCCESS;
1454}
1455
1456
1457/**
1458 * PGMR3PhysRegisterRam worker that initializes and links a RAM range.
1459 *
1460 * @param pVM Pointer to the VM.
1461 * @param pNew The new RAM range.
1462 * @param GCPhys The address of the RAM range.
1463 * @param GCPhysLast The last address of the RAM range.
1464 * @param RCPtrNew The RC address if the range is floating. NIL_RTRCPTR
1465 * if in HMA.
1466 * @param R0PtrNew Ditto for R0.
1467 * @param pszDesc The description.
1468 * @param pPrev The previous RAM range (for linking).
1469 */
1470static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
1471 RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
1472{
1473 /*
1474 * Initialize the range.
1475 */
1476 pNew->pSelfR0 = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
1477 pNew->pSelfRC = RCPtrNew != NIL_RTRCPTR ? RCPtrNew : MMHyperCCToRC(pVM, pNew);
1478 pNew->GCPhys = GCPhys;
1479 pNew->GCPhysLast = GCPhysLast;
1480 pNew->cb = GCPhysLast - GCPhys + 1;
1481 pNew->pszDesc = pszDesc;
1482 pNew->fFlags = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
1483 pNew->pvR3 = NULL;
1484 pNew->paLSPages = NULL;
1485
1486 uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
1487 RTGCPHYS iPage = cPages;
1488 while (iPage-- > 0)
1489 PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
1490
1491 /* Update the page count stats. */
1492 pVM->pgm.s.cZeroPages += cPages;
1493 pVM->pgm.s.cAllPages += cPages;
1494
1495 /*
1496 * Link it.
1497 */
1498 pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
1499}
1500
1501
1502/**
1503 * Relocate a floating RAM range.
1504 *
1505 * @copydoc FNPGMRELOCATE.
1506 */
1507static DECLCALLBACK(bool) pgmR3PhysRamRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser)
1508{
1509 PPGMRAMRANGE pRam = (PPGMRAMRANGE)pvUser;
1510 Assert(pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
1511 Assert(pRam->pSelfRC == GCPtrOld + PAGE_SIZE);
1512
1513 switch (enmMode)
1514 {
1515 case PGMRELOCATECALL_SUGGEST:
1516 return true;
1517
1518 case PGMRELOCATECALL_RELOCATE:
1519 {
1520 /*
1521 * Update myself, then relink all the ranges and flush the RC TLB.
1522 */
1523 pgmLock(pVM);
1524
1525 pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
1526
1527 pgmR3PhysRelinkRamRanges(pVM);
1528 for (unsigned i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
1529 pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
1530
1531 pgmUnlock(pVM);
1532 return true;
1533 }
1534
1535 default:
1536 AssertFailedReturn(false);
1537 }
1538}
1539
1540
1541/**
1542 * PGMR3PhysRegisterRam worker that registers a high chunk.
1543 *
1544 * @returns VBox status code.
1545 * @param pVM Pointer to the VM.
1546 * @param GCPhys The address of the RAM.
1547 * @param cRamPages The number of RAM pages to register.
1548 * @param cbChunk The size of the PGMRAMRANGE guest mapping.
1549 * @param iChunk The chunk number.
1550 * @param pszDesc The RAM range description.
1551 * @param ppPrev Previous RAM range pointer. In/Out.
1552 */
1553static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages,
1554 uint32_t cbChunk, uint32_t iChunk, const char *pszDesc,
1555 PPGMRAMRANGE *ppPrev)
1556{
1557 const char *pszDescChunk = iChunk == 0
1558 ? pszDesc
1559 : MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
1560 AssertReturn(pszDescChunk, VERR_NO_MEMORY);
1561
1562 /*
1563 * Allocate memory for the new chunk.
1564 */
1565 size_t const cChunkPages = RT_ALIGN_Z(RT_UOFFSETOF(PGMRAMRANGE, aPages[cRamPages]), PAGE_SIZE) >> PAGE_SHIFT;
1566 PSUPPAGE paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
1567 AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
1568 RTR0PTR R0PtrChunk = NIL_RTR0PTR;
1569 void *pvChunk = NULL;
1570 int rc = SUPR3PageAllocEx(cChunkPages, 0 /*fFlags*/, &pvChunk,
1571#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1572 HMIsEnabled(pVM) ? &R0PtrChunk : NULL,
1573#else
1574 NULL,
1575#endif
1576 paChunkPages);
1577 if (RT_SUCCESS(rc))
1578 {
1579#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1580 if (!HMIsEnabled(pVM))
1581 R0PtrChunk = NIL_RTR0PTR;
1582#else
1583 R0PtrChunk = (uintptr_t)pvChunk;
1584#endif
1585 memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
1586
1587 PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
1588
1589 /*
1590 * Create a mapping and map the pages into it.
1591 * We push these in below the HMA.
1592 */
1593 RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
1594 rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /*fFlags*/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
1595 if (RT_SUCCESS(rc))
1596 {
1597 pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
1598
1599 RTGCPTR const GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
1600 RTGCPTR GCPtrPage = GCPtrChunk;
1601 for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
1602 rc = PGMMap(pVM, GCPtrPage, paChunkPages[iPage].Phys, PAGE_SIZE, 0);
1603 if (RT_SUCCESS(rc))
1604 {
1605 /*
1606 * Ok, init and link the range.
1607 */
1608 pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
1609 (RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
1610 *ppPrev = pNew;
1611 }
1612 }
1613
1614 if (RT_FAILURE(rc))
1615 SUPR3PageFreeEx(pvChunk, cChunkPages);
1616 }
1617
1618 RTMemTmpFree(paChunkPages);
1619 return rc;
1620}
1621
1622
1623/**
1624 * Sets up a range RAM.
1625 *
1626 * This will check for conflicting registrations, make a resource
1627 * reservation for the memory (with GMM), and setup the per-page
1628 * tracking structures (PGMPAGE).
1629 *
1630 * @returns VBox status code.
1631 * @param pVM Pointer to the VM.
1632 * @param GCPhys The physical address of the RAM.
1633 * @param cb The size of the RAM.
1634 * @param pszDesc The description - not copied, so, don't free or change it.
1635 */
1636VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
1637{
1638 /*
1639 * Validate input.
1640 */
1641 Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
1642 AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
1643 AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
1644 AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
1645 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
1646 AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
1647 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1648 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
1649
1650 pgmLock(pVM);
1651
1652 /*
1653 * Find range location and check for conflicts.
1654 * (We don't lock here because the locking by EMT is only required on update.)
1655 */
1656 PPGMRAMRANGE pPrev = NULL;
1657 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
1658 while (pRam && GCPhysLast >= pRam->GCPhys)
1659 {
1660 if ( GCPhysLast >= pRam->GCPhys
1661 && GCPhys <= pRam->GCPhysLast)
1662 AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
1663 GCPhys, GCPhysLast, pszDesc,
1664 pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
1665 VERR_PGM_RAM_CONFLICT);
1666
1667 /* next */
1668 pPrev = pRam;
1669 pRam = pRam->pNextR3;
1670 }
1671
1672 /*
1673 * Register it with GMM (the API bitches).
1674 */
1675 const RTGCPHYS cPages = cb >> PAGE_SHIFT;
1676 int rc = MMR3IncreaseBaseReservation(pVM, cPages);
1677 if (RT_FAILURE(rc))
1678 {
1679 pgmUnlock(pVM);
1680 return rc;
1681 }
1682
1683 if ( GCPhys >= _4G
1684 && cPages > 256)
1685 {
1686 /*
1687 * The PGMRAMRANGE structures for the high memory can get very big.
1688 * In order to avoid SUPR3PageAllocEx allocation failures due to the
1689 * allocation size limit there and also to avoid being unable to find
1690 * guest mapping space for them, we split this memory up into 4MB in
1691 * (potential) raw-mode configs and 16MB chunks in forced AMD-V/VT-x
1692 * mode.
1693 *
1694 * The first and last page of each mapping are guard pages and marked
1695 * not-present. So, we've got 4186112 and 16769024 bytes available for
1696 * the PGMRAMRANGE structure.
1697 *
1698 * Note! The sizes used here will influence the saved state.
1699 */
1700 uint32_t cbChunk;
1701 uint32_t cPagesPerChunk;
1702 if (HMIsEnabled(pVM))
1703 {
1704 cbChunk = 16U*_1M;
1705 cPagesPerChunk = 1048048; /* max ~1048059 */
1706 AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 1048048 < 16U*_1M - PAGE_SIZE * 2);
1707 }
1708 else
1709 {
1710 cbChunk = 4U*_1M;
1711 cPagesPerChunk = 261616; /* max ~261627 */
1712 AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 261616 < 4U*_1M - PAGE_SIZE * 2);
1713 }
1714 AssertRelease(RT_UOFFSETOF(PGMRAMRANGE, aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
1715
1716 RTGCPHYS cPagesLeft = cPages;
1717 RTGCPHYS GCPhysChunk = GCPhys;
1718 uint32_t iChunk = 0;
1719 while (cPagesLeft > 0)
1720 {
1721 uint32_t cPagesInChunk = cPagesLeft;
1722 if (cPagesInChunk > cPagesPerChunk)
1723 cPagesInChunk = cPagesPerChunk;
1724
1725 rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, cbChunk, iChunk, pszDesc, &pPrev);
1726 AssertRCReturn(rc, rc);
1727
1728 /* advance */
1729 GCPhysChunk += (RTGCPHYS)cPagesInChunk << PAGE_SHIFT;
1730 cPagesLeft -= cPagesInChunk;
1731 iChunk++;
1732 }
1733 }
1734 else
1735 {
1736 /*
1737 * Allocate, initialize and link the new RAM range.
1738 */
1739 const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
1740 PPGMRAMRANGE pNew;
1741 rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
1742 AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
1743
1744 pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
1745 }
1746 pgmPhysInvalidatePageMapTLB(pVM);
1747 pgmUnlock(pVM);
1748
1749#ifdef VBOX_WITH_REM
1750 /*
1751 * Notify REM.
1752 */
1753 REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_RAM);
1754#endif
1755
1756 return VINF_SUCCESS;
1757}
1758
1759
1760/**
1761 * Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
1762 *
1763 * We do this late in the init process so that all the ROM and MMIO ranges have
1764 * been registered already and we don't go wasting memory on them.
1765 *
1766 * @returns VBox status code.
1767 *
1768 * @param pVM Pointer to the VM.
1769 */
1770int pgmR3PhysRamPreAllocate(PVM pVM)
1771{
1772 Assert(pVM->pgm.s.fRamPreAlloc);
1773 Log(("pgmR3PhysRamPreAllocate: enter\n"));
1774
1775 /*
1776 * Walk the RAM ranges and allocate all RAM pages, halt at
1777 * the first allocation error.
1778 */
1779 uint64_t cPages = 0;
1780 uint64_t NanoTS = RTTimeNanoTS();
1781 pgmLock(pVM);
1782 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1783 {
1784 PPGMPAGE pPage = &pRam->aPages[0];
1785 RTGCPHYS GCPhys = pRam->GCPhys;
1786 uint32_t cLeft = pRam->cb >> PAGE_SHIFT;
1787 while (cLeft-- > 0)
1788 {
1789 if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
1790 {
1791 switch (PGM_PAGE_GET_STATE(pPage))
1792 {
1793 case PGM_PAGE_STATE_ZERO:
1794 {
1795 int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
1796 if (RT_FAILURE(rc))
1797 {
1798 LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
1799 pgmUnlock(pVM);
1800 return rc;
1801 }
1802 cPages++;
1803 break;
1804 }
1805
1806 case PGM_PAGE_STATE_BALLOONED:
1807 case PGM_PAGE_STATE_ALLOCATED:
1808 case PGM_PAGE_STATE_WRITE_MONITORED:
1809 case PGM_PAGE_STATE_SHARED:
1810 /* nothing to do here. */
1811 break;
1812 }
1813 }
1814
1815 /* next */
1816 pPage++;
1817 GCPhys += PAGE_SIZE;
1818 }
1819 }
1820 pgmUnlock(pVM);
1821 NanoTS = RTTimeNanoTS() - NanoTS;
1822
1823 LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
1824 Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
1825 return VINF_SUCCESS;
1826}
1827
1828
1829/**
1830 * Checks shared page checksums.
1831 *
1832 * @param pVM Pointer to the shared VM structure.
1833 */
1834void pgmR3PhysAssertSharedPageChecksums(PVM pVM)
1835{
1836#ifdef VBOX_STRICT
1837 pgmLock(pVM);
1838
1839 if (pVM->pgm.s.cSharedPages > 0)
1840 {
1841 /*
1842 * Walk the ram ranges.
1843 */
1844 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1845 {
1846 uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1847 AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1848
1849 while (iPage-- > 0)
1850 {
1851 PPGMPAGE pPage = &pRam->aPages[iPage];
1852 if (PGM_PAGE_IS_SHARED(pPage))
1853 {
1854 uint32_t u32Checksum = pPage->s.u2Unused0 | ((uint32_t)pPage->s.u2Unused1 << 8);
1855 if (!u32Checksum)
1856 {
1857 RTGCPHYS GCPhysPage = pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
1858 void const *pvPage;
1859 int rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhysPage, &pvPage);
1860 if (RT_SUCCESS(rc))
1861 {
1862 uint32_t u32Checksum2 = RTCrc32(pvPage, PAGE_SIZE);
1863# if 0
1864 AssertMsg((u32Checksum2 & UINT32_C(0x00000303)) == u32Checksum, ("GCPhysPage=%RGp\n", GCPhysPage));
1865# else
1866 if ((u32Checksum2 & UINT32_C(0x00000303)) == u32Checksum)
1867 LogFlow(("shpg %#x @ %RGp %#x [OK]\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
1868 else
1869 AssertMsgFailed(("shpg %#x @ %RGp %#x\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
1870# endif
1871 }
1872 else
1873 AssertRC(rc);
1874 }
1875 }
1876
1877 } /* for each page */
1878
1879 } /* for each ram range */
1880 }
1881
1882 pgmUnlock(pVM);
1883#endif /* VBOX_STRICT */
1884 NOREF(pVM);
1885}
1886
1887
1888/**
1889 * Resets the physical memory state.
1890 *
1891 * ASSUMES that the caller owns the PGM lock.
1892 *
1893 * @returns VBox status code.
1894 * @param pVM Pointer to the VM.
1895 */
1896int pgmR3PhysRamReset(PVM pVM)
1897{
1898 PGM_LOCK_ASSERT_OWNER(pVM);
1899
1900 /* Reset the memory balloon. */
1901 int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
1902 AssertRC(rc);
1903
1904#ifdef VBOX_WITH_PAGE_SHARING
1905 /* Clear all registered shared modules. */
1906 pgmR3PhysAssertSharedPageChecksums(pVM);
1907 rc = GMMR3ResetSharedModules(pVM);
1908 AssertRC(rc);
1909#endif
1910 /* Reset counters. */
1911 pVM->pgm.s.cReusedSharedPages = 0;
1912 pVM->pgm.s.cBalloonedPages = 0;
1913
1914 return VINF_SUCCESS;
1915}
1916
1917
1918/**
1919 * Resets (zeros) the RAM after all devices and components have been reset.
1920 *
1921 * ASSUMES that the caller owns the PGM lock.
1922 *
1923 * @returns VBox status code.
1924 * @param pVM Pointer to the VM.
1925 */
1926int pgmR3PhysRamZeroAll(PVM pVM)
1927{
1928 PGM_LOCK_ASSERT_OWNER(pVM);
1929
1930 /*
1931 * We batch up pages that should be freed instead of calling GMM for
1932 * each and every one of them.
1933 */
1934 uint32_t cPendingPages = 0;
1935 PGMMFREEPAGESREQ pReq;
1936 int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1937 AssertLogRelRCReturn(rc, rc);
1938
1939 /*
1940 * Walk the ram ranges.
1941 */
1942 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
1943 {
1944 uint32_t iPage = pRam->cb >> PAGE_SHIFT;
1945 AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
1946
1947#ifndef NO_RAM_RESET
1948 if (!pVM->pgm.s.fRamPreAlloc)
1949#else
1950 if (0)
1951#endif
1952 {
1953 /* Replace all RAM pages by ZERO pages. */
1954 while (iPage-- > 0)
1955 {
1956 PPGMPAGE pPage = &pRam->aPages[iPage];
1957 switch (PGM_PAGE_GET_TYPE(pPage))
1958 {
1959 case PGMPAGETYPE_RAM:
1960 /* Do not replace pages part of a 2 MB continuous range
1961 with zero pages, but zero them instead. */
1962 if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE
1963 || PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
1964 {
1965 void *pvPage;
1966 rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
1967 AssertLogRelRCReturn(rc, rc);
1968 ASMMemZeroPage(pvPage);
1969 }
1970 else if (PGM_PAGE_IS_BALLOONED(pPage))
1971 {
1972 /* Turn into a zero page; the balloon status is lost when the VM reboots. */
1973 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1974 }
1975 else if (!PGM_PAGE_IS_ZERO(pPage))
1976 {
1977 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
1978 AssertLogRelRCReturn(rc, rc);
1979 }
1980 break;
1981
1982 case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
1983 case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone? I don't think VT-x copes with this code. */
1984 pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
1985 true /*fDoAccounting*/);
1986 break;
1987
1988 case PGMPAGETYPE_MMIO2:
1989 case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
1990 case PGMPAGETYPE_ROM:
1991 case PGMPAGETYPE_MMIO:
1992 break;
1993 default:
1994 AssertFailed();
1995 }
1996 } /* for each page */
1997 }
1998 else
1999 {
2000 /* Zero the memory. */
2001 while (iPage-- > 0)
2002 {
2003 PPGMPAGE pPage = &pRam->aPages[iPage];
2004 switch (PGM_PAGE_GET_TYPE(pPage))
2005 {
2006 case PGMPAGETYPE_RAM:
2007 switch (PGM_PAGE_GET_STATE(pPage))
2008 {
2009 case PGM_PAGE_STATE_ZERO:
2010 break;
2011
2012 case PGM_PAGE_STATE_BALLOONED:
2013 /* Turn into a zero page; the balloon status is lost when the VM reboots. */
2014 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
2015 break;
2016
2017 case PGM_PAGE_STATE_SHARED:
2018 case PGM_PAGE_STATE_WRITE_MONITORED:
2019 rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
2020 AssertLogRelRCReturn(rc, rc);
2021 /* no break */
2022
2023 case PGM_PAGE_STATE_ALLOCATED:
2024 {
2025 void *pvPage;
2026 rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
2027 AssertLogRelRCReturn(rc, rc);
2028#ifndef NO_RAM_RESET
2029 ASMMemZeroPage(pvPage);
2030#endif
2031 break;
2032 }
2033 }
2034 break;
2035
2036 case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2037 case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone? I don't think VT-x copes with this code. */
2038 pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
2039 true /*fDoAccounting*/);
2040 break;
2041
2042 case PGMPAGETYPE_MMIO2:
2043 case PGMPAGETYPE_ROM_SHADOW:
2044 case PGMPAGETYPE_ROM:
2045 case PGMPAGETYPE_MMIO:
2046 break;
2047 default:
2048 AssertFailed();
2049
2050 }
2051 } /* for each page */
2052 }
2053
2054 }
2055
2056 /*
2057 * Finish off any pages pending freeing.
2058 */
2059 if (cPendingPages)
2060 {
2061 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2062 AssertLogRelRCReturn(rc, rc);
2063 }
2064 GMMR3FreePagesCleanup(pReq);
2065 return VINF_SUCCESS;
2066}
2067
2068
2069/**
2070 * Frees all RAM during VM termination
2071 *
2072 * ASSUMES that the caller owns the PGM lock.
2073 *
2074 * @returns VBox status code.
2075 * @param pVM Pointer to the VM.
2076 */
2077int pgmR3PhysRamTerm(PVM pVM)
2078{
2079 PGM_LOCK_ASSERT_OWNER(pVM);
2080
2081 /* Reset the memory balloon. */
2082 int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
2083 AssertRC(rc);
2084
2085#ifdef VBOX_WITH_PAGE_SHARING
2086 /*
2087 * Clear all registered shared modules.
2088 */
2089 pgmR3PhysAssertSharedPageChecksums(pVM);
2090 rc = GMMR3ResetSharedModules(pVM);
2091 AssertRC(rc);
2092
2093 /*
2094 * Flush the handy pages updates to make sure no shared pages are hiding
2095 * in there. (No unlikely if the VM shuts down, apparently.)
2096 */
2097 rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_FLUSH_HANDY_PAGES, 0, NULL);
2098#endif
2099
2100 /*
2101 * We batch up pages that should be freed instead of calling GMM for
2102 * each and every one of them.
2103 */
2104 uint32_t cPendingPages = 0;
2105 PGMMFREEPAGESREQ pReq;
2106 rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2107 AssertLogRelRCReturn(rc, rc);
2108
2109 /*
2110 * Walk the ram ranges.
2111 */
2112 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3; pRam; pRam = pRam->pNextR3)
2113 {
2114 uint32_t iPage = pRam->cb >> PAGE_SHIFT;
2115 AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
2116
2117 while (iPage-- > 0)
2118 {
2119 PPGMPAGE pPage = &pRam->aPages[iPage];
2120 switch (PGM_PAGE_GET_TYPE(pPage))
2121 {
2122 case PGMPAGETYPE_RAM:
2123 /* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
2124 /** @todo change this to explicitly free private pages here. */
2125 if (PGM_PAGE_IS_SHARED(pPage))
2126 {
2127 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
2128 AssertLogRelRCReturn(rc, rc);
2129 }
2130 break;
2131
2132 case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2133 case PGMPAGETYPE_SPECIAL_ALIAS_MMIO:
2134 case PGMPAGETYPE_MMIO2:
2135 case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2136 case PGMPAGETYPE_ROM:
2137 case PGMPAGETYPE_MMIO:
2138 break;
2139 default:
2140 AssertFailed();
2141 }
2142 } /* for each page */
2143 }
2144
2145 /*
2146 * Finish off any pages pending freeing.
2147 */
2148 if (cPendingPages)
2149 {
2150 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2151 AssertLogRelRCReturn(rc, rc);
2152 }
2153 GMMR3FreePagesCleanup(pReq);
2154 return VINF_SUCCESS;
2155}
2156
2157
2158/**
2159 * This is the interface IOM is using to register an MMIO region.
2160 *
2161 * It will check for conflicts and ensure that a RAM range structure
2162 * is present before calling the PGMR3HandlerPhysicalRegister API to
2163 * register the callbacks.
2164 *
2165 * @returns VBox status code.
2166 *
2167 * @param pVM Pointer to the VM.
2168 * @param GCPhys The start of the MMIO region.
2169 * @param cb The size of the MMIO region.
2170 * @param pfnHandlerR3 The address of the ring-3 handler. (IOMR3MMIOHandler)
2171 * @param pvUserR3 The user argument for R3.
2172 * @param pfnHandlerR0 The address of the ring-0 handler. (IOMMMIOHandler)
2173 * @param pvUserR0 The user argument for R0.
2174 * @param pfnHandlerRC The address of the RC handler. (IOMMMIOHandler)
2175 * @param pvUserRC The user argument for RC.
2176 * @param pszDesc The description of the MMIO region.
2177 */
2178VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb,
2179 R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3, RTR3PTR pvUserR3,
2180 R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0, RTR0PTR pvUserR0,
2181 RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC, RTRCPTR pvUserRC,
2182 R3PTRTYPE(const char *) pszDesc)
2183{
2184 /*
2185 * Assert on some assumption.
2186 */
2187 VM_ASSERT_EMT(pVM);
2188 AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2189 AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2190 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2191 AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2192
2193 int rc = pgmLock(pVM);
2194 AssertRCReturn(rc, rc);
2195
2196 /*
2197 * Make sure there's a RAM range structure for the region.
2198 */
2199 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2200 bool fRamExists = false;
2201 PPGMRAMRANGE pRamPrev = NULL;
2202 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2203 while (pRam && GCPhysLast >= pRam->GCPhys)
2204 {
2205 if ( GCPhysLast >= pRam->GCPhys
2206 && GCPhys <= pRam->GCPhysLast)
2207 {
2208 /* Simplification: all within the same range. */
2209 AssertLogRelMsgReturnStmt( GCPhys >= pRam->GCPhys
2210 && GCPhysLast <= pRam->GCPhysLast,
2211 ("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
2212 GCPhys, GCPhysLast, pszDesc,
2213 pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2214 pgmUnlock(pVM),
2215 VERR_PGM_RAM_CONFLICT);
2216
2217 /* Check that it's all RAM or MMIO pages. */
2218 PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2219 uint32_t cLeft = cb >> PAGE_SHIFT;
2220 while (cLeft-- > 0)
2221 {
2222 AssertLogRelMsgReturnStmt( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2223 || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
2224 ("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
2225 GCPhys, GCPhysLast, pszDesc, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
2226 pgmUnlock(pVM),
2227 VERR_PGM_RAM_CONFLICT);
2228 pPage++;
2229 }
2230
2231 /* Looks good. */
2232 fRamExists = true;
2233 break;
2234 }
2235
2236 /* next */
2237 pRamPrev = pRam;
2238 pRam = pRam->pNextR3;
2239 }
2240 PPGMRAMRANGE pNew;
2241 if (fRamExists)
2242 {
2243 pNew = NULL;
2244
2245 /*
2246 * Make all the pages in the range MMIO/ZERO pages, freeing any
2247 * RAM pages currently mapped here. This might not be 100% correct
2248 * for PCI memory, but we're doing the same thing for MMIO2 pages.
2249 */
2250 rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
2251 AssertRCReturnStmt(rc, pgmUnlock(pVM), rc);
2252
2253 /* Force a PGM pool flush as guest ram references have been changed. */
2254 /** @todo not entirely SMP safe; assuming for now the guest takes
2255 * care of this internally (not touch mapped mmio while changing the
2256 * mapping). */
2257 PVMCPU pVCpu = VMMGetCpu(pVM);
2258 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
2259 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2260 }
2261 else
2262 {
2263
2264 /*
2265 * No RAM range, insert an ad hoc one.
2266 *
2267 * Note that we don't have to tell REM about this range because
2268 * PGMHandlerPhysicalRegisterEx will do that for us.
2269 */
2270 Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
2271
2272 const uint32_t cPages = cb >> PAGE_SHIFT;
2273 const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
2274 rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
2275 AssertLogRelMsgRCReturnStmt(rc, ("cbRamRange=%zu\n", cbRamRange), pgmUnlock(pVM), rc);
2276
2277 /* Initialize the range. */
2278 pNew->pSelfR0 = MMHyperCCToR0(pVM, pNew);
2279 pNew->pSelfRC = MMHyperCCToRC(pVM, pNew);
2280 pNew->GCPhys = GCPhys;
2281 pNew->GCPhysLast = GCPhysLast;
2282 pNew->cb = cb;
2283 pNew->pszDesc = pszDesc;
2284 pNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
2285 pNew->pvR3 = NULL;
2286 pNew->paLSPages = NULL;
2287
2288 uint32_t iPage = cPages;
2289 while (iPage-- > 0)
2290 PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
2291 Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
2292
2293 /* update the page count stats. */
2294 pVM->pgm.s.cPureMmioPages += cPages;
2295 pVM->pgm.s.cAllPages += cPages;
2296
2297 /* link it */
2298 pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
2299 }
2300
2301 /*
2302 * Register the access handler.
2303 */
2304 rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_MMIO, GCPhys, GCPhysLast,
2305 pfnHandlerR3, pvUserR3,
2306 pfnHandlerR0, pvUserR0,
2307 pfnHandlerRC, pvUserRC, pszDesc);
2308 if ( RT_FAILURE(rc)
2309 && !fRamExists)
2310 {
2311 pVM->pgm.s.cPureMmioPages -= cb >> PAGE_SHIFT;
2312 pVM->pgm.s.cAllPages -= cb >> PAGE_SHIFT;
2313
2314 /* remove the ad hoc range. */
2315 pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
2316 pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
2317 MMHyperFree(pVM, pRam);
2318 }
2319 pgmPhysInvalidatePageMapTLB(pVM);
2320
2321 pgmUnlock(pVM);
2322 return rc;
2323}
2324
2325
2326/**
2327 * This is the interface IOM is using to register an MMIO region.
2328 *
2329 * It will take care of calling PGMHandlerPhysicalDeregister and clean up
2330 * any ad hoc PGMRAMRANGE left behind.
2331 *
2332 * @returns VBox status code.
2333 * @param pVM Pointer to the VM.
2334 * @param GCPhys The start of the MMIO region.
2335 * @param cb The size of the MMIO region.
2336 */
2337VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
2338{
2339 VM_ASSERT_EMT(pVM);
2340
2341 int rc = pgmLock(pVM);
2342 AssertRCReturn(rc, rc);
2343
2344 /*
2345 * First deregister the handler, then check if we should remove the ram range.
2346 */
2347 rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
2348 if (RT_SUCCESS(rc))
2349 {
2350 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
2351 PPGMRAMRANGE pRamPrev = NULL;
2352 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2353 while (pRam && GCPhysLast >= pRam->GCPhys)
2354 {
2355 /** @todo We're being a bit too careful here. rewrite. */
2356 if ( GCPhysLast == pRam->GCPhysLast
2357 && GCPhys == pRam->GCPhys)
2358 {
2359 Assert(pRam->cb == cb);
2360
2361 /*
2362 * See if all the pages are dead MMIO pages.
2363 */
2364 uint32_t const cPages = cb >> PAGE_SHIFT;
2365 bool fAllMMIO = true;
2366 uint32_t iPage = 0;
2367 uint32_t cLeft = cPages;
2368 while (cLeft-- > 0)
2369 {
2370 PPGMPAGE pPage = &pRam->aPages[iPage];
2371 if ( !PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)
2372 /*|| not-out-of-action later */)
2373 {
2374 fAllMMIO = false;
2375 AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2376 break;
2377 }
2378 Assert( PGM_PAGE_IS_ZERO(pPage)
2379 || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
2380 || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO);
2381 pPage++;
2382 }
2383 if (fAllMMIO)
2384 {
2385 /*
2386 * Ad-hoc range, unlink and free it.
2387 */
2388 Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
2389 GCPhys, GCPhysLast, pRam->pszDesc));
2390
2391 pVM->pgm.s.cAllPages -= cPages;
2392 pVM->pgm.s.cPureMmioPages -= cPages;
2393
2394 pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
2395 pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
2396 MMHyperFree(pVM, pRam);
2397 break;
2398 }
2399 }
2400
2401 /*
2402 * Range match? It will all be within one range (see PGMAllHandler.cpp).
2403 */
2404 if ( GCPhysLast >= pRam->GCPhys
2405 && GCPhys <= pRam->GCPhysLast)
2406 {
2407 Assert(GCPhys >= pRam->GCPhys);
2408 Assert(GCPhysLast <= pRam->GCPhysLast);
2409
2410 /*
2411 * Turn the pages back into RAM pages.
2412 */
2413 uint32_t iPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
2414 uint32_t cLeft = cb >> PAGE_SHIFT;
2415 while (cLeft--)
2416 {
2417 PPGMPAGE pPage = &pRam->aPages[iPage];
2418 AssertMsg( (PGM_PAGE_IS_MMIO(pPage) && PGM_PAGE_IS_ZERO(pPage))
2419 || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
2420 || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO,
2421 ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
2422 if (PGM_PAGE_IS_MMIO_OR_ALIAS(pPage))
2423 PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_RAM);
2424 }
2425 break;
2426 }
2427
2428 /* next */
2429 pRamPrev = pRam;
2430 pRam = pRam->pNextR3;
2431 }
2432 }
2433
2434 /* Force a PGM pool flush as guest ram references have been changed. */
2435 /** @todo Not entirely SMP safe; assuming for now the guest takes care of
2436 * this internally (not touch mapped mmio while changing the mapping). */
2437 PVMCPU pVCpu = VMMGetCpu(pVM);
2438 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
2439 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2440
2441 pgmPhysInvalidatePageMapTLB(pVM);
2442 pgmPhysInvalidRamRangeTlbs(pVM);
2443 pgmUnlock(pVM);
2444 return rc;
2445}
2446
2447
2448/**
2449 * Locate a MMIO2 range.
2450 *
2451 * @returns Pointer to the MMIO2 range.
2452 * @param pVM Pointer to the VM.
2453 * @param pDevIns The device instance owning the region.
2454 * @param iRegion The region.
2455 */
2456DECLINLINE(PPGMMMIO2RANGE) pgmR3PhysMMIO2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2457{
2458 /*
2459 * Search the list.
2460 */
2461 for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
2462 if ( pCur->pDevInsR3 == pDevIns
2463 && pCur->iRegion == iRegion)
2464 return pCur;
2465 return NULL;
2466}
2467
2468
2469/**
2470 * Allocate and register an MMIO2 region.
2471 *
2472 * As mentioned elsewhere, MMIO2 is just RAM spelled differently. It's RAM
2473 * associated with a device. It is also non-shared memory with a permanent
2474 * ring-3 mapping and page backing (presently).
2475 *
2476 * A MMIO2 range may overlap with base memory if a lot of RAM is configured for
2477 * the VM, in which case we'll drop the base memory pages. Presently we will
2478 * make no attempt to preserve anything that happens to be present in the base
2479 * memory that is replaced, this is of course incorrect but it's too much
2480 * effort.
2481 *
2482 * @returns VBox status code.
2483 * @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
2484 * memory.
2485 * @retval VERR_ALREADY_EXISTS if the region already exists.
2486 *
2487 * @param pVM Pointer to the VM.
2488 * @param pDevIns The device instance owning the region.
2489 * @param iRegion The region number. If the MMIO2 memory is a PCI
2490 * I/O region this number has to be the number of that
2491 * region. Otherwise it can be any number safe
2492 * UINT8_MAX.
2493 * @param cb The size of the region. Must be page aligned.
2494 * @param fFlags Reserved for future use, must be zero.
2495 * @param ppv Where to store the pointer to the ring-3 mapping of
2496 * the memory.
2497 * @param pszDesc The description.
2498 */
2499VMMR3DECL(int) PGMR3PhysMMIO2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags,
2500 void **ppv, const char *pszDesc)
2501{
2502 /*
2503 * Validate input.
2504 */
2505 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2506 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2507 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2508 AssertPtrReturn(ppv, VERR_INVALID_POINTER);
2509 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2510 AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
2511 AssertReturn(pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion) == NULL, VERR_ALREADY_EXISTS);
2512 AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2513 AssertReturn(cb, VERR_INVALID_PARAMETER);
2514 AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
2515
2516 const uint32_t cPages = cb >> PAGE_SHIFT;
2517 AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
2518 AssertLogRelReturn(cPages <= PGM_MMIO2_MAX_PAGE_COUNT, VERR_NO_MEMORY);
2519
2520 /*
2521 * For the 2nd+ instance, mangle the description string so it's unique.
2522 */
2523 if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
2524 {
2525 pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
2526 if (!pszDesc)
2527 return VERR_NO_MEMORY;
2528 }
2529
2530 /*
2531 * Allocate an MMIO2 range ID (not freed on failure).
2532 * The zero ID is not used as it could be confused with NIL_GMM_PAGEID.
2533 */
2534 pgmLock(pVM);
2535 uint8_t idMmio2 = pVM->pgm.s.cMmio2Regions + 1;
2536 if (idMmio2 > PGM_MMIO2_MAX_RANGES)
2537 {
2538 pgmUnlock(pVM);
2539 AssertLogRelFailedReturn(VERR_PGM_TOO_MANY_MMIO2_RANGES);
2540 }
2541 pVM->pgm.s.cMmio2Regions = idMmio2;
2542 pgmUnlock(pVM);
2543
2544 /*
2545 * Try reserve and allocate the backing memory first as this is what is
2546 * most likely to fail.
2547 */
2548 int rc = MMR3AdjustFixedReservation(pVM, cPages, pszDesc);
2549 if (RT_SUCCESS(rc))
2550 {
2551 void *pvPages;
2552 PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(SUPPAGE));
2553 if (RT_SUCCESS(rc))
2554 rc = SUPR3PageAllocEx(cPages, 0 /*fFlags*/, &pvPages, NULL /*pR0Ptr*/, paPages);
2555 if (RT_SUCCESS(rc))
2556 {
2557 memset(pvPages, 0, cPages * PAGE_SIZE);
2558
2559 /*
2560 * Create the MMIO2 range record for it.
2561 */
2562 const size_t cbRange = RT_OFFSETOF(PGMMMIO2RANGE, RamRange.aPages[cPages]);
2563 PPGMMMIO2RANGE pNew;
2564 rc = MMR3HyperAllocOnceNoRel(pVM, cbRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
2565 AssertLogRelMsgRC(rc, ("cbRamRange=%zu\n", cbRange));
2566 if (RT_SUCCESS(rc))
2567 {
2568 pNew->pDevInsR3 = pDevIns;
2569 pNew->pvR3 = pvPages;
2570 //pNew->pNext = NULL;
2571 //pNew->fMapped = false;
2572 //pNew->fOverlapping = false;
2573 pNew->iRegion = iRegion;
2574 pNew->idSavedState = UINT8_MAX;
2575 pNew->idMmio2 = idMmio2;
2576 pNew->RamRange.pSelfR0 = MMHyperCCToR0(pVM, &pNew->RamRange);
2577 pNew->RamRange.pSelfRC = MMHyperCCToRC(pVM, &pNew->RamRange);
2578 pNew->RamRange.GCPhys = NIL_RTGCPHYS;
2579 pNew->RamRange.GCPhysLast = NIL_RTGCPHYS;
2580 pNew->RamRange.pszDesc = pszDesc;
2581 pNew->RamRange.cb = cb;
2582 pNew->RamRange.fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2;
2583 pNew->RamRange.pvR3 = pvPages;
2584 //pNew->RamRange.paLSPages = NULL;
2585
2586 uint32_t iPage = cPages;
2587 while (iPage-- > 0)
2588 {
2589 PGM_PAGE_INIT(&pNew->RamRange.aPages[iPage],
2590 paPages[iPage].Phys,
2591 PGM_MMIO2_PAGEID_MAKE(idMmio2, iPage),
2592 PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
2593 }
2594
2595 /* update page count stats */
2596 pVM->pgm.s.cAllPages += cPages;
2597 pVM->pgm.s.cPrivatePages += cPages;
2598
2599 /*
2600 * Link it into the list.
2601 * Since there is no particular order, just push it.
2602 */
2603 /** @todo we can save us the linked list now, just search the lookup table... */
2604 pgmLock(pVM);
2605 Assert(pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] == NULL);
2606 Assert(pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] == NIL_RTR0PTR);
2607 pNew->pNextR3 = pVM->pgm.s.pMmio2RangesR3;
2608 pVM->pgm.s.pMmio2RangesR3 = pNew;
2609 pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] = pNew;
2610 pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] = MMHyperCCToR0(pVM, pNew);
2611 pgmUnlock(pVM);
2612
2613 *ppv = pvPages;
2614 RTMemTmpFree(paPages);
2615 pgmPhysInvalidatePageMapTLB(pVM);
2616 return VINF_SUCCESS;
2617 }
2618
2619 SUPR3PageFreeEx(pvPages, cPages);
2620 }
2621 RTMemTmpFree(paPages);
2622 MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pszDesc);
2623 }
2624 if (pDevIns->iInstance > 0)
2625 MMR3HeapFree((void *)pszDesc);
2626 return rc;
2627}
2628
2629
2630/**
2631 * Deregisters and frees an MMIO2 region.
2632 *
2633 * Any physical (and virtual) access handlers registered for the region must
2634 * be deregistered before calling this function.
2635 *
2636 * @returns VBox status code.
2637 * @param pVM Pointer to the VM.
2638 * @param pDevIns The device instance owning the region.
2639 * @param iRegion The region. If it's UINT32_MAX it'll be a wildcard match.
2640 */
2641VMMR3DECL(int) PGMR3PhysMMIO2Deregister(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
2642{
2643 /*
2644 * Validate input.
2645 */
2646 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2647 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2648 AssertReturn(iRegion <= UINT8_MAX || iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
2649
2650 pgmLock(pVM);
2651 int rc = VINF_SUCCESS;
2652 unsigned cFound = 0;
2653 PPGMMMIO2RANGE pPrev = NULL;
2654 PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3;
2655 while (pCur)
2656 {
2657 if ( pCur->pDevInsR3 == pDevIns
2658 && ( iRegion == UINT32_MAX
2659 || pCur->iRegion == iRegion))
2660 {
2661 cFound++;
2662
2663 /*
2664 * Unmap it if it's mapped.
2665 */
2666 if (pCur->fMapped)
2667 {
2668 int rc2 = PGMR3PhysMMIO2Unmap(pVM, pCur->pDevInsR3, pCur->iRegion, pCur->RamRange.GCPhys);
2669 AssertRC(rc2);
2670 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2671 rc = rc2;
2672 }
2673
2674 /*
2675 * Unlink it
2676 */
2677 PPGMMMIO2RANGE pNext = pCur->pNextR3;
2678 if (pPrev)
2679 pPrev->pNextR3 = pNext;
2680 else
2681 pVM->pgm.s.pMmio2RangesR3 = pNext;
2682 pCur->pNextR3 = NULL;
2683
2684 uint8_t idMmio2 = pCur->idMmio2;
2685 Assert(pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] == pCur);
2686 pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] = NULL;
2687 pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] = NIL_RTR0PTR;
2688
2689 /*
2690 * Free the memory.
2691 */
2692 int rc2 = SUPR3PageFreeEx(pCur->pvR3, pCur->RamRange.cb >> PAGE_SHIFT);
2693 AssertRC(rc2);
2694 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2695 rc = rc2;
2696
2697 uint32_t const cPages = pCur->RamRange.cb >> PAGE_SHIFT;
2698 rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pCur->RamRange.pszDesc);
2699 AssertRC(rc2);
2700 if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
2701 rc = rc2;
2702
2703 /* we're leaking hyper memory here if done at runtime. */
2704#ifdef VBOX_STRICT
2705 VMSTATE const enmState = VMR3GetState(pVM);
2706 AssertMsg( enmState == VMSTATE_POWERING_OFF
2707 || enmState == VMSTATE_POWERING_OFF_LS
2708 || enmState == VMSTATE_OFF
2709 || enmState == VMSTATE_OFF_LS
2710 || enmState == VMSTATE_DESTROYING
2711 || enmState == VMSTATE_TERMINATED
2712 || enmState == VMSTATE_CREATING
2713 , ("%s\n", VMR3GetStateName(enmState)));
2714#endif
2715 /*rc = MMHyperFree(pVM, pCur);
2716 AssertRCReturn(rc, rc); - not safe, see the alloc call. */
2717
2718
2719 /* update page count stats */
2720 pVM->pgm.s.cAllPages -= cPages;
2721 pVM->pgm.s.cPrivatePages -= cPages;
2722
2723 /* next */
2724 pCur = pNext;
2725 }
2726 else
2727 {
2728 pPrev = pCur;
2729 pCur = pCur->pNextR3;
2730 }
2731 }
2732 pgmPhysInvalidatePageMapTLB(pVM);
2733 pgmUnlock(pVM);
2734 return !cFound && iRegion != UINT32_MAX ? VERR_NOT_FOUND : rc;
2735}
2736
2737
2738/**
2739 * Maps a MMIO2 region.
2740 *
2741 * This is done when a guest / the bios / state loading changes the
2742 * PCI config. The replacing of base memory has the same restrictions
2743 * as during registration, of course.
2744 *
2745 * @returns VBox status code.
2746 *
2747 * @param pVM Pointer to the VM.
2748 * @param pDevIns The device instance owning the region.
2749 * @param iRegion The index of the registered region.
2750 * @param GCPhys The guest-physical address to be remapped.
2751 */
2752VMMR3DECL(int) PGMR3PhysMMIO2Map(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2753{
2754 /*
2755 * Validate input
2756 */
2757 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2758 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2759 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2760 AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2761 AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2762 AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2763
2764 PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2765 AssertReturn(pCur, VERR_NOT_FOUND);
2766 AssertReturn(!pCur->fMapped, VERR_WRONG_ORDER);
2767 Assert(pCur->RamRange.GCPhys == NIL_RTGCPHYS);
2768 Assert(pCur->RamRange.GCPhysLast == NIL_RTGCPHYS);
2769
2770 const RTGCPHYS GCPhysLast = GCPhys + pCur->RamRange.cb - 1;
2771 AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2772
2773 /*
2774 * Find our location in the ram range list, checking for
2775 * restriction we don't bother implementing yet (partially overlapping).
2776 */
2777 bool fRamExists = false;
2778 PPGMRAMRANGE pRamPrev = NULL;
2779 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2780 while (pRam && GCPhysLast >= pRam->GCPhys)
2781 {
2782 if ( GCPhys <= pRam->GCPhysLast
2783 && GCPhysLast >= pRam->GCPhys)
2784 {
2785 /* completely within? */
2786 AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
2787 && GCPhysLast <= pRam->GCPhysLast,
2788 ("%RGp-%RGp (MMIO2/%s) falls partly outside %RGp-%RGp (%s)\n",
2789 GCPhys, GCPhysLast, pCur->RamRange.pszDesc,
2790 pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
2791 VERR_PGM_RAM_CONFLICT);
2792 fRamExists = true;
2793 break;
2794 }
2795
2796 /* next */
2797 pRamPrev = pRam;
2798 pRam = pRam->pNextR3;
2799 }
2800 if (fRamExists)
2801 {
2802 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2803 uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2804 while (cPagesLeft-- > 0)
2805 {
2806 AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
2807 ("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
2808 GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pCur->RamRange.pszDesc),
2809 VERR_PGM_RAM_CONFLICT);
2810 pPage++;
2811 }
2812 }
2813 Log(("PGMR3PhysMMIO2Map: %RGp-%RGp fRamExists=%RTbool %s\n",
2814 GCPhys, GCPhysLast, fRamExists, pCur->RamRange.pszDesc));
2815
2816 /*
2817 * Make the changes.
2818 */
2819 pgmLock(pVM);
2820
2821 pCur->RamRange.GCPhys = GCPhys;
2822 pCur->RamRange.GCPhysLast = GCPhysLast;
2823 pCur->fMapped = true;
2824 pCur->fOverlapping = fRamExists;
2825
2826 if (fRamExists)
2827 {
2828/** @todo use pgmR3PhysFreePageRange here. */
2829 uint32_t cPendingPages = 0;
2830 PGMMFREEPAGESREQ pReq;
2831 int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2832 AssertLogRelRCReturn(rc, rc);
2833
2834 /* replace the pages, freeing all present RAM pages. */
2835 PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2836 PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2837 uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2838 while (cPagesLeft-- > 0)
2839 {
2840 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
2841 AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
2842
2843 RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
2844 uint32_t const idPage = PGM_PAGE_GET_PAGEID(pPageSrc);
2845 PGM_PAGE_SET_PAGEID(pVM, pPageDst, idPage);
2846 PGM_PAGE_SET_HCPHYS(pVM, pPageDst, HCPhys);
2847 PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO2);
2848 PGM_PAGE_SET_STATE(pVM, pPageDst, PGM_PAGE_STATE_ALLOCATED);
2849 PGM_PAGE_SET_PDE_TYPE(pVM, pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
2850 PGM_PAGE_SET_PTE_INDEX(pVM, pPageDst, 0);
2851 PGM_PAGE_SET_TRACKING(pVM, pPageDst, 0);
2852
2853 pVM->pgm.s.cZeroPages--;
2854 GCPhys += PAGE_SIZE;
2855 pPageSrc++;
2856 pPageDst++;
2857 }
2858
2859 /* Flush physical page map TLB. */
2860 pgmPhysInvalidatePageMapTLB(pVM);
2861
2862 if (cPendingPages)
2863 {
2864 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2865 AssertLogRelRCReturn(rc, rc);
2866 }
2867 GMMR3FreePagesCleanup(pReq);
2868
2869 /* Force a PGM pool flush as guest ram references have been changed. */
2870 /** @todo not entirely SMP safe; assuming for now the guest takes care of
2871 * this internally (not touch mapped mmio while changing the mapping). */
2872 PVMCPU pVCpu = VMMGetCpu(pVM);
2873 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
2874 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2875
2876 pgmUnlock(pVM);
2877 }
2878 else
2879 {
2880 RTGCPHYS cb = pCur->RamRange.cb;
2881
2882 /* Clear the tracking data of pages we're going to reactivate. */
2883 PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
2884 uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2885 while (cPagesLeft-- > 0)
2886 {
2887 PGM_PAGE_SET_TRACKING(pVM, pPageSrc, 0);
2888 PGM_PAGE_SET_PTE_INDEX(pVM, pPageSrc, 0);
2889 pPageSrc++;
2890 }
2891
2892 /* link in the ram range */
2893 pgmR3PhysLinkRamRange(pVM, &pCur->RamRange, pRamPrev);
2894 pgmUnlock(pVM);
2895
2896#ifdef VBOX_WITH_REM
2897 REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_MMIO2);
2898#endif
2899 }
2900
2901 pgmPhysInvalidatePageMapTLB(pVM);
2902 return VINF_SUCCESS;
2903}
2904
2905
2906/**
2907 * Unmaps a MMIO2 region.
2908 *
2909 * This is done when a guest / the bios / state loading changes the
2910 * PCI config. The replacing of base memory has the same restrictions
2911 * as during registration, of course.
2912 */
2913VMMR3DECL(int) PGMR3PhysMMIO2Unmap(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
2914{
2915 /*
2916 * Validate input
2917 */
2918 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
2919 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
2920 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
2921 AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
2922 AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
2923 AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2924
2925 PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
2926 AssertReturn(pCur, VERR_NOT_FOUND);
2927 AssertReturn(pCur->fMapped, VERR_WRONG_ORDER);
2928 AssertReturn(pCur->RamRange.GCPhys == GCPhys, VERR_INVALID_PARAMETER);
2929 Assert(pCur->RamRange.GCPhysLast != NIL_RTGCPHYS);
2930
2931 Log(("PGMR3PhysMMIO2Unmap: %RGp-%RGp %s\n",
2932 pCur->RamRange.GCPhys, pCur->RamRange.GCPhysLast, pCur->RamRange.pszDesc));
2933
2934 /*
2935 * Unmap it.
2936 */
2937 pgmLock(pVM);
2938
2939#ifdef VBOX_WITH_REM
2940 RTGCPHYS GCPhysRangeREM;
2941 RTGCPHYS cbRangeREM;
2942 bool fInformREM;
2943#endif
2944 if (pCur->fOverlapping)
2945 {
2946 /* Restore the RAM pages we've replaced. */
2947 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
2948 while (pRam->GCPhys > pCur->RamRange.GCPhysLast)
2949 pRam = pRam->pNextR3;
2950
2951 PPGMPAGE pPageDst = &pRam->aPages[(pCur->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
2952 uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
2953 while (cPagesLeft-- > 0)
2954 {
2955 PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
2956 pVM->pgm.s.cZeroPages++;
2957 pPageDst++;
2958 }
2959
2960 /* Flush physical page map TLB. */
2961 pgmPhysInvalidatePageMapTLB(pVM);
2962#ifdef VBOX_WITH_REM
2963 GCPhysRangeREM = NIL_RTGCPHYS; /* shuts up gcc */
2964 cbRangeREM = RTGCPHYS_MAX; /* ditto */
2965 fInformREM = false;
2966#endif
2967 }
2968 else
2969 {
2970#ifdef VBOX_WITH_REM
2971 GCPhysRangeREM = pCur->RamRange.GCPhys;
2972 cbRangeREM = pCur->RamRange.cb;
2973 fInformREM = true;
2974#endif
2975 pgmR3PhysUnlinkRamRange(pVM, &pCur->RamRange);
2976 }
2977
2978 pCur->RamRange.GCPhys = NIL_RTGCPHYS;
2979 pCur->RamRange.GCPhysLast = NIL_RTGCPHYS;
2980 pCur->fOverlapping = false;
2981 pCur->fMapped = false;
2982
2983 /* Force a PGM pool flush as guest ram references have been changed. */
2984 /** @todo not entirely SMP safe; assuming for now the guest takes care
2985 * of this internally (not touch mapped mmio while changing the
2986 * mapping). */
2987 PVMCPU pVCpu = VMMGetCpu(pVM);
2988 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
2989 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2990
2991 pgmPhysInvalidatePageMapTLB(pVM);
2992 pgmPhysInvalidRamRangeTlbs(pVM);
2993 pgmUnlock(pVM);
2994
2995#ifdef VBOX_WITH_REM
2996 if (fInformREM)
2997 REMR3NotifyPhysRamDeregister(pVM, GCPhysRangeREM, cbRangeREM);
2998#endif
2999
3000 return VINF_SUCCESS;
3001}
3002
3003
3004/**
3005 * Checks if the given address is an MMIO2 base address or not.
3006 *
3007 * @returns true/false accordingly.
3008 * @param pVM Pointer to the VM.
3009 * @param pDevIns The owner of the memory, optional.
3010 * @param GCPhys The address to check.
3011 */
3012VMMR3DECL(bool) PGMR3PhysMMIO2IsBase(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
3013{
3014 /*
3015 * Validate input
3016 */
3017 VM_ASSERT_EMT_RETURN(pVM, false);
3018 AssertPtrReturn(pDevIns, false);
3019 AssertReturn(GCPhys != NIL_RTGCPHYS, false);
3020 AssertReturn(GCPhys != 0, false);
3021 AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), false);
3022
3023 /*
3024 * Search the list.
3025 */
3026 pgmLock(pVM);
3027 for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
3028 if (pCur->RamRange.GCPhys == GCPhys)
3029 {
3030 Assert(pCur->fMapped);
3031 pgmUnlock(pVM);
3032 return true;
3033 }
3034 pgmUnlock(pVM);
3035 return false;
3036}
3037
3038
3039/**
3040 * Gets the HC physical address of a page in the MMIO2 region.
3041 *
3042 * This is API is intended for MMHyper and shouldn't be called
3043 * by anyone else...
3044 *
3045 * @returns VBox status code.
3046 * @param pVM Pointer to the VM.
3047 * @param pDevIns The owner of the memory, optional.
3048 * @param iRegion The region.
3049 * @param off The page expressed an offset into the MMIO2 region.
3050 * @param pHCPhys Where to store the result.
3051 */
3052VMMR3DECL(int) PGMR3PhysMMIO2GetHCPhys(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, PRTHCPHYS pHCPhys)
3053{
3054 /*
3055 * Validate input
3056 */
3057 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3058 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3059 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
3060
3061 pgmLock(pVM);
3062 PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
3063 AssertReturn(pCur, VERR_NOT_FOUND);
3064 AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
3065
3066 PCPGMPAGE pPage = &pCur->RamRange.aPages[off >> PAGE_SHIFT];
3067 *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3068 pgmUnlock(pVM);
3069 return VINF_SUCCESS;
3070}
3071
3072
3073/**
3074 * Maps a portion of an MMIO2 region into kernel space (host).
3075 *
3076 * The kernel mapping will become invalid when the MMIO2 memory is deregistered
3077 * or the VM is terminated.
3078 *
3079 * @return VBox status code.
3080 *
3081 * @param pVM Pointer to the VM.
3082 * @param pDevIns The device owning the MMIO2 memory.
3083 * @param iRegion The region.
3084 * @param off The offset into the region. Must be page aligned.
3085 * @param cb The number of bytes to map. Must be page aligned.
3086 * @param pszDesc Mapping description.
3087 * @param pR0Ptr Where to store the R0 address.
3088 */
3089VMMR3DECL(int) PGMR3PhysMMIO2MapKernel(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
3090 const char *pszDesc, PRTR0PTR pR0Ptr)
3091{
3092 /*
3093 * Validate input.
3094 */
3095 VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3096 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3097 AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
3098
3099 PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
3100 AssertReturn(pCur, VERR_NOT_FOUND);
3101 AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
3102 AssertReturn(cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
3103 AssertReturn(off + cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
3104 NOREF(pszDesc);
3105
3106 /*
3107 * Pass the request on to the support library/driver.
3108 */
3109 int rc = SUPR3PageMapKernel(pCur->pvR3, off, cb, 0, pR0Ptr);
3110
3111 return rc;
3112}
3113
3114
3115/**
3116 * Worker for PGMR3PhysRomRegister.
3117 *
3118 * This is here to simplify lock management, i.e. the caller does all the
3119 * locking and we can simply return without needing to remember to unlock
3120 * anything first.
3121 *
3122 * @returns VBox status.
3123 * @param pVM Pointer to the VM.
3124 * @param pDevIns The device instance owning the ROM.
3125 * @param GCPhys First physical address in the range.
3126 * Must be page aligned!
3127 * @param cb The size of the range (in bytes).
3128 * Must be page aligned!
3129 * @param pvBinary Pointer to the binary data backing the ROM image.
3130 * @param cbBinary The size of the binary data pvBinary points to.
3131 * This must be less or equal to @a cb.
3132 * @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
3133 * and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
3134 * @param pszDesc Pointer to description string. This must not be freed.
3135 */
3136static int pgmR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
3137 const void *pvBinary, uint32_t cbBinary, uint32_t fFlags, const char *pszDesc)
3138{
3139 /*
3140 * Validate input.
3141 */
3142 AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3143 AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
3144 AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
3145 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
3146 AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3147 AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
3148 AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
3149 AssertReturn(!(fFlags & ~(PGMPHYS_ROM_FLAGS_SHADOWED | PGMPHYS_ROM_FLAGS_PERMANENT_BINARY)), VERR_INVALID_PARAMETER);
3150 VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
3151
3152 const uint32_t cPages = cb >> PAGE_SHIFT;
3153
3154 /*
3155 * Find the ROM location in the ROM list first.
3156 */
3157 PPGMROMRANGE pRomPrev = NULL;
3158 PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3;
3159 while (pRom && GCPhysLast >= pRom->GCPhys)
3160 {
3161 if ( GCPhys <= pRom->GCPhysLast
3162 && GCPhysLast >= pRom->GCPhys)
3163 AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
3164 GCPhys, GCPhysLast, pszDesc,
3165 pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
3166 VERR_PGM_RAM_CONFLICT);
3167 /* next */
3168 pRomPrev = pRom;
3169 pRom = pRom->pNextR3;
3170 }
3171
3172 /*
3173 * Find the RAM location and check for conflicts.
3174 *
3175 * Conflict detection is a bit different than for RAM
3176 * registration since a ROM can be located within a RAM
3177 * range. So, what we have to check for is other memory
3178 * types (other than RAM that is) and that we don't span
3179 * more than one RAM range (layz).
3180 */
3181 bool fRamExists = false;
3182 PPGMRAMRANGE pRamPrev = NULL;
3183 PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
3184 while (pRam && GCPhysLast >= pRam->GCPhys)
3185 {
3186 if ( GCPhys <= pRam->GCPhysLast
3187 && GCPhysLast >= pRam->GCPhys)
3188 {
3189 /* completely within? */
3190 AssertLogRelMsgReturn( GCPhys >= pRam->GCPhys
3191 && GCPhysLast <= pRam->GCPhysLast,
3192 ("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
3193 GCPhys, GCPhysLast, pszDesc,
3194 pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
3195 VERR_PGM_RAM_CONFLICT);
3196 fRamExists = true;
3197 break;
3198 }
3199
3200 /* next */
3201 pRamPrev = pRam;
3202 pRam = pRam->pNextR3;
3203 }
3204 if (fRamExists)
3205 {
3206 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3207 uint32_t cPagesLeft = cPages;
3208 while (cPagesLeft-- > 0)
3209 {
3210 AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
3211 ("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
3212 pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
3213 pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
3214 Assert(PGM_PAGE_IS_ZERO(pPage));
3215 pPage++;
3216 }
3217 }
3218
3219 /*
3220 * Update the base memory reservation if necessary.
3221 */
3222 uint32_t cExtraBaseCost = fRamExists ? 0 : cPages;
3223 if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3224 cExtraBaseCost += cPages;
3225 if (cExtraBaseCost)
3226 {
3227 int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
3228 if (RT_FAILURE(rc))
3229 return rc;
3230 }
3231
3232 /*
3233 * Allocate memory for the virgin copy of the RAM.
3234 */
3235 PGMMALLOCATEPAGESREQ pReq;
3236 int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
3237 AssertRCReturn(rc, rc);
3238
3239 for (uint32_t iPage = 0; iPage < cPages; iPage++)
3240 {
3241 pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
3242 pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
3243 pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
3244 }
3245
3246 rc = GMMR3AllocatePagesPerform(pVM, pReq);
3247 if (RT_FAILURE(rc))
3248 {
3249 GMMR3AllocatePagesCleanup(pReq);
3250 return rc;
3251 }
3252
3253 /*
3254 * Allocate the new ROM range and RAM range (if necessary).
3255 */
3256 PPGMROMRANGE pRomNew;
3257 rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
3258 if (RT_SUCCESS(rc))
3259 {
3260 PPGMRAMRANGE pRamNew = NULL;
3261 if (!fRamExists)
3262 rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), sizeof(PGMPAGE), MM_TAG_PGM_PHYS, (void **)&pRamNew);
3263 if (RT_SUCCESS(rc))
3264 {
3265 /*
3266 * Initialize and insert the RAM range (if required).
3267 */
3268 PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
3269 if (!fRamExists)
3270 {
3271 pRamNew->pSelfR0 = MMHyperCCToR0(pVM, pRamNew);
3272 pRamNew->pSelfRC = MMHyperCCToRC(pVM, pRamNew);
3273 pRamNew->GCPhys = GCPhys;
3274 pRamNew->GCPhysLast = GCPhysLast;
3275 pRamNew->cb = cb;
3276 pRamNew->pszDesc = pszDesc;
3277 pRamNew->fFlags = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
3278 pRamNew->pvR3 = NULL;
3279 pRamNew->paLSPages = NULL;
3280
3281 PPGMPAGE pPage = &pRamNew->aPages[0];
3282 for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3283 {
3284 PGM_PAGE_INIT(pPage,
3285 pReq->aPages[iPage].HCPhysGCPhys,
3286 pReq->aPages[iPage].idPage,
3287 PGMPAGETYPE_ROM,
3288 PGM_PAGE_STATE_ALLOCATED);
3289
3290 pRomPage->Virgin = *pPage;
3291 }
3292
3293 pVM->pgm.s.cAllPages += cPages;
3294 pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
3295 }
3296 else
3297 {
3298 PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3299 for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
3300 {
3301 PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_ROM);
3302 PGM_PAGE_SET_HCPHYS(pVM, pPage, pReq->aPages[iPage].HCPhysGCPhys);
3303 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
3304 PGM_PAGE_SET_PAGEID(pVM, pPage, pReq->aPages[iPage].idPage);
3305 PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
3306 PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
3307 PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
3308
3309 pRomPage->Virgin = *pPage;
3310 }
3311
3312 pRamNew = pRam;
3313
3314 pVM->pgm.s.cZeroPages -= cPages;
3315 }
3316 pVM->pgm.s.cPrivatePages += cPages;
3317
3318 /* Flush physical page map TLB. */
3319 pgmPhysInvalidatePageMapTLB(pVM);
3320
3321
3322 /*
3323 * !HACK ALERT! REM + (Shadowed) ROM ==> mess.
3324 *
3325 * If it's shadowed we'll register the handler after the ROM notification
3326 * so we get the access handler callbacks that we should. If it isn't
3327 * shadowed we'll do it the other way around to make REM use the built-in
3328 * ROM behavior and not the handler behavior (which is to route all access
3329 * to PGM atm).
3330 */
3331 if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3332 {
3333#ifdef VBOX_WITH_REM
3334 REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, true /* fShadowed */);
3335#endif
3336 rc = PGMR3HandlerPhysicalRegister(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE, GCPhys, GCPhysLast,
3337 pgmR3PhysRomWriteHandler, pRomNew,
3338 NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
3339 NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
3340 }
3341 else
3342 {
3343 rc = PGMR3HandlerPhysicalRegister(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE, GCPhys, GCPhysLast,
3344 pgmR3PhysRomWriteHandler, pRomNew,
3345 NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
3346 NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
3347#ifdef VBOX_WITH_REM
3348 REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, false /* fShadowed */);
3349#endif
3350 }
3351 if (RT_SUCCESS(rc))
3352 {
3353 /*
3354 * Copy the image over to the virgin pages.
3355 * This must be done after linking in the RAM range.
3356 */
3357 size_t cbBinaryLeft = cbBinary;
3358 PPGMPAGE pRamPage = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
3359 for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
3360 {
3361 void *pvDstPage;
3362 rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << PAGE_SHIFT), &pvDstPage);
3363 if (RT_FAILURE(rc))
3364 {
3365 VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
3366 break;
3367 }
3368 if (cbBinaryLeft >= PAGE_SIZE)
3369 {
3370 memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), PAGE_SIZE);
3371 cbBinaryLeft -= PAGE_SIZE;
3372 }
3373 else
3374 {
3375 ASMMemZeroPage(pvDstPage); /* (shouldn't be necessary, but can't hurt either) */
3376 if (cbBinaryLeft > 0)
3377 {
3378 memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), cbBinaryLeft);
3379 cbBinaryLeft = 0;
3380 }
3381 }
3382 }
3383 if (RT_SUCCESS(rc))
3384 {
3385 /*
3386 * Initialize the ROM range.
3387 * Note that the Virgin member of the pages has already been initialized above.
3388 */
3389 pRomNew->GCPhys = GCPhys;
3390 pRomNew->GCPhysLast = GCPhysLast;
3391 pRomNew->cb = cb;
3392 pRomNew->fFlags = fFlags;
3393 pRomNew->idSavedState = UINT8_MAX;
3394 pRomNew->cbOriginal = cbBinary;
3395#ifdef VBOX_STRICT
3396 pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
3397 ? pvBinary : RTMemDup(pvBinary, cbBinary);
3398#else
3399 pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY ? pvBinary : NULL;
3400#endif
3401 pRomNew->pszDesc = pszDesc;
3402
3403 for (unsigned iPage = 0; iPage < cPages; iPage++)
3404 {
3405 PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
3406 pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
3407 PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
3408 }
3409
3410 /* update the page count stats for the shadow pages. */
3411 if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3412 {
3413 pVM->pgm.s.cZeroPages += cPages;
3414 pVM->pgm.s.cAllPages += cPages;
3415 }
3416
3417 /*
3418 * Insert the ROM range, tell REM and return successfully.
3419 */
3420 pRomNew->pNextR3 = pRom;
3421 pRomNew->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
3422 pRomNew->pNextRC = pRom ? MMHyperCCToRC(pVM, pRom) : NIL_RTRCPTR;
3423
3424 if (pRomPrev)
3425 {
3426 pRomPrev->pNextR3 = pRomNew;
3427 pRomPrev->pNextR0 = MMHyperCCToR0(pVM, pRomNew);
3428 pRomPrev->pNextRC = MMHyperCCToRC(pVM, pRomNew);
3429 }
3430 else
3431 {
3432 pVM->pgm.s.pRomRangesR3 = pRomNew;
3433 pVM->pgm.s.pRomRangesR0 = MMHyperCCToR0(pVM, pRomNew);
3434 pVM->pgm.s.pRomRangesRC = MMHyperCCToRC(pVM, pRomNew);
3435 }
3436
3437 pgmPhysInvalidatePageMapTLB(pVM);
3438 GMMR3AllocatePagesCleanup(pReq);
3439 return VINF_SUCCESS;
3440 }
3441
3442 /* bail out */
3443
3444 int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
3445 AssertRC(rc2);
3446 }
3447
3448 if (!fRamExists)
3449 {
3450 pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
3451 MMHyperFree(pVM, pRamNew);
3452 }
3453 }
3454 MMHyperFree(pVM, pRomNew);
3455 }
3456
3457 /** @todo Purge the mapping cache or something... */
3458 GMMR3FreeAllocatedPages(pVM, pReq);
3459 GMMR3AllocatePagesCleanup(pReq);
3460 return rc;
3461}
3462
3463
3464/**
3465 * Registers a ROM image.
3466 *
3467 * Shadowed ROM images requires double the amount of backing memory, so,
3468 * don't use that unless you have to. Shadowing of ROM images is process
3469 * where we can select where the reads go and where the writes go. On real
3470 * hardware the chipset provides means to configure this. We provide
3471 * PGMR3PhysProtectROM() for this purpose.
3472 *
3473 * A read-only copy of the ROM image will always be kept around while we
3474 * will allocate RAM pages for the changes on demand (unless all memory
3475 * is configured to be preallocated).
3476 *
3477 * @returns VBox status.
3478 * @param pVM Pointer to the VM.
3479 * @param pDevIns The device instance owning the ROM.
3480 * @param GCPhys First physical address in the range.
3481 * Must be page aligned!
3482 * @param cb The size of the range (in bytes).
3483 * Must be page aligned!
3484 * @param pvBinary Pointer to the binary data backing the ROM image.
3485 * @param cbBinary The size of the binary data pvBinary points to.
3486 * This must be less or equal to @a cb.
3487 * @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
3488 * and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
3489 * @param pszDesc Pointer to description string. This must not be freed.
3490 *
3491 * @remark There is no way to remove the rom, automatically on device cleanup or
3492 * manually from the device yet. This isn't difficult in any way, it's
3493 * just not something we expect to be necessary for a while.
3494 */
3495VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
3496 const void *pvBinary, uint32_t cbBinary, uint32_t fFlags, const char *pszDesc)
3497{
3498 Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
3499 pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
3500 pgmLock(pVM);
3501 int rc = pgmR3PhysRomRegister(pVM, pDevIns, GCPhys, cb, pvBinary, cbBinary, fFlags, pszDesc);
3502 pgmUnlock(pVM);
3503 return rc;
3504}
3505
3506
3507/**
3508 * \#PF Handler callback for ROM write accesses.
3509 *
3510 * @returns VINF_SUCCESS if the handler have carried out the operation.
3511 * @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
3512 * @param pVM Pointer to the VM.
3513 * @param GCPhys The physical address the guest is writing to.
3514 * @param pvPhys The HC mapping of that address.
3515 * @param pvBuf What the guest is reading/writing.
3516 * @param cbBuf How much it's reading/writing.
3517 * @param enmAccessType The access type.
3518 * @param pvUser User argument.
3519 */
3520static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf,
3521 PGMACCESSTYPE enmAccessType, void *pvUser)
3522{
3523 PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
3524 const uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3525 Assert(iPage < (pRom->cb >> PAGE_SHIFT));
3526 PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3527 Log5(("pgmR3PhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
3528 NOREF(pvPhys);
3529
3530 if (enmAccessType == PGMACCESSTYPE_READ)
3531 {
3532 switch (pRomPage->enmProt)
3533 {
3534 /*
3535 * Take the default action.
3536 */
3537 case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3538 case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3539 case PGMROMPROT_READ_ROM_WRITE_RAM:
3540 case PGMROMPROT_READ_RAM_WRITE_RAM:
3541 return VINF_PGM_HANDLER_DO_DEFAULT;
3542
3543 default:
3544 AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3545 pRom->aPages[iPage].enmProt, iPage, GCPhys),
3546 VERR_IPE_NOT_REACHED_DEFAULT_CASE);
3547 }
3548 }
3549 else
3550 {
3551 Assert(enmAccessType == PGMACCESSTYPE_WRITE);
3552 switch (pRomPage->enmProt)
3553 {
3554 /*
3555 * Ignore writes.
3556 */
3557 case PGMROMPROT_READ_ROM_WRITE_IGNORE:
3558 case PGMROMPROT_READ_RAM_WRITE_IGNORE:
3559 return VINF_SUCCESS;
3560
3561 /*
3562 * Write to the RAM page.
3563 */
3564 case PGMROMPROT_READ_ROM_WRITE_RAM:
3565 case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's *way* simpler that way. */
3566 {
3567 /* This should be impossible now, pvPhys doesn't work cross page anylonger. */
3568 Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
3569
3570 /*
3571 * Take the lock, do lazy allocation, map the page and copy the data.
3572 *
3573 * Note that we have to bypass the mapping TLB since it works on
3574 * guest physical addresses and entering the shadow page would
3575 * kind of screw things up...
3576 */
3577 int rc = pgmLock(pVM);
3578 AssertRC(rc);
3579
3580 PPGMPAGE pShadowPage = &pRomPage->Shadow;
3581 if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
3582 {
3583 pShadowPage = pgmPhysGetPage(pVM, GCPhys);
3584 AssertLogRelReturn(pShadowPage, VERR_PGM_PHYS_PAGE_GET_IPE);
3585 }
3586
3587 void *pvDstPage;
3588 rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
3589 if (RT_SUCCESS(rc))
3590 {
3591 memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
3592 pRomPage->LiveSave.fWrittenTo = true;
3593 }
3594
3595 pgmUnlock(pVM);
3596 return rc;
3597 }
3598
3599 default:
3600 AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
3601 pRom->aPages[iPage].enmProt, iPage, GCPhys),
3602 VERR_IPE_NOT_REACHED_DEFAULT_CASE);
3603 }
3604 }
3605}
3606
3607
3608/**
3609 * Called by PGMR3MemSetup to reset the shadow, switch to the virgin, and verify
3610 * that the virgin part is untouched.
3611 *
3612 * This is done after the normal memory has been cleared.
3613 *
3614 * ASSUMES that the caller owns the PGM lock.
3615 *
3616 * @param pVM Pointer to the VM.
3617 */
3618int pgmR3PhysRomReset(PVM pVM)
3619{
3620 PGM_LOCK_ASSERT_OWNER(pVM);
3621 for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3622 {
3623 const uint32_t cPages = pRom->cb >> PAGE_SHIFT;
3624
3625 if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
3626 {
3627 /*
3628 * Reset the physical handler.
3629 */
3630 int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
3631 AssertRCReturn(rc, rc);
3632
3633 /*
3634 * What we do with the shadow pages depends on the memory
3635 * preallocation option. If not enabled, we'll just throw
3636 * out all the dirty pages and replace them by the zero page.
3637 */
3638 if (!pVM->pgm.s.fRamPreAlloc)
3639 {
3640 /* Free the dirty pages. */
3641 uint32_t cPendingPages = 0;
3642 PGMMFREEPAGESREQ pReq;
3643 rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
3644 AssertRCReturn(rc, rc);
3645
3646 for (uint32_t iPage = 0; iPage < cPages; iPage++)
3647 if ( !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
3648 && !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
3649 {
3650 Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
3651 rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
3652 pRom->GCPhys + (iPage << PAGE_SHIFT));
3653 AssertLogRelRCReturn(rc, rc);
3654 }
3655
3656 if (cPendingPages)
3657 {
3658 rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
3659 AssertLogRelRCReturn(rc, rc);
3660 }
3661 GMMR3FreePagesCleanup(pReq);
3662 }
3663 else
3664 {
3665 /* clear all the shadow pages. */
3666 for (uint32_t iPage = 0; iPage < cPages; iPage++)
3667 {
3668 if (PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow))
3669 continue;
3670 Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
3671 void *pvDstPage;
3672 const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3673 rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
3674 if (RT_FAILURE(rc))
3675 break;
3676 ASMMemZeroPage(pvDstPage);
3677 }
3678 AssertRCReturn(rc, rc);
3679 }
3680 }
3681
3682#ifdef VBOX_STRICT
3683 /*
3684 * Verify that the virgin page is unchanged if possible.
3685 */
3686 if (pRom->pvOriginal)
3687 {
3688 size_t cbSrcLeft = pRom->cbOriginal;
3689 uint8_t const *pbSrcPage = (uint8_t const *)pRom->pvOriginal;
3690 for (uint32_t iPage = 0; iPage < cPages && cbSrcLeft > 0; iPage++, pbSrcPage += PAGE_SIZE)
3691 {
3692 const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
3693 void const *pvDstPage;
3694 int rc = pgmPhysPageMapReadOnly(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPage);
3695 if (RT_FAILURE(rc))
3696 break;
3697
3698 if (memcmp(pvDstPage, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE)))
3699 {
3700# ifdef DEBUG_bird /* This is darn handy for EFI debugging w/ snapshots, should be made default later. */
3701 void *pvDstPageW;
3702 rc = pgmPhysPageMap(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPageW);
3703 AssertRCReturn(rc, rc);
3704 memcpy(pvDstPageW, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE));
3705# else
3706 LogRel(("pgmR3PhysRomReset: %RGp rom page changed (%s) - loaded saved state?\n",
3707 GCPhys, pRom->pszDesc));
3708# endif
3709 }
3710 cbSrcLeft -= RT_MIN(cbSrcLeft, PAGE_SIZE);
3711 }
3712 }
3713#endif
3714 }
3715
3716 return VINF_SUCCESS;
3717}
3718
3719
3720/**
3721 * Called by PGMR3Term to free resources.
3722 *
3723 * ASSUMES that the caller owns the PGM lock.
3724 *
3725 * @param pVM Pointer to the VM.
3726 */
3727void pgmR3PhysRomTerm(PVM pVM)
3728{
3729#ifdef RT_STRICT
3730 /*
3731 * Free the heap copy of the original bits.
3732 */
3733 for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3734 {
3735 if ( pRom->pvOriginal
3736 && !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
3737 {
3738 RTMemFree((void *)pRom->pvOriginal);
3739 pRom->pvOriginal = NULL;
3740 }
3741 }
3742#endif
3743}
3744
3745
3746/**
3747 * Change the shadowing of a range of ROM pages.
3748 *
3749 * This is intended for implementing chipset specific memory registers
3750 * and will not be very strict about the input. It will silently ignore
3751 * any pages that are not the part of a shadowed ROM.
3752 *
3753 * @returns VBox status code.
3754 * @retval VINF_PGM_SYNC_CR3
3755 *
3756 * @param pVM Pointer to the VM.
3757 * @param GCPhys Where to start. Page aligned.
3758 * @param cb How much to change. Page aligned.
3759 * @param enmProt The new ROM protection.
3760 */
3761VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
3762{
3763 /*
3764 * Check input
3765 */
3766 if (!cb)
3767 return VINF_SUCCESS;
3768 AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3769 AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3770 RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
3771 AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3772 AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
3773
3774 /*
3775 * Process the request.
3776 */
3777 pgmLock(pVM);
3778 int rc = VINF_SUCCESS;
3779 bool fFlushTLB = false;
3780 for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
3781 {
3782 if ( GCPhys <= pRom->GCPhysLast
3783 && GCPhysLast >= pRom->GCPhys
3784 && (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
3785 {
3786 /*
3787 * Iterate the relevant pages and make necessary the changes.
3788 */
3789 bool fChanges = false;
3790 uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
3791 ? pRom->cb >> PAGE_SHIFT
3792 : (GCPhysLast - pRom->GCPhys + 1) >> PAGE_SHIFT;
3793 for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
3794 iPage < cPages;
3795 iPage++)
3796 {
3797 PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
3798 if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
3799 {
3800 fChanges = true;
3801
3802 /* flush references to the page. */
3803 PPGMPAGE pRamPage = pgmPhysGetPage(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT));
3804 int rc2 = pgmPoolTrackUpdateGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage,
3805 true /*fFlushPTEs*/, &fFlushTLB);
3806 if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS || RT_FAILURE(rc2)))
3807 rc = rc2;
3808
3809 PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
3810 PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
3811
3812 *pOld = *pRamPage;
3813 *pRamPage = *pNew;
3814 /** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
3815 }
3816 pRomPage->enmProt = enmProt;
3817 }
3818
3819 /*
3820 * Reset the access handler if we made changes, no need
3821 * to optimize this.
3822 */
3823 if (fChanges)
3824 {
3825 int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
3826 if (RT_FAILURE(rc2))
3827 {
3828 pgmUnlock(pVM);
3829 AssertRC(rc);
3830 return rc2;
3831 }
3832 }
3833
3834 /* Advance - cb isn't updated. */
3835 GCPhys = pRom->GCPhys + (cPages << PAGE_SHIFT);
3836 }
3837 }
3838 pgmUnlock(pVM);
3839 if (fFlushTLB)
3840 PGM_INVL_ALL_VCPU_TLBS(pVM);
3841
3842 return rc;
3843}
3844
3845
3846/**
3847 * Sets the Address Gate 20 state.
3848 *
3849 * @param pVCpu Pointer to the VMCPU.
3850 * @param fEnable True if the gate should be enabled.
3851 * False if the gate should be disabled.
3852 */
3853VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
3854{
3855 LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
3856 if (pVCpu->pgm.s.fA20Enabled != fEnable)
3857 {
3858 pVCpu->pgm.s.fA20Enabled = fEnable;
3859 pVCpu->pgm.s.GCPhysA20Mask = ~((RTGCPHYS)!fEnable << 20);
3860#ifdef VBOX_WITH_REM
3861 REMR3A20Set(pVCpu->pVMR3, pVCpu, fEnable);
3862#endif
3863#ifdef PGM_WITH_A20
3864 pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL;
3865 VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3866 pgmR3RefreshShadowModeAfterA20Change(pVCpu);
3867 HMFlushTLB(pVCpu);
3868#endif
3869 STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cA20Changes);
3870 }
3871}
3872
3873
3874/**
3875 * Tree enumeration callback for dealing with age rollover.
3876 * It will perform a simple compression of the current age.
3877 */
3878static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
3879{
3880 /* Age compression - ASSUMES iNow == 4. */
3881 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3882 if (pChunk->iLastUsed >= UINT32_C(0xffffff00))
3883 pChunk->iLastUsed = 3;
3884 else if (pChunk->iLastUsed >= UINT32_C(0xfffff000))
3885 pChunk->iLastUsed = 2;
3886 else if (pChunk->iLastUsed)
3887 pChunk->iLastUsed = 1;
3888 else /* iLastUsed = 0 */
3889 pChunk->iLastUsed = 4;
3890
3891 NOREF(pvUser);
3892 return 0;
3893}
3894
3895
3896/**
3897 * The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
3898 */
3899typedef struct PGMR3PHYSCHUNKUNMAPCB
3900{
3901 PVM pVM; /**< Pointer to the VM. */
3902 PPGMCHUNKR3MAP pChunk; /**< The chunk to unmap. */
3903} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
3904
3905
3906/**
3907 * Callback used to find the mapping that's been unused for
3908 * the longest time.
3909 */
3910static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
3911{
3912 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
3913 PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
3914
3915 /*
3916 * Check for locks and compare when last used.
3917 */
3918 if (pChunk->cRefs)
3919 return 0;
3920 if (pChunk->cPermRefs)
3921 return 0;
3922 if ( pArg->pChunk
3923 && pChunk->iLastUsed >= pArg->pChunk->iLastUsed)
3924 return 0;
3925
3926 /*
3927 * Check that it's not in any of the TLBs.
3928 */
3929 PVM pVM = pArg->pVM;
3930 if ( pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(pChunk->Core.Key)].idChunk
3931 == pChunk->Core.Key)
3932 {
3933 pChunk = NULL;
3934 return 0;
3935 }
3936#ifdef VBOX_STRICT
3937 for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
3938 {
3939 Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk != pChunk);
3940 Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk != pChunk->Core.Key);
3941 }
3942#endif
3943
3944 for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
3945 if (pVM->pgm.s.PhysTlbHC.aEntries[i].pMap == pChunk)
3946 return 0;
3947
3948 pArg->pChunk = pChunk;
3949 return 0;
3950}
3951
3952
3953/**
3954 * Finds a good candidate for unmapping when the ring-3 mapping cache is full.
3955 *
3956 * The candidate will not be part of any TLBs, so no need to flush
3957 * anything afterwards.
3958 *
3959 * @returns Chunk id.
3960 * @param pVM Pointer to the VM.
3961 */
3962static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
3963{
3964 PGM_LOCK_ASSERT_OWNER(pVM);
3965
3966 /*
3967 * Enumerate the age tree starting with the left most node.
3968 */
3969 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3970 PGMR3PHYSCHUNKUNMAPCB Args;
3971 Args.pVM = pVM;
3972 Args.pChunk = NULL;
3973 RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
3974 Assert(Args.pChunk);
3975 if (Args.pChunk)
3976 {
3977 Assert(Args.pChunk->cRefs == 0);
3978 Assert(Args.pChunk->cPermRefs == 0);
3979 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3980 return Args.pChunk->Core.Key;
3981 }
3982
3983 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
3984 return INT32_MAX;
3985}
3986
3987
3988/**
3989 * Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
3990 *
3991 * This is only called on one of the EMTs while the other ones are waiting for
3992 * it to complete this function.
3993 *
3994 * @returns VINF_SUCCESS (VBox strict status code).
3995 * @param pVM Pointer to the VM.
3996 * @param pVCpu The VMCPU for the EMT we're being called on. Unused.
3997 * @param pvUser User pointer. Unused
3998 *
3999 */
4000static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
4001{
4002 int rc = VINF_SUCCESS;
4003 pgmLock(pVM);
4004 NOREF(pVCpu); NOREF(pvUser);
4005
4006 if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
4007 {
4008 /* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
4009 /** @todo also not really efficient to unmap a chunk that contains PD
4010 * or PT pages. */
4011 pgmR3PoolClearAllRendezvous(pVM, &pVM->aCpus[0], NULL /* no need to flush the REM TLB as we already did that above */);
4012
4013 /*
4014 * Request the ring-0 part to unmap a chunk to make space in the mapping cache.
4015 */
4016 GMMMAPUNMAPCHUNKREQ Req;
4017 Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
4018 Req.Hdr.cbReq = sizeof(Req);
4019 Req.pvR3 = NULL;
4020 Req.idChunkMap = NIL_GMM_CHUNKID;
4021 Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
4022 if (Req.idChunkUnmap != INT32_MAX)
4023 {
4024 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
4025 rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
4026 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
4027 if (RT_SUCCESS(rc))
4028 {
4029 /*
4030 * Remove the unmapped one.
4031 */
4032 PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
4033 AssertRelease(pUnmappedChunk);
4034 AssertRelease(!pUnmappedChunk->cRefs);
4035 AssertRelease(!pUnmappedChunk->cPermRefs);
4036 pUnmappedChunk->pv = NULL;
4037 pUnmappedChunk->Core.Key = UINT32_MAX;
4038#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4039 MMR3HeapFree(pUnmappedChunk);
4040#else
4041 MMR3UkHeapFree(pVM, pUnmappedChunk, MM_TAG_PGM_CHUNK_MAPPING);
4042#endif
4043 pVM->pgm.s.ChunkR3Map.c--;
4044 pVM->pgm.s.cUnmappedChunks++;
4045
4046 /*
4047 * Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses).
4048 */
4049 /** todo: we should not flush chunks which include cr3 mappings. */
4050 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
4051 {
4052 PPGMCPU pPGM = &pVM->aCpus[idCpu].pgm.s;
4053
4054 pPGM->pGst32BitPdR3 = NULL;
4055 pPGM->pGstPaePdptR3 = NULL;
4056 pPGM->pGstAmd64Pml4R3 = NULL;
4057#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4058 pPGM->pGst32BitPdR0 = NIL_RTR0PTR;
4059 pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
4060 pPGM->pGstAmd64Pml4R0 = NIL_RTR0PTR;
4061#endif
4062 for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
4063 {
4064 pPGM->apGstPaePDsR3[i] = NULL;
4065#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4066 pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
4067#endif
4068 }
4069
4070 /* Flush REM TLBs. */
4071 CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
4072 }
4073#ifdef VBOX_WITH_REM
4074 /* Flush REM translation blocks. */
4075 REMFlushTBs(pVM);
4076#endif
4077 }
4078 }
4079 }
4080 pgmUnlock(pVM);
4081 return rc;
4082}
4083
4084/**
4085 * Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
4086 *
4087 * @returns VBox status code.
4088 * @param pVM Pointer to the VM.
4089 */
4090void pgmR3PhysUnmapChunk(PVM pVM)
4091{
4092 int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
4093 AssertRC(rc);
4094}
4095
4096
4097/**
4098 * Maps the given chunk into the ring-3 mapping cache.
4099 *
4100 * This will call ring-0.
4101 *
4102 * @returns VBox status code.
4103 * @param pVM Pointer to the VM.
4104 * @param idChunk The chunk in question.
4105 * @param ppChunk Where to store the chunk tracking structure.
4106 *
4107 * @remarks Called from within the PGM critical section.
4108 * @remarks Can be called from any thread!
4109 */
4110int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
4111{
4112 int rc;
4113
4114 PGM_LOCK_ASSERT_OWNER(pVM);
4115
4116 /*
4117 * Move the chunk time forward.
4118 */
4119 pVM->pgm.s.ChunkR3Map.iNow++;
4120 if (pVM->pgm.s.ChunkR3Map.iNow == 0)
4121 {
4122 pVM->pgm.s.ChunkR3Map.iNow = 4;
4123 RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkAgeingRolloverCallback, NULL);
4124 }
4125
4126 /*
4127 * Allocate a new tracking structure first.
4128 */
4129#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4130 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
4131#else
4132 PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3UkHeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk), NULL);
4133#endif
4134 AssertReturn(pChunk, VERR_NO_MEMORY);
4135 pChunk->Core.Key = idChunk;
4136 pChunk->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
4137
4138 /*
4139 * Request the ring-0 part to map the chunk in question.
4140 */
4141 GMMMAPUNMAPCHUNKREQ Req;
4142 Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
4143 Req.Hdr.cbReq = sizeof(Req);
4144 Req.pvR3 = NULL;
4145 Req.idChunkMap = idChunk;
4146 Req.idChunkUnmap = NIL_GMM_CHUNKID;
4147
4148 /* Must be callable from any thread, so can't use VMMR3CallR0. */
4149 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
4150 rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
4151 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
4152 if (RT_SUCCESS(rc))
4153 {
4154 pChunk->pv = Req.pvR3;
4155
4156 /*
4157 * If we're running out of virtual address space, then we should
4158 * unmap another chunk.
4159 *
4160 * Currently, an unmap operation requires that all other virtual CPUs
4161 * are idling and not by chance making use of the memory we're
4162 * unmapping. So, we create an async unmap operation here.
4163 *
4164 * Now, when creating or restoring a saved state this wont work very
4165 * well since we may want to restore all guest RAM + a little something.
4166 * So, we have to do the unmap synchronously. Fortunately for us
4167 * though, during these operations the other virtual CPUs are inactive
4168 * and it should be safe to do this.
4169 */
4170 /** @todo Eventually we should lock all memory when used and do
4171 * map+unmap as one kernel call without any rendezvous or
4172 * other precautions. */
4173 if (pVM->pgm.s.ChunkR3Map.c + 1 >= pVM->pgm.s.ChunkR3Map.cMax)
4174 {
4175 switch (VMR3GetState(pVM))
4176 {
4177 case VMSTATE_LOADING:
4178 case VMSTATE_SAVING:
4179 {
4180 PVMCPU pVCpu = VMMGetCpu(pVM);
4181 if ( pVCpu
4182 && pVM->pgm.s.cDeprecatedPageLocks == 0)
4183 {
4184 pgmR3PhysUnmapChunkRendezvous(pVM, pVCpu, NULL);
4185 break;
4186 }
4187 /* fall thru */
4188 }
4189 default:
4190 rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
4191 AssertRC(rc);
4192 break;
4193 }
4194 }
4195
4196 /*
4197 * Update the tree. We must do this after any unmapping to make sure
4198 * the chunk we're going to return isn't unmapped by accident.
4199 */
4200 AssertPtr(Req.pvR3);
4201 bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
4202 AssertRelease(fRc);
4203 pVM->pgm.s.ChunkR3Map.c++;
4204 pVM->pgm.s.cMappedChunks++;
4205 }
4206 else
4207 {
4208 /** @todo this may fail because of /proc/sys/vm/max_map_count, so we
4209 * should probably restrict ourselves on linux. */
4210 AssertRC(rc);
4211#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
4212 MMR3HeapFree(pChunk);
4213#else
4214 MMR3UkHeapFree(pVM, pChunk, MM_TAG_PGM_CHUNK_MAPPING);
4215#endif
4216 pChunk = NULL;
4217 }
4218
4219 *ppChunk = pChunk;
4220 return rc;
4221}
4222
4223
4224/**
4225 * For VMMCALLRING3_PGM_MAP_CHUNK, considered internal.
4226 *
4227 * @returns see pgmR3PhysChunkMap.
4228 * @param pVM Pointer to the VM.
4229 * @param idChunk The chunk to map.
4230 */
4231VMMR3DECL(int) PGMR3PhysChunkMap(PVM pVM, uint32_t idChunk)
4232{
4233 PPGMCHUNKR3MAP pChunk;
4234 int rc;
4235
4236 pgmLock(pVM);
4237 rc = pgmR3PhysChunkMap(pVM, idChunk, &pChunk);
4238 pgmUnlock(pVM);
4239 return rc;
4240}
4241
4242
4243/**
4244 * Invalidates the TLB for the ring-3 mapping cache.
4245 *
4246 * @param pVM Pointer to the VM.
4247 */
4248VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
4249{
4250 pgmLock(pVM);
4251 for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
4252 {
4253 pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
4254 pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
4255 }
4256 /* The page map TLB references chunks, so invalidate that one too. */
4257 pgmPhysInvalidatePageMapTLB(pVM);
4258 pgmUnlock(pVM);
4259}
4260
4261
4262/**
4263 * Response to VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE to allocate a large
4264 * (2MB) page for use with a nested paging PDE.
4265 *
4266 * @returns The following VBox status codes.
4267 * @retval VINF_SUCCESS on success.
4268 * @retval VINF_EM_NO_MEMORY if we're out of memory.
4269 *
4270 * @param pVM Pointer to the VM.
4271 * @param GCPhys GC physical start address of the 2 MB range
4272 */
4273VMMR3DECL(int) PGMR3PhysAllocateLargeHandyPage(PVM pVM, RTGCPHYS GCPhys)
4274{
4275#ifdef PGM_WITH_LARGE_PAGES
4276 uint64_t u64TimeStamp1, u64TimeStamp2;
4277
4278 pgmLock(pVM);
4279
4280 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4281 u64TimeStamp1 = RTTimeMilliTS();
4282 int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE, 0, NULL);
4283 u64TimeStamp2 = RTTimeMilliTS();
4284 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
4285 if (RT_SUCCESS(rc))
4286 {
4287 Assert(pVM->pgm.s.cLargeHandyPages == 1);
4288
4289 uint32_t idPage = pVM->pgm.s.aLargeHandyPage[0].idPage;
4290 RTHCPHYS HCPhys = pVM->pgm.s.aLargeHandyPage[0].HCPhysGCPhys;
4291
4292 void *pv;
4293
4294 /* Map the large page into our address space.
4295 *
4296 * Note: assuming that within the 2 MB range:
4297 * - GCPhys + PAGE_SIZE = HCPhys + PAGE_SIZE (whole point of this exercise)
4298 * - user space mapping is continuous as well
4299 * - page id (GCPhys) + 1 = page id (GCPhys + PAGE_SIZE)
4300 */
4301 rc = pgmPhysPageMapByPageID(pVM, idPage, HCPhys, &pv);
4302 AssertLogRelMsg(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", idPage, HCPhys, rc));
4303
4304 if (RT_SUCCESS(rc))
4305 {
4306 /*
4307 * Clear the pages.
4308 */
4309 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4310 for (unsigned i = 0; i < _2M/PAGE_SIZE; i++)
4311 {
4312 ASMMemZeroPage(pv);
4313
4314 PPGMPAGE pPage;
4315 rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
4316 AssertRC(rc);
4317
4318 Assert(PGM_PAGE_IS_ZERO(pPage));
4319 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
4320 pVM->pgm.s.cZeroPages--;
4321
4322 /*
4323 * Do the PGMPAGE modifications.
4324 */
4325 pVM->pgm.s.cPrivatePages++;
4326 PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
4327 PGM_PAGE_SET_PAGEID(pVM, pPage, idPage);
4328 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
4329 PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE);
4330 PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4331 PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4332
4333 /* Somewhat dirty assumption that page ids are increasing. */
4334 idPage++;
4335
4336 HCPhys += PAGE_SIZE;
4337 GCPhys += PAGE_SIZE;
4338
4339 pv = (void *)((uintptr_t)pv + PAGE_SIZE);
4340
4341 Log3(("PGMR3PhysAllocateLargePage: idPage=%#x HCPhys=%RGp\n", idPage, HCPhys));
4342 }
4343 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
4344
4345 /* Flush all TLBs. */
4346 PGM_INVL_ALL_VCPU_TLBS(pVM);
4347 pgmPhysInvalidatePageMapTLB(pVM);
4348 }
4349 pVM->pgm.s.cLargeHandyPages = 0;
4350 }
4351
4352 if (RT_SUCCESS(rc))
4353 {
4354 static uint32_t cTimeOut = 0;
4355 uint64_t u64TimeStampDelta = u64TimeStamp2 - u64TimeStamp1;
4356
4357 if (u64TimeStampDelta > 100)
4358 {
4359 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatLargePageOverflow);
4360 if ( ++cTimeOut > 10
4361 || u64TimeStampDelta > 1000 /* more than one second forces an early retirement from allocating large pages. */)
4362 {
4363 /* If repeated attempts to allocate a large page takes more than 100 ms, then we fall back to normal 4k pages.
4364 * E.g. Vista 64 tries to move memory around, which takes a huge amount of time.
4365 */
4366 LogRel(("PGMR3PhysAllocateLargePage: allocating large pages takes too long (last attempt %d ms; nr of timeouts %d); DISABLE\n", u64TimeStampDelta, cTimeOut));
4367 PGMSetLargePageUsage(pVM, false);
4368 }
4369 }
4370 else
4371 if (cTimeOut > 0)
4372 cTimeOut--;
4373 }
4374
4375 pgmUnlock(pVM);
4376 return rc;
4377#else
4378 return VERR_NOT_IMPLEMENTED;
4379#endif /* PGM_WITH_LARGE_PAGES */
4380}
4381
4382
4383/**
4384 * Response to VM_FF_PGM_NEED_HANDY_PAGES and VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES.
4385 *
4386 * This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
4387 * signal and clear the out of memory condition. When contracted, this API is
4388 * used to try clear the condition when the user wants to resume.
4389 *
4390 * @returns The following VBox status codes.
4391 * @retval VINF_SUCCESS on success. FFs cleared.
4392 * @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
4393 * this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
4394 *
4395 * @param pVM Pointer to the VM.
4396 *
4397 * @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
4398 * in EM.cpp and shouldn't be propagated outside TRPM, HM, EM and
4399 * pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
4400 * handler.
4401 */
4402VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
4403{
4404 pgmLock(pVM);
4405
4406 /*
4407 * Allocate more pages, noting down the index of the first new page.
4408 */
4409 uint32_t iClear = pVM->pgm.s.cHandyPages;
4410 AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_PGM_HANDY_PAGE_IPE);
4411 Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
4412 int rcAlloc = VINF_SUCCESS;
4413 int rcSeed = VINF_SUCCESS;
4414 int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4415 while (rc == VERR_GMM_SEED_ME)
4416 {
4417 void *pvChunk;
4418 rcAlloc = rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
4419 if (RT_SUCCESS(rc))
4420 {
4421 rcSeed = rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
4422 if (RT_FAILURE(rc))
4423 SUPR3PageFree(pvChunk, GMM_CHUNK_SIZE >> PAGE_SHIFT);
4424 }
4425 if (RT_SUCCESS(rc))
4426 rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
4427 }
4428
4429 /* todo: we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
4430 if ( rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
4431 && pVM->pgm.s.cHandyPages > 0)
4432 {
4433 /* Still handy pages left, so don't panic. */
4434 rc = VINF_SUCCESS;
4435 }
4436
4437 if (RT_SUCCESS(rc))
4438 {
4439 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
4440 Assert(pVM->pgm.s.cHandyPages > 0);
4441 VM_FF_CLEAR(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4442 VM_FF_CLEAR(pVM, VM_FF_PGM_NO_MEMORY);
4443
4444#ifdef VBOX_STRICT
4445 uint32_t i;
4446 for (i = iClear; i < pVM->pgm.s.cHandyPages; i++)
4447 if ( pVM->pgm.s.aHandyPages[i].idPage == NIL_GMM_PAGEID
4448 || pVM->pgm.s.aHandyPages[i].idSharedPage != NIL_GMM_PAGEID
4449 || (pVM->pgm.s.aHandyPages[i].HCPhysGCPhys & PAGE_OFFSET_MASK))
4450 break;
4451 if (i != pVM->pgm.s.cHandyPages)
4452 {
4453 RTAssertMsg1Weak(NULL, __LINE__, __FILE__, __FUNCTION__);
4454 RTAssertMsg2Weak("i=%d iClear=%d cHandyPages=%d\n", i, iClear, pVM->pgm.s.cHandyPages);
4455 for (uint32_t j = iClear; j < pVM->pgm.s.cHandyPages; j++)
4456 RTAssertMsg2Add(("%03d: idPage=%d HCPhysGCPhys=%RHp idSharedPage=%d%\n", j,
4457 pVM->pgm.s.aHandyPages[j].idPage,
4458 pVM->pgm.s.aHandyPages[j].HCPhysGCPhys,
4459 pVM->pgm.s.aHandyPages[j].idSharedPage,
4460 j == i ? " <---" : ""));
4461 RTAssertPanic();
4462 }
4463#endif
4464 /*
4465 * Clear the pages.
4466 */
4467 while (iClear < pVM->pgm.s.cHandyPages)
4468 {
4469 PGMMPAGEDESC pPage = &pVM->pgm.s.aHandyPages[iClear];
4470 void *pv;
4471 rc = pgmPhysPageMapByPageID(pVM, pPage->idPage, pPage->HCPhysGCPhys, &pv);
4472 AssertLogRelMsgBreak(RT_SUCCESS(rc),
4473 ("%u/%u: idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n",
4474 iClear, pVM->pgm.s.cHandyPages, pPage->idPage, pPage->HCPhysGCPhys, rc));
4475 ASMMemZeroPage(pv);
4476 iClear++;
4477 Log3(("PGMR3PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
4478 }
4479 }
4480 else
4481 {
4482 uint64_t cAllocPages, cMaxPages, cBalloonPages;
4483
4484 /*
4485 * We should never get here unless there is a genuine shortage of
4486 * memory (or some internal error). Flag the error so the VM can be
4487 * suspended ASAP and the user informed. If we're totally out of
4488 * handy pages we will return failure.
4489 */
4490 /* Report the failure. */
4491 LogRel(("PGM: Failed to procure handy pages; rc=%Rrc rcAlloc=%Rrc rcSeed=%Rrc cHandyPages=%#x\n"
4492 " cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
4493 rc, rcAlloc, rcSeed,
4494 pVM->pgm.s.cHandyPages,
4495 pVM->pgm.s.cAllPages,
4496 pVM->pgm.s.cPrivatePages,
4497 pVM->pgm.s.cSharedPages,
4498 pVM->pgm.s.cZeroPages));
4499
4500 if (GMMR3QueryMemoryStats(pVM, &cAllocPages, &cMaxPages, &cBalloonPages) == VINF_SUCCESS)
4501 {
4502 LogRel(("GMM: Statistics:\n"
4503 " Allocated pages: %RX64\n"
4504 " Maximum pages: %RX64\n"
4505 " Ballooned pages: %RX64\n", cAllocPages, cMaxPages, cBalloonPages));
4506 }
4507
4508 if ( rc != VERR_NO_MEMORY
4509 && rc != VERR_NO_PHYS_MEMORY
4510 && rc != VERR_LOCK_FAILED)
4511 {
4512 for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4513 {
4514 LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
4515 i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
4516 pVM->pgm.s.aHandyPages[i].idSharedPage));
4517 uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
4518 if (idPage != NIL_GMM_PAGEID)
4519 {
4520 for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesXR3;
4521 pRam;
4522 pRam = pRam->pNextR3)
4523 {
4524 uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
4525 for (uint32_t iPage = 0; iPage < cPages; iPage++)
4526 if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
4527 LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
4528 pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
4529 }
4530 }
4531 }
4532 }
4533
4534 /* Set the FFs and adjust rc. */
4535 VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
4536 VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
4537 if ( rc == VERR_NO_MEMORY
4538 || rc == VERR_NO_PHYS_MEMORY
4539 || rc == VERR_LOCK_FAILED)
4540 rc = VINF_EM_NO_MEMORY;
4541 }
4542
4543 pgmUnlock(pVM);
4544 return rc;
4545}
4546
4547
4548/**
4549 * Frees the specified RAM page and replaces it with the ZERO page.
4550 *
4551 * This is used by ballooning, remapping MMIO2, RAM reset and state loading.
4552 *
4553 * @param pVM Pointer to the VM.
4554 * @param pReq Pointer to the request.
4555 * @param pcPendingPages Where the number of pages waiting to be freed are
4556 * kept. This will normally be incremented.
4557 * @param pPage Pointer to the page structure.
4558 * @param GCPhys The guest physical address of the page, if applicable.
4559 *
4560 * @remarks The caller must own the PGM lock.
4561 */
4562int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys)
4563{
4564 /*
4565 * Assert sanity.
4566 */
4567 PGM_LOCK_ASSERT_OWNER(pVM);
4568 if (RT_UNLIKELY( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
4569 && PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
4570 {
4571 AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4572 return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
4573 }
4574
4575 /** @todo What about ballooning of large pages??! */
4576 Assert( PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
4577 && PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED);
4578
4579 if ( PGM_PAGE_IS_ZERO(pPage)
4580 || PGM_PAGE_IS_BALLOONED(pPage))
4581 return VINF_SUCCESS;
4582
4583 const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
4584 Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
4585 if (RT_UNLIKELY( idPage == NIL_GMM_PAGEID
4586 || idPage > GMM_PAGEID_LAST
4587 || PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
4588 {
4589 AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
4590 return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
4591 }
4592
4593 /* update page count stats. */
4594 if (PGM_PAGE_IS_SHARED(pPage))
4595 pVM->pgm.s.cSharedPages--;
4596 else
4597 pVM->pgm.s.cPrivatePages--;
4598 pVM->pgm.s.cZeroPages++;
4599
4600 /* Deal with write monitored pages. */
4601 if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
4602 {
4603 PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
4604 pVM->pgm.s.cWrittenToPages++;
4605 }
4606
4607 /*
4608 * pPage = ZERO page.
4609 */
4610 PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg);
4611 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
4612 PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
4613 PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4614 PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
4615 PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
4616
4617 /* Flush physical page map TLB entry. */
4618 pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
4619
4620 /*
4621 * Make sure it's not in the handy page array.
4622 */
4623 for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
4624 {
4625 if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
4626 {
4627 pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
4628 break;
4629 }
4630 if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
4631 {
4632 pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
4633 break;
4634 }
4635 }
4636
4637 /*
4638 * Push it onto the page array.
4639 */
4640 uint32_t iPage = *pcPendingPages;
4641 Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
4642 *pcPendingPages += 1;
4643
4644 pReq->aPages[iPage].idPage = idPage;
4645
4646 if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
4647 return VINF_SUCCESS;
4648
4649 /*
4650 * Flush the pages.
4651 */
4652 int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
4653 if (RT_SUCCESS(rc))
4654 {
4655 GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
4656 *pcPendingPages = 0;
4657 }
4658 return rc;
4659}
4660
4661
4662/**
4663 * Converts a GC physical address to a HC ring-3 pointer, with some
4664 * additional checks.
4665 *
4666 * @returns VBox status code.
4667 * @retval VINF_SUCCESS on success.
4668 * @retval VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4669 * access handler of some kind.
4670 * @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4671 * accesses or is odd in any way.
4672 * @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4673 *
4674 * @param pVM Pointer to the VM.
4675 * @param GCPhys The GC physical address to convert. Since this is only
4676 * used for filling the REM TLB, the A20 mask must be
4677 * applied before calling this API.
4678 * @param fWritable Whether write access is required.
4679 * @param ppv Where to store the pointer corresponding to GCPhys on
4680 * success.
4681 */
4682VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
4683{
4684 pgmLock(pVM);
4685 PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
4686
4687 PPGMRAMRANGE pRam;
4688 PPGMPAGE pPage;
4689 int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4690 if (RT_SUCCESS(rc))
4691 {
4692 if (PGM_PAGE_IS_BALLOONED(pPage))
4693 rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4694 else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
4695 rc = VINF_SUCCESS;
4696 else
4697 {
4698 if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4699 rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4700 else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
4701 {
4702 /** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
4703 * in -norawr0 mode. */
4704 if (fWritable)
4705 rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4706 }
4707 else
4708 {
4709 /* Temporarily disabled physical handler(s), since the recompiler
4710 doesn't get notified when it's reset we'll have to pretend it's
4711 operating normally. */
4712 if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
4713 rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4714 else
4715 rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
4716 }
4717 }
4718 if (RT_SUCCESS(rc))
4719 {
4720 int rc2;
4721
4722 /* Make sure what we return is writable. */
4723 if (fWritable)
4724 switch (PGM_PAGE_GET_STATE(pPage))
4725 {
4726 case PGM_PAGE_STATE_ALLOCATED:
4727 break;
4728 case PGM_PAGE_STATE_BALLOONED:
4729 AssertFailed();
4730 break;
4731 case PGM_PAGE_STATE_ZERO:
4732 case PGM_PAGE_STATE_SHARED:
4733 if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
4734 break;
4735 case PGM_PAGE_STATE_WRITE_MONITORED:
4736 rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
4737 AssertLogRelRCReturn(rc2, rc2);
4738 break;
4739 }
4740
4741 /* Get a ring-3 mapping of the address. */
4742 PPGMPAGER3MAPTLBE pTlbe;
4743 rc2 = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
4744 AssertLogRelRCReturn(rc2, rc2);
4745 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4746 /** @todo mapping/locking hell; this isn't horribly efficient since
4747 * pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
4748
4749 Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] *ppv=%p\n", GCPhys, rc, pPage, *ppv));
4750 }
4751 else
4752 Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
4753
4754 /* else: handler catching all access, no pointer returned. */
4755 }
4756 else
4757 rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
4758
4759 pgmUnlock(pVM);
4760 return rc;
4761}
4762
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette