VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp@ 73009

Last change on this file since 73009 was 72600, checked in by vboxsync, 7 years ago

VMM: Eliminated VBOX_WITH_2ND_IEM_STEP.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 177.7 KB
Line 
1/* $Id: PGMAllPhys.cpp 72600 2018-06-18 13:40:48Z vboxsync $ */
2/** @file
3 * PGM - Page Manager and Monitor, Physical Memory Addressing.
4 */
5
6/*
7 * Copyright (C) 2006-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_PGM_PHYS
23#include <VBox/vmm/pgm.h>
24#include <VBox/vmm/trpm.h>
25#include <VBox/vmm/vmm.h>
26#include <VBox/vmm/iom.h>
27#include <VBox/vmm/em.h>
28#include <VBox/vmm/nem.h>
29#ifdef VBOX_WITH_REM
30# include <VBox/vmm/rem.h>
31#endif
32#include "PGMInternal.h"
33#include <VBox/vmm/vm.h>
34#include "PGMInline.h"
35#include <VBox/param.h>
36#include <VBox/err.h>
37#include <iprt/assert.h>
38#include <iprt/string.h>
39#include <iprt/asm-amd64-x86.h>
40#include <VBox/log.h>
41#ifdef IN_RING3
42# include <iprt/thread.h>
43#endif
44
45
46/*********************************************************************************************************************************
47* Defined Constants And Macros *
48*********************************************************************************************************************************/
49/** Enable the physical TLB. */
50#define PGM_WITH_PHYS_TLB
51
52/** @def PGM_HANDLER_PHYS_IS_VALID_STATUS
53 * Checks if valid physical access handler return code (normal handler, not PF).
54 *
55 * Checks if the given strict status code is one of the expected ones for a
56 * physical access handler in the current context.
57 *
58 * @returns true or false.
59 * @param a_rcStrict The status code.
60 * @param a_fWrite Whether it is a write or read being serviced.
61 *
62 * @remarks We wish to keep the list of statuses here as short as possible.
63 * When changing, please make sure to update the PGMPhysRead,
64 * PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too.
65 */
66#ifdef IN_RING3
67# define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
68 ( (a_rcStrict) == VINF_SUCCESS \
69 || (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT)
70#elif defined(IN_RING0) || defined(IN_RC)
71#define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
72 ( (a_rcStrict) == VINF_SUCCESS \
73 || (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT \
74 \
75 || (a_rcStrict) == ((a_fWrite) ? VINF_IOM_R3_MMIO_WRITE : VINF_IOM_R3_MMIO_READ) \
76 || (a_rcStrict) == VINF_IOM_R3_MMIO_READ_WRITE \
77 || ((a_rcStrict) == VINF_IOM_R3_MMIO_COMMIT_WRITE && (a_fWrite)) \
78 \
79 || ((a_fWrite) ? (a_rcStrict) == VINF_EM_RAW_EMULATE_IO_BLOCK : false) \
80 \
81 || (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR \
82 || (a_rcStrict) == VINF_EM_DBG_STOP \
83 || (a_rcStrict) == VINF_EM_DBG_EVENT \
84 || (a_rcStrict) == VINF_EM_DBG_BREAKPOINT \
85 || (a_rcStrict) == VINF_EM_OFF \
86 || (a_rcStrict) == VINF_EM_SUSPEND \
87 || (a_rcStrict) == VINF_EM_RESET \
88 )
89#else
90# error "Context?"
91#endif
92
93/** @def PGM_HANDLER_VIRT_IS_VALID_STATUS
94 * Checks if valid virtual access handler return code (normal handler, not PF).
95 *
96 * Checks if the given strict status code is one of the expected ones for a
97 * virtual access handler in the current context.
98 *
99 * @returns true or false.
100 * @param a_rcStrict The status code.
101 * @param a_fWrite Whether it is a write or read being serviced.
102 *
103 * @remarks We wish to keep the list of statuses here as short as possible.
104 * When changing, please make sure to update the PGMPhysRead,
105 * PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too.
106 */
107#ifdef IN_RING3
108# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
109 ( (a_rcStrict) == VINF_SUCCESS \
110 || (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT)
111#elif defined(IN_RING0)
112# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
113 (false /* no virtual handlers in ring-0! */ )
114#elif defined(IN_RC)
115# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
116 ( (a_rcStrict) == VINF_SUCCESS \
117 || (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT \
118 \
119 || ((a_fWrite) ? (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT : 0) \
120 || ((a_fWrite) ? (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT : 0) \
121 || ((a_fWrite) ? (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT : 0) \
122 || ((a_fWrite) ? (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT : 0) \
123 || ((a_fWrite) ? (a_rcStrict) == VINF_SELM_SYNC_GDT : 0) \
124 || ((a_fWrite) ? (a_rcStrict) == VINF_CSAM_PENDING_ACTION : 0) \
125 || (a_rcStrict) == VINF_PATM_CHECK_PATCH_PAGE \
126 \
127 || (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR \
128 || (a_rcStrict) == VINF_EM_DBG_STOP \
129 || (a_rcStrict) == VINF_EM_DBG_EVENT \
130 || (a_rcStrict) == VINF_EM_DBG_BREAKPOINT \
131 )
132#else
133# error "Context?"
134#endif
135
136
137
138#ifndef IN_RING3
139
140/**
141 * @callback_method_impl{FNPGMPHYSHANDLER,
142 * Dummy for forcing ring-3 handling of the access.}
143 */
144DECLEXPORT(VBOXSTRICTRC)
145pgmPhysHandlerRedirectToHC(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf,
146 PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
147{
148 NOREF(pVM); NOREF(pVCpu); NOREF(GCPhys); NOREF(pvPhys); NOREF(pvBuf); NOREF(cbBuf);
149 NOREF(enmAccessType); NOREF(enmOrigin); NOREF(pvUser);
150 return VINF_EM_RAW_EMULATE_INSTR;
151}
152
153
154/**
155 * @callback_method_impl{FNPGMRZPHYSPFHANDLER,
156 * Dummy for forcing ring-3 handling of the access.}
157 */
158VMMDECL(VBOXSTRICTRC) pgmPhysPfHandlerRedirectToHC(PVM pVM, PVMCPU pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame,
159 RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
160{
161 NOREF(pVM); NOREF(pVCpu); NOREF(uErrorCode); NOREF(pRegFrame); NOREF(pvFault); NOREF(GCPhysFault); NOREF(pvUser);
162 return VINF_EM_RAW_EMULATE_INSTR;
163}
164
165
166/**
167 * @callback_method_impl{FNPGMRZPHYSPFHANDLER,
168 * \#PF access handler callback for guest ROM range write access.}
169 *
170 * @remarks The @a pvUser argument points to the PGMROMRANGE.
171 */
172DECLEXPORT(VBOXSTRICTRC) pgmPhysRomWritePfHandler(PVM pVM, PVMCPU pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame,
173 RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
174{
175 int rc;
176 PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
177 uint32_t iPage = (GCPhysFault - pRom->GCPhys) >> PAGE_SHIFT;
178 NOREF(uErrorCode); NOREF(pvFault);
179
180 Assert(uErrorCode & X86_TRAP_PF_RW); /* This shall not be used for read access! */
181
182 Assert(iPage < (pRom->cb >> PAGE_SHIFT));
183 switch (pRom->aPages[iPage].enmProt)
184 {
185 case PGMROMPROT_READ_ROM_WRITE_IGNORE:
186 case PGMROMPROT_READ_RAM_WRITE_IGNORE:
187 {
188 /*
189 * If it's a simple instruction which doesn't change the cpu state
190 * we will simply skip it. Otherwise we'll have to defer it to REM.
191 */
192 uint32_t cbOp;
193 PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
194 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
195 if ( RT_SUCCESS(rc)
196 && pDis->uCpuMode == DISCPUMODE_32BIT /** @todo why does this matter? */
197 && !(pDis->fPrefix & (DISPREFIX_REPNE | DISPREFIX_REP | DISPREFIX_SEG)))
198 {
199 switch (pDis->bOpCode)
200 {
201 /** @todo Find other instructions we can safely skip, possibly
202 * adding this kind of detection to DIS or EM. */
203 case OP_MOV:
204 pRegFrame->rip += cbOp;
205 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestROMWriteHandled);
206 return VINF_SUCCESS;
207 }
208 }
209 break;
210 }
211
212 case PGMROMPROT_READ_RAM_WRITE_RAM:
213 pRom->aPages[iPage].LiveSave.fWrittenTo = true;
214 rc = PGMHandlerPhysicalPageTempOff(pVM, pRom->GCPhys, GCPhysFault & X86_PTE_PG_MASK);
215 AssertRC(rc);
216 break; /** @todo Must edit the shadow PT and restart the instruction, not use the interpreter! */
217
218 case PGMROMPROT_READ_ROM_WRITE_RAM:
219 /* Handle it in ring-3 because it's *way* easier there. */
220 pRom->aPages[iPage].LiveSave.fWrittenTo = true;
221 break;
222
223 default:
224 AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhysFault=%RGp\n",
225 pRom->aPages[iPage].enmProt, iPage, GCPhysFault),
226 VERR_IPE_NOT_REACHED_DEFAULT_CASE);
227 }
228
229 STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZGuestROMWriteUnhandled);
230 return VINF_EM_RAW_EMULATE_INSTR;
231}
232
233#endif /* !IN_RING3 */
234
235
236/**
237 * @callback_method_impl{FNPGMPHYSHANDLER,
238 * Access handler callback for ROM write accesses.}
239 *
240 * @remarks The @a pvUser argument points to the PGMROMRANGE.
241 */
242PGM_ALL_CB2_DECL(VBOXSTRICTRC)
243pgmPhysRomWriteHandler(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf,
244 PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
245{
246 PPGMROMRANGE pRom = (PPGMROMRANGE)pvUser;
247 const uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
248 Assert(iPage < (pRom->cb >> PAGE_SHIFT));
249 PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
250 Log5(("pgmPhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
251 NOREF(pVCpu); NOREF(pvPhys); NOREF(enmOrigin);
252
253 if (enmAccessType == PGMACCESSTYPE_READ)
254 {
255 switch (pRomPage->enmProt)
256 {
257 /*
258 * Take the default action.
259 */
260 case PGMROMPROT_READ_ROM_WRITE_IGNORE:
261 case PGMROMPROT_READ_RAM_WRITE_IGNORE:
262 case PGMROMPROT_READ_ROM_WRITE_RAM:
263 case PGMROMPROT_READ_RAM_WRITE_RAM:
264 return VINF_PGM_HANDLER_DO_DEFAULT;
265
266 default:
267 AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
268 pRom->aPages[iPage].enmProt, iPage, GCPhys),
269 VERR_IPE_NOT_REACHED_DEFAULT_CASE);
270 }
271 }
272 else
273 {
274 Assert(enmAccessType == PGMACCESSTYPE_WRITE);
275 switch (pRomPage->enmProt)
276 {
277 /*
278 * Ignore writes.
279 */
280 case PGMROMPROT_READ_ROM_WRITE_IGNORE:
281 case PGMROMPROT_READ_RAM_WRITE_IGNORE:
282 return VINF_SUCCESS;
283
284 /*
285 * Write to the RAM page.
286 */
287 case PGMROMPROT_READ_ROM_WRITE_RAM:
288 case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's *way* simpler that way. */
289 {
290 /* This should be impossible now, pvPhys doesn't work cross page anylonger. */
291 Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
292
293 /*
294 * Take the lock, do lazy allocation, map the page and copy the data.
295 *
296 * Note that we have to bypass the mapping TLB since it works on
297 * guest physical addresses and entering the shadow page would
298 * kind of screw things up...
299 */
300 int rc = pgmLock(pVM);
301 AssertRC(rc);
302
303 PPGMPAGE pShadowPage = &pRomPage->Shadow;
304 if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
305 {
306 pShadowPage = pgmPhysGetPage(pVM, GCPhys);
307 AssertLogRelReturn(pShadowPage, VERR_PGM_PHYS_PAGE_GET_IPE);
308 }
309
310 void *pvDstPage;
311 rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
312 if (RT_SUCCESS(rc))
313 {
314 memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
315 pRomPage->LiveSave.fWrittenTo = true;
316
317 AssertMsg( rc == VINF_SUCCESS
318 || ( rc == VINF_PGM_SYNC_CR3
319 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
320 , ("%Rrc\n", rc));
321 rc = VINF_SUCCESS;
322 }
323
324 pgmUnlock(pVM);
325 return rc;
326 }
327
328 default:
329 AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
330 pRom->aPages[iPage].enmProt, iPage, GCPhys),
331 VERR_IPE_NOT_REACHED_DEFAULT_CASE);
332 }
333 }
334}
335
336
337/**
338 * Invalidates the RAM range TLBs.
339 *
340 * @param pVM The cross context VM structure.
341 */
342void pgmPhysInvalidRamRangeTlbs(PVM pVM)
343{
344 pgmLock(pVM);
345 for (uint32_t i = 0; i < PGM_RAMRANGE_TLB_ENTRIES; i++)
346 {
347 pVM->pgm.s.apRamRangesTlbR3[i] = NIL_RTR3PTR;
348 pVM->pgm.s.apRamRangesTlbR0[i] = NIL_RTR0PTR;
349 pVM->pgm.s.apRamRangesTlbRC[i] = NIL_RTRCPTR;
350 }
351 pgmUnlock(pVM);
352}
353
354
355/**
356 * Tests if a value of type RTGCPHYS is negative if the type had been signed
357 * instead of unsigned.
358 *
359 * @returns @c true if negative, @c false if positive or zero.
360 * @param a_GCPhys The value to test.
361 * @todo Move me to iprt/types.h.
362 */
363#define RTGCPHYS_IS_NEGATIVE(a_GCPhys) ((a_GCPhys) & ((RTGCPHYS)1 << (sizeof(RTGCPHYS)*8 - 1)))
364
365
366/**
367 * Slow worker for pgmPhysGetRange.
368 *
369 * @copydoc pgmPhysGetRange
370 */
371PPGMRAMRANGE pgmPhysGetRangeSlow(PVM pVM, RTGCPHYS GCPhys)
372{
373 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
374
375 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
376 while (pRam)
377 {
378 RTGCPHYS off = GCPhys - pRam->GCPhys;
379 if (off < pRam->cb)
380 {
381 pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
382 return pRam;
383 }
384 if (RTGCPHYS_IS_NEGATIVE(off))
385 pRam = pRam->CTX_SUFF(pLeft);
386 else
387 pRam = pRam->CTX_SUFF(pRight);
388 }
389 return NULL;
390}
391
392
393/**
394 * Slow worker for pgmPhysGetRangeAtOrAbove.
395 *
396 * @copydoc pgmPhysGetRangeAtOrAbove
397 */
398PPGMRAMRANGE pgmPhysGetRangeAtOrAboveSlow(PVM pVM, RTGCPHYS GCPhys)
399{
400 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
401
402 PPGMRAMRANGE pLastLeft = NULL;
403 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
404 while (pRam)
405 {
406 RTGCPHYS off = GCPhys - pRam->GCPhys;
407 if (off < pRam->cb)
408 {
409 pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
410 return pRam;
411 }
412 if (RTGCPHYS_IS_NEGATIVE(off))
413 {
414 pLastLeft = pRam;
415 pRam = pRam->CTX_SUFF(pLeft);
416 }
417 else
418 pRam = pRam->CTX_SUFF(pRight);
419 }
420 return pLastLeft;
421}
422
423
424/**
425 * Slow worker for pgmPhysGetPage.
426 *
427 * @copydoc pgmPhysGetPage
428 */
429PPGMPAGE pgmPhysGetPageSlow(PVM pVM, RTGCPHYS GCPhys)
430{
431 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
432
433 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
434 while (pRam)
435 {
436 RTGCPHYS off = GCPhys - pRam->GCPhys;
437 if (off < pRam->cb)
438 {
439 pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
440 return &pRam->aPages[off >> PAGE_SHIFT];
441 }
442
443 if (RTGCPHYS_IS_NEGATIVE(off))
444 pRam = pRam->CTX_SUFF(pLeft);
445 else
446 pRam = pRam->CTX_SUFF(pRight);
447 }
448 return NULL;
449}
450
451
452/**
453 * Slow worker for pgmPhysGetPageEx.
454 *
455 * @copydoc pgmPhysGetPageEx
456 */
457int pgmPhysGetPageExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
458{
459 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
460
461 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
462 while (pRam)
463 {
464 RTGCPHYS off = GCPhys - pRam->GCPhys;
465 if (off < pRam->cb)
466 {
467 pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
468 *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
469 return VINF_SUCCESS;
470 }
471
472 if (RTGCPHYS_IS_NEGATIVE(off))
473 pRam = pRam->CTX_SUFF(pLeft);
474 else
475 pRam = pRam->CTX_SUFF(pRight);
476 }
477
478 *ppPage = NULL;
479 return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
480}
481
482
483/**
484 * Slow worker for pgmPhysGetPageAndRangeEx.
485 *
486 * @copydoc pgmPhysGetPageAndRangeEx
487 */
488int pgmPhysGetPageAndRangeExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
489{
490 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,RamRangeTlbMisses));
491
492 PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
493 while (pRam)
494 {
495 RTGCPHYS off = GCPhys - pRam->GCPhys;
496 if (off < pRam->cb)
497 {
498 pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
499 *ppRam = pRam;
500 *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
501 return VINF_SUCCESS;
502 }
503
504 if (RTGCPHYS_IS_NEGATIVE(off))
505 pRam = pRam->CTX_SUFF(pLeft);
506 else
507 pRam = pRam->CTX_SUFF(pRight);
508 }
509
510 *ppRam = NULL;
511 *ppPage = NULL;
512 return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
513}
514
515
516/**
517 * Checks if Address Gate 20 is enabled or not.
518 *
519 * @returns true if enabled.
520 * @returns false if disabled.
521 * @param pVCpu The cross context virtual CPU structure.
522 */
523VMMDECL(bool) PGMPhysIsA20Enabled(PVMCPU pVCpu)
524{
525 LogFlow(("PGMPhysIsA20Enabled %d\n", pVCpu->pgm.s.fA20Enabled));
526 return pVCpu->pgm.s.fA20Enabled;
527}
528
529
530/**
531 * Validates a GC physical address.
532 *
533 * @returns true if valid.
534 * @returns false if invalid.
535 * @param pVM The cross context VM structure.
536 * @param GCPhys The physical address to validate.
537 */
538VMMDECL(bool) PGMPhysIsGCPhysValid(PVM pVM, RTGCPHYS GCPhys)
539{
540 PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
541 return pPage != NULL;
542}
543
544
545/**
546 * Checks if a GC physical address is a normal page,
547 * i.e. not ROM, MMIO or reserved.
548 *
549 * @returns true if normal.
550 * @returns false if invalid, ROM, MMIO or reserved page.
551 * @param pVM The cross context VM structure.
552 * @param GCPhys The physical address to check.
553 */
554VMMDECL(bool) PGMPhysIsGCPhysNormal(PVM pVM, RTGCPHYS GCPhys)
555{
556 PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
557 return pPage
558 && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM;
559}
560
561
562/**
563 * Converts a GC physical address to a HC physical address.
564 *
565 * @returns VINF_SUCCESS on success.
566 * @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
567 * page but has no physical backing.
568 * @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
569 * GC physical address.
570 *
571 * @param pVM The cross context VM structure.
572 * @param GCPhys The GC physical address to convert.
573 * @param pHCPhys Where to store the HC physical address on success.
574 */
575VMM_INT_DECL(int) PGMPhysGCPhys2HCPhys(PVM pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
576{
577 pgmLock(pVM);
578 PPGMPAGE pPage;
579 int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
580 if (RT_SUCCESS(rc))
581 *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) | (GCPhys & PAGE_OFFSET_MASK);
582 pgmUnlock(pVM);
583 return rc;
584}
585
586
587/**
588 * Invalidates all page mapping TLBs.
589 *
590 * @param pVM The cross context VM structure.
591 */
592void pgmPhysInvalidatePageMapTLB(PVM pVM)
593{
594 pgmLock(pVM);
595 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatPageMapTlbFlushes);
596
597 /* Clear the shared R0/R3 TLB completely. */
598 for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
599 {
600 pVM->pgm.s.PhysTlbHC.aEntries[i].GCPhys = NIL_RTGCPHYS;
601 pVM->pgm.s.PhysTlbHC.aEntries[i].pPage = 0;
602 pVM->pgm.s.PhysTlbHC.aEntries[i].pMap = 0;
603 pVM->pgm.s.PhysTlbHC.aEntries[i].pv = 0;
604 }
605
606 /** @todo clear the RC TLB whenever we add it. */
607
608 pgmUnlock(pVM);
609}
610
611
612/**
613 * Invalidates a page mapping TLB entry
614 *
615 * @param pVM The cross context VM structure.
616 * @param GCPhys GCPhys entry to flush
617 */
618void pgmPhysInvalidatePageMapTLBEntry(PVM pVM, RTGCPHYS GCPhys)
619{
620 PGM_LOCK_ASSERT_OWNER(pVM);
621
622 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatPageMapTlbFlushEntry);
623
624#ifdef IN_RC
625 unsigned idx = PGM_PAGER3MAPTLB_IDX(GCPhys);
626 pVM->pgm.s.PhysTlbHC.aEntries[idx].GCPhys = NIL_RTGCPHYS;
627 pVM->pgm.s.PhysTlbHC.aEntries[idx].pPage = 0;
628 pVM->pgm.s.PhysTlbHC.aEntries[idx].pMap = 0;
629 pVM->pgm.s.PhysTlbHC.aEntries[idx].pv = 0;
630#else
631 /* Clear the shared R0/R3 TLB entry. */
632 PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
633 pTlbe->GCPhys = NIL_RTGCPHYS;
634 pTlbe->pPage = 0;
635 pTlbe->pMap = 0;
636 pTlbe->pv = 0;
637#endif
638
639 /** @todo clear the RC TLB whenever we add it. */
640}
641
642
643/**
644 * Makes sure that there is at least one handy page ready for use.
645 *
646 * This will also take the appropriate actions when reaching water-marks.
647 *
648 * @returns VBox status code.
649 * @retval VINF_SUCCESS on success.
650 * @retval VERR_EM_NO_MEMORY if we're really out of memory.
651 *
652 * @param pVM The cross context VM structure.
653 *
654 * @remarks Must be called from within the PGM critical section. It may
655 * nip back to ring-3/0 in some cases.
656 */
657static int pgmPhysEnsureHandyPage(PVM pVM)
658{
659 AssertMsg(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", pVM->pgm.s.cHandyPages));
660
661 /*
662 * Do we need to do anything special?
663 */
664#ifdef IN_RING3
665 if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_R3_ALLOC))
666#else
667 if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_RZ_TO_R3))
668#endif
669 {
670 /*
671 * Allocate pages only if we're out of them, or in ring-3, almost out.
672 */
673#ifdef IN_RING3
674 if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_R3_ALLOC)
675#else
676 if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_ALLOC)
677#endif
678 {
679 Log(("PGM: cHandyPages=%u out of %u -> allocate more; VM_FF_PGM_NO_MEMORY=%RTbool\n",
680 pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages), VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY) ));
681#ifdef IN_RING3
682 int rc = PGMR3PhysAllocateHandyPages(pVM);
683#else
684 int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES, 0);
685#endif
686 if (RT_UNLIKELY(rc != VINF_SUCCESS))
687 {
688 if (RT_FAILURE(rc))
689 return rc;
690 AssertMsgReturn(rc == VINF_EM_NO_MEMORY, ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
691 if (!pVM->pgm.s.cHandyPages)
692 {
693 LogRel(("PGM: no more handy pages!\n"));
694 return VERR_EM_NO_MEMORY;
695 }
696 Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES));
697 Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY));
698#ifdef IN_RING3
699# ifdef VBOX_WITH_REM
700 REMR3NotifyFF(pVM);
701# endif
702#else
703 VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); /* paranoia */
704#endif
705 }
706 AssertMsgReturn( pVM->pgm.s.cHandyPages > 0
707 && pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages),
708 ("%u\n", pVM->pgm.s.cHandyPages),
709 VERR_PGM_HANDY_PAGE_IPE);
710 }
711 else
712 {
713 if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_SET_FF)
714 VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
715#ifndef IN_RING3
716 if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_TO_R3)
717 {
718 Log(("PGM: VM_FF_TO_R3 - cHandyPages=%u out of %u\n", pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
719 VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
720 }
721#endif
722 }
723 }
724
725 return VINF_SUCCESS;
726}
727
728
729
730/**
731 * Replace a zero or shared page with new page that we can write to.
732 *
733 * @returns The following VBox status codes.
734 * @retval VINF_SUCCESS on success, pPage is modified.
735 * @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
736 * @retval VERR_EM_NO_MEMORY if we're totally out of memory.
737 *
738 * @todo Propagate VERR_EM_NO_MEMORY up the call tree.
739 *
740 * @param pVM The cross context VM structure.
741 * @param pPage The physical page tracking structure. This will
742 * be modified on success.
743 * @param GCPhys The address of the page.
744 *
745 * @remarks Must be called from within the PGM critical section. It may
746 * nip back to ring-3/0 in some cases.
747 *
748 * @remarks This function shouldn't really fail, however if it does
749 * it probably means we've screwed up the size of handy pages and/or
750 * the low-water mark. Or, that some device I/O is causing a lot of
751 * pages to be allocated while while the host is in a low-memory
752 * condition. This latter should be handled elsewhere and in a more
753 * controlled manner, it's on the @bugref{3170} todo list...
754 */
755int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
756{
757 LogFlow(("pgmPhysAllocPage: %R[pgmpage] %RGp\n", pPage, GCPhys));
758
759 /*
760 * Prereqs.
761 */
762 PGM_LOCK_ASSERT_OWNER(pVM);
763 AssertMsg(PGM_PAGE_IS_ZERO(pPage) || PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
764 Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
765
766# ifdef PGM_WITH_LARGE_PAGES
767 /*
768 * Try allocate a large page if applicable.
769 */
770 if ( PGMIsUsingLargePages(pVM)
771 && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
772 && !VM_IS_NEM_ENABLED(pVM)) /** @todo NEM: Implement large pages support. */
773 {
774 RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
775 PPGMPAGE pBasePage;
776
777 int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pBasePage);
778 AssertRCReturn(rc, rc); /* paranoia; can't happen. */
779 if (PGM_PAGE_GET_PDE_TYPE(pBasePage) == PGM_PAGE_PDE_TYPE_DONTCARE)
780 {
781 rc = pgmPhysAllocLargePage(pVM, GCPhys);
782 if (rc == VINF_SUCCESS)
783 return rc;
784 }
785 /* Mark the base as type page table, so we don't check over and over again. */
786 PGM_PAGE_SET_PDE_TYPE(pVM, pBasePage, PGM_PAGE_PDE_TYPE_PT);
787
788 /* fall back to 4KB pages. */
789 }
790# endif
791
792 /*
793 * Flush any shadow page table mappings of the page.
794 * When VBOX_WITH_NEW_LAZY_PAGE_ALLOC isn't defined, there shouldn't be any.
795 */
796 bool fFlushTLBs = false;
797 int rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhys, pPage, true /*fFlushTLBs*/, &fFlushTLBs);
798 AssertMsgReturn(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc), RT_FAILURE(rc) ? rc : VERR_IPE_UNEXPECTED_STATUS);
799
800 /*
801 * Ensure that we've got a page handy, take it and use it.
802 */
803 int rc2 = pgmPhysEnsureHandyPage(pVM);
804 if (RT_FAILURE(rc2))
805 {
806 if (fFlushTLBs)
807 PGM_INVL_ALL_VCPU_TLBS(pVM);
808 Assert(rc2 == VERR_EM_NO_MEMORY);
809 return rc2;
810 }
811 /* re-assert preconditions since pgmPhysEnsureHandyPage may do a context switch. */
812 PGM_LOCK_ASSERT_OWNER(pVM);
813 AssertMsg(PGM_PAGE_IS_ZERO(pPage) || PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
814 Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
815
816 uint32_t iHandyPage = --pVM->pgm.s.cHandyPages;
817 AssertMsg(iHandyPage < RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", iHandyPage));
818 Assert(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys != NIL_RTHCPHYS);
819 Assert(!(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
820 Assert(pVM->pgm.s.aHandyPages[iHandyPage].idPage != NIL_GMM_PAGEID);
821 Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
822
823 /*
824 * There are one or two action to be taken the next time we allocate handy pages:
825 * - Tell the GMM (global memory manager) what the page is being used for.
826 * (Speeds up replacement operations - sharing and defragmenting.)
827 * - If the current backing is shared, it must be freed.
828 */
829 const RTHCPHYS HCPhys = pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys;
830 pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
831
832 void const *pvSharedPage = NULL;
833 if (PGM_PAGE_IS_SHARED(pPage))
834 {
835 /* Mark this shared page for freeing/dereferencing. */
836 pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage = PGM_PAGE_GET_PAGEID(pPage);
837 Assert(PGM_PAGE_GET_PAGEID(pPage) != NIL_GMM_PAGEID);
838
839 Log(("PGM: Replaced shared page %#x at %RGp with %#x / %RHp\n", PGM_PAGE_GET_PAGEID(pPage),
840 GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
841 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageReplaceShared));
842 pVM->pgm.s.cSharedPages--;
843
844 /* Grab the address of the page so we can make a copy later on. (safe) */
845 rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvSharedPage);
846 AssertRC(rc);
847 }
848 else
849 {
850 Log2(("PGM: Replaced zero page %RGp with %#x / %RHp\n", GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
851 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
852 pVM->pgm.s.cZeroPages--;
853 }
854
855 /*
856 * Do the PGMPAGE modifications.
857 */
858 pVM->pgm.s.cPrivatePages++;
859 PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
860 PGM_PAGE_SET_PAGEID(pVM, pPage, pVM->pgm.s.aHandyPages[iHandyPage].idPage);
861 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
862 PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PT);
863 pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
864
865 /* Copy the shared page contents to the replacement page. */
866 if (pvSharedPage)
867 {
868 /* Get the virtual address of the new page. */
869 PGMPAGEMAPLOCK PgMpLck;
870 void *pvNewPage;
871 rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvNewPage, &PgMpLck); AssertRC(rc);
872 if (RT_SUCCESS(rc))
873 {
874 memcpy(pvNewPage, pvSharedPage, PAGE_SIZE); /** @todo todo write ASMMemCopyPage */
875 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
876 }
877 }
878
879 if ( fFlushTLBs
880 && rc != VINF_PGM_GCPHYS_ALIASED)
881 PGM_INVL_ALL_VCPU_TLBS(pVM);
882
883#ifndef IN_RC
884 /*
885 * Notify NEM about the mapping change for this page.
886 *
887 * Note! Shadow ROM pages are complicated as they can definitely be
888 * allocated while not visible, so play safe.
889 */
890 if (VM_IS_NEM_ENABLED(pVM))
891 {
892 PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
893 if ( enmType != PGMPAGETYPE_ROM_SHADOW
894 || pgmPhysGetPage(pVM, GCPhys) == pPage)
895 {
896 uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
897 rc2 = NEMHCNotifyPhysPageAllocated(pVM, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, HCPhys,
898 pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
899 if (RT_SUCCESS(rc))
900 PGM_PAGE_SET_NEM_STATE(pPage, u2State);
901 else
902 rc = rc2;
903 }
904 }
905#endif
906
907 return rc;
908}
909
910#ifdef PGM_WITH_LARGE_PAGES
911
912/**
913 * Replace a 2 MB range of zero pages with new pages that we can write to.
914 *
915 * @returns The following VBox status codes.
916 * @retval VINF_SUCCESS on success, pPage is modified.
917 * @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
918 * @retval VERR_EM_NO_MEMORY if we're totally out of memory.
919 *
920 * @todo Propagate VERR_EM_NO_MEMORY up the call tree.
921 *
922 * @param pVM The cross context VM structure.
923 * @param GCPhys The address of the page.
924 *
925 * @remarks Must be called from within the PGM critical section. It may
926 * nip back to ring-3/0 in some cases.
927 */
928int pgmPhysAllocLargePage(PVM pVM, RTGCPHYS GCPhys)
929{
930 RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
931 LogFlow(("pgmPhysAllocLargePage: %RGp base %RGp\n", GCPhys, GCPhysBase));
932 Assert(!VM_IS_NEM_ENABLED(pVM)); /** @todo NEM: Large page support. */
933
934 /*
935 * Prereqs.
936 */
937 PGM_LOCK_ASSERT_OWNER(pVM);
938 Assert(PGMIsUsingLargePages(pVM));
939
940 PPGMPAGE pFirstPage;
941 int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pFirstPage);
942 if ( RT_SUCCESS(rc)
943 && PGM_PAGE_GET_TYPE(pFirstPage) == PGMPAGETYPE_RAM)
944 {
945 unsigned uPDEType = PGM_PAGE_GET_PDE_TYPE(pFirstPage);
946
947 /* Don't call this function for already allocated pages. */
948 Assert(uPDEType != PGM_PAGE_PDE_TYPE_PDE);
949
950 if ( uPDEType == PGM_PAGE_PDE_TYPE_DONTCARE
951 && PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ZERO)
952 {
953 /* Lazy approach: check all pages in the 2 MB range.
954 * The whole range must be ram and unallocated. */
955 GCPhys = GCPhysBase;
956 unsigned iPage;
957 for (iPage = 0; iPage < _2M/PAGE_SIZE; iPage++)
958 {
959 PPGMPAGE pSubPage;
960 rc = pgmPhysGetPageEx(pVM, GCPhys, &pSubPage);
961 if ( RT_FAILURE(rc)
962 || PGM_PAGE_GET_TYPE(pSubPage) != PGMPAGETYPE_RAM /* Anything other than ram implies monitoring. */
963 || PGM_PAGE_GET_STATE(pSubPage) != PGM_PAGE_STATE_ZERO) /* Allocated, monitored or shared means we can't use a large page here */
964 {
965 LogFlow(("Found page %RGp with wrong attributes (type=%d; state=%d); cancel check. rc=%d\n", GCPhys, PGM_PAGE_GET_TYPE(pSubPage), PGM_PAGE_GET_STATE(pSubPage), rc));
966 break;
967 }
968 Assert(PGM_PAGE_GET_PDE_TYPE(pSubPage) == PGM_PAGE_PDE_TYPE_DONTCARE);
969 GCPhys += PAGE_SIZE;
970 }
971 if (iPage != _2M/PAGE_SIZE)
972 {
973 /* Failed. Mark as requiring a PT so we don't check the whole thing again in the future. */
974 STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRefused);
975 PGM_PAGE_SET_PDE_TYPE(pVM, pFirstPage, PGM_PAGE_PDE_TYPE_PT);
976 return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
977 }
978
979 /*
980 * Do the allocation.
981 */
982# ifdef IN_RING3
983 rc = PGMR3PhysAllocateLargeHandyPage(pVM, GCPhysBase);
984# else
985 rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE, GCPhysBase);
986# endif
987 if (RT_SUCCESS(rc))
988 {
989 Assert(PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ALLOCATED);
990 pVM->pgm.s.cLargePages++;
991 return VINF_SUCCESS;
992 }
993
994 /* If we fail once, it most likely means the host's memory is too
995 fragmented; don't bother trying again. */
996 LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
997 PGMSetLargePageUsage(pVM, false);
998 return rc;
999 }
1000 }
1001 return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
1002}
1003
1004
1005/**
1006 * Recheck the entire 2 MB range to see if we can use it again as a large page.
1007 *
1008 * @returns The following VBox status codes.
1009 * @retval VINF_SUCCESS on success, the large page can be used again
1010 * @retval VERR_PGM_INVALID_LARGE_PAGE_RANGE if it can't be reused
1011 *
1012 * @param pVM The cross context VM structure.
1013 * @param GCPhys The address of the page.
1014 * @param pLargePage Page structure of the base page
1015 */
1016int pgmPhysRecheckLargePage(PVM pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage)
1017{
1018 STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRecheck);
1019
1020 Assert(!VM_IS_NEM_ENABLED(pVM)); /** @todo NEM: Large page support. */
1021
1022 GCPhys &= X86_PDE2M_PAE_PG_MASK;
1023
1024 /* Check the base page. */
1025 Assert(PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED);
1026 if ( PGM_PAGE_GET_STATE(pLargePage) != PGM_PAGE_STATE_ALLOCATED
1027 || PGM_PAGE_GET_TYPE(pLargePage) != PGMPAGETYPE_RAM
1028 || PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
1029 {
1030 LogFlow(("pgmPhysRecheckLargePage: checks failed for base page %x %x %x\n", PGM_PAGE_GET_STATE(pLargePage), PGM_PAGE_GET_TYPE(pLargePage), PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage)));
1031 return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
1032 }
1033
1034 STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,IsValidLargePage), a);
1035 /* Check all remaining pages in the 2 MB range. */
1036 unsigned i;
1037 GCPhys += PAGE_SIZE;
1038 for (i = 1; i < _2M/PAGE_SIZE; i++)
1039 {
1040 PPGMPAGE pPage;
1041 int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1042 AssertRCBreak(rc);
1043
1044 if ( PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1045 || PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
1046 || PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
1047 || PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
1048 {
1049 LogFlow(("pgmPhysRecheckLargePage: checks failed for page %d; %x %x %x\n", i, PGM_PAGE_GET_STATE(pPage), PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)));
1050 break;
1051 }
1052
1053 GCPhys += PAGE_SIZE;
1054 }
1055 STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,IsValidLargePage), a);
1056
1057 if (i == _2M/PAGE_SIZE)
1058 {
1059 PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE);
1060 pVM->pgm.s.cLargePagesDisabled--;
1061 Log(("pgmPhysRecheckLargePage: page %RGp can be reused!\n", GCPhys - _2M));
1062 return VINF_SUCCESS;
1063 }
1064
1065 return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
1066}
1067
1068#endif /* PGM_WITH_LARGE_PAGES */
1069
1070
1071/**
1072 * Deal with a write monitored page.
1073 *
1074 * @returns VBox strict status code.
1075 *
1076 * @param pVM The cross context VM structure.
1077 * @param pPage The physical page tracking structure.
1078 * @param GCPhys The guest physical address of the page.
1079 * PGMPhysReleasePageMappingLock() passes NIL_RTGCPHYS in a
1080 * very unlikely situation where it is okay that we let NEM
1081 * fix the page access in a lazy fasion.
1082 *
1083 * @remarks Called from within the PGM critical section.
1084 */
1085void pgmPhysPageMakeWriteMonitoredWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
1086{
1087 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED);
1088 PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
1089 PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
1090 Assert(pVM->pgm.s.cMonitoredPages > 0);
1091 pVM->pgm.s.cMonitoredPages--;
1092 pVM->pgm.s.cWrittenToPages++;
1093
1094#ifndef IN_RC
1095 /*
1096 * Notify NEM about the protection change so we won't spin forever.
1097 *
1098 * Note! NEM need to be handle to lazily correct page protection as we cannot
1099 * really get it 100% right here it seems. The page pool does this too.
1100 */
1101 if (VM_IS_NEM_ENABLED(pVM) && GCPhys != NIL_RTGCPHYS)
1102 {
1103 uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
1104 PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
1105 NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage),
1106 pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
1107 PGM_PAGE_SET_NEM_STATE(pPage, u2State);
1108 }
1109#else
1110 RT_NOREF(GCPhys);
1111#endif
1112}
1113
1114
1115/**
1116 * Deal with pages that are not writable, i.e. not in the ALLOCATED state.
1117 *
1118 * @returns VBox strict status code.
1119 * @retval VINF_SUCCESS on success.
1120 * @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
1121 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1122 *
1123 * @param pVM The cross context VM structure.
1124 * @param pPage The physical page tracking structure.
1125 * @param GCPhys The address of the page.
1126 *
1127 * @remarks Called from within the PGM critical section.
1128 */
1129int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
1130{
1131 PGM_LOCK_ASSERT_OWNER(pVM);
1132 switch (PGM_PAGE_GET_STATE(pPage))
1133 {
1134 case PGM_PAGE_STATE_WRITE_MONITORED:
1135 pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
1136 RT_FALL_THRU();
1137 default: /* to shut up GCC */
1138 case PGM_PAGE_STATE_ALLOCATED:
1139 return VINF_SUCCESS;
1140
1141 /*
1142 * Zero pages can be dummy pages for MMIO or reserved memory,
1143 * so we need to check the flags before joining cause with
1144 * shared page replacement.
1145 */
1146 case PGM_PAGE_STATE_ZERO:
1147 if (PGM_PAGE_IS_MMIO(pPage))
1148 return VERR_PGM_PHYS_PAGE_RESERVED;
1149 RT_FALL_THRU();
1150 case PGM_PAGE_STATE_SHARED:
1151 return pgmPhysAllocPage(pVM, pPage, GCPhys);
1152
1153 /* Not allowed to write to ballooned pages. */
1154 case PGM_PAGE_STATE_BALLOONED:
1155 return VERR_PGM_PHYS_PAGE_BALLOONED;
1156 }
1157}
1158
1159
1160/**
1161 * Internal usage: Map the page specified by its GMM ID.
1162 *
1163 * This is similar to pgmPhysPageMap
1164 *
1165 * @returns VBox status code.
1166 *
1167 * @param pVM The cross context VM structure.
1168 * @param idPage The Page ID.
1169 * @param HCPhys The physical address (for RC).
1170 * @param ppv Where to store the mapping address.
1171 *
1172 * @remarks Called from within the PGM critical section. The mapping is only
1173 * valid while you are inside this section.
1174 */
1175int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv)
1176{
1177 /*
1178 * Validation.
1179 */
1180 PGM_LOCK_ASSERT_OWNER(pVM);
1181 AssertReturn(HCPhys && !(HCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
1182 const uint32_t idChunk = idPage >> GMM_CHUNKID_SHIFT;
1183 AssertReturn(idChunk != NIL_GMM_CHUNKID, VERR_INVALID_PARAMETER);
1184
1185#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1186 /*
1187 * Map it by HCPhys.
1188 */
1189 return pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), HCPhys, ppv RTLOG_COMMA_SRC_POS);
1190
1191#else
1192 /*
1193 * Find/make Chunk TLB entry for the mapping chunk.
1194 */
1195 PPGMCHUNKR3MAP pMap;
1196 PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
1197 if (pTlbe->idChunk == idChunk)
1198 {
1199 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbHits));
1200 pMap = pTlbe->pChunk;
1201 }
1202 else
1203 {
1204 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
1205
1206 /*
1207 * Find the chunk, map it if necessary.
1208 */
1209 pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
1210 if (pMap)
1211 pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
1212 else
1213 {
1214# ifdef IN_RING0
1215 int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
1216 AssertRCReturn(rc, rc);
1217 pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
1218 Assert(pMap);
1219# else
1220 int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
1221 if (RT_FAILURE(rc))
1222 return rc;
1223# endif
1224 }
1225
1226 /*
1227 * Enter it into the Chunk TLB.
1228 */
1229 pTlbe->idChunk = idChunk;
1230 pTlbe->pChunk = pMap;
1231 }
1232
1233 *ppv = (uint8_t *)pMap->pv + ((idPage &GMM_PAGEID_IDX_MASK) << PAGE_SHIFT);
1234 return VINF_SUCCESS;
1235#endif
1236}
1237
1238
1239/**
1240 * Maps a page into the current virtual address space so it can be accessed.
1241 *
1242 * @returns VBox status code.
1243 * @retval VINF_SUCCESS on success.
1244 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1245 *
1246 * @param pVM The cross context VM structure.
1247 * @param pPage The physical page tracking structure.
1248 * @param GCPhys The address of the page.
1249 * @param ppMap Where to store the address of the mapping tracking structure.
1250 * @param ppv Where to store the mapping address of the page. The page
1251 * offset is masked off!
1252 *
1253 * @remarks Called from within the PGM critical section.
1254 */
1255static int pgmPhysPageMapCommon(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv)
1256{
1257 PGM_LOCK_ASSERT_OWNER(pVM);
1258 NOREF(GCPhys);
1259
1260#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1261 /*
1262 * Just some sketchy GC/R0-darwin code.
1263 */
1264 *ppMap = NULL;
1265 RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
1266 Assert(HCPhys != pVM->pgm.s.HCPhysZeroPg);
1267 pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), HCPhys, ppv RTLOG_COMMA_SRC_POS);
1268 return VINF_SUCCESS;
1269
1270#else /* IN_RING3 || IN_RING0 */
1271
1272
1273 /*
1274 * Special cases: MMIO2, ZERO and specially aliased MMIO pages.
1275 */
1276 if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2
1277 || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO)
1278 {
1279 /* Decode the page id to a page in a MMIO2 ram range. */
1280 uint8_t idMmio2 = PGM_MMIO2_PAGEID_GET_MMIO2_ID(PGM_PAGE_GET_PAGEID(pPage));
1281 uint32_t iPage = PGM_MMIO2_PAGEID_GET_IDX(PGM_PAGE_GET_PAGEID(pPage));
1282 AssertLogRelMsgReturn((uint8_t)(idMmio2 - 1U) < RT_ELEMENTS(pVM->pgm.s.CTX_SUFF(apMmio2Ranges)),
1283 ("idMmio2=%u size=%u type=%u GCPHys=%#RGp Id=%u State=%u", idMmio2,
1284 RT_ELEMENTS(pVM->pgm.s.CTX_SUFF(apMmio2Ranges)), PGM_PAGE_GET_TYPE(pPage), GCPhys,
1285 pPage->s.idPage, pPage->s.uStateY),
1286 VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1287 PPGMREGMMIORANGE pMmio2Range = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[idMmio2 - 1];
1288 AssertLogRelReturn(pMmio2Range, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1289 AssertLogRelReturn(pMmio2Range->idMmio2 == idMmio2, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1290 AssertLogRelReturn(iPage < (pMmio2Range->RamRange.cb >> PAGE_SHIFT), VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
1291 *ppv = (uint8_t *)pMmio2Range->RamRange.pvR3 + ((uintptr_t)iPage << PAGE_SHIFT);
1292 *ppMap = NULL;
1293 return VINF_SUCCESS;
1294 }
1295
1296 const uint32_t idChunk = PGM_PAGE_GET_CHUNKID(pPage);
1297 if (idChunk == NIL_GMM_CHUNKID)
1298 {
1299 AssertMsgReturn(PGM_PAGE_GET_PAGEID(pPage) == NIL_GMM_PAGEID, ("pPage=%R[pgmpage]\n", pPage),
1300 VERR_PGM_PHYS_PAGE_MAP_IPE_1);
1301 if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
1302 {
1303 AssertMsgReturn(PGM_PAGE_IS_ZERO(pPage), ("pPage=%R[pgmpage]\n", pPage),
1304 VERR_PGM_PHYS_PAGE_MAP_IPE_3);
1305 AssertMsgReturn(PGM_PAGE_GET_HCPHYS(pPage)== pVM->pgm.s.HCPhysZeroPg, ("pPage=%R[pgmpage]\n", pPage),
1306 VERR_PGM_PHYS_PAGE_MAP_IPE_4);
1307 *ppv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
1308 }
1309 else
1310 *ppv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
1311 *ppMap = NULL;
1312 return VINF_SUCCESS;
1313 }
1314
1315 /*
1316 * Find/make Chunk TLB entry for the mapping chunk.
1317 */
1318 PPGMCHUNKR3MAP pMap;
1319 PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
1320 if (pTlbe->idChunk == idChunk)
1321 {
1322 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbHits));
1323 pMap = pTlbe->pChunk;
1324 AssertPtr(pMap->pv);
1325 }
1326 else
1327 {
1328 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
1329
1330 /*
1331 * Find the chunk, map it if necessary.
1332 */
1333 pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
1334 if (pMap)
1335 {
1336 AssertPtr(pMap->pv);
1337 pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
1338 }
1339 else
1340 {
1341#ifdef IN_RING0
1342 int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_MAP_CHUNK, idChunk);
1343 AssertRCReturn(rc, rc);
1344 pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
1345 Assert(pMap);
1346#else
1347 int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
1348 if (RT_FAILURE(rc))
1349 return rc;
1350#endif
1351 AssertPtr(pMap->pv);
1352 }
1353
1354 /*
1355 * Enter it into the Chunk TLB.
1356 */
1357 pTlbe->idChunk = idChunk;
1358 pTlbe->pChunk = pMap;
1359 }
1360
1361 *ppv = (uint8_t *)pMap->pv + (PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) << PAGE_SHIFT);
1362 *ppMap = pMap;
1363 return VINF_SUCCESS;
1364#endif /* IN_RING3 */
1365}
1366
1367
1368/**
1369 * Combination of pgmPhysPageMakeWritable and pgmPhysPageMapWritable.
1370 *
1371 * This is typically used is paths where we cannot use the TLB methods (like ROM
1372 * pages) or where there is no point in using them since we won't get many hits.
1373 *
1374 * @returns VBox strict status code.
1375 * @retval VINF_SUCCESS on success.
1376 * @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
1377 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1378 *
1379 * @param pVM The cross context VM structure.
1380 * @param pPage The physical page tracking structure.
1381 * @param GCPhys The address of the page.
1382 * @param ppv Where to store the mapping address of the page. The page
1383 * offset is masked off!
1384 *
1385 * @remarks Called from within the PGM critical section. The mapping is only
1386 * valid while you are inside section.
1387 */
1388int pgmPhysPageMakeWritableAndMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1389{
1390 int rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1391 if (RT_SUCCESS(rc))
1392 {
1393 AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* returned */, ("%Rrc\n", rc));
1394 PPGMPAGEMAP pMapIgnore;
1395 int rc2 = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
1396 if (RT_FAILURE(rc2)) /* preserve rc */
1397 rc = rc2;
1398 }
1399 return rc;
1400}
1401
1402
1403/**
1404 * Maps a page into the current virtual address space so it can be accessed for
1405 * both writing and reading.
1406 *
1407 * This is typically used is paths where we cannot use the TLB methods (like ROM
1408 * pages) or where there is no point in using them since we won't get many hits.
1409 *
1410 * @returns VBox status code.
1411 * @retval VINF_SUCCESS on success.
1412 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1413 *
1414 * @param pVM The cross context VM structure.
1415 * @param pPage The physical page tracking structure. Must be in the
1416 * allocated state.
1417 * @param GCPhys The address of the page.
1418 * @param ppv Where to store the mapping address of the page. The page
1419 * offset is masked off!
1420 *
1421 * @remarks Called from within the PGM critical section. The mapping is only
1422 * valid while you are inside section.
1423 */
1424int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1425{
1426 Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
1427 PPGMPAGEMAP pMapIgnore;
1428 return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
1429}
1430
1431
1432/**
1433 * Maps a page into the current virtual address space so it can be accessed for
1434 * reading.
1435 *
1436 * This is typically used is paths where we cannot use the TLB methods (like ROM
1437 * pages) or where there is no point in using them since we won't get many hits.
1438 *
1439 * @returns VBox status code.
1440 * @retval VINF_SUCCESS on success.
1441 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1442 *
1443 * @param pVM The cross context VM structure.
1444 * @param pPage The physical page tracking structure.
1445 * @param GCPhys The address of the page.
1446 * @param ppv Where to store the mapping address of the page. The page
1447 * offset is masked off!
1448 *
1449 * @remarks Called from within the PGM critical section. The mapping is only
1450 * valid while you are inside this section.
1451 */
1452int pgmPhysPageMapReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv)
1453{
1454 PPGMPAGEMAP pMapIgnore;
1455 return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, (void **)ppv);
1456}
1457
1458#if !defined(IN_RC) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1459
1460/**
1461 * Load a guest page into the ring-3 physical TLB.
1462 *
1463 * @returns VBox status code.
1464 * @retval VINF_SUCCESS on success
1465 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1466 * @param pPGM The PGM instance pointer.
1467 * @param GCPhys The guest physical address in question.
1468 */
1469int pgmPhysPageLoadIntoTlb(PVM pVM, RTGCPHYS GCPhys)
1470{
1471 PGM_LOCK_ASSERT_OWNER(pVM);
1472
1473 /*
1474 * Find the ram range and page and hand it over to the with-page function.
1475 * 99.8% of requests are expected to be in the first range.
1476 */
1477 PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
1478 if (!pPage)
1479 {
1480 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
1481 return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
1482 }
1483
1484 return pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys);
1485}
1486
1487
1488/**
1489 * Load a guest page into the ring-3 physical TLB.
1490 *
1491 * @returns VBox status code.
1492 * @retval VINF_SUCCESS on success
1493 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1494 *
1495 * @param pVM The cross context VM structure.
1496 * @param pPage Pointer to the PGMPAGE structure corresponding to
1497 * GCPhys.
1498 * @param GCPhys The guest physical address in question.
1499 */
1500int pgmPhysPageLoadIntoTlbWithPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
1501{
1502 PGM_LOCK_ASSERT_OWNER(pVM);
1503 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbMisses));
1504
1505 /*
1506 * Map the page.
1507 * Make a special case for the zero page as it is kind of special.
1508 */
1509 PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
1510 if ( !PGM_PAGE_IS_ZERO(pPage)
1511 && !PGM_PAGE_IS_BALLOONED(pPage))
1512 {
1513 void *pv;
1514 PPGMPAGEMAP pMap;
1515 int rc = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMap, &pv);
1516 if (RT_FAILURE(rc))
1517 return rc;
1518 pTlbe->pMap = pMap;
1519 pTlbe->pv = pv;
1520 Assert(!((uintptr_t)pTlbe->pv & PAGE_OFFSET_MASK));
1521 }
1522 else
1523 {
1524 AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == pVM->pgm.s.HCPhysZeroPg, ("%RGp/%R[pgmpage]\n", GCPhys, pPage));
1525 pTlbe->pMap = NULL;
1526 pTlbe->pv = pVM->pgm.s.CTXALLSUFF(pvZeroPg);
1527 }
1528#ifdef PGM_WITH_PHYS_TLB
1529 if ( PGM_PAGE_GET_TYPE(pPage) < PGMPAGETYPE_ROM_SHADOW
1530 || PGM_PAGE_GET_TYPE(pPage) > PGMPAGETYPE_ROM)
1531 pTlbe->GCPhys = GCPhys & X86_PTE_PAE_PG_MASK;
1532 else
1533 pTlbe->GCPhys = NIL_RTGCPHYS; /* ROM: Problematic because of the two pages. :-/ */
1534#else
1535 pTlbe->GCPhys = NIL_RTGCPHYS;
1536#endif
1537 pTlbe->pPage = pPage;
1538 return VINF_SUCCESS;
1539}
1540
1541#endif /* !IN_RC && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
1542
1543/**
1544 * Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
1545 * own the PGM lock and therefore not need to lock the mapped page.
1546 *
1547 * @returns VBox status code.
1548 * @retval VINF_SUCCESS on success.
1549 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1550 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1551 *
1552 * @param pVM The cross context VM structure.
1553 * @param GCPhys The guest physical address of the page that should be mapped.
1554 * @param pPage Pointer to the PGMPAGE structure for the page.
1555 * @param ppv Where to store the address corresponding to GCPhys.
1556 *
1557 * @internal
1558 * @deprecated Use pgmPhysGCPhys2CCPtrInternalEx.
1559 */
1560int pgmPhysGCPhys2CCPtrInternalDepr(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
1561{
1562 int rc;
1563 AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
1564 PGM_LOCK_ASSERT_OWNER(pVM);
1565 pVM->pgm.s.cDeprecatedPageLocks++;
1566
1567 /*
1568 * Make sure the page is writable.
1569 */
1570 if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1571 {
1572 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1573 if (RT_FAILURE(rc))
1574 return rc;
1575 AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1576 }
1577 Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1578
1579 /*
1580 * Get the mapping address.
1581 */
1582#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1583 void *pv;
1584 rc = pgmRZDynMapHCPageInlined(VMMGetCpu(pVM),
1585 PGM_PAGE_GET_HCPHYS(pPage),
1586 &pv
1587 RTLOG_COMMA_SRC_POS);
1588 if (RT_FAILURE(rc))
1589 return rc;
1590 *ppv = (void *)((uintptr_t)pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1591#else
1592 PPGMPAGEMAPTLBE pTlbe;
1593 rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1594 if (RT_FAILURE(rc))
1595 return rc;
1596 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1597#endif
1598 return VINF_SUCCESS;
1599}
1600
1601#if !defined(IN_RC) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1602
1603/**
1604 * Locks a page mapping for writing.
1605 *
1606 * @param pVM The cross context VM structure.
1607 * @param pPage The page.
1608 * @param pTlbe The mapping TLB entry for the page.
1609 * @param pLock The lock structure (output).
1610 */
1611DECLINLINE(void) pgmPhysPageMapLockForWriting(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
1612{
1613 PPGMPAGEMAP pMap = pTlbe->pMap;
1614 if (pMap)
1615 pMap->cRefs++;
1616
1617 unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
1618 if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
1619 {
1620 if (cLocks == 0)
1621 pVM->pgm.s.cWriteLockedPages++;
1622 PGM_PAGE_INC_WRITE_LOCKS(pPage);
1623 }
1624 else if (cLocks != PGM_PAGE_MAX_LOCKS)
1625 {
1626 PGM_PAGE_INC_WRITE_LOCKS(pPage);
1627 AssertMsgFailed(("%R[pgmpage] is entering permanent write locked state!\n", pPage));
1628 if (pMap)
1629 pMap->cRefs++; /* Extra ref to prevent it from going away. */
1630 }
1631
1632 pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_WRITE;
1633 pLock->pvMap = pMap;
1634}
1635
1636/**
1637 * Locks a page mapping for reading.
1638 *
1639 * @param pVM The cross context VM structure.
1640 * @param pPage The page.
1641 * @param pTlbe The mapping TLB entry for the page.
1642 * @param pLock The lock structure (output).
1643 */
1644DECLINLINE(void) pgmPhysPageMapLockForReading(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
1645{
1646 PPGMPAGEMAP pMap = pTlbe->pMap;
1647 if (pMap)
1648 pMap->cRefs++;
1649
1650 unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
1651 if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
1652 {
1653 if (cLocks == 0)
1654 pVM->pgm.s.cReadLockedPages++;
1655 PGM_PAGE_INC_READ_LOCKS(pPage);
1656 }
1657 else if (cLocks != PGM_PAGE_MAX_LOCKS)
1658 {
1659 PGM_PAGE_INC_READ_LOCKS(pPage);
1660 AssertMsgFailed(("%R[pgmpage] is entering permanent read locked state!\n", pPage));
1661 if (pMap)
1662 pMap->cRefs++; /* Extra ref to prevent it from going away. */
1663 }
1664
1665 pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_READ;
1666 pLock->pvMap = pMap;
1667}
1668
1669#endif /* !IN_RC && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
1670
1671
1672/**
1673 * Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
1674 * own the PGM lock and have access to the page structure.
1675 *
1676 * @returns VBox status code.
1677 * @retval VINF_SUCCESS on success.
1678 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1679 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1680 *
1681 * @param pVM The cross context VM structure.
1682 * @param GCPhys The guest physical address of the page that should be mapped.
1683 * @param pPage Pointer to the PGMPAGE structure for the page.
1684 * @param ppv Where to store the address corresponding to GCPhys.
1685 * @param pLock Where to store the lock information that
1686 * pgmPhysReleaseInternalPageMappingLock needs.
1687 *
1688 * @internal
1689 */
1690int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
1691{
1692 int rc;
1693 AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
1694 PGM_LOCK_ASSERT_OWNER(pVM);
1695
1696 /*
1697 * Make sure the page is writable.
1698 */
1699 if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1700 {
1701 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1702 if (RT_FAILURE(rc))
1703 return rc;
1704 AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1705 }
1706 Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1707
1708 /*
1709 * Do the job.
1710 */
1711#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1712 void *pv;
1713 PVMCPU pVCpu = VMMGetCpu(pVM);
1714 rc = pgmRZDynMapHCPageInlined(pVCpu,
1715 PGM_PAGE_GET_HCPHYS(pPage),
1716 &pv
1717 RTLOG_COMMA_SRC_POS);
1718 if (RT_FAILURE(rc))
1719 return rc;
1720 *ppv = (void *)((uintptr_t)pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1721 pLock->pvPage = pv;
1722 pLock->pVCpu = pVCpu;
1723
1724#else
1725 PPGMPAGEMAPTLBE pTlbe;
1726 rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1727 if (RT_FAILURE(rc))
1728 return rc;
1729 pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
1730 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1731#endif
1732 return VINF_SUCCESS;
1733}
1734
1735
1736/**
1737 * Internal version of PGMPhysGCPhys2CCPtrReadOnly that expects the caller to
1738 * own the PGM lock and have access to the page structure.
1739 *
1740 * @returns VBox status code.
1741 * @retval VINF_SUCCESS on success.
1742 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1743 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1744 *
1745 * @param pVM The cross context VM structure.
1746 * @param GCPhys The guest physical address of the page that should be mapped.
1747 * @param pPage Pointer to the PGMPAGE structure for the page.
1748 * @param ppv Where to store the address corresponding to GCPhys.
1749 * @param pLock Where to store the lock information that
1750 * pgmPhysReleaseInternalPageMappingLock needs.
1751 *
1752 * @internal
1753 */
1754int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock)
1755{
1756 AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
1757 PGM_LOCK_ASSERT_OWNER(pVM);
1758 Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
1759
1760 /*
1761 * Do the job.
1762 */
1763#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1764 void *pv;
1765 PVMCPU pVCpu = VMMGetCpu(pVM);
1766 int rc = pgmRZDynMapHCPageInlined(pVCpu,
1767 PGM_PAGE_GET_HCPHYS(pPage),
1768 &pv
1769 RTLOG_COMMA_SRC_POS); /** @todo add a read only flag? */
1770 if (RT_FAILURE(rc))
1771 return rc;
1772 *ppv = (void *)((uintptr_t)pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1773 pLock->pvPage = pv;
1774 pLock->pVCpu = pVCpu;
1775
1776#else
1777 PPGMPAGEMAPTLBE pTlbe;
1778 int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1779 if (RT_FAILURE(rc))
1780 return rc;
1781 pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
1782 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1783#endif
1784 return VINF_SUCCESS;
1785}
1786
1787
1788/**
1789 * Requests the mapping of a guest page into the current context.
1790 *
1791 * This API should only be used for very short term, as it will consume scarse
1792 * resources (R0 and GC) in the mapping cache. When you're done with the page,
1793 * call PGMPhysReleasePageMappingLock() ASAP to release it.
1794 *
1795 * This API will assume your intention is to write to the page, and will
1796 * therefore replace shared and zero pages. If you do not intend to modify
1797 * the page, use the PGMPhysGCPhys2CCPtrReadOnly() API.
1798 *
1799 * @returns VBox status code.
1800 * @retval VINF_SUCCESS on success.
1801 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1802 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1803 *
1804 * @param pVM The cross context VM structure.
1805 * @param GCPhys The guest physical address of the page that should be
1806 * mapped.
1807 * @param ppv Where to store the address corresponding to GCPhys.
1808 * @param pLock Where to store the lock information that
1809 * PGMPhysReleasePageMappingLock needs.
1810 *
1811 * @remarks The caller is responsible for dealing with access handlers.
1812 * @todo Add an informational return code for pages with access handlers?
1813 *
1814 * @remark Avoid calling this API from within critical sections (other than
1815 * the PGM one) because of the deadlock risk. External threads may
1816 * need to delegate jobs to the EMTs.
1817 * @remarks Only one page is mapped! Make no assumption about what's after or
1818 * before the returned page!
1819 * @thread Any thread.
1820 */
1821VMM_INT_DECL(int) PGMPhysGCPhys2CCPtr(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
1822{
1823 int rc = pgmLock(pVM);
1824 AssertRCReturn(rc, rc);
1825
1826#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1827 /*
1828 * Find the page and make sure it's writable.
1829 */
1830 PPGMPAGE pPage;
1831 rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1832 if (RT_SUCCESS(rc))
1833 {
1834 if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1835 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1836 if (RT_SUCCESS(rc))
1837 {
1838 AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1839
1840 PVMCPU pVCpu = VMMGetCpu(pVM);
1841 void *pv;
1842 rc = pgmRZDynMapHCPageInlined(pVCpu,
1843 PGM_PAGE_GET_HCPHYS(pPage),
1844 &pv
1845 RTLOG_COMMA_SRC_POS);
1846 if (RT_SUCCESS(rc))
1847 {
1848 AssertRCSuccess(rc);
1849
1850 pv = (void *)((uintptr_t)pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1851 *ppv = pv;
1852 pLock->pvPage = pv;
1853 pLock->pVCpu = pVCpu;
1854 }
1855 }
1856 }
1857
1858#else /* IN_RING3 || IN_RING0 */
1859 /*
1860 * Query the Physical TLB entry for the page (may fail).
1861 */
1862 PPGMPAGEMAPTLBE pTlbe;
1863 rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
1864 if (RT_SUCCESS(rc))
1865 {
1866 /*
1867 * If the page is shared, the zero page, or being write monitored
1868 * it must be converted to a page that's writable if possible.
1869 */
1870 PPGMPAGE pPage = pTlbe->pPage;
1871 if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
1872 {
1873 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
1874 if (RT_SUCCESS(rc))
1875 {
1876 AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
1877 rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
1878 }
1879 }
1880 if (RT_SUCCESS(rc))
1881 {
1882 /*
1883 * Now, just perform the locking and calculate the return address.
1884 */
1885 pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
1886 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1887 }
1888 }
1889
1890#endif /* IN_RING3 || IN_RING0 */
1891 pgmUnlock(pVM);
1892 return rc;
1893}
1894
1895
1896/**
1897 * Requests the mapping of a guest page into the current context.
1898 *
1899 * This API should only be used for very short term, as it will consume scarse
1900 * resources (R0 and GC) in the mapping cache. When you're done with the page,
1901 * call PGMPhysReleasePageMappingLock() ASAP to release it.
1902 *
1903 * @returns VBox status code.
1904 * @retval VINF_SUCCESS on success.
1905 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
1906 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
1907 *
1908 * @param pVM The cross context VM structure.
1909 * @param GCPhys The guest physical address of the page that should be
1910 * mapped.
1911 * @param ppv Where to store the address corresponding to GCPhys.
1912 * @param pLock Where to store the lock information that
1913 * PGMPhysReleasePageMappingLock needs.
1914 *
1915 * @remarks The caller is responsible for dealing with access handlers.
1916 * @todo Add an informational return code for pages with access handlers?
1917 *
1918 * @remarks Avoid calling this API from within critical sections (other than
1919 * the PGM one) because of the deadlock risk.
1920 * @remarks Only one page is mapped! Make no assumption about what's after or
1921 * before the returned page!
1922 * @thread Any thread.
1923 */
1924VMM_INT_DECL(int) PGMPhysGCPhys2CCPtrReadOnly(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
1925{
1926 int rc = pgmLock(pVM);
1927 AssertRCReturn(rc, rc);
1928
1929#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1930 /*
1931 * Find the page and make sure it's readable.
1932 */
1933 PPGMPAGE pPage;
1934 rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1935 if (RT_SUCCESS(rc))
1936 {
1937 if (RT_UNLIKELY(PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)))
1938 rc = VERR_PGM_PHYS_PAGE_RESERVED;
1939 else
1940 {
1941 PVMCPU pVCpu = VMMGetCpu(pVM);
1942 void *pv;
1943 rc = pgmRZDynMapHCPageInlined(pVCpu,
1944 PGM_PAGE_GET_HCPHYS(pPage),
1945 &pv
1946 RTLOG_COMMA_SRC_POS); /** @todo add a read only flag? */
1947 if (RT_SUCCESS(rc))
1948 {
1949 AssertRCSuccess(rc);
1950
1951 pv = (void *)((uintptr_t)pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1952 *ppv = pv;
1953 pLock->pvPage = pv;
1954 pLock->pVCpu = pVCpu;
1955 }
1956 }
1957 }
1958
1959#else /* IN_RING3 || IN_RING0 */
1960 /*
1961 * Query the Physical TLB entry for the page (may fail).
1962 */
1963 PPGMPAGEMAPTLBE pTlbe;
1964 rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
1965 if (RT_SUCCESS(rc))
1966 {
1967 /* MMIO pages doesn't have any readable backing. */
1968 PPGMPAGE pPage = pTlbe->pPage;
1969 if (RT_UNLIKELY(PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)))
1970 rc = VERR_PGM_PHYS_PAGE_RESERVED;
1971 else
1972 {
1973 /*
1974 * Now, just perform the locking and calculate the return address.
1975 */
1976 pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
1977 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
1978 }
1979 }
1980
1981#endif /* IN_RING3 || IN_RING0 */
1982 pgmUnlock(pVM);
1983 return rc;
1984}
1985
1986
1987/**
1988 * Requests the mapping of a guest page given by virtual address into the current context.
1989 *
1990 * This API should only be used for very short term, as it will consume
1991 * scarse resources (R0 and GC) in the mapping cache. When you're done
1992 * with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
1993 *
1994 * This API will assume your intention is to write to the page, and will
1995 * therefore replace shared and zero pages. If you do not intend to modify
1996 * the page, use the PGMPhysGCPtr2CCPtrReadOnly() API.
1997 *
1998 * @returns VBox status code.
1999 * @retval VINF_SUCCESS on success.
2000 * @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
2001 * @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
2002 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2003 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
2004 *
2005 * @param pVCpu The cross context virtual CPU structure.
2006 * @param GCPtr The guest physical address of the page that should be
2007 * mapped.
2008 * @param ppv Where to store the address corresponding to GCPhys.
2009 * @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
2010 *
2011 * @remark Avoid calling this API from within critical sections (other than
2012 * the PGM one) because of the deadlock risk.
2013 * @thread EMT
2014 */
2015VMM_INT_DECL(int) PGMPhysGCPtr2CCPtr(PVMCPU pVCpu, RTGCPTR GCPtr, void **ppv, PPGMPAGEMAPLOCK pLock)
2016{
2017 VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
2018 RTGCPHYS GCPhys;
2019 int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
2020 if (RT_SUCCESS(rc))
2021 rc = PGMPhysGCPhys2CCPtr(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
2022 return rc;
2023}
2024
2025
2026/**
2027 * Requests the mapping of a guest page given by virtual address into the current context.
2028 *
2029 * This API should only be used for very short term, as it will consume
2030 * scarse resources (R0 and GC) in the mapping cache. When you're done
2031 * with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
2032 *
2033 * @returns VBox status code.
2034 * @retval VINF_SUCCESS on success.
2035 * @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
2036 * @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
2037 * @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2038 * @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
2039 *
2040 * @param pVCpu The cross context virtual CPU structure.
2041 * @param GCPtr The guest physical address of the page that should be
2042 * mapped.
2043 * @param ppv Where to store the address corresponding to GCPtr.
2044 * @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
2045 *
2046 * @remark Avoid calling this API from within critical sections (other than
2047 * the PGM one) because of the deadlock risk.
2048 * @thread EMT
2049 */
2050VMM_INT_DECL(int) PGMPhysGCPtr2CCPtrReadOnly(PVMCPU pVCpu, RTGCPTR GCPtr, void const **ppv, PPGMPAGEMAPLOCK pLock)
2051{
2052 VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
2053 RTGCPHYS GCPhys;
2054 int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
2055 if (RT_SUCCESS(rc))
2056 rc = PGMPhysGCPhys2CCPtrReadOnly(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
2057 return rc;
2058}
2059
2060
2061/**
2062 * Release the mapping of a guest page.
2063 *
2064 * This is the counter part of PGMPhysGCPhys2CCPtr, PGMPhysGCPhys2CCPtrReadOnly
2065 * PGMPhysGCPtr2CCPtr and PGMPhysGCPtr2CCPtrReadOnly.
2066 *
2067 * @param pVM The cross context VM structure.
2068 * @param pLock The lock structure initialized by the mapping function.
2069 */
2070VMMDECL(void) PGMPhysReleasePageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock)
2071{
2072#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
2073 Assert(pLock->pvPage != NULL);
2074 Assert(pLock->pVCpu == VMMGetCpu(pVM)); RT_NOREF_PV(pVM);
2075 PGM_DYNMAP_UNUSED_HINT(pLock->pVCpu, pLock->pvPage);
2076 pLock->pVCpu = NULL;
2077 pLock->pvPage = NULL;
2078
2079#else
2080 PPGMPAGEMAP pMap = (PPGMPAGEMAP)pLock->pvMap;
2081 PPGMPAGE pPage = (PPGMPAGE)(pLock->uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
2082 bool fWriteLock = (pLock->uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
2083
2084 pLock->uPageAndType = 0;
2085 pLock->pvMap = NULL;
2086
2087 pgmLock(pVM);
2088 if (fWriteLock)
2089 {
2090 unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
2091 Assert(cLocks > 0);
2092 if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
2093 {
2094 if (cLocks == 1)
2095 {
2096 Assert(pVM->pgm.s.cWriteLockedPages > 0);
2097 pVM->pgm.s.cWriteLockedPages--;
2098 }
2099 PGM_PAGE_DEC_WRITE_LOCKS(pPage);
2100 }
2101
2102 if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED)
2103 { /* probably extremely likely */ }
2104 else
2105 pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, NIL_RTGCPHYS);
2106 }
2107 else
2108 {
2109 unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
2110 Assert(cLocks > 0);
2111 if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
2112 {
2113 if (cLocks == 1)
2114 {
2115 Assert(pVM->pgm.s.cReadLockedPages > 0);
2116 pVM->pgm.s.cReadLockedPages--;
2117 }
2118 PGM_PAGE_DEC_READ_LOCKS(pPage);
2119 }
2120 }
2121
2122 if (pMap)
2123 {
2124 Assert(pMap->cRefs >= 1);
2125 pMap->cRefs--;
2126 }
2127 pgmUnlock(pVM);
2128#endif /* IN_RING3 */
2129}
2130
2131
2132/**
2133 * Release the internal mapping of a guest page.
2134 *
2135 * This is the counter part of pgmPhysGCPhys2CCPtrInternalEx and
2136 * pgmPhysGCPhys2CCPtrInternalReadOnly.
2137 *
2138 * @param pVM The cross context VM structure.
2139 * @param pLock The lock structure initialized by the mapping function.
2140 *
2141 * @remarks Caller must hold the PGM lock.
2142 */
2143void pgmPhysReleaseInternalPageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock)
2144{
2145 PGM_LOCK_ASSERT_OWNER(pVM);
2146 PGMPhysReleasePageMappingLock(pVM, pLock); /* lazy for now */
2147}
2148
2149
2150/**
2151 * Converts a GC physical address to a HC ring-3 pointer.
2152 *
2153 * @returns VINF_SUCCESS on success.
2154 * @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
2155 * page but has no physical backing.
2156 * @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
2157 * GC physical address.
2158 * @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
2159 * a dynamic ram chunk boundary
2160 *
2161 * @param pVM The cross context VM structure.
2162 * @param GCPhys The GC physical address to convert.
2163 * @param pR3Ptr Where to store the R3 pointer on success.
2164 *
2165 * @deprecated Avoid when possible!
2166 */
2167int pgmPhysGCPhys2R3Ptr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
2168{
2169/** @todo this is kind of hacky and needs some more work. */
2170#ifndef DEBUG_sandervl
2171 VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
2172#endif
2173
2174 Log(("pgmPhysGCPhys2R3Ptr(,%RGp,): dont use this API!\n", GCPhys)); /** @todo eliminate this API! */
2175#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
2176 NOREF(pVM); NOREF(pR3Ptr); RT_NOREF_PV(GCPhys);
2177 AssertFailedReturn(VERR_NOT_IMPLEMENTED);
2178#else
2179 pgmLock(pVM);
2180
2181 PPGMRAMRANGE pRam;
2182 PPGMPAGE pPage;
2183 int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
2184 if (RT_SUCCESS(rc))
2185 rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
2186
2187 pgmUnlock(pVM);
2188 Assert(rc <= VINF_SUCCESS);
2189 return rc;
2190#endif
2191}
2192
2193#if 0 /*defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)*/
2194
2195/**
2196 * Maps and locks a guest CR3 or PD (PAE) page.
2197 *
2198 * @returns VINF_SUCCESS on success.
2199 * @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
2200 * page but has no physical backing.
2201 * @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
2202 * GC physical address.
2203 * @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
2204 * a dynamic ram chunk boundary
2205 *
2206 * @param pVM The cross context VM structure.
2207 * @param GCPhys The GC physical address to convert.
2208 * @param pR3Ptr Where to store the R3 pointer on success. This may or
2209 * may not be valid in ring-0 depending on the
2210 * VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 build option.
2211 *
2212 * @remarks The caller must own the PGM lock.
2213 */
2214int pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
2215{
2216
2217 PPGMRAMRANGE pRam;
2218 PPGMPAGE pPage;
2219 int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
2220 if (RT_SUCCESS(rc))
2221 rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
2222 Assert(rc <= VINF_SUCCESS);
2223 return rc;
2224}
2225
2226
2227int pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
2228{
2229
2230}
2231
2232#endif
2233
2234/**
2235 * Converts a guest pointer to a GC physical address.
2236 *
2237 * This uses the current CR3/CR0/CR4 of the guest.
2238 *
2239 * @returns VBox status code.
2240 * @param pVCpu The cross context virtual CPU structure.
2241 * @param GCPtr The guest pointer to convert.
2242 * @param pGCPhys Where to store the GC physical address.
2243 */
2244VMMDECL(int) PGMPhysGCPtr2GCPhys(PVMCPU pVCpu, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
2245{
2246 int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, pGCPhys);
2247 if (pGCPhys && RT_SUCCESS(rc))
2248 *pGCPhys |= (RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK;
2249 return rc;
2250}
2251
2252
2253/**
2254 * Converts a guest pointer to a HC physical address.
2255 *
2256 * This uses the current CR3/CR0/CR4 of the guest.
2257 *
2258 * @returns VBox status code.
2259 * @param pVCpu The cross context virtual CPU structure.
2260 * @param GCPtr The guest pointer to convert.
2261 * @param pHCPhys Where to store the HC physical address.
2262 */
2263VMM_INT_DECL(int) PGMPhysGCPtr2HCPhys(PVMCPU pVCpu, RTGCPTR GCPtr, PRTHCPHYS pHCPhys)
2264{
2265 PVM pVM = pVCpu->CTX_SUFF(pVM);
2266 RTGCPHYS GCPhys;
2267 int rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtr, NULL, &GCPhys);
2268 if (RT_SUCCESS(rc))
2269 rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys | ((RTGCUINTPTR)GCPtr & PAGE_OFFSET_MASK), pHCPhys);
2270 return rc;
2271}
2272
2273
2274
2275#undef LOG_GROUP
2276#define LOG_GROUP LOG_GROUP_PGM_PHYS_ACCESS
2277
2278
2279#if defined(IN_RING3) && defined(SOME_UNUSED_FUNCTION)
2280/**
2281 * Cache PGMPhys memory access
2282 *
2283 * @param pVM The cross context VM structure.
2284 * @param pCache Cache structure pointer
2285 * @param GCPhys GC physical address
2286 * @param pbHC HC pointer corresponding to physical page
2287 *
2288 * @thread EMT.
2289 */
2290static void pgmPhysCacheAdd(PVM pVM, PGMPHYSCACHE *pCache, RTGCPHYS GCPhys, uint8_t *pbR3)
2291{
2292 uint32_t iCacheIndex;
2293
2294 Assert(VM_IS_EMT(pVM));
2295
2296 GCPhys = PHYS_PAGE_ADDRESS(GCPhys);
2297 pbR3 = (uint8_t *)PAGE_ADDRESS(pbR3);
2298
2299 iCacheIndex = ((GCPhys >> PAGE_SHIFT) & PGM_MAX_PHYSCACHE_ENTRIES_MASK);
2300
2301 ASMBitSet(&pCache->aEntries, iCacheIndex);
2302
2303 pCache->Entry[iCacheIndex].GCPhys = GCPhys;
2304 pCache->Entry[iCacheIndex].pbR3 = pbR3;
2305}
2306#endif /* IN_RING3 */
2307
2308
2309/**
2310 * Deals with reading from a page with one or more ALL access handlers.
2311 *
2312 * @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3.
2313 * See PGM_HANDLER_PHYS_IS_VALID_STATUS and
2314 * PGM_HANDLER_VIRT_IS_VALID_STATUS for details.
2315 *
2316 * @param pVM The cross context VM structure.
2317 * @param pPage The page descriptor.
2318 * @param GCPhys The physical address to start reading at.
2319 * @param pvBuf Where to put the bits we read.
2320 * @param cb How much to read - less or equal to a page.
2321 * @param enmOrigin The origin of this call.
2322 */
2323static VBOXSTRICTRC pgmPhysReadHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void *pvBuf, size_t cb,
2324 PGMACCESSORIGIN enmOrigin)
2325{
2326 /*
2327 * The most frequent access here is MMIO and shadowed ROM.
2328 * The current code ASSUMES all these access handlers covers full pages!
2329 */
2330
2331 /*
2332 * Whatever we do we need the source page, map it first.
2333 */
2334 PGMPAGEMAPLOCK PgMpLck;
2335 const void *pvSrc = NULL;
2336 int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvSrc, &PgMpLck);
2337/** @todo Check how this can work for MMIO pages? */
2338 if (RT_FAILURE(rc))
2339 {
2340 AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
2341 GCPhys, pPage, rc));
2342 memset(pvBuf, 0xff, cb);
2343 return VINF_SUCCESS;
2344 }
2345
2346 VBOXSTRICTRC rcStrict = VINF_PGM_HANDLER_DO_DEFAULT;
2347
2348 /*
2349 * Deal with any physical handlers.
2350 */
2351 PVMCPU pVCpu = VMMGetCpu(pVM);
2352 PPGMPHYSHANDLER pPhys = NULL;
2353 if ( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL
2354 || PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
2355 {
2356 pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2357 AssertReleaseMsg(pPhys, ("GCPhys=%RGp cb=%#x\n", GCPhys, cb));
2358 Assert(GCPhys >= pPhys->Core.Key && GCPhys <= pPhys->Core.KeyLast);
2359 Assert((pPhys->Core.Key & PAGE_OFFSET_MASK) == 0);
2360 Assert((pPhys->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
2361#ifndef IN_RING3
2362 if (enmOrigin != PGMACCESSORIGIN_IEM)
2363 {
2364 /* Cannot reliably handle informational status codes in this context */
2365 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2366 return VERR_PGM_PHYS_WR_HIT_HANDLER;
2367 }
2368#endif
2369 PFNPGMPHYSHANDLER pfnHandler = PGMPHYSHANDLER_GET_TYPE(pVM, pPhys)->CTX_SUFF(pfnHandler); Assert(pfnHandler);
2370 void *pvUser = pPhys->CTX_SUFF(pvUser);
2371
2372 Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cb, pPage, R3STRING(pPhys->pszDesc) ));
2373 STAM_PROFILE_START(&pPhys->Stat, h);
2374 PGM_LOCK_ASSERT_OWNER(pVM);
2375
2376 /* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2377 pgmUnlock(pVM);
2378 rcStrict = pfnHandler(pVM, pVCpu, GCPhys, (void *)pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, enmOrigin, pvUser);
2379 pgmLock(pVM);
2380
2381#ifdef VBOX_WITH_STATISTICS
2382 pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2383 if (pPhys)
2384 STAM_PROFILE_STOP(&pPhys->Stat, h);
2385#else
2386 pPhys = NULL; /* might not be valid anymore. */
2387#endif
2388 AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, false),
2389 ("rcStrict=%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys));
2390 if ( rcStrict != VINF_PGM_HANDLER_DO_DEFAULT
2391 && !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
2392 {
2393 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2394 return rcStrict;
2395 }
2396 }
2397
2398#if !defined(IN_RING0) && defined(VBOX_WITH_RAW_MODE)
2399 /*
2400 * Deal with any virtual handlers.
2401 */
2402 if (PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) == PGM_PAGE_HNDL_VIRT_STATE_ALL)
2403 {
2404 unsigned iPage;
2405 PPGMVIRTHANDLER pVirt = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &iPage);
2406 AssertReleaseMsg(pVirt, ("GCPhys=%RGp cb=%#x\n", GCPhys, cb));
2407 Assert((pVirt->Core.Key & PAGE_OFFSET_MASK) == 0);
2408 Assert((pVirt->Core.KeyLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
2409 Assert(GCPhys >= pVirt->aPhysToVirt[iPage].Core.Key && GCPhys <= pVirt->aPhysToVirt[iPage].Core.KeyLast);
2410
2411# ifndef IN_RING3
2412 if (enmOrigin != PGMACCESSORIGIN_IEM)
2413 {
2414 /* Cannot reliably handle informational status codes in this context */
2415 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2416 return VERR_PGM_PHYS_WR_HIT_HANDLER;
2417 }
2418# endif
2419 PPGMVIRTHANDLERTYPEINT pVirtType = PGMVIRTANDLER_GET_TYPE(pVM, pVirt);
2420 if (!pPhys)
2421 Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] virt %s\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc) ));
2422 else
2423 Log(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys/virt %s/%s\n", GCPhys, cb, pPage, R3STRING(pVirt->pszDesc), R3STRING(pPhys->pszDesc) ));
2424 RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2425 + (iPage << PAGE_SHIFT)
2426 + (GCPhys & PAGE_OFFSET_MASK);
2427
2428 STAM_PROFILE_START(&pVirt->Stat, h);
2429 VBOXSTRICTRC rcStrict2 = pVirtType->CTX_SUFF(pfnHandler)(pVM, pVCpu, GCPtr, (void *)pvSrc, pvBuf, cb,
2430 PGMACCESSTYPE_READ, enmOrigin, pVirt->CTX_SUFF(pvUser));
2431 STAM_PROFILE_STOP(&pVirt->Stat, h);
2432
2433 /* Merge status codes. */
2434 if (rcStrict2 == VINF_SUCCESS)
2435 {
2436 if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
2437 rcStrict = VINF_SUCCESS;
2438 }
2439 else if (rcStrict2 != VINF_PGM_HANDLER_DO_DEFAULT)
2440 {
2441 AssertLogRelMsg(PGM_HANDLER_VIRT_IS_VALID_STATUS(rcStrict2, false),
2442 ("rcStrict2=%Rrc (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n",
2443 VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pVirt->pszDesc));
2444 if (!PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
2445 {
2446 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2447 return rcStrict2;
2448 }
2449 if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
2450 rcStrict = rcStrict2;
2451 else
2452 PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
2453 }
2454 }
2455#endif /* !IN_RING0 && VBOX_WITH_RAW_MODE */
2456
2457 /*
2458 * Take the default action.
2459 */
2460 if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
2461 {
2462 memcpy(pvBuf, pvSrc, cb);
2463 rcStrict = VINF_SUCCESS;
2464 }
2465 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2466 return rcStrict;
2467}
2468
2469
2470/**
2471 * Read physical memory.
2472 *
2473 * This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
2474 * want to ignore those.
2475 *
2476 * @returns Strict VBox status code in raw-mode and ring-0, normal VBox status
2477 * code in ring-3. Use PGM_PHYS_RW_IS_SUCCESS to check.
2478 * @retval VINF_SUCCESS in all context - read completed.
2479 *
2480 * @retval VINF_EM_OFF in RC and R0 - read completed.
2481 * @retval VINF_EM_SUSPEND in RC and R0 - read completed.
2482 * @retval VINF_EM_RESET in RC and R0 - read completed.
2483 * @retval VINF_EM_HALT in RC and R0 - read completed.
2484 * @retval VINF_SELM_SYNC_GDT in RC only - read completed.
2485 *
2486 * @retval VINF_EM_DBG_STOP in RC and R0 - read completed.
2487 * @retval VINF_EM_DBG_BREAKPOINT in RC and R0 - read completed.
2488 * @retval VINF_EM_RAW_EMULATE_INSTR in RC and R0 only.
2489 *
2490 * @retval VINF_IOM_R3_MMIO_READ in RC and R0.
2491 * @retval VINF_IOM_R3_MMIO_READ_WRITE in RC and R0.
2492 *
2493 * @retval VINF_PATM_CHECK_PATCH_PAGE in RC only.
2494 *
2495 * @retval VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that
2496 * haven't been cleared for strict status codes yet.
2497 *
2498 * @param pVM The cross context VM structure.
2499 * @param GCPhys Physical address start reading from.
2500 * @param pvBuf Where to put the read bits.
2501 * @param cbRead How many bytes to read.
2502 * @param enmOrigin The origin of this call.
2503 */
2504VMMDECL(VBOXSTRICTRC) PGMPhysRead(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead, PGMACCESSORIGIN enmOrigin)
2505{
2506 AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
2507 LogFlow(("PGMPhysRead: %RGp %d\n", GCPhys, cbRead));
2508
2509 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysRead));
2510 STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysReadBytes), cbRead);
2511
2512 pgmLock(pVM);
2513
2514 /*
2515 * Copy loop on ram ranges.
2516 */
2517 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
2518 PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
2519 for (;;)
2520 {
2521 /* Inside range or not? */
2522 if (pRam && GCPhys >= pRam->GCPhys)
2523 {
2524 /*
2525 * Must work our way thru this page by page.
2526 */
2527 RTGCPHYS off = GCPhys - pRam->GCPhys;
2528 while (off < pRam->cb)
2529 {
2530 unsigned iPage = off >> PAGE_SHIFT;
2531 PPGMPAGE pPage = &pRam->aPages[iPage];
2532 size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
2533 if (cb > cbRead)
2534 cb = cbRead;
2535
2536 /*
2537 * Normal page? Get the pointer to it.
2538 */
2539 if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
2540 && !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
2541 {
2542 /*
2543 * Get the pointer to the page.
2544 */
2545 PGMPAGEMAPLOCK PgMpLck;
2546 const void *pvSrc;
2547 int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
2548 if (RT_SUCCESS(rc))
2549 {
2550 memcpy(pvBuf, pvSrc, cb);
2551 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2552 }
2553 else
2554 {
2555 AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
2556 pRam->GCPhys + off, pPage, rc));
2557 memset(pvBuf, 0xff, cb);
2558 }
2559 }
2560 /*
2561 * Have ALL/MMIO access handlers.
2562 */
2563 else
2564 {
2565 VBOXSTRICTRC rcStrict2 = pgmPhysReadHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin);
2566 if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
2567 PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
2568 else
2569 {
2570 memset(pvBuf, 0xff, cb);
2571 pgmUnlock(pVM);
2572 return rcStrict2;
2573 }
2574 }
2575
2576 /* next page */
2577 if (cb >= cbRead)
2578 {
2579 pgmUnlock(pVM);
2580 return rcStrict;
2581 }
2582 cbRead -= cb;
2583 off += cb;
2584 pvBuf = (char *)pvBuf + cb;
2585 } /* walk pages in ram range. */
2586
2587 GCPhys = pRam->GCPhysLast + 1;
2588 }
2589 else
2590 {
2591 LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
2592
2593 /*
2594 * Unassigned address space.
2595 */
2596 size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
2597 if (cb >= cbRead)
2598 {
2599 memset(pvBuf, 0xff, cbRead);
2600 break;
2601 }
2602 memset(pvBuf, 0xff, cb);
2603
2604 cbRead -= cb;
2605 pvBuf = (char *)pvBuf + cb;
2606 GCPhys += cb;
2607 }
2608
2609 /* Advance range if necessary. */
2610 while (pRam && GCPhys > pRam->GCPhysLast)
2611 pRam = pRam->CTX_SUFF(pNext);
2612 } /* Ram range walk */
2613
2614 pgmUnlock(pVM);
2615 return rcStrict;
2616}
2617
2618
2619/**
2620 * Deals with writing to a page with one or more WRITE or ALL access handlers.
2621 *
2622 * @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3.
2623 * See PGM_HANDLER_PHYS_IS_VALID_STATUS and
2624 * PGM_HANDLER_VIRT_IS_VALID_STATUS for details.
2625 *
2626 * @param pVM The cross context VM structure.
2627 * @param pPage The page descriptor.
2628 * @param GCPhys The physical address to start writing at.
2629 * @param pvBuf What to write.
2630 * @param cbWrite How much to write - less or equal to a page.
2631 * @param enmOrigin The origin of this call.
2632 */
2633static VBOXSTRICTRC pgmPhysWriteHandler(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const *pvBuf, size_t cbWrite,
2634 PGMACCESSORIGIN enmOrigin)
2635{
2636 PGMPAGEMAPLOCK PgMpLck;
2637 void *pvDst = NULL;
2638 VBOXSTRICTRC rcStrict;
2639
2640 /*
2641 * Give priority to physical handlers (like #PF does).
2642 *
2643 * Hope for a lonely physical handler first that covers the whole
2644 * write area. This should be a pretty frequent case with MMIO and
2645 * the heavy usage of full page handlers in the page pool.
2646 */
2647 PVMCPU pVCpu = VMMGetCpu(pVM);
2648 if ( !PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage)
2649 || PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage) /* screw virtual handlers on MMIO pages */)
2650 {
2651 PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
2652 if (pCur)
2653 {
2654 Assert(GCPhys >= pCur->Core.Key && GCPhys <= pCur->Core.KeyLast);
2655#ifndef IN_RING3
2656 if (enmOrigin != PGMACCESSORIGIN_IEM)
2657 /* Cannot reliably handle informational status codes in this context */
2658 return VERR_PGM_PHYS_WR_HIT_HANDLER;
2659#endif
2660 size_t cbRange = pCur->Core.KeyLast - GCPhys + 1;
2661 if (cbRange > cbWrite)
2662 cbRange = cbWrite;
2663
2664 Assert(PGMPHYSHANDLER_GET_TYPE(pVM, pCur)->CTX_SUFF(pfnHandler));
2665 Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n",
2666 GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
2667 if (!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
2668 rcStrict = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
2669 else
2670 rcStrict = VINF_SUCCESS;
2671 if (RT_SUCCESS(rcStrict))
2672 {
2673 PFNPGMPHYSHANDLER pfnHandler = PGMPHYSHANDLER_GET_TYPE(pVM, pCur)->CTX_SUFF(pfnHandler);
2674 void *pvUser = pCur->CTX_SUFF(pvUser);
2675 STAM_PROFILE_START(&pCur->Stat, h);
2676
2677 /* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2678 PGM_LOCK_ASSERT_OWNER(pVM);
2679 pgmUnlock(pVM);
2680 rcStrict = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, pvUser);
2681 pgmLock(pVM);
2682
2683#ifdef VBOX_WITH_STATISTICS
2684 pCur = pgmHandlerPhysicalLookup(pVM, GCPhys);
2685 if (pCur)
2686 STAM_PROFILE_STOP(&pCur->Stat, h);
2687#else
2688 pCur = NULL; /* might not be valid anymore. */
2689#endif
2690 if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
2691 {
2692 if (pvDst)
2693 memcpy(pvDst, pvBuf, cbRange);
2694 rcStrict = VINF_SUCCESS;
2695 }
2696 else
2697 AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, true),
2698 ("rcStrict=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n",
2699 VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pCur ? R3STRING(pCur->pszDesc) : ""));
2700 }
2701 else
2702 AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2703 GCPhys, pPage, VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
2704 if (RT_LIKELY(cbRange == cbWrite) || !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
2705 {
2706 if (pvDst)
2707 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2708 return rcStrict;
2709 }
2710
2711 /* more fun to be had below */
2712 cbWrite -= cbRange;
2713 GCPhys += cbRange;
2714 pvBuf = (uint8_t *)pvBuf + cbRange;
2715 pvDst = (uint8_t *)pvDst + cbRange;
2716 }
2717 else /* The handler is somewhere else in the page, deal with it below. */
2718 rcStrict = VINF_SUCCESS;
2719 Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)); /* MMIO handlers are all PAGE_SIZEed! */
2720 }
2721#if !defined(IN_RING0) && defined(VBOX_WITH_RAW_MODE)
2722 /*
2723 * A virtual handler without any interfering physical handlers.
2724 * Hopefully it'll cover the whole write.
2725 */
2726 else if (!PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage))
2727 {
2728 unsigned iPage;
2729 PPGMVIRTHANDLER pVirt = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &iPage);
2730 if (pVirt)
2731 {
2732# ifndef IN_RING3
2733 if (enmOrigin != PGMACCESSORIGIN_IEM)
2734 /* Cannot reliably handle informational status codes in this context */
2735 return VERR_PGM_PHYS_WR_HIT_HANDLER;
2736# endif
2737 PPGMVIRTHANDLERTYPEINT pVirtType = PGMVIRTANDLER_GET_TYPE(pVM, pVirt);
2738 size_t cbRange = (PAGE_OFFSET_MASK & pVirt->Core.KeyLast) - (PAGE_OFFSET_MASK & GCPhys) + 1;
2739 if (cbRange > cbWrite)
2740 cbRange = cbWrite;
2741
2742 Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] virt %s\n",
2743 GCPhys, cbRange, pPage, R3STRING(pVirt->pszDesc) ));
2744 rcStrict = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
2745 if (RT_SUCCESS(rcStrict))
2746 {
2747 Assert(pVirtType->CTX_SUFF(pfnHandler));
2748 RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2749 + (iPage << PAGE_SHIFT)
2750 + (GCPhys & PAGE_OFFSET_MASK);
2751
2752 STAM_PROFILE_START(&pVirt->Stat, h);
2753 rcStrict = pVirtType->CTX_SUFF(pfnHandler)(pVM, pVCpu, GCPtr, pvDst, (void *)pvBuf, cbRange,
2754 PGMACCESSTYPE_WRITE, enmOrigin, pVirt->CTX_SUFF(pvUser));
2755 STAM_PROFILE_STOP(&pVirt->Stat, h);
2756 if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
2757 {
2758 memcpy(pvDst, pvBuf, cbRange);
2759 rcStrict = VINF_SUCCESS;
2760 }
2761 else
2762 AssertLogRelMsg(PGM_HANDLER_VIRT_IS_VALID_STATUS(rcStrict, true),
2763 ("rcStrict=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n",
2764 VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, R3STRING(pVirt->pszDesc)));
2765 }
2766 else
2767 AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
2768 GCPhys, pPage, VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
2769 if (RT_LIKELY(cbRange == cbWrite) || !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
2770 {
2771 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2772 return rcStrict;
2773 }
2774
2775 /* more fun to be had below */
2776 cbWrite -= cbRange;
2777 GCPhys += cbRange;
2778 pvBuf = (uint8_t *)pvBuf + cbRange;
2779 pvDst = (uint8_t *)pvDst + cbRange;
2780 }
2781 else /* The handler is somewhere else in the page, deal with it below. */
2782 rcStrict = VINF_SUCCESS;
2783 }
2784#endif /* !IN_RING0 && VBOX_WITH_RAW_MODE */
2785 else
2786 rcStrict = VINF_SUCCESS;
2787
2788
2789 /*
2790 * Deal with all the odd ends.
2791 */
2792 Assert(rcStrict != VINF_PGM_HANDLER_DO_DEFAULT);
2793
2794 /* We need a writable destination page. */
2795 if (!pvDst)
2796 {
2797 int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
2798 AssertLogRelMsgReturn(RT_SUCCESS(rc2),
2799 ("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n", GCPhys, pPage, rc2),
2800 rc2);
2801 }
2802
2803 /* The loop state (big + ugly). */
2804#if !defined(IN_RING0) && defined(VBOX_WITH_RAW_MODE)
2805 unsigned iVirtPage = 0;
2806 PPGMVIRTHANDLER pVirt = NULL;
2807 uint32_t offVirt = PAGE_SIZE;
2808 uint32_t offVirtLast = PAGE_SIZE;
2809 bool fMoreVirt = PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage);
2810#else
2811 uint32_t const offVirt = UINT32_MAX;
2812#endif
2813
2814 PPGMPHYSHANDLER pPhys = NULL;
2815 uint32_t offPhys = PAGE_SIZE;
2816 uint32_t offPhysLast = PAGE_SIZE;
2817 bool fMorePhys = PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage);
2818
2819 /* The loop. */
2820 for (;;)
2821 {
2822#if !defined(IN_RING0) && defined(VBOX_WITH_RAW_MODE)
2823 /*
2824 * Find the closest handler at or above GCPhys.
2825 */
2826 if (fMoreVirt && !pVirt)
2827 {
2828 pVirt = pgmHandlerVirtualFindByPhysAddr(pVM, GCPhys, &iVirtPage);
2829 if (pVirt)
2830 {
2831 offVirt = 0;
2832 offVirtLast = (pVirt->aPhysToVirt[iVirtPage].Core.KeyLast & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2833 }
2834 else
2835 {
2836 PPGMPHYS2VIRTHANDLER pVirtPhys;
2837 pVirtPhys = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysToVirtHandlers,
2838 GCPhys, true /* fAbove */);
2839 if ( pVirtPhys
2840 && (pVirtPhys->Core.Key >> PAGE_SHIFT) == (GCPhys >> PAGE_SHIFT))
2841 {
2842 /* ASSUME that pVirtPhys only covers one page. */
2843 Assert((pVirtPhys->Core.Key >> PAGE_SHIFT) == (pVirtPhys->Core.KeyLast >> PAGE_SHIFT));
2844 Assert(pVirtPhys->Core.Key > GCPhys);
2845
2846 pVirt = (PPGMVIRTHANDLER)((uintptr_t)pVirtPhys + pVirtPhys->offVirtHandler);
2847 iVirtPage = pVirtPhys - &pVirt->aPhysToVirt[0]; Assert(iVirtPage == 0);
2848 offVirt = (pVirtPhys->Core.Key & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2849 offVirtLast = (pVirtPhys->Core.KeyLast & PAGE_OFFSET_MASK) - (GCPhys & PAGE_OFFSET_MASK);
2850 }
2851 else
2852 {
2853 pVirt = NULL;
2854 fMoreVirt = false;
2855 offVirt = offVirtLast = PAGE_SIZE;
2856 }
2857 }
2858 }
2859#endif
2860
2861 if (fMorePhys && !pPhys)
2862 {
2863 pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2864 if (pPhys)
2865 {
2866 offPhys = 0;
2867 offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
2868 }
2869 else
2870 {
2871 pPhys = (PPGMPHYSHANDLER)RTAvlroGCPhysGetBestFit(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers,
2872 GCPhys, true /* fAbove */);
2873 if ( pPhys
2874 && pPhys->Core.Key <= GCPhys + (cbWrite - 1))
2875 {
2876 offPhys = pPhys->Core.Key - GCPhys;
2877 offPhysLast = pPhys->Core.KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
2878 }
2879 else
2880 {
2881 pPhys = NULL;
2882 fMorePhys = false;
2883 offPhys = offPhysLast = PAGE_SIZE;
2884 }
2885 }
2886 }
2887
2888 /*
2889 * Handle access to space without handlers (that's easy).
2890 */
2891 VBOXSTRICTRC rcStrict2 = VINF_PGM_HANDLER_DO_DEFAULT;
2892 uint32_t cbRange = (uint32_t)cbWrite;
2893 if (offPhys != 0 && offVirt != 0)
2894 {
2895 if (cbRange > offPhys)
2896 cbRange = offPhys;
2897 if (cbRange > offVirt)
2898 cbRange = offVirt;
2899 Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] miss\n", GCPhys, cbRange, pPage));
2900 }
2901 /*
2902 * Physical handler.
2903 */
2904 else if (!offPhys && offVirt)
2905 {
2906#ifndef IN_RING3
2907 if (enmOrigin != PGMACCESSORIGIN_IEM)
2908 /* Cannot reliably handle informational status codes in this context */
2909 return VERR_PGM_PHYS_WR_HIT_HANDLER;
2910#endif
2911 if (cbRange > offPhysLast + 1)
2912 cbRange = offPhysLast + 1;
2913 if (cbRange > offVirt)
2914 cbRange = offVirt;
2915
2916 PFNPGMPHYSHANDLER pfnHandler = PGMPHYSHANDLER_GET_TYPE(pVM, pPhys)->CTX_SUFF(pfnHandler);
2917 void *pvUser = pPhys->CTX_SUFF(pvUser);
2918
2919 Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc) ));
2920 STAM_PROFILE_START(&pPhys->Stat, h);
2921
2922 /* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2923 PGM_LOCK_ASSERT_OWNER(pVM);
2924 pgmUnlock(pVM);
2925 rcStrict2 = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, pvUser);
2926 pgmLock(pVM);
2927
2928#ifdef VBOX_WITH_STATISTICS
2929 pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
2930 if (pPhys)
2931 STAM_PROFILE_STOP(&pPhys->Stat, h);
2932#else
2933 pPhys = NULL; /* might not be valid anymore. */
2934#endif
2935 AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict2, true),
2936 ("rcStrict2=%Rrc (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n", VBOXSTRICTRC_VAL(rcStrict2),
2937 VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pPhys ? R3STRING(pPhys->pszDesc) : ""));
2938 }
2939#if !defined(IN_RING0) && defined(VBOX_WITH_RAW_MODE)
2940 /*
2941 * Virtual handler.
2942 */
2943 else if (offPhys && !offVirt)
2944 {
2945# ifndef IN_RING3
2946 if (enmOrigin != PGMACCESSORIGIN_IEM)
2947 /* Cannot reliably handle informational status codes in this context */
2948 return VERR_PGM_PHYS_WR_HIT_HANDLER;
2949# endif
2950 if (cbRange > offVirtLast + 1)
2951 cbRange = offVirtLast + 1;
2952 if (cbRange > offPhys)
2953 cbRange = offPhys;
2954
2955 PPGMVIRTHANDLERTYPEINT pVirtType = PGMVIRTANDLER_GET_TYPE(pVM, pVirt);
2956 Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pVirt->pszDesc) ));
2957 Assert(pVirtType->CTX_SUFF(pfnHandler));
2958 RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
2959 + (iVirtPage << PAGE_SHIFT)
2960 + (GCPhys & PAGE_OFFSET_MASK);
2961 STAM_PROFILE_START(&pVirt->Stat, h);
2962 rcStrict2 = pVirtType->CTX_SUFF(pfnHandler)(pVM, pVCpu, GCPtr, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE,
2963 enmOrigin, pVirt->CTX_SUFF(pvUser));
2964 STAM_PROFILE_STOP(&pVirt->Stat, h);
2965 AssertLogRelMsg(PGM_HANDLER_VIRT_IS_VALID_STATUS(rcStrict2, true),
2966 ("rcStrict2=%Rrc (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n", VBOXSTRICTRC_VAL(rcStrict2),
2967 VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pPhys ? R3STRING(pPhys->pszDesc) : ""));
2968 pVirt = NULL;
2969 }
2970 /*
2971 * Both... give the physical one priority.
2972 */
2973 else
2974 {
2975# ifndef IN_RING3
2976 if (enmOrigin != PGMACCESSORIGIN_IEM)
2977 /* Cannot reliably handle informational status codes in this context */
2978 return VERR_PGM_PHYS_WR_HIT_HANDLER;
2979# endif
2980 Assert(!offPhys && !offVirt);
2981 if (cbRange > offVirtLast + 1)
2982 cbRange = offVirtLast + 1;
2983 if (cbRange > offPhysLast + 1)
2984 cbRange = offPhysLast + 1;
2985
2986 PPGMVIRTHANDLERTYPEINT pVirtType = PGMVIRTANDLER_GET_TYPE(pVM, pVirt);
2987 if (pVirtType->pfnHandlerR3)
2988 Log(("pgmPhysWriteHandler: overlapping phys and virt handlers at %RGp %R[pgmpage]; cbRange=%#x\n", GCPhys, pPage, cbRange));
2989 Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys/virt %s/%s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc), R3STRING(pVirt->pszDesc) ));
2990
2991 PFNPGMPHYSHANDLER pfnHandler = PGMPHYSHANDLER_GET_TYPE(pVM, pPhys)->CTX_SUFF(pfnHandler);
2992 void *pvUser = pPhys->CTX_SUFF(pvUser);
2993 STAM_PROFILE_START(&pPhys->Stat, h);
2994
2995 /* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
2996 PGM_LOCK_ASSERT_OWNER(pVM);
2997 pgmUnlock(pVM);
2998 rcStrict2 = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, pvUser);
2999 pgmLock(pVM);
3000
3001# ifdef VBOX_WITH_STATISTICS
3002 pPhys = pgmHandlerPhysicalLookup(pVM, GCPhys);
3003 if (pPhys)
3004 STAM_PROFILE_STOP(&pPhys->Stat, h);
3005# else
3006 pPhys = NULL; /* might not be valid anymore. */
3007# endif
3008 AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict2, true),
3009 ("rcStrict2=%Rrc (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n", VBOXSTRICTRC_VAL(rcStrict2),
3010 VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pPhys ? R3STRING(pPhys->pszDesc) : ""));
3011 if (rcStrict2 == VINF_PGM_HANDLER_DO_DEFAULT || PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
3012 {
3013 Assert(pVirtType->CTX_SUFF(pfnHandler));
3014 RTGCUINTPTR GCPtr = ((RTGCUINTPTR)pVirt->Core.Key & PAGE_BASE_GC_MASK)
3015 + (iVirtPage << PAGE_SHIFT)
3016 + (GCPhys & PAGE_OFFSET_MASK);
3017 pvUser = pVirt->CTX_SUFF(pvUser);
3018
3019 STAM_PROFILE_START(&pVirt->Stat, h2);
3020 VBOXSTRICTRC rcStrict3 = pVirtType->CTX_SUFF(pfnHandler)(pVM, pVCpu, GCPtr, pvDst, (void *)pvBuf, cbRange,
3021 PGMACCESSTYPE_WRITE, enmOrigin, pvUser);
3022 STAM_PROFILE_STOP(&pVirt->Stat, h2);
3023
3024 /* Merge the 3rd status into the 2nd. */
3025 if (rcStrict3 == VINF_SUCCESS)
3026 {
3027 if (rcStrict2 == VINF_PGM_HANDLER_DO_DEFAULT)
3028 rcStrict2 = VINF_SUCCESS;
3029 }
3030 else if (rcStrict3 != VINF_PGM_HANDLER_DO_DEFAULT)
3031 {
3032 AssertLogRelMsg(PGM_HANDLER_VIRT_IS_VALID_STATUS(rcStrict3, true),
3033 ("rcStrict3=%Rrc (rcStrict2=%Rrc) (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n",
3034 VBOXSTRICTRC_VAL(rcStrict3), VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict),
3035 GCPhys, pPage, R3STRING(pVirt->pszDesc) ));
3036 if (rcStrict2 == VINF_PGM_HANDLER_DO_DEFAULT)
3037 rcStrict2 = rcStrict3;
3038 else if (!PGM_PHYS_RW_IS_SUCCESS(rcStrict3))
3039 rcStrict2 = rcStrict3;
3040 else
3041 PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict2, rcStrict3);
3042 }
3043 }
3044 pPhys = NULL;
3045 pVirt = NULL;
3046 }
3047#endif /* !IN_RING0 && VBOX_WITH_RAW_MODE */
3048
3049
3050 /*
3051 * Execute the default action and merge the status codes.
3052 */
3053 if (rcStrict2 == VINF_PGM_HANDLER_DO_DEFAULT)
3054 {
3055 memcpy(pvDst, pvBuf, cbRange);
3056 rcStrict2 = VINF_SUCCESS;
3057 }
3058 else if (!PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
3059 {
3060 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
3061 return rcStrict2;
3062 }
3063 else
3064 PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
3065
3066 /*
3067 * Advance if we've got more stuff to do.
3068 */
3069 if (cbRange >= cbWrite)
3070 {
3071 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
3072 return rcStrict;
3073 }
3074
3075
3076 cbWrite -= cbRange;
3077 GCPhys += cbRange;
3078 pvBuf = (uint8_t *)pvBuf + cbRange;
3079 pvDst = (uint8_t *)pvDst + cbRange;
3080
3081 offPhys -= cbRange;
3082 offPhysLast -= cbRange;
3083#if !defined(IN_RING0) && defined(VBOX_WITH_RAW_MODE)
3084 offVirt -= cbRange;
3085 offVirtLast -= cbRange;
3086#endif
3087 }
3088}
3089
3090
3091/**
3092 * Write to physical memory.
3093 *
3094 * This API respects access handlers and MMIO. Use PGMPhysSimpleWriteGCPhys() if you
3095 * want to ignore those.
3096 *
3097 * @returns Strict VBox status code in raw-mode and ring-0, normal VBox status
3098 * code in ring-3. Use PGM_PHYS_RW_IS_SUCCESS to check.
3099 * @retval VINF_SUCCESS in all context - write completed.
3100 *
3101 * @retval VINF_EM_OFF in RC and R0 - write completed.
3102 * @retval VINF_EM_SUSPEND in RC and R0 - write completed.
3103 * @retval VINF_EM_RESET in RC and R0 - write completed.
3104 * @retval VINF_EM_HALT in RC and R0 - write completed.
3105 * @retval VINF_SELM_SYNC_GDT in RC only - write completed.
3106 *
3107 * @retval VINF_EM_DBG_STOP in RC and R0 - write completed.
3108 * @retval VINF_EM_DBG_BREAKPOINT in RC and R0 - write completed.
3109 * @retval VINF_EM_RAW_EMULATE_INSTR in RC and R0 only.
3110 *
3111 * @retval VINF_IOM_R3_MMIO_WRITE in RC and R0.
3112 * @retval VINF_IOM_R3_MMIO_READ_WRITE in RC and R0.
3113 * @retval VINF_IOM_R3_MMIO_COMMIT_WRITE in RC and R0.
3114 *
3115 * @retval VINF_EM_RAW_EMULATE_IO_BLOCK in R0 only.
3116 *
3117 * @retval VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT in RC only - write completed.
3118 * @retval VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT in RC only.
3119 * @retval VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT in RC only.
3120 * @retval VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT in RC only.
3121 * @retval VINF_CSAM_PENDING_ACTION in RC only.
3122 * @retval VINF_PATM_CHECK_PATCH_PAGE in RC only.
3123 *
3124 * @retval VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that
3125 * haven't been cleared for strict status codes yet.
3126 *
3127 *
3128 * @param pVM The cross context VM structure.
3129 * @param GCPhys Physical address to write to.
3130 * @param pvBuf What to write.
3131 * @param cbWrite How many bytes to write.
3132 * @param enmOrigin Who is calling.
3133 */
3134VMMDECL(VBOXSTRICTRC) PGMPhysWrite(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, PGMACCESSORIGIN enmOrigin)
3135{
3136 AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMPhysWrite after pgmR3Save()! enmOrigin=%d\n", enmOrigin));
3137 AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
3138 LogFlow(("PGMPhysWrite: %RGp %d\n", GCPhys, cbWrite));
3139
3140 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysWrite));
3141 STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysWriteBytes), cbWrite);
3142
3143 pgmLock(pVM);
3144
3145 /*
3146 * Copy loop on ram ranges.
3147 */
3148 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
3149 PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
3150 for (;;)
3151 {
3152 /* Inside range or not? */
3153 if (pRam && GCPhys >= pRam->GCPhys)
3154 {
3155 /*
3156 * Must work our way thru this page by page.
3157 */
3158 RTGCPTR off = GCPhys - pRam->GCPhys;
3159 while (off < pRam->cb)
3160 {
3161 RTGCPTR iPage = off >> PAGE_SHIFT;
3162 PPGMPAGE pPage = &pRam->aPages[iPage];
3163 size_t cb = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
3164 if (cb > cbWrite)
3165 cb = cbWrite;
3166
3167 /*
3168 * Normal page? Get the pointer to it.
3169 */
3170 if ( !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
3171 && !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
3172 {
3173 PGMPAGEMAPLOCK PgMpLck;
3174 void *pvDst;
3175 int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
3176 if (RT_SUCCESS(rc))
3177 {
3178 Assert(!PGM_PAGE_IS_BALLOONED(pPage));
3179 memcpy(pvDst, pvBuf, cb);
3180 pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
3181 }
3182 /* Ignore writes to ballooned pages. */
3183 else if (!PGM_PAGE_IS_BALLOONED(pPage))
3184 AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
3185 pRam->GCPhys + off, pPage, rc));
3186 }
3187 /*
3188 * Active WRITE or ALL access handlers.
3189 */
3190 else
3191 {
3192 VBOXSTRICTRC rcStrict2 = pgmPhysWriteHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin);
3193 if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
3194 PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
3195 else
3196 {
3197 pgmUnlock(pVM);
3198 return rcStrict2;
3199 }
3200 }
3201
3202 /* next page */
3203 if (cb >= cbWrite)
3204 {
3205 pgmUnlock(pVM);
3206 return rcStrict;
3207 }
3208
3209 cbWrite -= cb;
3210 off += cb;
3211 pvBuf = (const char *)pvBuf + cb;
3212 } /* walk pages in ram range */
3213
3214 GCPhys = pRam->GCPhysLast + 1;
3215 }
3216 else
3217 {
3218 /*
3219 * Unassigned address space, skip it.
3220 */
3221 if (!pRam)
3222 break;
3223 size_t cb = pRam->GCPhys - GCPhys;
3224 if (cb >= cbWrite)
3225 break;
3226 cbWrite -= cb;
3227 pvBuf = (const char *)pvBuf + cb;
3228 GCPhys += cb;
3229 }
3230
3231 /* Advance range if necessary. */
3232 while (pRam && GCPhys > pRam->GCPhysLast)
3233 pRam = pRam->CTX_SUFF(pNext);
3234 } /* Ram range walk */
3235
3236 pgmUnlock(pVM);
3237 return rcStrict;
3238}
3239
3240
3241/**
3242 * Read from guest physical memory by GC physical address, bypassing
3243 * MMIO and access handlers.
3244 *
3245 * @returns VBox status code.
3246 * @param pVM The cross context VM structure.
3247 * @param pvDst The destination address.
3248 * @param GCPhysSrc The source address (GC physical address).
3249 * @param cb The number of bytes to read.
3250 */
3251VMMDECL(int) PGMPhysSimpleReadGCPhys(PVM pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb)
3252{
3253 /*
3254 * Treat the first page as a special case.
3255 */
3256 if (!cb)
3257 return VINF_SUCCESS;
3258
3259 /* map the 1st page */
3260 void const *pvSrc;
3261 PGMPAGEMAPLOCK Lock;
3262 int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
3263 if (RT_FAILURE(rc))
3264 return rc;
3265
3266 /* optimize for the case where access is completely within the first page. */
3267 size_t cbPage = PAGE_SIZE - (GCPhysSrc & PAGE_OFFSET_MASK);
3268 if (RT_LIKELY(cb <= cbPage))
3269 {
3270 memcpy(pvDst, pvSrc, cb);
3271 PGMPhysReleasePageMappingLock(pVM, &Lock);
3272 return VINF_SUCCESS;
3273 }
3274
3275 /* copy to the end of the page. */
3276 memcpy(pvDst, pvSrc, cbPage);
3277 PGMPhysReleasePageMappingLock(pVM, &Lock);
3278 GCPhysSrc += cbPage;
3279 pvDst = (uint8_t *)pvDst + cbPage;
3280 cb -= cbPage;
3281
3282 /*
3283 * Page by page.
3284 */
3285 for (;;)
3286 {
3287 /* map the page */
3288 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
3289 if (RT_FAILURE(rc))
3290 return rc;
3291
3292 /* last page? */
3293 if (cb <= PAGE_SIZE)
3294 {
3295 memcpy(pvDst, pvSrc, cb);
3296 PGMPhysReleasePageMappingLock(pVM, &Lock);
3297 return VINF_SUCCESS;
3298 }
3299
3300 /* copy the entire page and advance */
3301 memcpy(pvDst, pvSrc, PAGE_SIZE);
3302 PGMPhysReleasePageMappingLock(pVM, &Lock);
3303 GCPhysSrc += PAGE_SIZE;
3304 pvDst = (uint8_t *)pvDst + PAGE_SIZE;
3305 cb -= PAGE_SIZE;
3306 }
3307 /* won't ever get here. */
3308}
3309
3310
3311/**
3312 * Write to guest physical memory referenced by GC pointer.
3313 * Write memory to GC physical address in guest physical memory.
3314 *
3315 * This will bypass MMIO and access handlers.
3316 *
3317 * @returns VBox status code.
3318 * @param pVM The cross context VM structure.
3319 * @param GCPhysDst The GC physical address of the destination.
3320 * @param pvSrc The source buffer.
3321 * @param cb The number of bytes to write.
3322 */
3323VMMDECL(int) PGMPhysSimpleWriteGCPhys(PVM pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb)
3324{
3325 LogFlow(("PGMPhysSimpleWriteGCPhys: %RGp %zu\n", GCPhysDst, cb));
3326
3327 /*
3328 * Treat the first page as a special case.
3329 */
3330 if (!cb)
3331 return VINF_SUCCESS;
3332
3333 /* map the 1st page */
3334 void *pvDst;
3335 PGMPAGEMAPLOCK Lock;
3336 int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
3337 if (RT_FAILURE(rc))
3338 return rc;
3339
3340 /* optimize for the case where access is completely within the first page. */
3341 size_t cbPage = PAGE_SIZE - (GCPhysDst & PAGE_OFFSET_MASK);
3342 if (RT_LIKELY(cb <= cbPage))
3343 {
3344 memcpy(pvDst, pvSrc, cb);
3345 PGMPhysReleasePageMappingLock(pVM, &Lock);
3346 return VINF_SUCCESS;
3347 }
3348
3349 /* copy to the end of the page. */
3350 memcpy(pvDst, pvSrc, cbPage);
3351 PGMPhysReleasePageMappingLock(pVM, &Lock);
3352 GCPhysDst += cbPage;
3353 pvSrc = (const uint8_t *)pvSrc + cbPage;
3354 cb -= cbPage;
3355
3356 /*
3357 * Page by page.
3358 */
3359 for (;;)
3360 {
3361 /* map the page */
3362 rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
3363 if (RT_FAILURE(rc))
3364 return rc;
3365
3366 /* last page? */
3367 if (cb <= PAGE_SIZE)
3368 {
3369 memcpy(pvDst, pvSrc, cb);
3370 PGMPhysReleasePageMappingLock(pVM, &Lock);
3371 return VINF_SUCCESS;
3372 }
3373
3374 /* copy the entire page and advance */
3375 memcpy(pvDst, pvSrc, PAGE_SIZE);
3376 PGMPhysReleasePageMappingLock(pVM, &Lock);
3377 GCPhysDst += PAGE_SIZE;
3378 pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
3379 cb -= PAGE_SIZE;
3380 }
3381 /* won't ever get here. */
3382}
3383
3384
3385/**
3386 * Read from guest physical memory referenced by GC pointer.
3387 *
3388 * This function uses the current CR3/CR0/CR4 of the guest and will
3389 * bypass access handlers and not set any accessed bits.
3390 *
3391 * @returns VBox status code.
3392 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
3393 * @param pvDst The destination address.
3394 * @param GCPtrSrc The source address (GC pointer).
3395 * @param cb The number of bytes to read.
3396 */
3397VMMDECL(int) PGMPhysSimpleReadGCPtr(PVMCPU pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
3398{
3399 PVM pVM = pVCpu->CTX_SUFF(pVM);
3400/** @todo fix the macro / state handling: VMCPU_ASSERT_EMT_OR_GURU(pVCpu); */
3401
3402 /*
3403 * Treat the first page as a special case.
3404 */
3405 if (!cb)
3406 return VINF_SUCCESS;
3407
3408 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleRead));
3409 STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleReadBytes), cb);
3410
3411 /* Take the PGM lock here, because many called functions take the lock for a very short period. That's counter-productive
3412 * when many VCPUs are fighting for the lock.
3413 */
3414 pgmLock(pVM);
3415
3416 /* map the 1st page */
3417 void const *pvSrc;
3418 PGMPAGEMAPLOCK Lock;
3419 int rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
3420 if (RT_FAILURE(rc))
3421 {
3422 pgmUnlock(pVM);
3423 return rc;
3424 }
3425
3426 /* optimize for the case where access is completely within the first page. */
3427 size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
3428 if (RT_LIKELY(cb <= cbPage))
3429 {
3430 memcpy(pvDst, pvSrc, cb);
3431 PGMPhysReleasePageMappingLock(pVM, &Lock);
3432 pgmUnlock(pVM);
3433 return VINF_SUCCESS;
3434 }
3435
3436 /* copy to the end of the page. */
3437 memcpy(pvDst, pvSrc, cbPage);
3438 PGMPhysReleasePageMappingLock(pVM, &Lock);
3439 GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + cbPage);
3440 pvDst = (uint8_t *)pvDst + cbPage;
3441 cb -= cbPage;
3442
3443 /*
3444 * Page by page.
3445 */
3446 for (;;)
3447 {
3448 /* map the page */
3449 rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
3450 if (RT_FAILURE(rc))
3451 {
3452 pgmUnlock(pVM);
3453 return rc;
3454 }
3455
3456 /* last page? */
3457 if (cb <= PAGE_SIZE)
3458 {
3459 memcpy(pvDst, pvSrc, cb);
3460 PGMPhysReleasePageMappingLock(pVM, &Lock);
3461 pgmUnlock(pVM);
3462 return VINF_SUCCESS;
3463 }
3464
3465 /* copy the entire page and advance */
3466 memcpy(pvDst, pvSrc, PAGE_SIZE);
3467 PGMPhysReleasePageMappingLock(pVM, &Lock);
3468 GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + PAGE_SIZE);
3469 pvDst = (uint8_t *)pvDst + PAGE_SIZE;
3470 cb -= PAGE_SIZE;
3471 }
3472 /* won't ever get here. */
3473}
3474
3475
3476/**
3477 * Write to guest physical memory referenced by GC pointer.
3478 *
3479 * This function uses the current CR3/CR0/CR4 of the guest and will
3480 * bypass access handlers and not set dirty or accessed bits.
3481 *
3482 * @returns VBox status code.
3483 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
3484 * @param GCPtrDst The destination address (GC pointer).
3485 * @param pvSrc The source address.
3486 * @param cb The number of bytes to write.
3487 */
3488VMMDECL(int) PGMPhysSimpleWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
3489{
3490 PVM pVM = pVCpu->CTX_SUFF(pVM);
3491 VMCPU_ASSERT_EMT(pVCpu);
3492
3493 /*
3494 * Treat the first page as a special case.
3495 */
3496 if (!cb)
3497 return VINF_SUCCESS;
3498
3499 STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleWrite));
3500 STAM_COUNTER_ADD(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysSimpleWriteBytes), cb);
3501
3502 /* map the 1st page */
3503 void *pvDst;
3504 PGMPAGEMAPLOCK Lock;
3505 int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3506 if (RT_FAILURE(rc))
3507 return rc;
3508
3509 /* optimize for the case where access is completely within the first page. */
3510 size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
3511 if (RT_LIKELY(cb <= cbPage))
3512 {
3513 memcpy(pvDst, pvSrc, cb);
3514 PGMPhysReleasePageMappingLock(pVM, &Lock);
3515 return VINF_SUCCESS;
3516 }
3517
3518 /* copy to the end of the page. */
3519 memcpy(pvDst, pvSrc, cbPage);
3520 PGMPhysReleasePageMappingLock(pVM, &Lock);
3521 GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
3522 pvSrc = (const uint8_t *)pvSrc + cbPage;
3523 cb -= cbPage;
3524
3525 /*
3526 * Page by page.
3527 */
3528 for (;;)
3529 {
3530 /* map the page */
3531 rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3532 if (RT_FAILURE(rc))
3533 return rc;
3534
3535 /* last page? */
3536 if (cb <= PAGE_SIZE)
3537 {
3538 memcpy(pvDst, pvSrc, cb);
3539 PGMPhysReleasePageMappingLock(pVM, &Lock);
3540 return VINF_SUCCESS;
3541 }
3542
3543 /* copy the entire page and advance */
3544 memcpy(pvDst, pvSrc, PAGE_SIZE);
3545 PGMPhysReleasePageMappingLock(pVM, &Lock);
3546 GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
3547 pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
3548 cb -= PAGE_SIZE;
3549 }
3550 /* won't ever get here. */
3551}
3552
3553
3554/**
3555 * Write to guest physical memory referenced by GC pointer and update the PTE.
3556 *
3557 * This function uses the current CR3/CR0/CR4 of the guest and will
3558 * bypass access handlers but will set any dirty and accessed bits in the PTE.
3559 *
3560 * If you don't want to set the dirty bit, use PGMPhysSimpleWriteGCPtr().
3561 *
3562 * @returns VBox status code.
3563 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
3564 * @param GCPtrDst The destination address (GC pointer).
3565 * @param pvSrc The source address.
3566 * @param cb The number of bytes to write.
3567 */
3568VMMDECL(int) PGMPhysSimpleDirtyWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
3569{
3570 PVM pVM = pVCpu->CTX_SUFF(pVM);
3571 VMCPU_ASSERT_EMT(pVCpu);
3572
3573 /*
3574 * Treat the first page as a special case.
3575 * Btw. this is the same code as in PGMPhyssimpleWriteGCPtr excep for the PGMGstModifyPage.
3576 */
3577 if (!cb)
3578 return VINF_SUCCESS;
3579
3580 /* map the 1st page */
3581 void *pvDst;
3582 PGMPAGEMAPLOCK Lock;
3583 int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3584 if (RT_FAILURE(rc))
3585 return rc;
3586
3587 /* optimize for the case where access is completely within the first page. */
3588 size_t cbPage = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
3589 if (RT_LIKELY(cb <= cbPage))
3590 {
3591 memcpy(pvDst, pvSrc, cb);
3592 PGMPhysReleasePageMappingLock(pVM, &Lock);
3593 rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc);
3594 return VINF_SUCCESS;
3595 }
3596
3597 /* copy to the end of the page. */
3598 memcpy(pvDst, pvSrc, cbPage);
3599 PGMPhysReleasePageMappingLock(pVM, &Lock);
3600 rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc);
3601 GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
3602 pvSrc = (const uint8_t *)pvSrc + cbPage;
3603 cb -= cbPage;
3604
3605 /*
3606 * Page by page.
3607 */
3608 for (;;)
3609 {
3610 /* map the page */
3611 rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
3612 if (RT_FAILURE(rc))
3613 return rc;
3614
3615 /* last page? */
3616 if (cb <= PAGE_SIZE)
3617 {
3618 memcpy(pvDst, pvSrc, cb);
3619 PGMPhysReleasePageMappingLock(pVM, &Lock);
3620 rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc);
3621 return VINF_SUCCESS;
3622 }
3623
3624 /* copy the entire page and advance */
3625 memcpy(pvDst, pvSrc, PAGE_SIZE);
3626 PGMPhysReleasePageMappingLock(pVM, &Lock);
3627 rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc);
3628 GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + PAGE_SIZE);
3629 pvSrc = (const uint8_t *)pvSrc + PAGE_SIZE;
3630 cb -= PAGE_SIZE;
3631 }
3632 /* won't ever get here. */
3633}
3634
3635
3636/**
3637 * Read from guest physical memory referenced by GC pointer.
3638 *
3639 * This function uses the current CR3/CR0/CR4 of the guest and will
3640 * respect access handlers and set accessed bits.
3641 *
3642 * @returns Strict VBox status, see PGMPhysRead for details.
3643 * @retval VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the
3644 * specified virtual address.
3645 *
3646 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
3647 * @param pvDst The destination address.
3648 * @param GCPtrSrc The source address (GC pointer).
3649 * @param cb The number of bytes to read.
3650 * @param enmOrigin Who is calling.
3651 * @thread EMT(pVCpu)
3652 */
3653VMMDECL(VBOXSTRICTRC) PGMPhysReadGCPtr(PVMCPU pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb, PGMACCESSORIGIN enmOrigin)
3654{
3655 RTGCPHYS GCPhys;
3656 uint64_t fFlags;
3657 int rc;
3658 PVM pVM = pVCpu->CTX_SUFF(pVM);
3659 VMCPU_ASSERT_EMT(pVCpu);
3660
3661 /*
3662 * Anything to do?
3663 */
3664 if (!cb)
3665 return VINF_SUCCESS;
3666
3667 LogFlow(("PGMPhysReadGCPtr: %RGv %zu\n", GCPtrSrc, cb));
3668
3669 /*
3670 * Optimize reads within a single page.
3671 */
3672 if (((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
3673 {
3674 /* Convert virtual to physical address + flags */
3675 rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
3676 AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
3677 GCPhys |= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
3678
3679 /* mark the guest page as accessed. */
3680 if (!(fFlags & X86_PTE_A))
3681 {
3682 rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
3683 AssertRC(rc);
3684 }
3685
3686 return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin);
3687 }
3688
3689 /*
3690 * Page by page.
3691 */
3692 for (;;)
3693 {
3694 /* Convert virtual to physical address + flags */
3695 rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrSrc, &fFlags, &GCPhys);
3696 AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
3697 GCPhys |= (RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK;
3698
3699 /* mark the guest page as accessed. */
3700 if (!(fFlags & X86_PTE_A))
3701 {
3702 rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
3703 AssertRC(rc);
3704 }
3705
3706 /* copy */
3707 size_t cbRead = PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & PAGE_OFFSET_MASK);
3708 if (cbRead < cb)
3709 {
3710 VBOXSTRICTRC rcStrict = PGMPhysRead(pVM, GCPhys, pvDst, cbRead, enmOrigin);
3711 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3712 { /* likely */ }
3713 else
3714 return rcStrict;
3715 }
3716 else /* Last page (cbRead is PAGE_SIZE, we only need cb!) */
3717 return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin);
3718
3719 /* next */
3720 Assert(cb > cbRead);
3721 cb -= cbRead;
3722 pvDst = (uint8_t *)pvDst + cbRead;
3723 GCPtrSrc += cbRead;
3724 }
3725}
3726
3727
3728/**
3729 * Write to guest physical memory referenced by GC pointer.
3730 *
3731 * This function uses the current CR3/CR0/CR4 of the guest and will
3732 * respect access handlers and set dirty and accessed bits.
3733 *
3734 * @returns Strict VBox status, see PGMPhysWrite for details.
3735 * @retval VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the
3736 * specified virtual address.
3737 *
3738 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
3739 * @param GCPtrDst The destination address (GC pointer).
3740 * @param pvSrc The source address.
3741 * @param cb The number of bytes to write.
3742 * @param enmOrigin Who is calling.
3743 */
3744VMMDECL(VBOXSTRICTRC) PGMPhysWriteGCPtr(PVMCPU pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb, PGMACCESSORIGIN enmOrigin)
3745{
3746 RTGCPHYS GCPhys;
3747 uint64_t fFlags;
3748 int rc;
3749 PVM pVM = pVCpu->CTX_SUFF(pVM);
3750 VMCPU_ASSERT_EMT(pVCpu);
3751
3752 /*
3753 * Anything to do?
3754 */
3755 if (!cb)
3756 return VINF_SUCCESS;
3757
3758 LogFlow(("PGMPhysWriteGCPtr: %RGv %zu\n", GCPtrDst, cb));
3759
3760 /*
3761 * Optimize writes within a single page.
3762 */
3763 if (((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK) + cb <= PAGE_SIZE)
3764 {
3765 /* Convert virtual to physical address + flags */
3766 rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
3767 AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
3768 GCPhys |= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
3769
3770 /* Mention when we ignore X86_PTE_RW... */
3771 if (!(fFlags & X86_PTE_RW))
3772 Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
3773
3774 /* Mark the guest page as accessed and dirty if necessary. */
3775 if ((fFlags & (X86_PTE_A | X86_PTE_D)) != (X86_PTE_A | X86_PTE_D))
3776 {
3777 rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D));
3778 AssertRC(rc);
3779 }
3780
3781 return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin);
3782 }
3783
3784 /*
3785 * Page by page.
3786 */
3787 for (;;)
3788 {
3789 /* Convert virtual to physical address + flags */
3790 rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, (RTGCUINTPTR)GCPtrDst, &fFlags, &GCPhys);
3791 AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
3792 GCPhys |= (RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK;
3793
3794 /* Mention when we ignore X86_PTE_RW... */
3795 if (!(fFlags & X86_PTE_RW))
3796 Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
3797
3798 /* Mark the guest page as accessed and dirty if necessary. */
3799 if ((fFlags & (X86_PTE_A | X86_PTE_D)) != (X86_PTE_A | X86_PTE_D))
3800 {
3801 rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D));
3802 AssertRC(rc);
3803 }
3804
3805 /* copy */
3806 size_t cbWrite = PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & PAGE_OFFSET_MASK);
3807 if (cbWrite < cb)
3808 {
3809 VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM, GCPhys, pvSrc, cbWrite, enmOrigin);
3810 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3811 { /* likely */ }
3812 else
3813 return rcStrict;
3814 }
3815 else /* Last page (cbWrite is PAGE_SIZE, we only need cb!) */
3816 return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin);
3817
3818 /* next */
3819 Assert(cb > cbWrite);
3820 cb -= cbWrite;
3821 pvSrc = (uint8_t *)pvSrc + cbWrite;
3822 GCPtrDst += cbWrite;
3823 }
3824}
3825
3826
3827/**
3828 * Performs a read of guest virtual memory for instruction emulation.
3829 *
3830 * This will check permissions, raise exceptions and update the access bits.
3831 *
3832 * The current implementation will bypass all access handlers. It may later be
3833 * changed to at least respect MMIO.
3834 *
3835 *
3836 * @returns VBox status code suitable to scheduling.
3837 * @retval VINF_SUCCESS if the read was performed successfully.
3838 * @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
3839 * @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
3840 *
3841 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
3842 * @param pCtxCore The context core.
3843 * @param pvDst Where to put the bytes we've read.
3844 * @param GCPtrSrc The source address.
3845 * @param cb The number of bytes to read. Not more than a page.
3846 *
3847 * @remark This function will dynamically map physical pages in GC. This may unmap
3848 * mappings done by the caller. Be careful!
3849 */
3850VMMDECL(int) PGMPhysInterpretedRead(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb)
3851{
3852 PVM pVM = pVCpu->CTX_SUFF(pVM);
3853 Assert(cb <= PAGE_SIZE);
3854 VMCPU_ASSERT_EMT(pVCpu);
3855
3856/** @todo r=bird: This isn't perfect!
3857 * -# It's not checking for reserved bits being 1.
3858 * -# It's not correctly dealing with the access bit.
3859 * -# It's not respecting MMIO memory or any other access handlers.
3860 */
3861 /*
3862 * 1. Translate virtual to physical. This may fault.
3863 * 2. Map the physical address.
3864 * 3. Do the read operation.
3865 * 4. Set access bits if required.
3866 */
3867 int rc;
3868 unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
3869 if (cb <= cb1)
3870 {
3871 /*
3872 * Not crossing pages.
3873 */
3874 RTGCPHYS GCPhys;
3875 uint64_t fFlags;
3876 rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
3877 if (RT_SUCCESS(rc))
3878 {
3879 /** @todo we should check reserved bits ... */
3880 PGMPAGEMAPLOCK PgMpLck;
3881 void const *pvSrc;
3882 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvSrc, &PgMpLck);
3883 switch (rc)
3884 {
3885 case VINF_SUCCESS:
3886 Log(("PGMPhysInterpretedRead: pvDst=%p pvSrc=%p cb=%d\n", pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb));
3887 memcpy(pvDst, (uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
3888 PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
3889 break;
3890 case VERR_PGM_PHYS_PAGE_RESERVED:
3891 case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3892 memset(pvDst, 0xff, cb);
3893 break;
3894 default:
3895 Assert(RT_FAILURE_NP(rc));
3896 return rc;
3897 }
3898
3899 /** @todo access bit emulation isn't 100% correct. */
3900 if (!(fFlags & X86_PTE_A))
3901 {
3902 rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3903 AssertRC(rc);
3904 }
3905 return VINF_SUCCESS;
3906 }
3907 }
3908 else
3909 {
3910 /*
3911 * Crosses pages.
3912 */
3913 size_t cb2 = cb - cb1;
3914 uint64_t fFlags1;
3915 RTGCPHYS GCPhys1;
3916 uint64_t fFlags2;
3917 RTGCPHYS GCPhys2;
3918 rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
3919 if (RT_SUCCESS(rc))
3920 {
3921 rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
3922 if (RT_SUCCESS(rc))
3923 {
3924 /** @todo we should check reserved bits ... */
3925 AssertMsgFailed(("cb=%d cb1=%d cb2=%d GCPtrSrc=%RGv\n", cb, cb1, cb2, GCPtrSrc));
3926 PGMPAGEMAPLOCK PgMpLck;
3927 void const *pvSrc1;
3928 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys1, &pvSrc1, &PgMpLck);
3929 switch (rc)
3930 {
3931 case VINF_SUCCESS:
3932 memcpy(pvDst, (uint8_t *)pvSrc1 + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
3933 PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
3934 break;
3935 case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3936 memset(pvDst, 0xff, cb1);
3937 break;
3938 default:
3939 Assert(RT_FAILURE_NP(rc));
3940 return rc;
3941 }
3942
3943 void const *pvSrc2;
3944 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys2, &pvSrc2, &PgMpLck);
3945 switch (rc)
3946 {
3947 case VINF_SUCCESS:
3948 memcpy((uint8_t *)pvDst + cb1, pvSrc2, cb2);
3949 PGMPhysReleasePageMappingLock(pVM, &PgMpLck);
3950 break;
3951 case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
3952 memset((uint8_t *)pvDst + cb1, 0xff, cb2);
3953 break;
3954 default:
3955 Assert(RT_FAILURE_NP(rc));
3956 return rc;
3957 }
3958
3959 if (!(fFlags1 & X86_PTE_A))
3960 {
3961 rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3962 AssertRC(rc);
3963 }
3964 if (!(fFlags2 & X86_PTE_A))
3965 {
3966 rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
3967 AssertRC(rc);
3968 }
3969 return VINF_SUCCESS;
3970 }
3971 }
3972 }
3973
3974 /*
3975 * Raise a #PF.
3976 */
3977 uint32_t uErr;
3978
3979 /* Get the current privilege level. */
3980 uint32_t cpl = CPUMGetGuestCPL(pVCpu);
3981 switch (rc)
3982 {
3983 case VINF_SUCCESS:
3984 uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD | X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
3985 break;
3986
3987 case VERR_PAGE_NOT_PRESENT:
3988 case VERR_PAGE_TABLE_NOT_PRESENT:
3989 uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
3990 break;
3991
3992 default:
3993 AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
3994 return rc;
3995 }
3996 Log(("PGMPhysInterpretedRead: GCPtrSrc=%RGv cb=%#x -> #PF(%#x)\n", GCPtrSrc, cb, uErr));
3997 return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrSrc);
3998}
3999
4000
4001/**
4002 * Performs a read of guest virtual memory for instruction emulation.
4003 *
4004 * This will check permissions, raise exceptions and update the access bits.
4005 *
4006 * The current implementation will bypass all access handlers. It may later be
4007 * changed to at least respect MMIO.
4008 *
4009 *
4010 * @returns VBox status code suitable to scheduling.
4011 * @retval VINF_SUCCESS if the read was performed successfully.
4012 * @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
4013 * @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
4014 *
4015 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
4016 * @param pCtxCore The context core.
4017 * @param pvDst Where to put the bytes we've read.
4018 * @param GCPtrSrc The source address.
4019 * @param cb The number of bytes to read. Not more than a page.
4020 * @param fRaiseTrap If set the trap will be raised on as per spec, if clear
4021 * an appropriate error status will be returned (no
4022 * informational at all).
4023 *
4024 *
4025 * @remarks Takes the PGM lock.
4026 * @remarks A page fault on the 2nd page of the access will be raised without
4027 * writing the bits on the first page since we're ASSUMING that the
4028 * caller is emulating an instruction access.
4029 * @remarks This function will dynamically map physical pages in GC. This may
4030 * unmap mappings done by the caller. Be careful!
4031 */
4032VMMDECL(int) PGMPhysInterpretedReadNoHandlers(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, void *pvDst, RTGCUINTPTR GCPtrSrc, size_t cb,
4033 bool fRaiseTrap)
4034{
4035 PVM pVM = pVCpu->CTX_SUFF(pVM);
4036 Assert(cb <= PAGE_SIZE);
4037 VMCPU_ASSERT_EMT(pVCpu);
4038
4039 /*
4040 * 1. Translate virtual to physical. This may fault.
4041 * 2. Map the physical address.
4042 * 3. Do the read operation.
4043 * 4. Set access bits if required.
4044 */
4045 int rc;
4046 unsigned cb1 = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
4047 if (cb <= cb1)
4048 {
4049 /*
4050 * Not crossing pages.
4051 */
4052 RTGCPHYS GCPhys;
4053 uint64_t fFlags;
4054 rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags, &GCPhys);
4055 if (RT_SUCCESS(rc))
4056 {
4057 if (1) /** @todo we should check reserved bits ... */
4058 {
4059 const void *pvSrc;
4060 PGMPAGEMAPLOCK Lock;
4061 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvSrc, &Lock);
4062 switch (rc)
4063 {
4064 case VINF_SUCCESS:
4065 Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d\n",
4066 pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb));
4067 memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
4068 PGMPhysReleasePageMappingLock(pVM, &Lock);
4069 break;
4070 case VERR_PGM_PHYS_PAGE_RESERVED:
4071 case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
4072 memset(pvDst, 0xff, cb);
4073 break;
4074 default:
4075 AssertMsgFailed(("%Rrc\n", rc));
4076 AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4077 return rc;
4078 }
4079
4080 if (!(fFlags & X86_PTE_A))
4081 {
4082 /** @todo access bit emulation isn't 100% correct. */
4083 rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
4084 AssertRC(rc);
4085 }
4086 return VINF_SUCCESS;
4087 }
4088 }
4089 }
4090 else
4091 {
4092 /*
4093 * Crosses pages.
4094 */
4095 size_t cb2 = cb - cb1;
4096 uint64_t fFlags1;
4097 RTGCPHYS GCPhys1;
4098 uint64_t fFlags2;
4099 RTGCPHYS GCPhys2;
4100 rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc, &fFlags1, &GCPhys1);
4101 if (RT_SUCCESS(rc))
4102 {
4103 rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrSrc + cb1, &fFlags2, &GCPhys2);
4104 if (RT_SUCCESS(rc))
4105 {
4106 if (1) /** @todo we should check reserved bits ... */
4107 {
4108 const void *pvSrc;
4109 PGMPAGEMAPLOCK Lock;
4110 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys1, &pvSrc, &Lock);
4111 switch (rc)
4112 {
4113 case VINF_SUCCESS:
4114 Log(("PGMPhysInterpretedReadNoHandlers: pvDst=%p pvSrc=%p (%RGv) cb=%d [2]\n",
4115 pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), GCPtrSrc, cb1));
4116 memcpy(pvDst, (const uint8_t *)pvSrc + (GCPtrSrc & PAGE_OFFSET_MASK), cb1);
4117 PGMPhysReleasePageMappingLock(pVM, &Lock);
4118 break;
4119 case VERR_PGM_PHYS_PAGE_RESERVED:
4120 case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
4121 memset(pvDst, 0xff, cb1);
4122 break;
4123 default:
4124 AssertMsgFailed(("%Rrc\n", rc));
4125 AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4126 return rc;
4127 }
4128
4129 rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys2, &pvSrc, &Lock);
4130 switch (rc)
4131 {
4132 case VINF_SUCCESS:
4133 memcpy((uint8_t *)pvDst + cb1, pvSrc, cb2);
4134 PGMPhysReleasePageMappingLock(pVM, &Lock);
4135 break;
4136 case VERR_PGM_PHYS_PAGE_RESERVED:
4137 case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
4138 memset((uint8_t *)pvDst + cb1, 0xff, cb2);
4139 break;
4140 default:
4141 AssertMsgFailed(("%Rrc\n", rc));
4142 AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4143 return rc;
4144 }
4145
4146 if (!(fFlags1 & X86_PTE_A))
4147 {
4148 rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
4149 AssertRC(rc);
4150 }
4151 if (!(fFlags2 & X86_PTE_A))
4152 {
4153 rc = PGMGstModifyPage(pVCpu, GCPtrSrc + cb1, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
4154 AssertRC(rc);
4155 }
4156 return VINF_SUCCESS;
4157 }
4158 /* sort out which page */
4159 }
4160 else
4161 GCPtrSrc += cb1; /* fault on 2nd page */
4162 }
4163 }
4164
4165 /*
4166 * Raise a #PF if we're allowed to do that.
4167 */
4168 /* Calc the error bits. */
4169 uint32_t cpl = CPUMGetGuestCPL(pVCpu);
4170 uint32_t uErr;
4171 switch (rc)
4172 {
4173 case VINF_SUCCESS:
4174 uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD | X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
4175 rc = VERR_ACCESS_DENIED;
4176 break;
4177
4178 case VERR_PAGE_NOT_PRESENT:
4179 case VERR_PAGE_TABLE_NOT_PRESENT:
4180 uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
4181 break;
4182
4183 default:
4184 AssertMsgFailed(("rc=%Rrc GCPtrSrc=%RGv cb=%#x\n", rc, GCPtrSrc, cb));
4185 AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4186 return rc;
4187 }
4188 if (fRaiseTrap)
4189 {
4190 Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrSrc, cb, uErr));
4191 return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrSrc);
4192 }
4193 Log(("PGMPhysInterpretedReadNoHandlers: GCPtrSrc=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrSrc, cb, uErr));
4194 return rc;
4195}
4196
4197
4198/**
4199 * Performs a write to guest virtual memory for instruction emulation.
4200 *
4201 * This will check permissions, raise exceptions and update the dirty and access
4202 * bits.
4203 *
4204 * @returns VBox status code suitable to scheduling.
4205 * @retval VINF_SUCCESS if the read was performed successfully.
4206 * @retval VINF_EM_RAW_GUEST_TRAP if an exception was raised but not dispatched yet.
4207 * @retval VINF_TRPM_XCPT_DISPATCHED if an exception was raised and dispatched.
4208 *
4209 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
4210 * @param pCtxCore The context core.
4211 * @param GCPtrDst The destination address.
4212 * @param pvSrc What to write.
4213 * @param cb The number of bytes to write. Not more than a page.
4214 * @param fRaiseTrap If set the trap will be raised on as per spec, if clear
4215 * an appropriate error status will be returned (no
4216 * informational at all).
4217 *
4218 * @remarks Takes the PGM lock.
4219 * @remarks A page fault on the 2nd page of the access will be raised without
4220 * writing the bits on the first page since we're ASSUMING that the
4221 * caller is emulating an instruction access.
4222 * @remarks This function will dynamically map physical pages in GC. This may
4223 * unmap mappings done by the caller. Be careful!
4224 */
4225VMMDECL(int) PGMPhysInterpretedWriteNoHandlers(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, RTGCPTR GCPtrDst, const void *pvSrc,
4226 size_t cb, bool fRaiseTrap)
4227{
4228 Assert(cb <= PAGE_SIZE);
4229 PVM pVM = pVCpu->CTX_SUFF(pVM);
4230 VMCPU_ASSERT_EMT(pVCpu);
4231
4232 /*
4233 * 1. Translate virtual to physical. This may fault.
4234 * 2. Map the physical address.
4235 * 3. Do the write operation.
4236 * 4. Set access bits if required.
4237 */
4238 /** @todo Since this method is frequently used by EMInterpret or IOM
4239 * upon a write fault to an write access monitored page, we can
4240 * reuse the guest page table walking from the \#PF code. */
4241 int rc;
4242 unsigned cb1 = PAGE_SIZE - (GCPtrDst & PAGE_OFFSET_MASK);
4243 if (cb <= cb1)
4244 {
4245 /*
4246 * Not crossing pages.
4247 */
4248 RTGCPHYS GCPhys;
4249 uint64_t fFlags;
4250 rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst, &fFlags, &GCPhys);
4251 if (RT_SUCCESS(rc))
4252 {
4253 if ( (fFlags & X86_PTE_RW) /** @todo Also check reserved bits. */
4254 || ( !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
4255 && CPUMGetGuestCPL(pVCpu) <= 2) ) /** @todo it's 2, right? Check cpl check below as well. */
4256 {
4257 void *pvDst;
4258 PGMPAGEMAPLOCK Lock;
4259 rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, &pvDst, &Lock);
4260 switch (rc)
4261 {
4262 case VINF_SUCCESS:
4263 Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
4264 (uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc, cb));
4265 memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb);
4266 PGMPhysReleasePageMappingLock(pVM, &Lock);
4267 break;
4268 case VERR_PGM_PHYS_PAGE_RESERVED:
4269 case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
4270 /* bit bucket */
4271 break;
4272 default:
4273 AssertMsgFailed(("%Rrc\n", rc));
4274 AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4275 return rc;
4276 }
4277
4278 if (!(fFlags & (X86_PTE_A | X86_PTE_D)))
4279 {
4280 /** @todo dirty & access bit emulation isn't 100% correct. */
4281 rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D));
4282 AssertRC(rc);
4283 }
4284 return VINF_SUCCESS;
4285 }
4286 rc = VERR_ACCESS_DENIED;
4287 }
4288 }
4289 else
4290 {
4291 /*
4292 * Crosses pages.
4293 */
4294 size_t cb2 = cb - cb1;
4295 uint64_t fFlags1;
4296 RTGCPHYS GCPhys1;
4297 uint64_t fFlags2;
4298 RTGCPHYS GCPhys2;
4299 rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst, &fFlags1, &GCPhys1);
4300 if (RT_SUCCESS(rc))
4301 {
4302 rc = PGM_GST_PFN(GetPage,pVCpu)(pVCpu, GCPtrDst + cb1, &fFlags2, &GCPhys2);
4303 if (RT_SUCCESS(rc))
4304 {
4305 if ( ( (fFlags1 & X86_PTE_RW) /** @todo Also check reserved bits. */
4306 && (fFlags2 & X86_PTE_RW))
4307 || ( !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP)
4308 && CPUMGetGuestCPL(pVCpu) <= 2) )
4309 {
4310 void *pvDst;
4311 PGMPAGEMAPLOCK Lock;
4312 rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys1, &pvDst, &Lock);
4313 switch (rc)
4314 {
4315 case VINF_SUCCESS:
4316 Log(("PGMPhysInterpretedWriteNoHandlers: pvDst=%p (%RGv) pvSrc=%p cb=%d\n",
4317 (uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), GCPtrDst, pvSrc, cb1));
4318 memcpy((uint8_t *)pvDst + (GCPtrDst & PAGE_OFFSET_MASK), pvSrc, cb1);
4319 PGMPhysReleasePageMappingLock(pVM, &Lock);
4320 break;
4321 case VERR_PGM_PHYS_PAGE_RESERVED:
4322 case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
4323 /* bit bucket */
4324 break;
4325 default:
4326 AssertMsgFailed(("%Rrc\n", rc));
4327 AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4328 return rc;
4329 }
4330
4331 rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys2, &pvDst, &Lock);
4332 switch (rc)
4333 {
4334 case VINF_SUCCESS:
4335 memcpy(pvDst, (const uint8_t *)pvSrc + cb1, cb2);
4336 PGMPhysReleasePageMappingLock(pVM, &Lock);
4337 break;
4338 case VERR_PGM_PHYS_PAGE_RESERVED:
4339 case VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS:
4340 /* bit bucket */
4341 break;
4342 default:
4343 AssertMsgFailed(("%Rrc\n", rc));
4344 AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4345 return rc;
4346 }
4347
4348 if (!(fFlags1 & (X86_PTE_A | X86_PTE_RW)))
4349 {
4350 rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, (X86_PTE_A | X86_PTE_RW), ~(uint64_t)(X86_PTE_A | X86_PTE_RW));
4351 AssertRC(rc);
4352 }
4353 if (!(fFlags2 & (X86_PTE_A | X86_PTE_RW)))
4354 {
4355 rc = PGMGstModifyPage(pVCpu, GCPtrDst + cb1, 1, (X86_PTE_A | X86_PTE_RW), ~(uint64_t)(X86_PTE_A | X86_PTE_RW));
4356 AssertRC(rc);
4357 }
4358 return VINF_SUCCESS;
4359 }
4360 if ((fFlags1 & (X86_PTE_RW)) == X86_PTE_RW)
4361 GCPtrDst += cb1; /* fault on the 2nd page. */
4362 rc = VERR_ACCESS_DENIED;
4363 }
4364 else
4365 GCPtrDst += cb1; /* fault on the 2nd page. */
4366 }
4367 }
4368
4369 /*
4370 * Raise a #PF if we're allowed to do that.
4371 */
4372 /* Calc the error bits. */
4373 uint32_t uErr;
4374 uint32_t cpl = CPUMGetGuestCPL(pVCpu);
4375 switch (rc)
4376 {
4377 case VINF_SUCCESS:
4378 uErr = (cpl >= 2) ? X86_TRAP_PF_RSVD | X86_TRAP_PF_US : X86_TRAP_PF_RSVD;
4379 rc = VERR_ACCESS_DENIED;
4380 break;
4381
4382 case VERR_ACCESS_DENIED:
4383 uErr = (cpl >= 2) ? X86_TRAP_PF_RW | X86_TRAP_PF_US : X86_TRAP_PF_RW;
4384 break;
4385
4386 case VERR_PAGE_NOT_PRESENT:
4387 case VERR_PAGE_TABLE_NOT_PRESENT:
4388 uErr = (cpl >= 2) ? X86_TRAP_PF_US : 0;
4389 break;
4390
4391 default:
4392 AssertMsgFailed(("rc=%Rrc GCPtrDst=%RGv cb=%#x\n", rc, GCPtrDst, cb));
4393 AssertReturn(RT_FAILURE(rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
4394 return rc;
4395 }
4396 if (fRaiseTrap)
4397 {
4398 Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> Raised #PF(%#x)\n", GCPtrDst, cb, uErr));
4399 return TRPMRaiseXcptErrCR2(pVCpu, pCtxCore, X86_XCPT_PF, uErr, GCPtrDst);
4400 }
4401 Log(("PGMPhysInterpretedWriteNoHandlers: GCPtrDst=%RGv cb=%#x -> #PF(%#x) [!raised]\n", GCPtrDst, cb, uErr));
4402 return rc;
4403}
4404
4405
4406/**
4407 * Return the page type of the specified physical address.
4408 *
4409 * @returns The page type.
4410 * @param pVM The cross context VM structure.
4411 * @param GCPhys Guest physical address
4412 */
4413VMM_INT_DECL(PGMPAGETYPE) PGMPhysGetPageType(PVM pVM, RTGCPHYS GCPhys)
4414{
4415 pgmLock(pVM);
4416 PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
4417 PGMPAGETYPE enmPgType = pPage ? (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage) : PGMPAGETYPE_INVALID;
4418 pgmUnlock(pVM);
4419
4420 return enmPgType;
4421}
4422
4423
4424/**
4425 * Converts a GC physical address to a HC ring-3 pointer, with some
4426 * additional checks.
4427 *
4428 * @returns VBox status code (no informational statuses).
4429 *
4430 * @param pVM The cross context VM structure.
4431 * @param pVCpu The cross context virtual CPU structure of the
4432 * calling EMT.
4433 * @param GCPhys The GC physical address to convert. This API mask
4434 * the A20 line when necessary.
4435 * @param puTlbPhysRev Where to read the physical TLB revision. Needs to
4436 * be done while holding the PGM lock.
4437 * @param ppb Where to store the pointer corresponding to GCPhys
4438 * on success.
4439 * @param pfTlb The TLB flags and revision. We only add stuff.
4440 *
4441 * @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr and
4442 * PGMPhysIemGCPhys2Ptr.
4443 *
4444 * @thread EMT(pVCpu).
4445 */
4446VMM_INT_DECL(int) PGMPhysIemGCPhys2PtrNoLock(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, uint64_t const volatile *puTlbPhysRev,
4447#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4448 R3PTRTYPE(uint8_t *) *ppb,
4449#else
4450 R3R0PTRTYPE(uint8_t *) *ppb,
4451#endif
4452 uint64_t *pfTlb)
4453{
4454 PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
4455 Assert(!(GCPhys & X86_PAGE_OFFSET_MASK));
4456
4457 pgmLock(pVM);
4458
4459 PPGMRAMRANGE pRam;
4460 PPGMPAGE pPage;
4461 int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4462 if (RT_SUCCESS(rc))
4463 {
4464 if (!PGM_PAGE_IS_BALLOONED(pPage))
4465 {
4466 if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
4467 {
4468 if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
4469 {
4470 /*
4471 * No access handler.
4472 */
4473 switch (PGM_PAGE_GET_STATE(pPage))
4474 {
4475 case PGM_PAGE_STATE_ALLOCATED:
4476 *pfTlb |= *puTlbPhysRev;
4477 break;
4478 case PGM_PAGE_STATE_BALLOONED:
4479 AssertFailed();
4480 RT_FALL_THRU();
4481 case PGM_PAGE_STATE_ZERO:
4482 case PGM_PAGE_STATE_SHARED:
4483 case PGM_PAGE_STATE_WRITE_MONITORED:
4484 *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE;
4485 break;
4486 }
4487#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4488 *pfTlb |= PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
4489 *ppb = NULL;
4490#else
4491 PPGMPAGER3MAPTLBE pTlbe;
4492 rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
4493 AssertLogRelRCReturn(rc, rc);
4494 *ppb = (uint8_t *)pTlbe->pv;
4495#endif
4496 }
4497 else if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
4498 {
4499 /*
4500 * MMIO or similar all access handler: Catch all access.
4501 */
4502 *pfTlb |= *puTlbPhysRev
4503 | PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
4504 *ppb = NULL;
4505 }
4506 else
4507 {
4508 /*
4509 * Write access handler: Catch write accesses if active.
4510 */
4511 if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
4512 *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE;
4513 else
4514 switch (PGM_PAGE_GET_STATE(pPage))
4515 {
4516 case PGM_PAGE_STATE_ALLOCATED:
4517 *pfTlb |= *puTlbPhysRev;
4518 break;
4519 case PGM_PAGE_STATE_BALLOONED:
4520 AssertFailed();
4521 RT_FALL_THRU();
4522 case PGM_PAGE_STATE_ZERO:
4523 case PGM_PAGE_STATE_SHARED:
4524 case PGM_PAGE_STATE_WRITE_MONITORED:
4525 *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE;
4526 break;
4527 }
4528#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4529 *pfTlb |= PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
4530 *ppb = NULL;
4531#else
4532 PPGMPAGER3MAPTLBE pTlbe;
4533 rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
4534 AssertLogRelRCReturn(rc, rc);
4535 *ppb = (uint8_t *)pTlbe->pv;
4536#endif
4537 }
4538 }
4539 else
4540 {
4541 /* Alias MMIO: For now, we catch all access. */
4542 *pfTlb |= *puTlbPhysRev
4543 | PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
4544 *ppb = NULL;
4545 }
4546 }
4547 else
4548 {
4549 /* Ballooned: Shouldn't get here, but we read zero page via PGMPhysRead and writes goes to /dev/null. */
4550 *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
4551 *ppb = NULL;
4552 }
4553 Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp *ppb=%p *pfTlb=%#RX64 pPage=%R[pgmpage]\n", GCPhys, *ppb, *pfTlb, pPage));
4554 }
4555 else
4556 {
4557 *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
4558 *ppb = NULL;
4559 Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp *ppb=%p *pfTlb=%#RX64 (rc=%Rrc)\n", GCPhys, *ppb, *pfTlb, rc));
4560 }
4561
4562 pgmUnlock(pVM);
4563 return VINF_SUCCESS;
4564}
4565
4566
4567/**
4568 * Converts a GC physical address to a HC ring-3 pointer, with some
4569 * additional checks.
4570 *
4571 * @returns VBox status code (no informational statuses).
4572 * @retval VINF_SUCCESS on success.
4573 * @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4574 * access handler of some kind.
4575 * @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4576 * accesses or is odd in any way.
4577 * @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4578 *
4579 * @param pVM The cross context VM structure.
4580 * @param pVCpu The cross context virtual CPU structure of the
4581 * calling EMT.
4582 * @param GCPhys The GC physical address to convert. This API mask
4583 * the A20 line when necessary.
4584 * @param fWritable Whether write access is required.
4585 * @param fByPassHandlers Whether to bypass access handlers.
4586 * @param ppv Where to store the pointer corresponding to GCPhys
4587 * on success.
4588 * @param pLock
4589 *
4590 * @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr.
4591 * @thread EMT(pVCpu).
4592 */
4593VMM_INT_DECL(int) PGMPhysIemGCPhys2Ptr(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers,
4594 void **ppv, PPGMPAGEMAPLOCK pLock)
4595{
4596 PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
4597
4598 pgmLock(pVM);
4599
4600 PPGMRAMRANGE pRam;
4601 PPGMPAGE pPage;
4602 int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4603 if (RT_SUCCESS(rc))
4604 {
4605 if (PGM_PAGE_IS_BALLOONED(pPage))
4606 rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4607 else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
4608 rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4609 else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
4610 || (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
4611 rc = VINF_SUCCESS;
4612 else
4613 {
4614 if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4615 {
4616 Assert(!fByPassHandlers || PGM_PAGE_IS_MMIO(pPage));
4617 rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4618 }
4619 else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
4620 {
4621 Assert(!fByPassHandlers);
4622 rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4623 }
4624 }
4625 if (RT_SUCCESS(rc))
4626 {
4627 int rc2;
4628
4629 /* Make sure what we return is writable. */
4630 if (fWritable)
4631 switch (PGM_PAGE_GET_STATE(pPage))
4632 {
4633 case PGM_PAGE_STATE_ALLOCATED:
4634 break;
4635 case PGM_PAGE_STATE_BALLOONED:
4636 AssertFailed();
4637 break;
4638 case PGM_PAGE_STATE_ZERO:
4639 case PGM_PAGE_STATE_SHARED:
4640 case PGM_PAGE_STATE_WRITE_MONITORED:
4641 rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
4642 AssertLogRelRCReturn(rc2, rc2);
4643 break;
4644 }
4645
4646#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4647 void *pv;
4648 rc = pgmRZDynMapHCPageInlined(pVCpu,
4649 PGM_PAGE_GET_HCPHYS(pPage),
4650 &pv
4651 RTLOG_COMMA_SRC_POS);
4652 if (RT_FAILURE(rc))
4653 return rc;
4654 *ppv = (void *)((uintptr_t)pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4655 pLock->pvPage = pv;
4656 pLock->pVCpu = pVCpu;
4657
4658#else
4659 /* Get a ring-3 mapping of the address. */
4660 PPGMPAGER3MAPTLBE pTlbe;
4661 rc2 = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
4662 AssertLogRelRCReturn(rc2, rc2);
4663
4664 /* Lock it and calculate the address. */
4665 if (fWritable)
4666 pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
4667 else
4668 pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
4669 *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
4670#endif
4671
4672 Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] *ppv=%p\n", GCPhys, rc, pPage, *ppv));
4673 }
4674 else
4675 Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
4676
4677 /* else: handler catching all access, no pointer returned. */
4678 }
4679 else
4680 rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
4681
4682 pgmUnlock(pVM);
4683 return rc;
4684}
4685
4686
4687/**
4688 * Checks if the give GCPhys page requires special handling for the given access
4689 * because it's MMIO or otherwise monitored.
4690 *
4691 * @returns VBox status code (no informational statuses).
4692 * @retval VINF_SUCCESS on success.
4693 * @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
4694 * access handler of some kind.
4695 * @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
4696 * accesses or is odd in any way.
4697 * @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
4698 *
4699 * @param pVM The cross context VM structure.
4700 * @param GCPhys The GC physical address to convert. Since this is
4701 * only used for filling the REM TLB, the A20 mask must
4702 * be applied before calling this API.
4703 * @param fWritable Whether write access is required.
4704 * @param fByPassHandlers Whether to bypass access handlers.
4705 *
4706 * @remarks This is a watered down version PGMPhysIemGCPhys2Ptr and really just
4707 * a stop gap thing that should be removed once there is a better TLB
4708 * for virtual address accesses.
4709 */
4710VMM_INT_DECL(int) PGMPhysIemQueryAccess(PVM pVM, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers)
4711{
4712 pgmLock(pVM);
4713 PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
4714
4715 PPGMRAMRANGE pRam;
4716 PPGMPAGE pPage;
4717 int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
4718 if (RT_SUCCESS(rc))
4719 {
4720 if (PGM_PAGE_IS_BALLOONED(pPage))
4721 rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4722 else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
4723 rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4724 else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
4725 || (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
4726 rc = VINF_SUCCESS;
4727 else
4728 {
4729 if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
4730 {
4731 Assert(!fByPassHandlers || PGM_PAGE_IS_MMIO(pPage));
4732 rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
4733 }
4734 else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
4735 {
4736 Assert(!fByPassHandlers);
4737 rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
4738 }
4739 }
4740 }
4741
4742 pgmUnlock(pVM);
4743 return rc;
4744}
4745
4746#ifndef IN_RC
4747
4748/**
4749 * Interface used by NEM to check what to do on a memory access exit.
4750 *
4751 * @returns VBox status code.
4752 * @param pVM The cross context VM structure.
4753 * @param pVCpu The cross context per virtual CPU structure.
4754 * Optional.
4755 * @param GCPhys The guest physical address.
4756 * @param fMakeWritable Whether to try make the page writable or not. If it
4757 * cannot be made writable, NEM_PAGE_PROT_WRITE won't
4758 * be returned and the return code will be unaffected
4759 * @param pInfo Where to return the page information. This is
4760 * initialized even on failure.
4761 * @param pfnChecker Page in-sync checker callback. Optional.
4762 * @param pvUser User argument to pass to pfnChecker.
4763 */
4764VMM_INT_DECL(int) PGMPhysNemPageInfoChecker(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, bool fMakeWritable, PPGMPHYSNEMPAGEINFO pInfo,
4765 PFNPGMPHYSNEMCHECKPAGE pfnChecker, void *pvUser)
4766{
4767 pgmLock(pVM);
4768
4769 PPGMPAGE pPage;
4770 int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
4771 if (RT_SUCCESS(rc))
4772 {
4773 /* Try make it writable if requested. */
4774 if (fMakeWritable)
4775 switch (PGM_PAGE_GET_STATE(pPage))
4776 {
4777 case PGM_PAGE_STATE_SHARED:
4778 case PGM_PAGE_STATE_WRITE_MONITORED:
4779 case PGM_PAGE_STATE_ZERO:
4780 rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
4781 if (rc == VERR_PGM_PHYS_PAGE_RESERVED)
4782 rc = VINF_SUCCESS;
4783 break;
4784 }
4785
4786 /* Fill in the info. */
4787 pInfo->HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4788 pInfo->u2NemState = PGM_PAGE_GET_NEM_STATE(pPage);
4789 pInfo->fHasHandlers = PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) ? 1 : 0;
4790 PGMPAGETYPE const enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
4791 pInfo->enmType = enmType;
4792 pInfo->fNemProt = pgmPhysPageCalcNemProtection(pPage, enmType);
4793 switch (PGM_PAGE_GET_STATE(pPage))
4794 {
4795 case PGM_PAGE_STATE_ALLOCATED:
4796 pInfo->fZeroPage = 0;
4797 break;
4798
4799 case PGM_PAGE_STATE_ZERO:
4800 pInfo->fZeroPage = 1;
4801 break;
4802
4803 case PGM_PAGE_STATE_WRITE_MONITORED:
4804 pInfo->fZeroPage = 0;
4805 break;
4806
4807 case PGM_PAGE_STATE_SHARED:
4808 pInfo->fZeroPage = 0;
4809 break;
4810
4811 case PGM_PAGE_STATE_BALLOONED:
4812 pInfo->fZeroPage = 1;
4813 break;
4814
4815 default:
4816 pInfo->fZeroPage = 1;
4817 AssertFailedStmt(rc = VERR_PGM_PHYS_PAGE_GET_IPE);
4818 }
4819
4820 /* Call the checker and update NEM state. */
4821 if (pfnChecker)
4822 {
4823 rc = pfnChecker(pVM, pVCpu, GCPhys, pInfo, pvUser);
4824 PGM_PAGE_SET_NEM_STATE(pPage, pInfo->u2NemState);
4825 }
4826
4827 /* Done. */
4828 pgmUnlock(pVM);
4829 }
4830 else
4831 {
4832 pgmUnlock(pVM);
4833
4834 pInfo->HCPhys = NIL_RTHCPHYS;
4835 pInfo->fNemProt = NEM_PAGE_PROT_NONE;
4836 pInfo->u2NemState = 0;
4837 pInfo->fHasHandlers = 0;
4838 pInfo->fZeroPage = 0;
4839 pInfo->enmType = PGMPAGETYPE_INVALID;
4840 }
4841
4842 return rc;
4843}
4844
4845
4846/**
4847 * NEM helper that performs @a pfnCallback on pages with NEM state @a uMinState
4848 * or higher.
4849 *
4850 * @returns VBox status code from callback.
4851 * @param pVM The cross context VM structure.
4852 * @param pVCpu The cross context per CPU structure. This is
4853 * optional as its only for passing to callback.
4854 * @param uMinState The minimum NEM state value to call on.
4855 * @param pfnCallback The callback function.
4856 * @param pvUser User argument for the callback.
4857 */
4858VMM_INT_DECL(int) PGMPhysNemEnumPagesByState(PVM pVM, PVMCPU pVCpu, uint8_t uMinState,
4859 PFNPGMPHYSNEMENUMCALLBACK pfnCallback, void *pvUser)
4860{
4861 /*
4862 * Just brute force this problem.
4863 */
4864 pgmLock(pVM);
4865 int rc = VINF_SUCCESS;
4866 for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX); pRam; pRam = pRam->CTX_SUFF(pNext))
4867 {
4868 uint32_t const cPages = pRam->cb >> X86_PAGE_SHIFT;
4869 for (uint32_t iPage = 0; iPage < cPages; iPage++)
4870 {
4871 uint8_t u2State = PGM_PAGE_GET_NEM_STATE(&pRam->aPages[iPage]);
4872 if (u2State < uMinState)
4873 { /* likely */ }
4874 else
4875 {
4876 rc = pfnCallback(pVM, pVCpu, pRam->GCPhys + ((RTGCPHYS)iPage << X86_PAGE_SHIFT), &u2State, pvUser);
4877 if (RT_SUCCESS(rc))
4878 PGM_PAGE_SET_NEM_STATE(&pRam->aPages[iPage], u2State);
4879 else
4880 break;
4881 }
4882 }
4883 }
4884 pgmUnlock(pVM);
4885
4886 return rc;
4887
4888}
4889
4890#endif /* !IN_RC */
4891
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette