IOMAllMmioNew.cpp@ 93628

Last change on this file since 93628 was 93554, checked in by vboxsync, 3 years ago
VMM: Changed PAGE_SIZE -> GUEST_PAGE_SIZE / HOST_PAGE_SIZE, PAGE_SHIFT -> GUEST_PAGE_SHIFT / HOST_PAGE_SHIFT, and PAGE_OFFSET_MASK -> GUEST_PAGE_OFFSET_MASK / HOST_PAGE_OFFSET_MASK. Also removed most usage of ASMMemIsZeroPage and ASMMemZeroPage since the host and guest page size doesn't need to be the same any more. Some work left to do in the page pool code. bugref:9898
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 50.7 KB

Line
1	/* $Id: IOMAllMmioNew.cpp 93554 2022-02-02 22:57:02Z vboxsync $ */
2	/** @file
3	* IOM - Input / Output Monitor - Any Context, MMIO & String I/O.
4	*/
5
6	/*
7	* Copyright (C) 2006-2022 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_IOM_MMIO
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <VBox/vmm/iom.h>
25	#include <VBox/vmm/cpum.h>
26	#include <VBox/vmm/pgm.h>
27	#include <VBox/vmm/selm.h>
28	#include <VBox/vmm/mm.h>
29	#include <VBox/vmm/em.h>
30	#include <VBox/vmm/pgm.h>
31	#include <VBox/vmm/trpm.h>
32	#include <VBox/vmm/iem.h>
33	#include "IOMInternal.h"
34	#include <VBox/vmm/vmcc.h>
35	#include <VBox/vmm/vmm.h>
36	#include <VBox/vmm/hm.h>
37	#include "IOMInline.h"
38
39	#include <VBox/dis.h>
40	#include <VBox/disopcode.h>
41	#include <VBox/vmm/pdmdev.h>
42	#include <VBox/param.h>
43	#include <VBox/err.h>
44	#include <iprt/assert.h>
45	#include <VBox/log.h>
46	#include <iprt/asm.h>
47	#include <iprt/string.h>
48
49
50	/*********************************************************************************************************************************
51	* Defined Constants And Macros *
52	*********************************************************************************************************************************/
53	/** @def IOM_MMIO_STATS_COMMA_DECL
54	* Parameter list declaration for statistics entry pointer. */
55	/** @def IOM_MMIO_STATS_COMMA_ARG
56	* Statistics entry pointer argument. */
57	#if defined(VBOX_WITH_STATISTICS) \|\| defined(DOXYGEN_RUNNING)
58	# define IOM_MMIO_STATS_COMMA_DECL , PIOMMMIOSTATSENTRY pStats
59	# define IOM_MMIO_STATS_COMMA_ARG , pStats
60	#else
61	# define IOM_MMIO_STATS_COMMA_DECL
62	# define IOM_MMIO_STATS_COMMA_ARG
63	#endif
64
65
66
67	#ifndef IN_RING3
68	/**
69	* Defers a pending MMIO write to ring-3.
70	*
71	* @returns VINF_IOM_R3_MMIO_COMMIT_WRITE
72	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
73	* @param GCPhys The write address.
74	* @param pvBuf The bytes being written.
75	* @param cbBuf How many bytes.
76	* @param idxRegEntry The MMIO registration index (handle) if available,
77	* otherwise UINT32_MAX.
78	*/
79	static VBOXSTRICTRC iomMmioRing3WritePending(PVMCPU pVCpu, RTGCPHYS GCPhys, void const *pvBuf, size_t cbBuf,
80	uint32_t idxRegEntry)
81	{
82	Log5(("iomMmioRing3WritePending: %RGp LB %#x (idx=%#x)\n", GCPhys, cbBuf, idxRegEntry));
83	if (pVCpu->iom.s.PendingMmioWrite.cbValue == 0)
84	{
85	pVCpu->iom.s.PendingMmioWrite.GCPhys = GCPhys;
86	AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
87	pVCpu->iom.s.PendingMmioWrite.cbValue = (uint32_t)cbBuf;
88	pVCpu->iom.s.PendingMmioWrite.idxMmioRegionHint = idxRegEntry;
89	memcpy(pVCpu->iom.s.PendingMmioWrite.abValue, pvBuf, cbBuf);
90	}
91	else
92	{
93	/*
94	* Join with pending if adjecent.
95	*
96	* This may happen if the stack overflows into MMIO territory and RSP/ESP/SP
97	* isn't aligned. IEM will bounce buffer the access and do one write for each
98	* page. We get here when the 2nd page part is written.
99	*/
100	uint32_t const cbOldValue = pVCpu->iom.s.PendingMmioWrite.cbValue;
101	AssertMsgReturn(GCPhys == pVCpu->iom.s.PendingMmioWrite.GCPhys + cbOldValue,
102	("pending %RGp LB %#x; incoming %RGp LB %#x\n",
103	pVCpu->iom.s.PendingMmioWrite.GCPhys, cbOldValue, GCPhys, cbBuf),
104	VERR_IOM_MMIO_IPE_1);
105	AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue) - cbOldValue, VERR_IOM_MMIO_IPE_2);
106	pVCpu->iom.s.PendingMmioWrite.cbValue = cbOldValue + (uint32_t)cbBuf;
107	memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[cbOldValue], pvBuf, cbBuf);
108	}
109
110	VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
111	return VINF_IOM_R3_MMIO_COMMIT_WRITE;
112	}
113	#endif
114
115
116	/**
117	* Deals with complicated MMIO writes.
118	*
119	* Complicated means unaligned or non-dword/qword sized accesses depending on
120	* the MMIO region's access mode flags.
121	*
122	* @returns Strict VBox status code. Any EM scheduling status code,
123	* VINF_IOM_R3_MMIO_WRITE, VINF_IOM_R3_MMIO_READ_WRITE or
124	* VINF_IOM_R3_MMIO_READ may be returned.
125	*
126	* @param pVM The cross context VM structure.
127	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
128	* @param pRegEntry The MMIO entry for the current context.
129	* @param GCPhys The physical address to start writing.
130	* @param offRegion MMIO region offset corresponding to @a GCPhys.
131	* @param pvValue Where to store the value.
132	* @param cbValue The size of the value to write.
133	* @param pStats Pointer to the statistics (never NULL).
134	*/
135	static VBOXSTRICTRC iomMmioDoComplicatedWrite(PVM pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry,
136	RTGCPHYS GCPhys, RTGCPHYS offRegion,
137	void const *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
138	{
139	AssertReturn( (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) != IOMMMIO_FLAGS_WRITE_PASSTHRU
140	&& (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) <= IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING,
141	VERR_IOM_MMIO_IPE_1);
142	AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
143	RTGCPHYS const GCPhysStart = GCPhys; NOREF(GCPhysStart);
144	bool const fReadMissing = (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_READ_MISSING
145	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING;
146	RT_NOREF_PV(pVCpu); /* ring-3 */
147
148	/*
149	* Do debug stop if requested.
150	*/
151	VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
152	#ifdef VBOX_STRICT
153	if (!(pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_WRITE))
154	{ /* likely */ }
155	else
156	{
157	# ifdef IN_RING3
158	LogRel(("IOM: Complicated write %#x byte at %RGp to %s, initiating debugger intervention\n", cbValue, GCPhys,
159	R3STRING(pRegEntry->pszDesc)));
160	rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
161	"Complicated write %#x byte at %RGp to %s\n", cbValue, GCPhys, pRegEntry->pszDesc);
162	if (rc == VERR_DBGF_NOT_ATTACHED)
163	rc = VINF_SUCCESS;
164	# else
165	return VINF_IOM_R3_MMIO_WRITE;
166	# endif
167	}
168	#endif
169
170	STAM_COUNTER_INC(&pStats->ComplicatedWrites);
171
172	/*
173	* Check if we should ignore the write.
174	*/
175	if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD)
176	{
177	Assert(cbValue != 4 \|\| (GCPhys & 3));
178	return VINF_SUCCESS;
179	}
180	if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD_QWORD)
181	{
182	Assert((cbValue != 4 && cbValue != 8) \|\| (GCPhys & (cbValue - 1)));
183	return VINF_SUCCESS;
184	}
185
186	/*
187	* Split and conquer.
188	*/
189	for (;;)
190	{
191	unsigned const offAccess = GCPhys & 3;
192	unsigned cbThisPart = 4 - offAccess;
193	if (cbThisPart > cbValue)
194	cbThisPart = cbValue;
195
196	/*
197	* Get the missing bits (if any).
198	*/
199	uint32_t u32MissingValue = 0;
200	if (fReadMissing && cbThisPart != 4)
201	{
202	VBOXSTRICTRC rc2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
203	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
204	? offRegion & ~(RTGCPHYS)3 : (GCPhys & ~(RTGCPHYS)3),
205	&u32MissingValue, sizeof(u32MissingValue));
206	switch (VBOXSTRICTRC_VAL(rc2))
207	{
208	case VINF_SUCCESS:
209	break;
210	case VINF_IOM_MMIO_UNUSED_FF:
211	STAM_COUNTER_INC(&pStats->FFor00Reads);
212	u32MissingValue = UINT32_C(0xffffffff);
213	break;
214	case VINF_IOM_MMIO_UNUSED_00:
215	STAM_COUNTER_INC(&pStats->FFor00Reads);
216	u32MissingValue = 0;
217	break;
218	#ifndef IN_RING3
219	case VINF_IOM_R3_MMIO_READ:
220	case VINF_IOM_R3_MMIO_READ_WRITE:
221	case VINF_IOM_R3_MMIO_WRITE:
222	LogFlow(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
223	rc2 = iomMmioRing3WritePending(pVCpu, GCPhys, pvValue, cbValue, pRegEntry->idxSelf);
224	if (rc == VINF_SUCCESS \|\| rc2 < rc)
225	rc = rc2;
226	return rc;
227	#endif
228	default:
229	if (RT_FAILURE(rc2))
230	{
231	Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
232	return rc2;
233	}
234	AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rc2)), VERR_IPE_UNEXPECTED_INFO_STATUS);
235	if (rc == VINF_SUCCESS \|\| rc2 < rc)
236	rc = rc2;
237	break;
238	}
239	}
240
241	/*
242	* Merge missing and given bits.
243	*/
244	uint32_t u32GivenMask;
245	uint32_t u32GivenValue;
246	switch (cbThisPart)
247	{
248	case 1:
249	u32GivenValue = (uint8_t const )pvValue;
250	u32GivenMask = UINT32_C(0x000000ff);
251	break;
252	case 2:
253	u32GivenValue = (uint16_t const )pvValue;
254	u32GivenMask = UINT32_C(0x0000ffff);
255	break;
256	case 3:
257	u32GivenValue = RT_MAKE_U32_FROM_U8(((uint8_t const )pvValue)[0], ((uint8_t const )pvValue)[1],
258	((uint8_t const *)pvValue)[2], 0);
259	u32GivenMask = UINT32_C(0x00ffffff);
260	break;
261	case 4:
262	u32GivenValue = (uint32_t const )pvValue;
263	u32GivenMask = UINT32_C(0xffffffff);
264	break;
265	default:
266	AssertFailedReturn(VERR_IOM_MMIO_IPE_3);
267	}
268	if (offAccess)
269	{
270	u32GivenValue <<= offAccess * 8;
271	u32GivenMask <<= offAccess * 8;
272	}
273
274	uint32_t u32Value = (u32MissingValue & ~u32GivenMask)
275	\| (u32GivenValue & u32GivenMask);
276
277	/*
278	* Do DWORD write to the device.
279	*/
280	VBOXSTRICTRC rc2 = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
281	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
282	? offRegion & ~(RTGCPHYS)3 : GCPhys & ~(RTGCPHYS)3,
283	&u32Value, sizeof(u32Value));
284	switch (VBOXSTRICTRC_VAL(rc2))
285	{
286	case VINF_SUCCESS:
287	break;
288	#ifndef IN_RING3
289	case VINF_IOM_R3_MMIO_READ:
290	case VINF_IOM_R3_MMIO_READ_WRITE:
291	case VINF_IOM_R3_MMIO_WRITE:
292	Log3(("iomMmioDoComplicatedWrite: deferring GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
293	AssertReturn(pVCpu->iom.s.PendingMmioWrite.cbValue == 0, VERR_IOM_MMIO_IPE_1);
294	AssertReturn(cbValue + (GCPhys & 3) <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
295	pVCpu->iom.s.PendingMmioWrite.GCPhys = GCPhys & ~(RTGCPHYS)3;
296	pVCpu->iom.s.PendingMmioWrite.cbValue = cbValue + (GCPhys & 3);
297	(uint32_t )pVCpu->iom.s.PendingMmioWrite.abValue = u32Value;
298	if (cbValue > cbThisPart)
299	memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[4],
300	(uint8_t const *)pvValue + cbThisPart, cbValue - cbThisPart);
301	VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
302	if (rc == VINF_SUCCESS)
303	rc = VINF_IOM_R3_MMIO_COMMIT_WRITE;
304	return rc;
305	#endif
306	default:
307	if (RT_FAILURE(rc2))
308	{
309	Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
310	return rc2;
311	}
312	AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rc2)), VERR_IPE_UNEXPECTED_INFO_STATUS);
313	if (rc == VINF_SUCCESS \|\| rc2 < rc)
314	rc = rc2;
315	break;
316	}
317
318	/*
319	* Advance.
320	*/
321	cbValue -= cbThisPart;
322	if (!cbValue)
323	break;
324	GCPhys += cbThisPart;
325	offRegion += cbThisPart;
326	pvValue = (uint8_t const *)pvValue + cbThisPart;
327	}
328
329	return rc;
330	}
331
332
333
334
335	/**
336	* Wrapper which does the write.
337	*/
338	DECLINLINE(VBOXSTRICTRC) iomMmioDoWrite(PVMCC pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry,
339	RTGCPHYS GCPhys, RTGCPHYS offRegion,
340	const void *pvData, uint32_t cb IOM_MMIO_STATS_COMMA_DECL)
341	{
342	VBOXSTRICTRC rcStrict;
343	if (RT_LIKELY(pRegEntry->pfnWriteCallback))
344	{
345	if ( (cb == 4 && !(GCPhys & 3))
346	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_PASSTHRU
347	\|\| (cb == 8 && !(GCPhys & 7) && IOMMMIO_DOES_WRITE_MODE_ALLOW_QWORD(pRegEntry->fFlags)) )
348	rcStrict = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
349	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) ? offRegion : GCPhys, pvData, cb);
350	else
351	rcStrict = iomMmioDoComplicatedWrite(pVM, pVCpu, pRegEntry, GCPhys, offRegion, pvData, cb IOM_MMIO_STATS_COMMA_ARG);
352	}
353	else
354	rcStrict = VINF_SUCCESS;
355	return rcStrict;
356	}
357
358
359	#ifdef IN_RING3
360	/**
361	* Helper for IOMR3ProcessForceFlag() that lives here to utilize iomMmioDoWrite et al.
362	*/
363	VBOXSTRICTRC iomR3MmioCommitWorker(PVM pVM, PVMCPU pVCpu, PIOMMMIOENTRYR3 pRegEntry, RTGCPHYS offRegion)
364	{
365	# ifdef VBOX_WITH_STATISTICS
366	STAM_PROFILE_START(UnusedMacroArg, Prf);
367	PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry);
368	# endif
369	PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
370	int rc = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VERR_IGNORED);
371	AssertRCReturn(rc, rc);
372
373	VBOXSTRICTRC rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, pVCpu->iom.s.PendingMmioWrite.GCPhys, offRegion,
374	pVCpu->iom.s.PendingMmioWrite.abValue, pVCpu->iom.s.PendingMmioWrite.cbValue
375	IOM_MMIO_STATS_COMMA_ARG);
376
377	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
378	STAM_PROFILE_STOP(&pStats->ProfWriteR3, Prf);
379	return rcStrict;
380	}
381	#endif /* IN_RING3 */
382
383
384	/**
385	* Deals with complicated MMIO reads.
386	*
387	* Complicated means unaligned or non-dword/qword sized accesses depending on
388	* the MMIO region's access mode flags.
389	*
390	* @returns Strict VBox status code. Any EM scheduling status code,
391	* VINF_IOM_R3_MMIO_READ, VINF_IOM_R3_MMIO_READ_WRITE or
392	* VINF_IOM_R3_MMIO_WRITE may be returned.
393	*
394	* @param pVM The cross context VM structure.
395	* @param pRegEntry The MMIO entry for the current context.
396	* @param GCPhys The physical address to start reading.
397	* @param offRegion MMIO region offset corresponding to @a GCPhys.
398	* @param pvValue Where to store the value.
399	* @param cbValue The size of the value to read.
400	* @param pStats Pointer to the statistics (never NULL).
401	*/
402	static VBOXSTRICTRC iomMMIODoComplicatedRead(PVM pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys, RTGCPHYS offRegion,
403	void *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
404	{
405	AssertReturn( (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD
406	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD,
407	VERR_IOM_MMIO_IPE_1);
408	AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
409	#ifdef LOG_ENABLED
410	RTGCPHYS const GCPhysStart = GCPhys;
411	#endif
412
413	/*
414	* Do debug stop if requested.
415	*/
416	VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
417	#ifdef VBOX_STRICT
418	if (pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_READ)
419	{
420	# ifdef IN_RING3
421	rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
422	"Complicated read %#x byte at %RGp to %s\n", cbValue, GCPhys, R3STRING(pRegEntry->pszDesc));
423	if (rc == VERR_DBGF_NOT_ATTACHED)
424	rc = VINF_SUCCESS;
425	# else
426	return VINF_IOM_R3_MMIO_READ;
427	# endif
428	}
429	#endif
430
431	STAM_COUNTER_INC(&pStats->ComplicatedReads);
432
433	/*
434	* Split and conquer.
435	*/
436	for (;;)
437	{
438	/*
439	* Do DWORD read from the device.
440	*/
441	uint32_t u32Value;
442	VBOXSTRICTRC rcStrict2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
443	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
444	? offRegion & ~(RTGCPHYS)3 : GCPhys & ~(RTGCPHYS)3,
445	&u32Value, sizeof(u32Value));
446	switch (VBOXSTRICTRC_VAL(rcStrict2))
447	{
448	case VINF_SUCCESS:
449	break;
450	case VINF_IOM_MMIO_UNUSED_FF:
451	STAM_COUNTER_INC(&pStats->FFor00Reads);
452	u32Value = UINT32_C(0xffffffff);
453	break;
454	case VINF_IOM_MMIO_UNUSED_00:
455	STAM_COUNTER_INC(&pStats->FFor00Reads);
456	u32Value = 0;
457	break;
458	case VINF_IOM_R3_MMIO_READ:
459	case VINF_IOM_R3_MMIO_READ_WRITE:
460	case VINF_IOM_R3_MMIO_WRITE:
461	/** @todo What if we've split a transfer and already read
462	* something? Since reads can have sideeffects we could be
463	* kind of screwed here... */
464	LogFlow(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
465	GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
466	return rcStrict2;
467	default:
468	if (RT_FAILURE(rcStrict2))
469	{
470	Log(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
471	GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
472	return rcStrict2;
473	}
474	AssertMsgReturn(rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)),
475	VERR_IPE_UNEXPECTED_INFO_STATUS);
476	if (rc == VINF_SUCCESS \|\| rcStrict2 < rc)
477	rc = rcStrict2;
478	break;
479	}
480	u32Value >>= (GCPhys & 3) * 8;
481
482	/*
483	* Write what we've read.
484	*/
485	unsigned cbThisPart = 4 - (GCPhys & 3);
486	if (cbThisPart > cbValue)
487	cbThisPart = cbValue;
488
489	switch (cbThisPart)
490	{
491	case 1:
492	(uint8_t )pvValue = (uint8_t)u32Value;
493	break;
494	case 2:
495	(uint16_t )pvValue = (uint16_t)u32Value;
496	break;
497	case 3:
498	((uint8_t *)pvValue)[0] = RT_BYTE1(u32Value);
499	((uint8_t *)pvValue)[1] = RT_BYTE2(u32Value);
500	((uint8_t *)pvValue)[2] = RT_BYTE3(u32Value);
501	break;
502	case 4:
503	(uint32_t )pvValue = u32Value;
504	break;
505	}
506
507	/*
508	* Advance.
509	*/
510	cbValue -= cbThisPart;
511	if (!cbValue)
512	break;
513	GCPhys += cbThisPart;
514	offRegion += cbThisPart;
515	pvValue = (uint8_t *)pvValue + cbThisPart;
516	}
517
518	return rc;
519	}
520
521
522	/**
523	* Implements VINF_IOM_MMIO_UNUSED_FF.
524	*
525	* @returns VINF_SUCCESS.
526	* @param pvValue Where to store the zeros.
527	* @param cbValue How many bytes to read.
528	* @param pStats Pointer to the statistics (never NULL).
529	*/
530	static int iomMMIODoReadFFs(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
531	{
532	switch (cbValue)
533	{
534	case 1: (uint8_t )pvValue = UINT8_C(0xff); break;
535	case 2: (uint16_t )pvValue = UINT16_C(0xffff); break;
536	case 4: (uint32_t )pvValue = UINT32_C(0xffffffff); break;
537	case 8: (uint64_t )pvValue = UINT64_C(0xffffffffffffffff); break;
538	default:
539	{
540	uint8_t pb = (uint8_t )pvValue;
541	while (cbValue--)
542	*pb++ = UINT8_C(0xff);
543	break;
544	}
545	}
546	STAM_COUNTER_INC(&pStats->FFor00Reads);
547	return VINF_SUCCESS;
548	}
549
550
551	/**
552	* Implements VINF_IOM_MMIO_UNUSED_00.
553	*
554	* @returns VINF_SUCCESS.
555	* @param pvValue Where to store the zeros.
556	* @param cbValue How many bytes to read.
557	* @param pStats Pointer to the statistics (never NULL).
558	*/
559	static int iomMMIODoRead00s(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
560	{
561	switch (cbValue)
562	{
563	case 1: (uint8_t )pvValue = UINT8_C(0x00); break;
564	case 2: (uint16_t )pvValue = UINT16_C(0x0000); break;
565	case 4: (uint32_t )pvValue = UINT32_C(0x00000000); break;
566	case 8: (uint64_t )pvValue = UINT64_C(0x0000000000000000); break;
567	default:
568	{
569	uint8_t pb = (uint8_t )pvValue;
570	while (cbValue--)
571	*pb++ = UINT8_C(0x00);
572	break;
573	}
574	}
575	STAM_COUNTER_INC(&pStats->FFor00Reads);
576	return VINF_SUCCESS;
577	}
578
579
580	/**
581	* Wrapper which does the read.
582	*/
583	DECLINLINE(VBOXSTRICTRC) iomMmioDoRead(PVMCC pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys, RTGCPHYS offRegion,
584	void *pvValue, uint32_t cbValue IOM_MMIO_STATS_COMMA_DECL)
585	{
586	VBOXSTRICTRC rcStrict;
587	if (RT_LIKELY(pRegEntry->pfnReadCallback))
588	{
589	if ( ( cbValue == 4
590	&& !(GCPhys & 3))
591	\|\| (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_PASSTHRU
592	\|\| ( cbValue == 8
593	&& !(GCPhys & 7)
594	&& (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD ) )
595	rcStrict = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
596	!(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) ? offRegion : GCPhys, pvValue, cbValue);
597	else
598	rcStrict = iomMMIODoComplicatedRead(pVM, pRegEntry, GCPhys, offRegion, pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG);
599	}
600	else
601	rcStrict = VINF_IOM_MMIO_UNUSED_FF;
602	if (rcStrict != VINF_SUCCESS)
603	{
604	switch (VBOXSTRICTRC_VAL(rcStrict))
605	{
606	case VINF_IOM_MMIO_UNUSED_FF: rcStrict = iomMMIODoReadFFs(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
607	case VINF_IOM_MMIO_UNUSED_00: rcStrict = iomMMIODoRead00s(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
608	}
609	}
610	return rcStrict;
611	}
612
613	#ifndef IN_RING3
614
615	/**
616	* Checks if we can handle an MMIO \#PF in R0/RC.
617	*/
618	DECLINLINE(bool) iomMmioCanHandlePfInRZ(PVMCC pVM, uint32_t uErrorCode, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
619	{
620	if (pRegEntry->cbRegion > 0)
621	{
622	if ( pRegEntry->pfnWriteCallback
623	&& pRegEntry->pfnReadCallback)
624	return true;
625
626	PIOMMMIOENTRYR3 const pRegEntryR3 = &pVM->iomr0.s.paMmioRing3Regs[pRegEntry->idxSelf];
627	if ( uErrorCode == UINT32_MAX
628	? pRegEntryR3->pfnWriteCallback \|\| pRegEntryR3->pfnReadCallback
629	: uErrorCode & X86_TRAP_PF_RW
630	? !pRegEntry->pfnWriteCallback && pRegEntryR3->pfnWriteCallback
631	: !pRegEntry->pfnReadCallback && pRegEntryR3->pfnReadCallback)
632	return false;
633
634	return true;
635	}
636	return false;
637	}
638
639
640	/**
641	* Common worker for the \#PF handler and IOMMMIOPhysHandler (APIC+VT-x).
642	*
643	* @returns VBox status code (appropriate for GC return).
644	* @param pVM The cross context VM structure.
645	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
646	* @param uErrorCode CPU Error code. This is UINT32_MAX when we don't have
647	* any error code (the EPT misconfig hack).
648	* @param GCPhysFault The GC physical address corresponding to pvFault.
649	* @param pRegEntry The MMIO entry for the current context.
650	*/
651	DECLINLINE(VBOXSTRICTRC) iomMmioCommonPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode,
652	RTGCPHYS GCPhysFault, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
653	{
654	Log(("iomMmioCommonPfHandler: GCPhysFault=%RGp uErr=%#x rip=%RGv\n", GCPhysFault, uErrorCode, CPUMGetGuestRIP(pVCpu) ));
655	RT_NOREF(GCPhysFault, uErrorCode);
656
657	VBOXSTRICTRC rcStrict;
658
659	#ifndef IN_RING3
660	/*
661	* Should we defer the request right away? This isn't usually the case, so
662	* do the simple test first and the try deal with uErrorCode being N/A.
663	*/
664	PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
665	if (RT_LIKELY( pDevIns
666	&& iomMmioCanHandlePfInRZ(pVM, uErrorCode, pRegEntry)))
667	{
668	/*
669	* Enter the device critsect prior to engaging IOM in case of lock contention.
670	* Note! Perhaps not a good move?
671	*/
672	rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_MMIO_READ_WRITE);
673	if (rcStrict == VINF_SUCCESS)
674	{
675	#endif /* !IN_RING3 */
676
677	/*
678	* Let IEM call us back via iomMmioHandler.
679	*/
680	rcStrict = IEMExecOne(pVCpu);
681
682	#ifndef IN_RING3
683	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
684	#endif
685	if (RT_SUCCESS(rcStrict))
686	{ /* likely */ }
687	else if ( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
688	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
689	{
690	Log(("IOM: Hit unsupported IEM feature!\n"));
691	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
692	}
693	#ifndef IN_RING3
694	return rcStrict;
695	}
696	STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
697	}
698	else
699	rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
700
701	# ifdef VBOX_WITH_STATISTICS
702	if (rcStrict == VINF_IOM_R3_MMIO_READ_WRITE)
703	{
704	PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry);
705	if (uErrorCode & X86_TRAP_PF_RW)
706	{
707	STAM_COUNTER_INC(&pStats->WriteRZToR3);
708	STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
709	}
710	else
711	{
712	STAM_COUNTER_INC(&pStats->ReadRZToR3);
713	STAM_COUNTER_INC(&pVM->iom.s.StatMmioReadsR0ToR3);
714	}
715	}
716	# endif
717	#else /* IN_RING3 */
718	RT_NOREF(pVM, pRegEntry);
719	#endif /* IN_RING3 */
720	return rcStrict;
721	}
722
723
724	/**
725	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
726	* \#PF access handler callback for MMIO pages.}
727	*
728	* @remarks The @a pvUser argument is the MMIO handle.
729	*/
730	DECLEXPORT(VBOXSTRICTRC) iomMmioPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTXCORE pCtxCore,
731	RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
732	{
733	STAM_PROFILE_START(&pVM->iom.s.StatMmioPfHandler, Prf);
734	LogFlow(("iomMmioPfHandlerNew: GCPhys=%RGp uErr=%#x pvFault=%RGv rip=%RGv\n",
735	GCPhysFault, (uint32_t)uErrorCode, pvFault, (RTGCPTR)pCtxCore->rip));
736	RT_NOREF(pvFault, pCtxCore);
737
738	/* Translate the MMIO handle to a registration entry for the current context. */
739	AssertReturn((uintptr_t)pvUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
740	# ifdef IN_RING0
741	AssertReturn((uintptr_t)pvUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
742	CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iomr0.s.paMmioRegs[(uintptr_t)pvUser];
743	# else
744	CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iom.s.paMmioRegs[(uintptr_t)pvUser];
745	# endif
746
747	VBOXSTRICTRC rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
748
749	STAM_PROFILE_STOP(&pVM->iom.s.StatMmioPfHandler, Prf);
750	return rcStrict;
751	}
752
753	#endif /* !IN_RING3 */
754
755	#ifdef IN_RING0
756	/**
757	* Physical access handler for MMIO ranges.
758	*
759	* This is actually only used by VT-x for APIC page accesses.
760	*
761	* @returns VBox status code (appropriate for GC return).
762	* @param pVM The cross context VM structure.
763	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
764	* @param uErrorCode CPU Error code.
765	* @param GCPhysFault The GC physical address.
766	*/
767	VMM_INT_DECL(VBOXSTRICTRC) IOMR0MmioPhysHandler(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, RTGCPHYS GCPhysFault)
768	{
769	STAM_PROFILE_START(&pVM->iom.s.StatMmioPhysHandler, Prf);
770
771	/*
772	* We don't have a range here, so look it up before calling the common function.
773	*/
774	VBOXSTRICTRC rcStrict = IOM_LOCK_SHARED(pVM);
775	if (RT_SUCCESS(rcStrict))
776	{
777	RTGCPHYS offRegion;
778	CTX_SUFF(PIOMMMIOENTRY) pRegEntry = iomMmioGetEntry(pVM, GCPhysFault, &offRegion, &pVCpu->iom.s.idxMmioLastPhysHandler);
779	IOM_UNLOCK_SHARED(pVM);
780	if (RT_LIKELY(pRegEntry))
781	rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
782	else
783	rcStrict = VERR_IOM_MMIO_RANGE_NOT_FOUND;
784	}
785	else if (rcStrict == VERR_SEM_BUSY)
786	rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
787
788	STAM_PROFILE_STOP(&pVM->iom.s.StatMmioPhysHandler, Prf);
789	return rcStrict;
790	}
791	#endif /* IN_RING0 */
792
793
794	/**
795	* @callback_method_impl{FNPGMPHYSHANDLER, MMIO page accesses}
796	*
797	* @remarks The @a pvUser argument is the MMIO handle.
798	*/
799	PGM_ALL_CB2_DECL(VBOXSTRICTRC) iomMmioHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, void pvPhys, void pvBuf,
800	size_t cbBuf, PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, void *pvUser)
801	{
802	STAM_PROFILE_START(UnusedMacroArg, Prf);
803	STAM_COUNTER_INC(&pVM->iom.s.CTX_SUFF(StatMmioHandler));
804	Log4(("iomMmioHandlerNew: GCPhysFault=%RGp cbBuf=%#x enmAccessType=%d enmOrigin=%d pvUser=%p\n", GCPhysFault, cbBuf, enmAccessType, enmOrigin, pvUser));
805
806	Assert(enmAccessType == PGMACCESSTYPE_READ \|\| enmAccessType == PGMACCESSTYPE_WRITE);
807	AssertMsg(cbBuf >= 1, ("%zu\n", cbBuf));
808	NOREF(pvPhys); NOREF(enmOrigin);
809
810	#ifdef IN_RING3
811	int const rcToRing3 = VERR_IOM_MMIO_IPE_3;
812	#else
813	int const rcToRing3 = enmAccessType == PGMACCESSTYPE_READ ? VINF_IOM_R3_MMIO_READ : VINF_IOM_R3_MMIO_WRITE;
814	#endif
815
816	/*
817	* Translate pvUser to an MMIO registration table entry. We can do this
818	* without any locking as the data is static after VM creation.
819	*/
820	AssertReturn((uintptr_t)pvUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
821	#ifdef IN_RING0
822	AssertReturn((uintptr_t)pvUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
823	CTX_SUFF(PIOMMMIOENTRY) const pRegEntry = &pVM->iomr0.s.paMmioRegs[(uintptr_t)pvUser];
824	PIOMMMIOENTRYR3 const pRegEntryR3 = &pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser];
825	#else
826	CTX_SUFF(PIOMMMIOENTRY) const pRegEntry = &pVM->iom.s.paMmioRegs[(uintptr_t)pvUser];
827	#endif
828	#ifdef VBOX_WITH_STATISTICS
829	PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry); /* (Works even without ring-0 device setup.) */
830	#endif
831	PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
832
833	#ifdef VBOX_STRICT
834	/*
835	* Assert the right entry in strict builds. This may yield a false positive
836	* for SMP VMs if we're unlucky and the guest isn't well behaved.
837	*/
838	# ifdef IN_RING0
839	Assert(pRegEntry && (GCPhysFault - pRegEntryR3->GCPhysMapping < pRegEntryR3->cbRegion \|\| !pRegEntryR3->fMapped));
840	# else
841	Assert(pRegEntry && (GCPhysFault - pRegEntry->GCPhysMapping < pRegEntry->cbRegion \|\| !pRegEntry->fMapped));
842	# endif
843	#endif
844
845	#ifndef IN_RING3
846	/*
847	* If someone is doing FXSAVE, FXRSTOR, XSAVE, XRSTOR or other stuff dealing with
848	* large amounts of data, just go to ring-3 where we don't need to deal with partial
849	* successes. No chance any of these will be problematic read-modify-write stuff.
850	*
851	* Also drop back if the ring-0 registration entry isn't actually used.
852	*/
853	if ( RT_LIKELY(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue))
854	&& pRegEntry->cbRegion != 0
855	&& ( enmAccessType == PGMACCESSTYPE_READ
856	? pRegEntry->pfnReadCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser].pfnReadCallback == NULL
857	: pRegEntry->pfnWriteCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser].pfnWriteCallback == NULL)
858	&& pDevIns )
859	{ /* likely */ }
860	else
861	{
862	Log4(("iomMmioHandlerNew: to ring-3: to-big=%RTbool zero-size=%RTbool no-callback=%RTbool pDevIns=%p hRegion=%p\n",
863	!(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue)), !(pRegEntry->cbRegion != 0),
864	!( enmAccessType == PGMACCESSTYPE_READ
865	? pRegEntry->pfnReadCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser].pfnReadCallback == NULL
866	: pRegEntry->pfnWriteCallback != NULL \|\| pVM->iomr0.s.paMmioRing3Regs[(uintptr_t)pvUser].pfnWriteCallback == NULL),
867	pDevIns, pvUser));
868	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3);
869	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatMmioReadsR0ToR3 : &pVM->iom.s.StatMmioWritesR0ToR3);
870	return rcToRing3;
871	}
872	#endif /* !IN_RING3 */
873
874	/*
875	* If we've got an offset that's outside the region, defer to ring-3 if we
876	* can, or pretend there is nothing there. This shouldn't happen, but can
877	* if we're unlucky with an SMP VM and the guest isn't behaving very well.
878	*/
879	#ifdef IN_RING0
880	RTGCPHYS const GCPhysMapping = pRegEntryR3->GCPhysMapping;
881	#else
882	RTGCPHYS const GCPhysMapping = pRegEntry->GCPhysMapping;
883	#endif
884	RTGCPHYS const offRegion = GCPhysFault - GCPhysMapping;
885	if (RT_LIKELY(offRegion < pRegEntry->cbRegion && GCPhysMapping != NIL_RTGCPHYS))
886	{ /* likely */ }
887	else
888	{
889	STAM_REL_COUNTER_INC(&pVM->iom.s.StatMmioStaleMappings);
890	LogRelMax(64, ("iomMmioHandlerNew: Stale access at %#RGp to range #%#x currently residing at %RGp LB %RGp\n",
891	GCPhysFault, pRegEntry->idxSelf, GCPhysMapping, pRegEntry->cbRegion));
892	#ifdef IN_RING3
893	if (enmAccessType == PGMACCESSTYPE_READ)
894	iomMMIODoReadFFs(pvBuf, cbBuf IOM_MMIO_STATS_COMMA_ARG);
895	return VINF_SUCCESS;
896	#else
897	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3);
898	STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatMmioReadsR0ToR3 : &pVM->iom.s.StatMmioWritesR0ToR3);
899	return rcToRing3;
900	#endif
901	}
902
903	/*
904	* Perform locking and the access.
905	*
906	* Writes requiring a return to ring-3 are buffered by IOM so IEM can
907	* commit the instruction.
908	*
909	* Note! We may end up locking the device even when the relevant callback is
910	* NULL. This is supposed to be an unlikely case, so not optimized yet.
911	*/
912	VBOXSTRICTRC rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), rcToRing3);
913	if (rcStrict == VINF_SUCCESS)
914	{
915	if (enmAccessType == PGMACCESSTYPE_READ)
916	{
917	/*
918	* Read.
919	*/
920	rcStrict = iomMmioDoRead(pVM, pRegEntry, GCPhysFault, offRegion, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
921
922	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
923	#ifndef IN_RING3
924	if (rcStrict == VINF_IOM_R3_MMIO_READ)
925	{
926	STAM_COUNTER_INC(&pStats->ReadRZToR3);
927	STAM_COUNTER_INC(&pVM->iom.s.StatMmioReadsR0ToR3);
928	}
929	else
930	#endif
931	STAM_COUNTER_INC(&pStats->Reads);
932	STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfRead), Prf);
933	}
934	else
935	{
936	/*
937	* Write.
938	*/
939	rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, GCPhysFault, offRegion, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
940	PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
941	#ifndef IN_RING3
942	if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
943	rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
944	if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
945	{
946	STAM_COUNTER_INC(&pStats->WriteRZToR3);
947	STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
948	}
949	else if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
950	{
951	STAM_COUNTER_INC(&pStats->CommitRZToR3);
952	STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsR0ToR3);
953	}
954	else
955	#endif
956	STAM_COUNTER_INC(&pStats->Writes);
957	STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfWrite), Prf);
958	}
959
960	/*
961	* Check the return code.
962	*/
963	#ifdef IN_RING3
964	AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc - Access type %d - %RGp - %s\n",
965	VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pRegEntry->pszDesc));
966	#else
967	AssertMsg( rcStrict == VINF_SUCCESS
968	\|\| rcStrict == rcToRing3
969	\|\| (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE && enmAccessType == PGMACCESSTYPE_WRITE)
970	\|\| rcStrict == VINF_EM_DBG_STOP
971	\|\| rcStrict == VINF_EM_DBG_EVENT
972	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
973	\|\| rcStrict == VINF_EM_OFF
974	\|\| rcStrict == VINF_EM_SUSPEND
975	\|\| rcStrict == VINF_EM_RESET
976	//\|\| rcStrict == VINF_EM_HALT /* ?? */
977	//\|\| rcStrict == VINF_EM_NO_MEMORY /* ?? */
978	, ("%Rrc - Access type %d - %RGp - %s #%u\n",
979	VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pDevIns->pReg->szName, pDevIns->iInstance));
980	#endif
981	}
982	/*
983	* Deal with enter-critsect failures.
984	*/
985	#ifndef IN_RING3
986	else if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
987	{
988	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
989	rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
990	if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
991	{
992	STAM_COUNTER_INC(&pStats->CommitRZToR3);
993	STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsR0ToR3);
994	}
995	else
996	{
997	STAM_COUNTER_INC(&pStats->WriteRZToR3);
998	STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
999	}
1000	STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
1001	}
1002	else if (rcStrict == VINF_IOM_R3_MMIO_READ)
1003	{
1004	Assert(enmAccessType == PGMACCESSTYPE_READ);
1005	STAM_COUNTER_INC(&pStats->ReadRZToR3);
1006	STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
1007	}
1008	#endif
1009	else
1010	AssertMsg(RT_FAILURE_NP(rcStrict), ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
1011	return rcStrict;
1012	}
1013
1014
1015	/**
1016	* Mapping an MMIO2 page in place of an MMIO page for direct access.
1017	*
1018	* This is a special optimization used by the VGA device. Call
1019	* IOMMmioResetRegion() to undo the mapping.
1020	*
1021	* @returns VBox status code. This API may return VINF_SUCCESS even if no
1022	* remapping is made.
1023	* @retval VERR_SEM_BUSY in ring-0 if we cannot get the IOM lock.
1024	*
1025	* @param pVM The cross context VM structure.
1026	* @param pDevIns The device instance @a hRegion and @a hMmio2 are
1027	* associated with.
1028	* @param hRegion The handle to the MMIO region.
1029	* @param offRegion The offset into @a hRegion of the page to be
1030	* remapped.
1031	* @param hMmio2 The MMIO2 handle.
1032	* @param offMmio2 Offset into @a hMmio2 of the page to be use for the
1033	* mapping.
1034	* @param fPageFlags Page flags to set. Must be (X86_PTE_RW \| X86_PTE_P)
1035	* for the time being.
1036	*/
1037	VMMDECL(int) IOMMmioMapMmio2Page(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, RTGCPHYS offRegion,
1038	uint64_t hMmio2, RTGCPHYS offMmio2, uint64_t fPageFlags)
1039	{
1040	/* Currently only called from the VGA device during MMIO. */
1041	Log(("IOMMmioMapMmio2Page %#RX64/%RGp -> %#RX64/%RGp flags=%RX64\n", hRegion, offRegion, hMmio2, offMmio2, fPageFlags));
1042	AssertReturn(fPageFlags == (X86_PTE_RW \| X86_PTE_P), VERR_INVALID_PARAMETER);
1043	AssertReturn(pDevIns, VERR_INVALID_POINTER);
1044
1045	/** @todo Why is this restricted to protected mode??? Try it in all modes! */
1046	PVMCPUCC pVCpu = VMMGetCpu(pVM);
1047
1048	/* This currently only works in real mode, protected mode without paging or with nested paging. */
1049	/** @todo NEM: MMIO page aliasing. */
1050	if ( !HMIsEnabled(pVM) /* useless without VT-x/AMD-V */
1051	\|\| ( CPUMIsGuestInPagedProtectedMode(pVCpu)
1052	&& !HMIsNestedPagingActive(pVM)))
1053	return VINF_SUCCESS; /* ignore / /* @todo return some indicator if we fail here */
1054
1055	/*
1056	* Translate the handle into an entry and check the region offset.
1057	*/
1058	AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
1059	#ifdef IN_RING0
1060	AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
1061	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
1062	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1063	AssertReturn(offRegion < pVM->iomr0.s.paMmioRegs[hRegion].cbRegion, VERR_OUT_OF_RANGE);
1064	AssertReturn( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
1065	\|\| ( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
1066	&& pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
1067	#else
1068	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
1069	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1070	AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
1071	#endif
1072	AssertReturn(offRegion < pRegEntry->cbRegion, VERR_OUT_OF_RANGE);
1073	Assert((pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK) == 0);
1074
1075	/*
1076	* When getting and using the mapping address, we must sit on the IOM lock
1077	* to prevent remapping. Shared suffices as we change nothing.
1078	*/
1079	int rc = IOM_LOCK_SHARED(pVM);
1080	if (rc == VINF_SUCCESS)
1081	{
1082	RTGCPHYS const GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
1083	if (GCPhys != NIL_RTGCPHYS)
1084	{
1085	Assert(!(GCPhys & GUEST_PAGE_OFFSET_MASK));
1086
1087	/*
1088	* Do the aliasing; page align the addresses since PGM is picky.
1089	*/
1090	rc = PGMHandlerPhysicalPageAliasMmio2(pVM, GCPhys, GCPhys + (offRegion & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK),
1091	pDevIns, hMmio2, offMmio2);
1092	}
1093	else
1094	AssertFailedStmt(rc = VERR_IOM_MMIO_REGION_NOT_MAPPED);
1095
1096	IOM_UNLOCK_SHARED(pVM);
1097	}
1098
1099	/** @todo either ditch this or replace it with something that works in the
1100	* nested case, since we really only care about nested paging! */
1101	#if 0
1102	/*
1103	* Modify the shadow page table. Since it's an MMIO page it won't be present and we
1104	* can simply prefetch it.
1105	*
1106	* Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page.
1107	*/
1108	# if 0 /* The assertion is wrong for the PGM_SYNC_CLEAR_PGM_POOL and VINF_PGM_HANDLER_ALREADY_ALIASED cases. */
1109	# ifdef VBOX_STRICT
1110	uint64_t fFlags;
1111	RTHCPHYS HCPhys;
1112	rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys);
1113	Assert(rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1114	# endif
1115	# endif
1116	rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys);
1117	Assert(rc == VINF_SUCCESS \|\| rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1118	#endif
1119	return rc;
1120	}
1121
1122
1123	#ifdef IN_RING0 /* VT-x ring-0 only, move to IOMR0Mmio.cpp later. */
1124	/**
1125	* Mapping a HC page in place of an MMIO page for direct access.
1126	*
1127	* This is a special optimization used by the APIC in the VT-x case. This VT-x
1128	* code uses PGMHandlerPhysicalReset rather than IOMMmioResetRegion() to undo
1129	* the effects here.
1130	*
1131	* @todo Make VT-x usage more consistent.
1132	*
1133	* @returns VBox status code.
1134	*
1135	* @param pVM The cross context VM structure.
1136	* @param pVCpu The cross context virtual CPU structure.
1137	* @param GCPhys The address of the MMIO page to be changed.
1138	* @param HCPhys The address of the host physical page.
1139	* @param fPageFlags Page flags to set. Must be (X86_PTE_RW \| X86_PTE_P)
1140	* for the time being.
1141	*/
1142	VMMR0_INT_DECL(int) IOMR0MmioMapMmioHCPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint64_t fPageFlags)
1143	{
1144	/* Currently only called from VT-x code during a page fault. */
1145	Log(("IOMR0MmioMapMmioHCPage %RGp -> %RGp flags=%RX64\n", GCPhys, HCPhys, fPageFlags));
1146
1147	AssertReturn(fPageFlags == (X86_PTE_RW \| X86_PTE_P), VERR_INVALID_PARAMETER);
1148	/** @todo NEM: MMIO page aliasing?? */
1149	Assert(HMIsEnabled(pVM));
1150
1151	# ifdef VBOX_STRICT
1152	/*
1153	* Check input address (it's HM calling, not the device, so no region handle).
1154	*/
1155	int rcSem = IOM_LOCK_SHARED(pVM);
1156	if (rcSem == VINF_SUCCESS)
1157	{
1158	RTGCPHYS offIgn;
1159	uint16_t idxIgn = UINT16_MAX;
1160	PIOMMMIOENTRYR0 pRegEntry = iomMmioGetEntry(pVM, GCPhys, &offIgn, &idxIgn);
1161	IOM_UNLOCK_SHARED(pVM);
1162	Assert(pRegEntry);
1163	Assert(pRegEntry && !(pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK));
1164	}
1165	# endif
1166
1167	/*
1168	* Do the aliasing; page align the addresses since PGM is picky.
1169	*/
1170	GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1171	HCPhys &= ~(RTHCPHYS)GUEST_PAGE_OFFSET_MASK;
1172
1173	int rc = PGMHandlerPhysicalPageAliasHC(pVM, GCPhys, GCPhys, HCPhys);
1174	AssertRCReturn(rc, rc);
1175
1176	/** @todo either ditch this or replace it with something that works in the
1177	* nested case, since we really only care about nested paging! */
1178
1179	/*
1180	* Modify the shadow page table. Since it's an MMIO page it won't be present and we
1181	* can simply prefetch it.
1182	*
1183	* Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page.
1184	*/
1185	rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys);
1186	Assert(rc == VINF_SUCCESS \|\| rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1187	return VINF_SUCCESS;
1188	}
1189	#endif
1190
1191
1192	/**
1193	* Reset a previously modified MMIO region; restore the access flags.
1194	*
1195	* This undoes the effects of IOMMmioMapMmio2Page() and is currently only
1196	* intended for some ancient VGA hack. However, it would be great to extend it
1197	* beyond VT-x and/or nested-paging.
1198	*
1199	* @returns VBox status code.
1200	*
1201	* @param pVM The cross context VM structure.
1202	* @param pDevIns The device instance @a hRegion is associated with.
1203	* @param hRegion The handle to the MMIO region.
1204	*/
1205	VMMDECL(int) IOMMmioResetRegion(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion)
1206	{
1207	Log(("IOMMMIOResetRegion %#RX64\n", hRegion));
1208	AssertReturn(pDevIns, VERR_INVALID_POINTER);
1209
1210	/** @todo Get rid of this this real/protected or nested paging restriction,
1211	* it probably shouldn't be here and would be nasty when the CPU
1212	* changes mode while we have the hack enabled... */
1213	PVMCPUCC pVCpu = VMMGetCpu(pVM);
1214
1215	/* This currently only works in real mode, protected mode without paging or with nested paging. */
1216	/** @todo NEM: MMIO page aliasing. */
1217	if ( !HMIsEnabled(pVM) /* useless without VT-x/AMD-V */
1218	\|\| ( CPUMIsGuestInPagedProtectedMode(pVCpu)
1219	&& !HMIsNestedPagingActive(pVM)))
1220	return VINF_SUCCESS; /* ignore */
1221
1222	/*
1223	* Translate the handle into an entry and mapping address for PGM.
1224	* We have to take the lock to safely access the mapping address here.
1225	*/
1226	AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
1227	#ifdef IN_RING0
1228	AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
1229	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
1230	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1231	AssertReturn( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
1232	\|\| ( pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
1233	&& pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
1234	#else
1235	PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
1236	AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
1237	AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
1238	#endif
1239	Assert((pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK) == 0);
1240
1241	int rcSem = IOM_LOCK_SHARED(pVM);
1242	RTGCPHYS GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
1243	if (rcSem == VINF_SUCCESS)
1244	IOM_UNLOCK_SHARED(pVM);
1245
1246	Assert(!(GCPhys & GUEST_PAGE_OFFSET_MASK));
1247	Assert(!(pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK));
1248
1249	/*
1250	* Call PGM to do the job work.
1251	*
1252	* After the call, all the pages should be non-present, unless there is
1253	* a page pool flush pending (unlikely).
1254	*/
1255	int rc = PGMHandlerPhysicalReset(pVM, GCPhys);
1256	AssertRC(rc);
1257
1258	# ifdef VBOX_STRICT
1259	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
1260	{
1261	RTGCPHYS cb = pRegEntry->cbRegion;
1262	while (cb)
1263	{
1264	uint64_t fFlags;
1265	RTHCPHYS HCPhys;
1266	rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys);
1267	Assert(rc == VERR_PAGE_NOT_PRESENT \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT);
1268	cb -= RT_MIN(GUEST_PAGE_SIZE, HOST_PAGE_SIZE);
1269	GCPhys += RT_MIN(GUEST_PAGE_SIZE, HOST_PAGE_SIZE);
1270	}
1271	}
1272	# endif
1273	return rc;
1274	}
1275

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp@ 93628

Download in other formats: