Virtio_1_0.cpp@ 83748

Last change on this file since 83748 was 83664, checked in by vboxsync, 5 years ago
Network/DevVirtioNet_1_0.cpp: Now have round trip networking working. Can ssh login into and out of VM guest. This is contingent on a 'kick' hack in Virtio_1_0.cpp that needs investigation. See BugRef 8651, Comment #65
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 88.0 KB

Line
1	/* $Id: Virtio_1_0.cpp 83664 2020-04-10 07:58:57Z vboxsync $ */
2	/** @file
3	* Virtio_1_0 - Virtio Common (PCI, feature & config mgt, queue mgt & proxy, notification mgt)
4	*/
5
6	/*
7	* Copyright (C) 2009-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_DEV_VIRTIO
23
24	#include <VBox/log.h>
25	#include <VBox/msi.h>
26	#include <VBox/AssertGuest.h>
27	#include <iprt/param.h>
28	#include <iprt/assert.h>
29	#include <iprt/uuid.h>
30	#include <iprt/mem.h>
31	#include <iprt/assert.h>
32	#include <iprt/sg.h>
33	#include <iprt/string.h>
34	#include <VBox/vmm/pdmdev.h>
35	#include "Virtio_1_0.h"
36
37
38	/*********************************************************************************************************************************
39	* Defined Constants And Macros *
40	*********************************************************************************************************************************/
41	#define INSTANCE(a_pVirtio) ((a_pVirtio)->szInstance)
42	#define VIRTQNAME(a_pVirtio, a_idxQueue) ((a_pVirtio)->virtqState[(a_idxQueue)].szVirtqName)
43	#define IS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
44
45	/**
46	* This macro returns true if the @a a_offAccess and access length (@a
47	* a_cbAccess) are within the range of the mapped capability struct described by
48	* @a a_LocCapData.
49	*
50	* @param[in] a_offAccess The offset into the MMIO bar of the access.
51	* @param[in] a_cbAccess The access size.
52	* @param[out] a_offIntraVar The variable to return the intra-capability
53	* offset into. ASSUMES this is uint32_t.
54	* @param[in] a_LocCapData The capability location info.
55	*/
56	#define MATCHES_VIRTIO_CAP_STRUCT(a_offAccess, a_cbAccess, a_offIntraVar, a_LocCapData) \
57	( ((a_offIntraVar) = (uint32_t)((a_offAccess) - (a_LocCapData).offMmio)) < (uint32_t)(a_LocCapData).cbMmio \
58	&& (a_offIntraVar) + (uint32_t)(a_cbAccess) <= (uint32_t)(a_LocCapData).cbMmio )
59
60
61	/** Marks the start of the virtio saved state (just for sanity). */
62	#define VIRTIO_SAVEDSTATE_MARKER UINT64_C(0x1133557799bbddff)
63	/** The current saved state version for the virtio core. */
64	#define VIRTIO_SAVEDSTATE_VERSION UINT32_C(1)
65
66
67	/*********************************************************************************************************************************
68	* Structures and Typedefs *
69	*********************************************************************************************************************************/
70	/**
71	* virtq related structs
72	* (struct names follow VirtIO 1.0 spec, typedef use VBox style)
73	*/
74	typedef struct virtq_desc
75	{
76	uint64_t GCPhysBuf; /*< addr GC Phys. address of buffer /
77	uint32_t cb; /*< len Buffer length /
78	uint16_t fFlags; /*< flags Buffer specific flags /
79	uint16_t uDescIdxNext; /*< next Idx set if VIRTIO_DESC_F_NEXT /
80	} VIRTQ_DESC_T, *PVIRTQ_DESC_T;
81
82	typedef struct virtq_avail
83	{
84	uint16_t fFlags; /*< flags avail ring drv to dev flags /
85	uint16_t uIdx; /*< idx Index of next free ring slot /
86	uint16_t auRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: avail drv to dev bufs /
87	/* uint16_t uUsedEventIdx; - used_event (if VIRTQ_USED_F_EVENT_IDX) */
88	} VIRTQ_AVAIL_T, *PVIRTQ_AVAIL_T;
89
90	typedef struct virtq_used_elem
91	{
92	uint32_t uDescIdx; /*< idx Start of used desc chain /
93	uint32_t cbElem; /*< len Total len of used desc chain /
94	} VIRTQ_USED_ELEM_T;
95
96	typedef struct virt_used
97	{
98	uint16_t fFlags; /*< flags used ring host-to-guest flags /
99	uint16_t uIdx; /*< idx Index of next ring slot /
100	VIRTQ_USED_ELEM_T aRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: used dev to drv bufs /
101	/* uint16_t uAvailEventIdx; - avail_event if (VIRTQ_USED_F_EVENT_IDX) */
102	} VIRTQ_USED_T, *PVIRTQ_USED_T;
103
104
105	const char *virtioCoreGetStateChangeText(VIRTIOVMSTATECHANGED enmState)
106	{
107	switch (enmState)
108	{
109	case kvirtIoVmStateChangedReset: return "VM RESET";
110	case kvirtIoVmStateChangedSuspend: return "VM SUSPEND";
111	case kvirtIoVmStateChangedPowerOff: return "VM POWER OFF";
112	case kvirtIoVmStateChangedResume: return "VM RESUME";
113	default: return "<BAD ENUM>";
114	}
115	}
116
117	/* Internal Functions */
118
119	static void virtioNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, bool fForce);
120	static int virtioKick(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uVec, bool fForce);
121
122	/** @name Internal queue operations
123	* @{ */
124
125	#if 0 /* unused */
126	DECLINLINE(int) virtqIsEventNeeded(uint16_t uEventIdx, uint16_t uDescIdxNew, uint16_t uDescIdxOld)
127	{
128	return (uint16_t)(uDescIdxNew - uEventIdx - 1) < (uint16_t)(uDescIdxNew - uDescIdxOld);
129	}
130	#endif
131
132	/**
133	* Accessor for virtq descriptor
134	*/
135	#ifdef IN_RING3
136	DECLINLINE(void) virtioReadDesc(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
137	uint32_t idxDesc, PVIRTQ_DESC_T pDesc)
138	{
139	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
140	uint16_t const cQueueItems = RT_MAX(pVirtio->uQueueSize[idxQueue], 1); /* Make sure to avoid div-by-zero. */
141	PDMDevHlpPCIPhysRead(pDevIns,
142	pVirtio->aGCPhysQueueDesc[idxQueue] + sizeof(VIRTQ_DESC_T) * (idxDesc % cQueueItems),
143	pDesc, sizeof(VIRTQ_DESC_T));
144	}
145	#endif
146
147	/**
148	* Accessors for virtq avail ring
149	*/
150	#ifdef IN_RING3
151	DECLINLINE(uint16_t) virtioReadAvailDescIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint32_t availIdx)
152	{
153	uint16_t uDescIdx;
154	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
155	uint16_t const cQueueItems = RT_MAX(pVirtio->uQueueSize[idxQueue], 1); /* Make sure to avoid div-by-zero. */
156	PDMDevHlpPCIPhysRead(pDevIns,
157	pVirtio->aGCPhysQueueAvail[idxQueue]
158	+ RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[availIdx % cQueueItems]),
159	&uDescIdx, sizeof(uDescIdx));
160	return uDescIdx;
161	}
162	#endif
163
164	DECLINLINE(uint16_t) virtioReadAvailRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
165	{
166	uint16_t uIdx = 0;
167	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
168	PDMDevHlpPCIPhysRead(pDevIns,
169	pVirtio->aGCPhysQueueAvail[idxQueue] + RT_UOFFSETOF(VIRTQ_AVAIL_T, uIdx),
170	&uIdx, sizeof(uIdx));
171	return uIdx;
172	}
173
174	DECLINLINE(bool) virtqIsEmpty(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
175	{
176	uint16_t uAvailGuest = virtioReadAvailRingIdx(pDevIns, pVirtio, idxQueue);
177	bool fEmpty = uAvailGuest == pVirtio->virtqState[idxQueue].uAvailIdx;
178
179	Log6Func(("%s: uAvailGuest=%u uAvailIdx=%u. (%s)\n",
180	VIRTQNAME(pVirtio, idxQueue), uAvailGuest, pVirtio->virtqState[idxQueue].uAvailIdx,
181	fEmpty ? "Queue empty" : "Queue has available descriptors"));
182	return fEmpty;
183	}
184
185	#if 0 /* unused - Will be used when VIRTIO_F_EVENT_IDX optional feature is implemented, VirtIO 1.0, 2.4.7 */
186	DECLINLINE(uint16_t) virtioReadAvailFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
187	{
188	uint16_t fFlags;
189	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
190	PDMDevHlpPCIPhysRead(pDevIns,
191	pVirtio->aGCPhysQueueAvail[idxQueue] + RT_UOFFSETOF(VIRTQ_AVAIL_T, fFlags),
192	&fFlags, sizeof(fFlags));
193	return fFlags;
194	}
195	#endif
196
197	#ifdef IN_RING3
198	DECLINLINE(uint16_t) virtioReadAvailUsedEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
199	{
200	uint16_t uUsedEventIdx;
201	/* VirtIO 1.0 uUsedEventIdx (used_event) immediately follows ring */
202	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
203	PDMDevHlpPCIPhysRead(pDevIns,
204	pVirtio->aGCPhysQueueAvail[idxQueue] + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtio->uQueueSize[idxQueue]]),
205	&uUsedEventIdx, sizeof(uUsedEventIdx));
206	return uUsedEventIdx;
207	}
208	#endif
209
210	/** @} */
211
212	/** @name Accessors for virtq used ring
213	* @{
214	*/
215
216	#ifdef IN_RING3
217	DECLINLINE(void) virtioWriteUsedElem(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
218	uint32_t usedIdx, uint32_t uDescIdx, uint32_t uLen)
219	{
220	VIRTQ_USED_ELEM_T elem = { uDescIdx, uLen };
221	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
222	uint16_t const cQueueItems = RT_MAX(pVirtio->uQueueSize[idxQueue], 1); /* Make sure to avoid div-by-zero. */
223	PDMDevHlpPCIPhysWrite(pDevIns,
224	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[usedIdx % cQueueItems]),
225	&elem, sizeof(elem));
226	}
227
228	DECLINLINE(void) virtioWriteUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint16_t fFlags)
229	{
230	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
231	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
232	PDMDevHlpPCIPhysWrite(pDevIns,
233	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
234	&fFlags, sizeof(fFlags));
235	}
236	#endif
237
238	DECLINLINE(void) virtioWriteUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint16_t uIdx)
239	{
240	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
241	PDMDevHlpPCIPhysWrite(pDevIns,
242	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
243	&uIdx, sizeof(uIdx));
244	}
245
246	#ifdef LOG_ENABLED
247	DECLINLINE(uint16_t) virtioReadUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
248	{
249	uint16_t uIdx = 0;
250	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
251	PDMDevHlpPCIPhysRead(pDevIns,
252	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
253	&uIdx, sizeof(uIdx));
254	return uIdx;
255	}
256	#endif
257
258	DECLINLINE(uint16_t) virtioReadUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
259	{
260	uint16_t fFlags = 0;
261	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
262	PDMDevHlpPCIPhysRead(pDevIns,
263	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
264	&fFlags, sizeof(fFlags));
265	return fFlags;
266	}
267
268	#if 0 /* unused - May be used when VIRTIO_F_EVENT_IDX optional feature is implemented VirtIO 1.0, 2.4.9.2*/
269	DECLINLINE(void) virtioWriteUsedAvailEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint32_t uAvailEventIdx)
270	{
271	/** VirtIO 1.0 uAvailEventIdx (avail_event) immediately follows ring */
272	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
273	PDMDevHlpPCIPhysWrite(pDevIns,
274	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[pVirtio->uQueueSize[idxQueue]]),
275	&uAvailEventIdx, sizeof(uAvailEventIdx));
276	}
277	#endif
278
279
280	/** @} */
281
282	void virtioCoreSgBufInit(PVIRTIOSGBUF pGcSgBuf, PVIRTIOSGSEG paSegs, size_t cSegs)
283	{
284	AssertPtr(pGcSgBuf);
285	Assert( (cSegs > 0 && VALID_PTR(paSegs)) \|\| (!cSegs && !paSegs));
286	Assert(cSegs < (~(unsigned)0 >> 1));
287
288	pGcSgBuf->paSegs = paSegs;
289	pGcSgBuf->cSegs = (unsigned)cSegs;
290	pGcSgBuf->idxSeg = 0;
291	if (cSegs && paSegs)
292	{
293	pGcSgBuf->gcPhysCur = paSegs[0].gcPhys;
294	pGcSgBuf->cbSegLeft = paSegs[0].cbSeg;
295	}
296	else
297	{
298	pGcSgBuf->gcPhysCur = 0;
299	pGcSgBuf->cbSegLeft = 0;
300	}
301	}
302
303	static RTGCPHYS virtioCoreSgBufGet(PVIRTIOSGBUF pGcSgBuf, size_t *pcbData)
304	{
305	size_t cbData;
306	RTGCPHYS pGcBuf;
307
308	/* Check that the S/G buffer has memory left. */
309	if (RT_LIKELY(pGcSgBuf->idxSeg < pGcSgBuf->cSegs && pGcSgBuf->cbSegLeft))
310	{ /* likely */ }
311	else
312	{
313	*pcbData = 0;
314	return 0;
315	}
316
317	AssertMsg( pGcSgBuf->cbSegLeft <= 128 * _1M
318	&& (RTGCPHYS)pGcSgBuf->gcPhysCur >= (RTGCPHYS)pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys
319	&& (RTGCPHYS)pGcSgBuf->gcPhysCur + pGcSgBuf->cbSegLeft <=
320	(RTGCPHYS)pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys + pGcSgBuf->paSegs[pGcSgBuf->idxSeg].cbSeg,
321	("pGcSgBuf->idxSeg=%d pGcSgBuf->cSegs=%d pGcSgBuf->gcPhysCur=%p pGcSgBuf->cbSegLeft=%zd "
322	"pGcSgBuf->paSegs[%d].gcPhys=%p pGcSgBuf->paSegs[%d].cbSeg=%zd\n",
323	pGcSgBuf->idxSeg, pGcSgBuf->cSegs, pGcSgBuf->gcPhysCur, pGcSgBuf->cbSegLeft,
324	pGcSgBuf->idxSeg, pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys, pGcSgBuf->idxSeg,
325	pGcSgBuf->paSegs[pGcSgBuf->idxSeg].cbSeg));
326
327	cbData = RT_MIN(*pcbData, pGcSgBuf->cbSegLeft);
328	pGcBuf = pGcSgBuf->gcPhysCur;
329	pGcSgBuf->cbSegLeft -= cbData;
330	if (!pGcSgBuf->cbSegLeft)
331	{
332	pGcSgBuf->idxSeg++;
333
334	if (pGcSgBuf->idxSeg < pGcSgBuf->cSegs)
335	{
336	pGcSgBuf->gcPhysCur = pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys;
337	pGcSgBuf->cbSegLeft = pGcSgBuf->paSegs[pGcSgBuf->idxSeg].cbSeg;
338	}
339	*pcbData = cbData;
340	}
341	else
342	pGcSgBuf->gcPhysCur = pGcSgBuf->gcPhysCur + cbData;
343
344	return pGcBuf;
345	}
346
347	void virtioCoreSgBufReset(PVIRTIOSGBUF pGcSgBuf)
348	{
349	AssertPtrReturnVoid(pGcSgBuf);
350
351	pGcSgBuf->idxSeg = 0;
352	if (pGcSgBuf->cSegs)
353	{
354	pGcSgBuf->gcPhysCur = pGcSgBuf->paSegs[0].gcPhys;
355	pGcSgBuf->cbSegLeft = pGcSgBuf->paSegs[0].cbSeg;
356	}
357	else
358	{
359	pGcSgBuf->gcPhysCur = 0;
360	pGcSgBuf->cbSegLeft = 0;
361	}
362	}
363
364	RTGCPHYS virtioCoreSgBufAdvance(PVIRTIOSGBUF pGcSgBuf, size_t cbAdvance)
365	{
366	AssertReturn(pGcSgBuf, 0);
367
368	size_t cbLeft = cbAdvance;
369	while (cbLeft)
370	{
371	size_t cbThisAdvance = cbLeft;
372	virtioCoreSgBufGet(pGcSgBuf, &cbThisAdvance);
373	if (!cbThisAdvance)
374	break;
375
376	cbLeft -= cbThisAdvance;
377	}
378	return cbAdvance - cbLeft;
379	}
380
381	RTGCPHYS virtioCoreSgBufGetNextSegment(PVIRTIOSGBUF pGcSgBuf, size_t *pcbSeg)
382	{
383	AssertReturn(pGcSgBuf, 0);
384	AssertPtrReturn(pcbSeg, 0);
385
386	if (!*pcbSeg)
387	*pcbSeg = pGcSgBuf->cbSegLeft;
388
389	return virtioCoreSgBufGet(pGcSgBuf, pcbSeg);
390	}
391
392	#ifdef LOG_ENABLED
393
394	/**
395	* Does a formatted hex dump using Log(()), recommend using VIRTIO_HEX_DUMP() macro to
396	* control enabling of logging efficiently.
397	*
398	* @param pv pointer to buffer to dump contents of
399	* @param cb count of characters to dump from buffer
400	* @param uBase base address of per-row address prefixing of hex output
401	* @param pszTitle Optional title. If present displays title that lists
402	* provided text with value of cb to indicate size next to it.
403	*/
404	void virtioCoreHexDump(uint8_t pv, uint32_t cb, uint32_t uBase, const char pszTitle)
405	{
406	if (pszTitle)
407	Log(("%s [%d bytes]:\n", pszTitle, cb));
408	for (uint32_t row = 0; row < RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
409	{
410	Log(("%04x: ", row * 16 + uBase)); /* line address */
411	for (uint8_t col = 0; col < 16; col++)
412	{
413	uint32_t idx = row * 16 + col;
414	if (idx >= cb)
415	Log(("-- %s", (col + 1) % 8 ? "" : " "));
416	else
417	Log(("%02x %s", pv[idx], (col + 1) % 8 ? "" : " "));
418	}
419	for (uint32_t idx = row * 16; idx < row * 16 + 16; idx++)
420	Log(("%c", (idx >= cb) ? ' ' : (pv[idx] >= 0x20 && pv[idx] <= 0x7e ? pv[idx] : '.')));
421	Log(("\n"));
422	}
423	Log(("\n"));
424	RT_NOREF2(uBase, pv);
425	}
426
427	/**
428	* Do a hex dump of memory in guest physical context
429	*
430	* @param gcPhys pointer to buffer to dump contents of
431	* @param cb count of characters to dump from buffer
432	* @param uBase base address of per-row address prefixing of hex output
433	* @param pszTitle Optional title. If present displays title that lists
434	* provided text with value of cb to indicate size next to it.
435	*/
436	void virtioCoreGcPhysHexDump(PPDMDEVINS pDevIns, RTGCPHYS gcPhys, uint32_t cb, uint32_t uBase, const char *pszTitle)
437	{
438	if (pszTitle)
439	Log(("%s [%d bytes]:\n", pszTitle, cb));
440	for (uint32_t row = 0; row < RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
441	{
442	uint8_t c;
443	Log(("%04x: ", row * 16 + uBase)); /* line address */
444	for (uint8_t col = 0; col < 16; col++)
445	{
446	uint32_t idx = row * 16 + col;
447	PDMDevHlpPCIPhysRead(pDevIns, gcPhys + idx, &c, 1);
448	if (idx >= cb)
449	Log(("-- %s", (col + 1) % 8 ? "" : " "));
450	else
451	Log(("%02x %s", c, (col + 1) % 8 ? "" : " "));
452	}
453	for (uint32_t idx = row * 16; idx < row * 16 + 16; idx++)
454	{
455	PDMDevHlpPCIPhysRead(pDevIns, gcPhys + idx, &c, 1);
456	Log(("%c", (idx >= cb) ? ' ' : (c >= 0x20 && c <= 0x7e ? c : '.')));
457	}
458	Log(("\n"));
459	}
460	Log(("\n"));
461	RT_NOREF(uBase);
462	}
463	#endif /* LOG_ENABLED */
464
465	/**
466	* Log memory-mapped I/O input or output value.
467	*
468	* This is designed to be invoked by macros that can make contextual assumptions
469	* (e.g. implicitly derive MACRO parameters from the invoking function). It is exposed
470	* for the VirtIO client doing the device-specific implementation in order to log in a
471	* similar fashion accesses to the device-specific MMIO configuration structure. Macros
472	* that leverage this function are found in virtioCommonCfgAccessed() and can be
473	* used as an example of how to use this effectively for the device-specific
474	* code.
475	*
476	* @param pszFunc To avoid displaying this function's name via __FUNCTION__ or LogFunc()
477	* @param pszMember Name of struct member
478	* @param pv pointer to value
479	* @param cb size of value
480	* @param uOffset offset into member where value starts
481	* @param fWrite True if write I/O
482	* @param fHasIndex True if the member is indexed
483	* @param idx The index if fHasIndex
484	*/
485	void virtioCoreLogMappedIoValue(const char pszFunc, const char pszMember, uint32_t uMemberSize,
486	const void *pv, uint32_t cb, uint32_t uOffset, int fWrite,
487	int fHasIndex, uint32_t idx)
488	{
489	if (!LogIs6Enabled())
490	return;
491
492	char szIdx[16];
493	if (fHasIndex)
494	RTStrPrintf(szIdx, sizeof(szIdx), "[%d]", idx);
495	else
496	szIdx[0] = '\0';
497
498	if (cb == 1 \|\| cb == 2 \|\| cb == 4 \|\| cb == 8)
499	{
500	char szDepiction[64];
501	size_t cchDepiction;
502	if (uOffset != 0 \|\| cb != uMemberSize) /* display bounds if partial member access */
503	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s[%d:%d]",
504	pszMember, szIdx, uOffset, uOffset + cb - 1);
505	else
506	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s", pszMember, szIdx);
507
508	/* padding */
509	if (cchDepiction < 30)
510	szDepiction[cchDepiction++] = ' ';
511	while (cchDepiction < 30)
512	szDepiction[cchDepiction++] = '.';
513	szDepiction[cchDepiction] = '\0';
514
515	RTUINT64U uValue;
516	uValue.u = 0;
517	memcpy(uValue.au8, pv, cb);
518	Log6(("%s: Guest %s %s %#0*RX64\n",
519	pszFunc, fWrite ? "wrote" : "read ", szDepiction, 2 + cb * 2, uValue.u));
520	}
521	else /* odd number or oversized access, ... log inline hex-dump style */
522	{
523	Log6(("%s: Guest %s %s%s[%d:%d]: %.*Rhxs\n",
524	pszFunc, fWrite ? "wrote" : "read ", pszMember,
525	szIdx, uOffset, uOffset + cb, cb, pv));
526	}
527	RT_NOREF2(fWrite, pszFunc);
528	}
529
530
531	/**
532	* Makes the MMIO-mapped Virtio uDeviceStatus registers non-cryptic
533	*/
534	DECLINLINE(void) virtioLogDeviceStatus(uint8_t bStatus)
535	{
536	if (bStatus == 0)
537	Log6(("RESET"));
538	else
539	{
540	int primed = 0;
541	if (bStatus & VIRTIO_STATUS_ACKNOWLEDGE)
542	Log6(("%sACKNOWLEDGE", primed++ ? "" : ""));
543	if (bStatus & VIRTIO_STATUS_DRIVER)
544	Log6(("%sDRIVER", primed++ ? " \| " : ""));
545	if (bStatus & VIRTIO_STATUS_FEATURES_OK)
546	Log6(("%sFEATURES_OK", primed++ ? " \| " : ""));
547	if (bStatus & VIRTIO_STATUS_DRIVER_OK)
548	Log6(("%sDRIVER_OK", primed++ ? " \| " : ""));
549	if (bStatus & VIRTIO_STATUS_FAILED)
550	Log6(("%sFAILED", primed++ ? " \| " : ""));
551	if (bStatus & VIRTIO_STATUS_DEVICE_NEEDS_RESET)
552	Log6(("%sNEEDS_RESET", primed++ ? " \| " : ""));
553	(void)primed;
554	}
555	}
556
557	#ifdef IN_RING3
558	/**
559	* Allocate client context for client to work with VirtIO-provided with queue
560	*
561	* @param pVirtio Pointer to the shared virtio state.
562	* @param idxQueue Queue number
563	* @param pcszName Name to give queue
564	*
565	* @returns VBox status code.
566	*/
567	int virtioCoreR3QueueAttach(PVIRTIOCORE pVirtio, uint16_t idxQueue, const char *pcszName)
568	{
569	LogFunc(("%s\n", pcszName));
570	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
571	pVirtq->uAvailIdx = 0;
572	pVirtq->uUsedIdx = 0;
573	pVirtq->fEventThresholdReached = false;
574	RTStrCopy(pVirtq->szVirtqName, sizeof(pVirtq->szVirtqName), pcszName);
575	return VINF_SUCCESS;
576	}
577	#endif /* IN_RING3 */
578
579
580	/**
581	* Check if the associated queue is empty
582	*
583	* @param pDevIns The device instance (for reading).
584	* @param pVirtio Pointer to the shared virtio state.
585	* @param idxQueue Queue number
586	*
587	* @retval true Queue is empty or unavailable.
588	* @retval false Queue is available and has entries
589	*/
590	bool virtioCoreQueueIsEmpty(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
591	{
592	if (pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
593	return virtqIsEmpty(pDevIns, pVirtio, idxQueue);
594	LogFunc(("VirtIO not ready: Returning 'true' for queue empty\n"));
595	return true;
596	}
597
598	#ifdef IN_RING3
599
600
601	int virtioCoreR3DescChainGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
602	uint16_t uHeadIdx, PPVIRTIO_DESC_CHAIN_T ppDescChain)
603	{
604	AssertReturn(ppDescChain, VERR_INVALID_POINTER);
605	*ppDescChain = NULL;
606
607	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
608
609	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
610
611	AssertMsgReturn(IS_DRIVER_OK(pVirtio) && pVirtio->uQueueEnable[idxQueue],
612	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
613
614	uint16_t uDescIdx = uHeadIdx;
615
616	Log6Func(("%s DESC CHAIN: (head) desc_idx=%u\n", pVirtq->szVirtqName, uHeadIdx));
617	RT_NOREF(pVirtq);
618
619	/*
620	* Allocate and initialize the descriptor chain structure.
621	*/
622	PVIRTIO_DESC_CHAIN_T pDescChain = (PVIRTIO_DESC_CHAIN_T)RTMemAllocZ(sizeof(VIRTIO_DESC_CHAIN_T));
623	AssertReturn(pDescChain, VERR_NO_MEMORY);
624	pDescChain->u32Magic = VIRTIO_DESC_CHAIN_MAGIC;
625	pDescChain->cRefs = 1;
626	pDescChain->uHeadIdx = uHeadIdx;
627	*ppDescChain = pDescChain;
628
629	/*
630	* Gather segments.
631	*/
632	VIRTQ_DESC_T desc;
633
634	uint32_t cbIn = 0;
635	uint32_t cbOut = 0;
636	uint32_t cSegsIn = 0;
637	uint32_t cSegsOut = 0;
638	PVIRTIOSGSEG paSegsIn = pDescChain->aSegsIn;
639	PVIRTIOSGSEG paSegsOut = pDescChain->aSegsOut;
640
641	do
642	{
643	PVIRTIOSGSEG pSeg;
644
645	/*
646	* Malicious guests may go beyond paSegsIn or paSegsOut boundaries by linking
647	* several descriptors into a loop. Since there is no legitimate way to get a sequences of
648	* linked descriptors exceeding the total number of descriptors in the ring (see @bugref{8620}),
649	* the following aborts I/O if breach and employs a simple log throttling algorithm to notify.
650	*/
651	if (cSegsIn + cSegsOut >= VIRTQ_MAX_SIZE)
652	{
653	static volatile uint32_t s_cMessages = 0;
654	static volatile uint32_t s_cThreshold = 1;
655	if (ASMAtomicIncU32(&s_cMessages) == ASMAtomicReadU32(&s_cThreshold))
656	{
657	LogRelMax(64, ("Too many linked descriptors; check if the guest arranges descriptors in a loop.\n"));
658	if (ASMAtomicReadU32(&s_cMessages) != 1)
659	LogRelMax(64, ("(the above error has occured %u times so far)\n", ASMAtomicReadU32(&s_cMessages)));
660	ASMAtomicWriteU32(&s_cThreshold, ASMAtomicReadU32(&s_cThreshold) * 10);
661	}
662	break;
663	}
664	RT_UNTRUSTED_VALIDATED_FENCE();
665
666	virtioReadDesc(pDevIns, pVirtio, idxQueue, uDescIdx, &desc);
667
668	if (desc.fFlags & VIRTQ_DESC_F_WRITE)
669	{
670	Log6Func(("%s IN desc_idx=%u seg=%u addr=%RGp cb=%u\n", VIRTQNAME(pVirtio, idxQueue), uDescIdx, cSegsIn, desc.GCPhysBuf, desc.cb));
671	cbIn += desc.cb;
672	pSeg = &paSegsIn[cSegsIn++];
673	}
674	else
675	{
676	Log6Func(("%s OUT desc_idx=%u seg=%u addr=%RGp cb=%u\n", VIRTQNAME(pVirtio, idxQueue), uDescIdx, cSegsOut, desc.GCPhysBuf, desc.cb));
677	cbOut += desc.cb;
678	pSeg = &paSegsOut[cSegsOut++];
679	}
680
681	pSeg->gcPhys = desc.GCPhysBuf;
682	pSeg->cbSeg = desc.cb;
683
684	uDescIdx = desc.uDescIdxNext;
685	} while (desc.fFlags & VIRTQ_DESC_F_NEXT);
686
687	/*
688	* Add segments to the descriptor chain structure.
689	*/
690	if (cSegsIn)
691	{
692	virtioCoreSgBufInit(&pDescChain->SgBufIn, paSegsIn, cSegsIn);
693	pDescChain->pSgPhysReturn = &pDescChain->SgBufIn;
694	pDescChain->cbPhysReturn = cbIn;
695	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsIn, cSegsIn);
696	}
697
698	if (cSegsOut)
699	{
700	virtioCoreSgBufInit(&pDescChain->SgBufOut, paSegsOut, cSegsOut);
701	pDescChain->pSgPhysSend = &pDescChain->SgBufOut;
702	pDescChain->cbPhysSend = cbOut;
703	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsOut, cSegsOut);
704	}
705
706	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsAllocated);
707	Log6Func(("%s -- segs OUT: %u (%u bytes) IN: %u (%u bytes) --\n", pVirtq->szVirtqName, cSegsOut, cbOut, cSegsIn, cbIn));
708
709	return VINF_SUCCESS;
710	}
711
712
713	/**
714	* Retains a reference to the given descriptor chain.
715	*
716	* @returns New reference count.
717	* @retval UINT32_MAX on invalid parameter.
718	* @param pDescChain The descriptor chain to reference.
719	*/
720	uint32_t virtioCoreR3DescChainRetain(PVIRTIO_DESC_CHAIN_T pDescChain)
721	{
722	AssertReturn(pDescChain, UINT32_MAX);
723	AssertReturn(pDescChain->u32Magic == VIRTIO_DESC_CHAIN_MAGIC, UINT32_MAX);
724	uint32_t cRefs = ASMAtomicIncU32(&pDescChain->cRefs);
725	Assert(cRefs > 1);
726	Assert(cRefs < 16);
727	return cRefs;
728	}
729
730
731	/**
732	* Releases a reference to the given descriptor chain.
733	*
734	* @returns New reference count.
735	* @retval 0 if freed or invalid parameter.
736	* @param pVirtio Pointer to the shared virtio state.
737	* @param pDescChain The descriptor chain to reference. NULL is quietly
738	* ignored (returns 0).
739	*/
740	uint32_t virtioCoreR3DescChainRelease(PVIRTIOCORE pVirtio, PVIRTIO_DESC_CHAIN_T pDescChain)
741	{
742	if (!pDescChain)
743	return 0;
744	AssertReturn(pDescChain, 0);
745	AssertReturn(pDescChain->u32Magic == VIRTIO_DESC_CHAIN_MAGIC, 0);
746	uint32_t cRefs = ASMAtomicDecU32(&pDescChain->cRefs);
747	Assert(cRefs < 16);
748	if (cRefs == 0)
749	{
750	pDescChain->u32Magic = ~VIRTIO_DESC_CHAIN_MAGIC;
751	RTMemFree(pDescChain);
752	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsFreed);
753	}
754	return cRefs;
755	}
756
757
758	/*
759	* Notifies guest (via ISR or MSI-X) of device configuration change
760	*
761	* @param pVirtio Pointer to the shared virtio state.
762	*/
763	void virtioCoreNotifyConfigChanged(PVIRTIOCORE pVirtio)
764	{
765	virtioKick(pVirtio->pDevIns, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig, false);
766	}
767
768	/**
769	* Enable or Disable notification for the specified queue
770	*
771	* @param pVirtio Pointer to the shared virtio state.
772	* @param idxQueue Queue number
773	* @param fEnabled Selects notification mode (enabled or disabled)
774	*/
775	void virtioCoreQueueSetNotify(PVIRTIOCORE pVirtio, uint16_t idxQueue, bool fEnabled)
776	{
777	if (pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
778	{
779	uint16_t fFlags = virtioReadUsedRingFlags(pVirtio->pDevIns, pVirtio, idxQueue);
780
781	if (fEnabled)
782	fFlags &= ~ VIRTQ_USED_F_NO_NOTIFY;
783	else
784	fFlags \|= VIRTQ_USED_F_NO_NOTIFY;
785
786	virtioWriteUsedRingFlags(pVirtio->pDevIns, pVirtio, idxQueue, fFlags);
787	}
788	}
789
790	/**
791	* Initiate orderly reset procedure. This is an exposed API for clients that might need it.
792	* Invoked by client to reset the device and driver (see VirtIO 1.0 section 2.1.1/2.1.2)
793	*
794	* @param pVirtio Pointer to the virtio state.
795	*/
796	void virtioCoreResetAll(PVIRTIOCORE pVirtio)
797	{
798	LogFunc(("\n"));
799	pVirtio->uDeviceStatus \|= VIRTIO_STATUS_DEVICE_NEEDS_RESET;
800	if (pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
801	{
802	pVirtio->fGenUpdatePending = true;
803	virtioKick(pVirtio->pDevIns, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig, false /* fForce */);
804	}
805	}
806	/**
807	* Get count of new (e.g. pending) elements in available ring.
808	*
809	* @param pDevIns The device instance.
810	* @param pVirtio Pointer to the shared virtio state.
811	* @param idxQueue Queue number
812	*
813	* @returns how many entries have been added to ring as a delta of the consumer's
814	* avail index and the queue's guest-side current avail index.
815	*/
816	int virtioCoreR3QueuePendingCount(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
817	{
818	uint16_t uAvailRingIdx = virtioReadAvailRingIdx(pDevIns, pVirtio, idxQueue);
819	uint16_t uNextAvailIdx = pVirtio->virtqState[idxQueue].uAvailIdx;
820	uint16_t uDelta = uAvailRingIdx - uNextAvailIdx;
821	if (uAvailRingIdx > uNextAvailIdx)
822	return uDelta;
823	return VIRTQ_MAX_CNT + uDelta;
824	}
825	/**
826	* Fetches descriptor chain using avail ring of indicated queue and converts the descriptor
827	* chain into its OUT (to device) and IN to guest components, but does NOT remove it from
828	* the 'avail' queue. I.e. doesn't advance the index. This can be used with virtioQueueSkip(),
829	* which does advance the avail index. Together they facilitate a mechanism that allows
830	* work with a queue element (descriptor chain) to be aborted if necessary, by not advancing
831	* the pointer, or, upon success calling the skip function (above) to move to the next element.
832	*
833	* Additionally it converts the OUT desc chain data to a contiguous virtual
834	* memory buffer for easy consumption by the caller. The caller must return the
835	* descriptor chain pointer via virtioCoreR3QueuePut() and then call virtioCoreQueueSync()
836	* at some point to return the data to the guest and complete the transaction.
837	*
838	* @param pDevIns The device instance.
839	* @param pVirtio Pointer to the shared virtio state.
840	* @param idxQueue Queue number
841	* @param ppDescChain Address to store pointer to descriptor chain that contains the
842	* pre-processed transaction information pulled from the virtq.
843	*
844	* @returns VBox status code:
845	* @retval VINF_SUCCESS Success
846	* @retval VERR_INVALID_STATE VirtIO not in ready state (asserted).
847	* @retval VERR_NOT_AVAILABLE If the queue is empty.
848	*/
849
850	int virtioCoreR3QueuePeek(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
851	PPVIRTIO_DESC_CHAIN_T ppDescChain)
852	{
853	return virtioCoreR3QueueGet(pDevIns, pVirtio, idxQueue, ppDescChain, false);
854	}
855
856	/**
857	* Skip the next entry in the specified queue (typically used with virtioCoreR3QueuePeek())
858	*
859	* @param pVirtio Pointer to the virtio state.
860	* @param idxQueue Index of queue
861	*/
862	int virtioCoreR3QueueSkip(PVIRTIOCORE pVirtio, uint16_t idxQueue)
863	{
864	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
865	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
866
867	AssertMsgReturn(IS_DRIVER_OK(pVirtio) && pVirtio->uQueueEnable[idxQueue],
868	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
869
870	if (virtioCoreQueueIsEmpty(pVirtio->pDevIns, pVirtio, idxQueue))
871	return VERR_NOT_AVAILABLE;
872
873	Log2Func(("%s avail_idx=%u\n", pVirtq->szVirtqName, pVirtq->uAvailIdx));
874	pVirtq->uAvailIdx++;
875
876	return VINF_SUCCESS;
877	}
878
879	/**
880	* Fetches descriptor chain using avail ring of indicated queue and converts the descriptor
881	* chain into its OUT (to device) and IN to guest components.
882	*
883	* Additionally it converts the OUT desc chain data to a contiguous virtual
884	* memory buffer for easy consumption by the caller. The caller must return the
885	* descriptor chain pointer via virtioCoreR3QueuePut() and then call virtioCoreQueueSync()
886	* at some point to return the data to the guest and complete the transaction.
887	*
888	* @param pDevIns The device instance.
889	* @param pVirtio Pointer to the shared virtio state.
890	* @param idxQueue Queue number
891	* @param ppDescChain Address to store pointer to descriptor chain that contains the
892	* pre-processed transaction information pulled from the virtq.
893	* Returned reference must be released by calling
894	* virtioCoreR3DescChainRelease().
895	* @param fRemove flags whether to remove desc chain from queue (false = peek)
896	*
897	* @returns VBox status code:
898	* @retval VINF_SUCCESS Success
899	* @retval VERR_INVALID_STATE VirtIO not in ready state (asserted).
900	* @retval VERR_NOT_AVAILABLE If the queue is empty.
901	*/
902	int virtioCoreR3QueueGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
903	PPVIRTIO_DESC_CHAIN_T ppDescChain, bool fRemove)
904	{
905	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
906
907	if (virtqIsEmpty(pDevIns, pVirtio, idxQueue))
908	return VERR_NOT_AVAILABLE;
909
910	uint16_t uHeadIdx = virtioReadAvailDescIdx(pDevIns, pVirtio, idxQueue, pVirtq->uAvailIdx);
911
912	if (fRemove)
913	pVirtq->uAvailIdx++;
914
915	int rc = virtioCoreR3DescChainGet(pDevIns, pVirtio, idxQueue, uHeadIdx, ppDescChain);
916	return rc;
917	}
918
919	/**
920	* Returns data to the guest to complete a transaction initiated by virtQueueGet().
921	*
922	* The caller passes in a pointer to a scatter-gather buffer of virtual memory segments
923	* and a pointer to the descriptor chain context originally derived from the pulled
924	* queue entry, and this function will write the virtual memory s/g buffer into the
925	* guest's physical memory free the descriptor chain. The caller handles the freeing
926	* (as needed) of the virtual memory buffer.
927	*
928	* @note This does a write-ahead to the used ring of the guest's queue. The data
929	* written won't be seen by the guest until the next call to virtioCoreQueueSync()
930	*
931	*
932	* @param pDevIns The device instance (for reading).
933	* @param pVirtio Pointer to the shared virtio state.
934	* @param idxQueue Queue number
935	*
936	* @param pSgVirtReturn Points to scatter-gather buffer of virtual memory
937	* segments the caller is returning to the guest.
938	*
939	* @param pDescChain This contains the context of the scatter-gather
940	* buffer originally pulled from the queue.
941	*
942	* @param fFence If true, put up copy fence (memory barrier) after
943	* copying to guest phys. mem.
944	*
945	* @returns VBox status code.
946	* @retval VINF_SUCCESS Success
947	* @retval VERR_INVALID_STATE VirtIO not in ready state
948	* @retval VERR_NOT_AVAILABLE Queue is empty
949	*
950	* @note This function will not release any reference to pDescChain. The
951	* caller must take care of that.
952	*/
953	int virtioCoreR3QueuePut(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, PRTSGBUF pSgVirtReturn,
954	PVIRTIO_DESC_CHAIN_T pDescChain, bool fFence)
955	{
956	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
957	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
958	PVIRTIOSGBUF pSgPhysReturn = pDescChain->pSgPhysReturn;
959
960	Assert(pDescChain->u32Magic == VIRTIO_DESC_CHAIN_MAGIC);
961	Assert(pDescChain->cRefs > 0);
962
963	AssertMsgReturn(IS_DRIVER_OK(pVirtio) /&& pVirtio->uQueueEnable[idxQueue]/,
964	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
965
966	Log6Func(("Copying client data to %s, desc chain (head desc_idx %d)\n",
967	VIRTQNAME(pVirtio, idxQueue), virtioReadUsedRingIdx(pDevIns, pVirtio, idxQueue)));
968
969	/* Copy s/g buf (virtual memory) to guest phys mem (IN direction). */
970
971	size_t cbCopy = 0, cbTotal = 0, cbRemain = 0;
972
973	if (pSgVirtReturn)
974	{
975	cbRemain = cbTotal = RTSgBufCalcTotalLength(pSgVirtReturn);
976	virtioCoreSgBufReset(pSgPhysReturn); /* Reset ptr because req data may have already been written */
977	while (cbRemain)
978	{
979	PVIRTIOSGSEG paSeg = &pSgPhysReturn->paSegs[pSgPhysReturn->idxSeg];
980	/** @todo r=bird: Shouldn't this be: RT_MIN(pSgVirtReturn->cbSegLeft, pSgPhysReturn->cbSegLeft); */
981	cbCopy = RT_MIN(pSgVirtReturn->cbSegLeft, paSeg->cbSeg - (size_t)(pSgPhysReturn->gcPhysCur - paSeg->gcPhys));
982	Assert(cbCopy > 0); /** @todo r=bird: There is no check that there is sufficient space in the output
983	* buffer (pSgPhysReturn), so we might loop here forever if the caller is careless,
984	* right? I'm pretty sure virtioScsiR3SendEvent could do more checks. */
985	PDMDevHlpPhysWrite(pDevIns, (RTGCPHYS)pSgPhysReturn->gcPhysCur, pSgVirtReturn->pvSegCur, cbCopy);
986	RTSgBufAdvance(pSgVirtReturn, cbCopy);
987	virtioCoreSgBufAdvance(pSgPhysReturn, cbCopy);
988	cbRemain -= cbCopy;
989	}
990
991	if (fFence)
992	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); /* needed? */
993
994	Assert(!(cbCopy >> 32));
995	}
996
997	/* If this write-ahead crosses threshold where the driver wants to get an event flag it */
998	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
999	if (pVirtq->uUsedIdx == virtioReadAvailUsedEvent(pDevIns, pVirtio, idxQueue))
1000	pVirtq->fEventThresholdReached = true;
1001
1002	/*
1003	* Place used buffer's descriptor in used ring but don't update used ring's slot index.
1004	* That will be done with a subsequent client call to virtioCoreQueueSync() */
1005	virtioWriteUsedElem(pDevIns, pVirtio, idxQueue, pVirtq->uUsedIdx++, pDescChain->uHeadIdx, (uint32_t)cbTotal);
1006
1007	if (pSgVirtReturn)
1008	Log6Func((".... Copied %zu bytes in %d segs to %u byte buffer, residual=%zu\n",
1009	cbTotal - cbRemain, pSgVirtReturn->cSegs, pDescChain->cbPhysReturn, pDescChain->cbPhysReturn - cbTotal));
1010
1011	Log6Func(("Write ahead used_idx=%u, %s used_idx=%u\n",
1012	pVirtq->uUsedIdx, VIRTQNAME(pVirtio, idxQueue), virtioReadUsedRingIdx(pDevIns, pVirtio, idxQueue)));
1013
1014	return VINF_SUCCESS;
1015	}
1016
1017	#endif /* IN_RING3 */
1018
1019	/**
1020	* Updates the indicated virtq's "used ring" descriptor index to match the
1021	* current write-head index, thus exposing the data added to the used ring by all
1022	* virtioCoreR3QueuePut() calls since the last sync. This should be called after one or
1023	* more virtioCoreR3QueuePut() calls to inform the guest driver there is data in the queue.
1024	* Explicit notifications (e.g. interrupt or MSI-X) will be sent to the guest,
1025	* depending on VirtIO features negotiated and conditions, otherwise the guest
1026	* will detect the update by polling. (see VirtIO 1.0
1027	* specification, Section 2.4 "Virtqueues").
1028	*
1029	* @param pDevIns The device instance.
1030	* @param pVirtio Pointer to the shared virtio state.
1031	* @param idxQueue Queue number
1032	*
1033	* @returns VBox status code.
1034	* @retval VINF_SUCCESS Success
1035	* @retval VERR_INVALID_STATE VirtIO not in ready state
1036	*/
1037	int virtioCoreQueueSync(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
1038	{
1039	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1040	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
1041
1042	AssertMsgReturn(IS_DRIVER_OK(pVirtio) && pVirtio->uQueueEnable[idxQueue],
1043	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1044
1045	Log6Func(("Updating %s used_idx from %u to %u\n",
1046	VIRTQNAME(pVirtio, idxQueue), virtioReadUsedRingIdx(pDevIns, pVirtio, idxQueue), pVirtq->uUsedIdx));
1047
1048	virtioWriteUsedRingIdx(pDevIns, pVirtio, idxQueue, pVirtq->uUsedIdx);
1049	virtioNotifyGuestDriver(pDevIns, pVirtio, idxQueue, false);
1050
1051	return VINF_SUCCESS;
1052	}
1053
1054	#ifdef IN_RING3
1055	/**
1056	*/
1057	static void virtioR3QueueNotified(PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, uint16_t idxQueue, uint16_t uNotifyIdx)
1058	{
1059	/* See VirtIO 1.0, section 4.1.5.2 It implies that idxQueue and uNotifyIdx should match.
1060	* Disregarding this notification may cause throughput to stop, however there's no way to know
1061	* which was queue was intended for wake-up if the two parameters disagree. */
1062
1063	AssertMsg(uNotifyIdx == idxQueue,
1064	("Guest kicked virtq %d's notify addr w/non-corresponding virtq idx %d\n",
1065	idxQueue, uNotifyIdx));
1066	RT_NOREF(uNotifyIdx);
1067
1068	AssertReturnVoid(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1069	Log6Func(("%s\n", pVirtio->virtqState[idxQueue].szVirtqName));
1070
1071	/* Inform client */
1072	pVirtioCC->pfnQueueNotified(pVirtio, pVirtioCC, idxQueue);
1073	}
1074	#endif /* IN_RING3 */
1075
1076	/**
1077	* Trigger MSI-X or INT# interrupt to notify guest of data added to used ring of
1078	* the specified virtq, depending on the interrupt configuration of the device
1079	* and depending on negotiated and realtime constraints flagged by the guest driver.
1080	*
1081	* See VirtIO 1.0 specification (section 2.4.7).
1082	*
1083	* @param pDevIns The device instance.
1084	* @param pVirtio Pointer to the shared virtio state.
1085	* @param idxQueue Queue to check for guest interrupt handling preference
1086	* @param fForce Overrides idxQueue, forcing notification regardless of driver's
1087	* notification preferences. This is a safeguard to prevent
1088	* stalls upon resuming the VM. VirtIO 1.0 specification Section 4.1.5.5
1089	* indicates spurious interrupts are harmless to guest driver's state,
1090	* as they only cause the guest driver to [re]scan queues for work to do.
1091	*/
1092	static void virtioNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, bool fForce)
1093	{
1094
1095	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1096	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
1097
1098	AssertMsgReturnVoid(IS_DRIVER_OK(pVirtio), ("Guest driver not in ready state.\n"));
1099	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1100	{
1101	if (pVirtq->fEventThresholdReached)
1102	{
1103	virtioKick(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtio->uQueueMsixVector[idxQueue], fForce);
1104	pVirtq->fEventThresholdReached = false;
1105	return;
1106	}
1107	Log6Func(("...skipping interrupt: VIRTIO_F_EVENT_IDX set but threshold not reached\n"));
1108	}
1109	else
1110	{
1111	/** If guest driver hasn't suppressed interrupts, interrupt */
1112	if (fForce \|\| !(virtioReadUsedRingFlags(pDevIns, pVirtio, idxQueue) & VIRTQ_AVAIL_F_NO_INTERRUPT))
1113	{
1114	virtioKick(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtio->uQueueMsixVector[idxQueue], fForce);
1115	return;
1116	}
1117	/** TEMPORARY DEBUGGING HACK, EVALUATE AND REMOVE */
1118	virtioKick(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtio->uQueueMsixVector[idxQueue], fForce);
1119
1120	Log6Func(("...skipping interrupt. Guest flagged VIRTQ_AVAIL_F_NO_INTERRUPT for queue\n"));
1121	}
1122	}
1123
1124	/**
1125	* Raise interrupt or MSI-X
1126	*
1127	* @param pDevIns The device instance.
1128	* @param pVirtio Pointer to the shared virtio state.
1129	* @param uCause Interrupt cause bit mask to set in PCI ISR port.
1130	* @param uVec MSI-X vector, if enabled
1131	* @param uForce True of out-of-band
1132	*/
1133	static int virtioKick(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uMsixVector, bool fForce)
1134	{
1135	if (fForce)
1136	Log6Func(("reason: resumed after suspend\n"));
1137	else
1138	if (uCause == VIRTIO_ISR_VIRTQ_INTERRUPT)
1139	Log6Func(("reason: buffer added to 'used' ring.\n"));
1140	else
1141	if (uCause == VIRTIO_ISR_DEVICE_CONFIG)
1142	Log6Func(("reason: device config change\n"));
1143
1144	if (!pVirtio->fMsiSupport)
1145	{
1146	pVirtio->uISR \|= uCause;
1147	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_HIGH);
1148	}
1149	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1150	PDMDevHlpPCISetIrq(pDevIns, uMsixVector, 1);
1151	return VINF_SUCCESS;
1152	}
1153
1154	/**
1155	* Lower interrupt (Called when guest reads ISR and when resetting)
1156	*
1157	* @param pDevIns The device instance.
1158	*/
1159	static void virtioLowerInterrupt(PPDMDEVINS pDevIns, uint16_t uMsixVector)
1160	{
1161	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1162	if (!pVirtio->fMsiSupport)
1163	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_LOW);
1164	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1165	PDMDevHlpPCISetIrq(pDevIns, pVirtio->uMsixConfig, PDM_IRQ_LEVEL_LOW);
1166	}
1167
1168	#ifdef IN_RING3
1169	static void virtioResetQueue(PVIRTIOCORE pVirtio, uint16_t idxQueue)
1170	{
1171	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1172	PVIRTQSTATE pVirtQ = &pVirtio->virtqState[idxQueue];
1173	pVirtQ->uAvailIdx = 0;
1174	pVirtQ->uUsedIdx = 0;
1175	pVirtio->uQueueEnable[idxQueue] = false;
1176	pVirtio->uQueueSize[idxQueue] = VIRTQ_MAX_SIZE;
1177	pVirtio->uQueueNotifyOff[idxQueue] = idxQueue;
1178	pVirtio->uQueueMsixVector[idxQueue] = idxQueue + 2;
1179
1180	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1181	pVirtio->uQueueMsixVector[idxQueue] = VIRTIO_MSI_NO_VECTOR;
1182
1183	virtioLowerInterrupt(pVirtio->pDevIns, pVirtio->uQueueMsixVector[idxQueue]);
1184	}
1185
1186	static void virtioResetDevice(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
1187	{
1188	Log2Func(("\n"));
1189	pVirtio->uDeviceFeaturesSelect = 0;
1190	pVirtio->uDriverFeaturesSelect = 0;
1191	pVirtio->uConfigGeneration = 0;
1192	pVirtio->uDeviceStatus = 0;
1193	pVirtio->uISR = 0;
1194
1195	if (!pVirtio->fMsiSupport)
1196	virtioLowerInterrupt(pDevIns, 0);
1197	else
1198	{
1199	virtioLowerInterrupt(pDevIns, pVirtio->uMsixConfig);
1200	for (int i = 0; i < VIRTQ_MAX_CNT; i++)
1201	{
1202	virtioLowerInterrupt(pDevIns, pVirtio->uQueueMsixVector[i]);
1203	pVirtio->uQueueMsixVector[i];
1204	}
1205	}
1206
1207	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1208	pVirtio->uMsixConfig = VIRTIO_MSI_NO_VECTOR;
1209
1210	for (uint16_t idxQueue = 0; idxQueue < VIRTQ_MAX_CNT; idxQueue++)
1211	virtioResetQueue(pVirtio, idxQueue);
1212	}
1213
1214	/**
1215	* Invoked by this implementation when guest driver resets the device.
1216	* The driver itself will not until the device has read the status change.
1217	*/
1218	static void virtioGuestR3WasReset(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1219	{
1220	LogFunc(("Guest reset the device\n"));
1221
1222	/* Let the client know */
1223	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 0);
1224	virtioResetDevice(pDevIns, pVirtio);
1225	}
1226	#endif /* IN_RING3 */
1227
1228	/**
1229	* Handle accesses to Common Configuration capability
1230	*
1231	* @returns VBox status code
1232	*
1233	* @param pDevIns The device instance.
1234	* @param pVirtio Pointer to the shared virtio state.
1235	* @param pVirtioCC Pointer to the current context virtio state.
1236	* @param fWrite Set if write access, clear if read access.
1237	* @param offCfg The common configuration capability offset.
1238	* @param cb Number of bytes to read or write
1239	* @param pv Pointer to location to write to or read from
1240	*/
1241	static int virtioCommonCfgAccessed(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1242	int fWrite, uint32_t offCfg, unsigned cb, void *pv)
1243	{
1244	/**
1245	* This macro resolves to boolean true if the implied parameters, offCfg and cb,
1246	* match the field offset and size of a field in the Common Cfg struct, (or if
1247	* it is a 64-bit field, if it accesses either 32-bit part as a 32-bit access)
1248	* This is mandated by section 4.1.3.1 of the VirtIO 1.0 specification)
1249	*
1250	* @param member Member of VIRTIO_PCI_COMMON_CFG_T
1251	* @param offCfg Implied parameter: Offset into VIRTIO_PCI_COMMON_CFG_T
1252	* @param cb Implied parameter: Number of bytes to access
1253	* @result true or false
1254	*/
1255	#define MATCH_COMMON_CFG(member) \
1256	( ( RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member) == 8 \
1257	&& ( offCfg == RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member) \
1258	\|\| offCfg == RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member) + sizeof(uint32_t)) \
1259	&& cb == sizeof(uint32_t)) \
1260	\|\| ( offCfg == RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member) \
1261	&& cb == RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member)) )
1262
1263	#ifdef LOG_ENABLED
1264	# define LOG_COMMON_CFG_ACCESS(member, a_offIntra) \
1265	if (LogIs7Enabled()) { \
1266	virtioCoreLogMappedIoValue(__FUNCTION__, #member, RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member), \
1267	pv, cb, a_offIntra, fWrite, false, 0); \
1268	}
1269	# define LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, a_offIntra) \
1270	if (LogIs7Enabled()) { \
1271	virtioCoreLogMappedIoValue(__FUNCTION__, #member, RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member), \
1272	pv, cb, a_offIntra, fWrite, true, idx); \
1273	}
1274	#else
1275	# define LOG_COMMON_CFG_ACCESS(member, a_offIntra) do { } while (0)
1276	# define LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, a_offIntra) do { } while (0)
1277	#endif
1278
1279	#define COMMON_CFG_ACCESSOR(member) \
1280	do \
1281	{ \
1282	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1283	if (fWrite) \
1284	memcpy((char )&pVirtio->member + offIntra, (const char )pv, cb); \
1285	else \
1286	memcpy(pv, (const char *)&pVirtio->member + offIntra, cb); \
1287	LOG_COMMON_CFG_ACCESS(member, offIntra); \
1288	} while(0)
1289
1290	#define COMMON_CFG_ACCESSOR_INDEXED(member, idx) \
1291	do \
1292	{ \
1293	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1294	if (fWrite) \
1295	memcpy((char *)&pVirtio->member[idx] + offIntra, pv, cb); \
1296	else \
1297	memcpy(pv, (const char *)&pVirtio->member[idx] + offIntra, cb); \
1298	LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, offIntra); \
1299	} while(0)
1300
1301	#define COMMON_CFG_ACCESSOR_READONLY(member) \
1302	do \
1303	{ \
1304	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1305	if (fWrite) \
1306	LogFunc(("Guest attempted to write readonly virtio_pci_common_cfg.%s\n", #member)); \
1307	else \
1308	{ \
1309	memcpy(pv, (const char *)&pVirtio->member + offIntra, cb); \
1310	LOG_COMMON_CFG_ACCESS(member, offIntra); \
1311	} \
1312	} while(0)
1313
1314	#define COMMON_CFG_ACCESSOR_INDEXED_READONLY(member, idx) \
1315	do \
1316	{ \
1317	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1318	if (fWrite) \
1319	LogFunc(("Guest attempted to write readonly virtio_pci_common_cfg.%s[%d]\n", #member, idx)); \
1320	else \
1321	{ \
1322	memcpy(pv, (char const *)&pVirtio->member[idx] + offIntra, cb); \
1323	LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, offIntra); \
1324	} \
1325	} while(0)
1326
1327
1328	int rc = VINF_SUCCESS;
1329	uint64_t val;
1330	if (MATCH_COMMON_CFG(uDeviceFeatures))
1331	{
1332	if (fWrite) /* Guest WRITE pCommonCfg>uDeviceFeatures */
1333	{
1334	LogFunc(("Guest attempted to write readonly virtio_pci_common_cfg.device_feature\n"));
1335	return VINF_SUCCESS;
1336	}
1337	else /* Guest READ pCommonCfg->uDeviceFeatures */
1338	{
1339	switch (pVirtio->uDeviceFeaturesSelect)
1340	{
1341	case 0:
1342	val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1343	memcpy(pv, &val, cb);
1344	LOG_COMMON_CFG_ACCESS(uDeviceFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDeviceFeatures));
1345	break;
1346	case 1:
1347	val = pVirtio->uDeviceFeatures >> 32;
1348	memcpy(pv, &val, cb);
1349	LOG_COMMON_CFG_ACCESS(uDeviceFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDeviceFeatures) + 4);
1350	break;
1351	default:
1352	LogFunc(("Guest read uDeviceFeatures with out of range selector (%#x), returning 0\n",
1353	pVirtio->uDeviceFeaturesSelect));
1354	return VINF_IOM_MMIO_UNUSED_00;
1355	}
1356	}
1357	}
1358	else if (MATCH_COMMON_CFG(uDriverFeatures))
1359	{
1360	if (fWrite) /* Guest WRITE pCommonCfg->udriverFeatures */
1361	{
1362	switch (pVirtio->uDriverFeaturesSelect)
1363	{
1364	case 0:
1365	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1366	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures));
1367	break;
1368	case 1:
1369	memcpy((char *)&pVirtio->uDriverFeatures + sizeof(uint32_t), pv, cb);
1370	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures) + 4);
1371	break;
1372	default:
1373	LogFunc(("Guest wrote uDriverFeatures with out of range selector (%#x), returning 0\n",
1374	pVirtio->uDriverFeaturesSelect));
1375	return VINF_SUCCESS;
1376	}
1377	}
1378	else /* Guest READ pCommonCfg->udriverFeatures */
1379	{
1380	switch (pVirtio->uDriverFeaturesSelect)
1381	{
1382	case 0:
1383	val = pVirtio->uDriverFeatures & 0xffffffff;
1384	memcpy(pv, &val, cb);
1385	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures));
1386	break;
1387	case 1:
1388	val = (pVirtio->uDriverFeatures >> 32) & 0xffffffff;
1389	memcpy(pv, &val, cb);
1390	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures) + 4);
1391	break;
1392	default:
1393	LogFunc(("Guest read uDriverFeatures with out of range selector (%#x), returning 0\n",
1394	pVirtio->uDriverFeaturesSelect));
1395	return VINF_IOM_MMIO_UNUSED_00;
1396	}
1397	}
1398	}
1399	else if (MATCH_COMMON_CFG(uNumQueues))
1400	{
1401	if (fWrite)
1402	{
1403	Log2Func(("Guest attempted to write readonly virtio_pci_common_cfg.num_queues\n"));
1404	return VINF_SUCCESS;
1405	}
1406	else
1407	{
1408	(uint16_t )pv = VIRTQ_MAX_CNT;
1409	LOG_COMMON_CFG_ACCESS(uNumQueues, 0);
1410	}
1411	}
1412	else if (MATCH_COMMON_CFG(uDeviceStatus))
1413	{
1414	if (fWrite) /* Guest WRITE pCommonCfg->uDeviceStatus */
1415	{
1416	uint8_t const fNewStatus = (uint8_t )pv;
1417	Log7Func(("Guest wrote uDeviceStatus (%#x, was %#x, change #%x) ................ (",
1418	fNewStatus, pVirtio->uDeviceStatus, fNewStatus ^ pVirtio->uDeviceStatus));
1419	if (LogIs7Enabled())
1420	virtioLogDeviceStatus(fNewStatus);
1421	Log7((")\n"));
1422
1423	/* If the status changed or we were reset, we need to go to ring-3 as
1424	it requires notifying the parent device. */
1425	bool const fStatusChanged = (fNewStatus & VIRTIO_STATUS_DRIVER_OK)
1426	!= (pVirtio->uPrevDeviceStatus & VIRTIO_STATUS_DRIVER_OK);
1427	#ifndef IN_RING3
1428	if (fStatusChanged \|\| fNewStatus == 0)
1429	{
1430	Log6Func(("=>ring3\n"));
1431	return VINF_IOM_R3_MMIO_WRITE;
1432	}
1433	#endif
1434	pVirtio->uDeviceStatus = fNewStatus;
1435
1436	#ifdef IN_RING3
1437	/*
1438	* Notify client only if status actually changed from last time and when we're reset.
1439	*/
1440	if (pVirtio->uDeviceStatus == 0)
1441	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1442	if (fStatusChanged)
1443	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, fNewStatus & VIRTIO_STATUS_DRIVER_OK);
1444	#endif
1445	/*
1446	* Save the current status for the next write so we can see what changed.
1447	*/
1448	pVirtio->uPrevDeviceStatus = pVirtio->uDeviceStatus;
1449	}
1450	else /* Guest READ pCommonCfg->uDeviceStatus */
1451	{
1452	Log7Func(("Guest read uDeviceStatus ................ ("));
1453	(uint8_t )pv = pVirtio->uDeviceStatus;
1454	if (LogIs7Enabled())
1455	virtioLogDeviceStatus(pVirtio->uDeviceStatus);
1456	Log7((")\n"));
1457	}
1458	}
1459	else
1460	if (MATCH_COMMON_CFG(uMsixConfig))
1461	COMMON_CFG_ACCESSOR(uMsixConfig);
1462	else
1463	if (MATCH_COMMON_CFG(uDeviceFeaturesSelect))
1464	COMMON_CFG_ACCESSOR(uDeviceFeaturesSelect);
1465	else
1466	if (MATCH_COMMON_CFG(uDriverFeaturesSelect))
1467	COMMON_CFG_ACCESSOR(uDriverFeaturesSelect);
1468	else
1469	if (MATCH_COMMON_CFG(uConfigGeneration))
1470	COMMON_CFG_ACCESSOR_READONLY(uConfigGeneration);
1471	else
1472	if (MATCH_COMMON_CFG(uQueueSelect))
1473	COMMON_CFG_ACCESSOR(uQueueSelect);
1474	else
1475	if (MATCH_COMMON_CFG(uQueueSize))
1476	COMMON_CFG_ACCESSOR_INDEXED(uQueueSize, pVirtio->uQueueSelect);
1477	else
1478	if (MATCH_COMMON_CFG(uQueueMsixVector))
1479	COMMON_CFG_ACCESSOR_INDEXED(uQueueMsixVector, pVirtio->uQueueSelect);
1480	else
1481	if (MATCH_COMMON_CFG(uQueueEnable))
1482	COMMON_CFG_ACCESSOR_INDEXED(uQueueEnable, pVirtio->uQueueSelect);
1483	else
1484	if (MATCH_COMMON_CFG(uQueueNotifyOff))
1485	COMMON_CFG_ACCESSOR_INDEXED_READONLY(uQueueNotifyOff, pVirtio->uQueueSelect);
1486	else
1487	if (MATCH_COMMON_CFG(aGCPhysQueueDesc))
1488	COMMON_CFG_ACCESSOR_INDEXED(aGCPhysQueueDesc, pVirtio->uQueueSelect);
1489	else
1490	if (MATCH_COMMON_CFG(aGCPhysQueueAvail))
1491	COMMON_CFG_ACCESSOR_INDEXED(aGCPhysQueueAvail, pVirtio->uQueueSelect);
1492	else
1493	if (MATCH_COMMON_CFG(aGCPhysQueueUsed))
1494	COMMON_CFG_ACCESSOR_INDEXED(aGCPhysQueueUsed, pVirtio->uQueueSelect);
1495	else
1496	{
1497	Log2Func(("Bad guest %s access to virtio_pci_common_cfg: offCfg=%#x (%d), cb=%d\n",
1498	fWrite ? "write" : "read ", offCfg, offCfg, cb));
1499	return fWrite ? VINF_SUCCESS : VINF_IOM_MMIO_UNUSED_00;
1500	}
1501
1502	#undef COMMON_CFG_ACCESSOR_READONLY
1503	#undef COMMON_CFG_ACCESSOR_INDEXED_READONLY
1504	#undef COMMON_CFG_ACCESSOR_INDEXED
1505	#undef COMMON_CFG_ACCESSOR
1506	#undef LOG_COMMON_CFG_ACCESS_INDEXED
1507	#undef LOG_COMMON_CFG_ACCESS
1508	#undef MATCH_COMMON_CFG
1509	#ifndef IN_RING3
1510	RT_NOREF(pDevIns, pVirtioCC);
1511	#endif
1512	return rc;
1513	}
1514
1515	/**
1516	* @callback_method_impl{FNIOMMMIONEWREAD,
1517	* Memory mapped I/O Handler for PCI Capabilities read operations.}
1518	*
1519	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
1520	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is limited to cb == 1, cb == 2, or cb==4 type reads.
1521	*
1522	*/
1523	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioRead(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void pv, unsigned cb)
1524	{
1525	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1526	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1527	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
1528	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
1529
1530	uint32_t offIntra;
1531	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocDeviceCap))
1532	{
1533	#ifdef IN_RING3
1534	/*
1535	* Callback to client to manage device-specific configuration.
1536	*/
1537	VBOXSTRICTRC rcStrict = pVirtioCC->pfnDevCapRead(pDevIns, offIntra, pv, cb);
1538
1539	/*
1540	* Additionally, anytime any part of the device-specific configuration (which our client maintains)
1541	* is READ it needs to be checked to see if it changed since the last time any part was read, in
1542	* order to maintain the config generation (see VirtIO 1.0 spec, section 4.1.4.3.1)
1543	*/
1544	bool fDevSpecificFieldChanged = !!memcmp(pVirtioCC->pbDevSpecificCfg + offIntra,
1545	pVirtioCC->pbPrevDevSpecificCfg + offIntra,
1546	RT_MIN(cb, pVirtioCC->cbDevSpecificCfg - offIntra));
1547
1548	memcpy(pVirtioCC->pbPrevDevSpecificCfg, pVirtioCC->pbDevSpecificCfg, pVirtioCC->cbDevSpecificCfg);
1549
1550	if (pVirtio->fGenUpdatePending \|\| fDevSpecificFieldChanged)
1551	{
1552	++pVirtio->uConfigGeneration;
1553	Log6Func(("Bumped cfg. generation to %d because %s%s\n",
1554	pVirtio->uConfigGeneration,
1555	fDevSpecificFieldChanged ? "<dev cfg changed> " : "",
1556	pVirtio->fGenUpdatePending ? "<update was pending>" : ""));
1557	pVirtio->fGenUpdatePending = false;
1558	}
1559
1560	virtioLowerInterrupt(pDevIns, 0);
1561	return rcStrict;
1562	#else
1563	return VINF_IOM_R3_MMIO_READ;
1564	#endif
1565	}
1566
1567	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocCommonCfgCap))
1568	return virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, false /* fWrite */, offIntra, cb, pv);
1569
1570	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocIsrCap) && cb == sizeof(uint8_t))
1571	{
1572	(uint8_t )pv = pVirtio->uISR;
1573	Log6Func(("Read and clear ISR\n"));
1574	pVirtio->uISR = 0; /* VirtIO specification requires reads of ISR to clear it */
1575	virtioLowerInterrupt(pDevIns, 0);
1576	return VINF_SUCCESS;
1577	}
1578
1579	ASSERT_GUEST_MSG_FAILED(("Bad read access to mapped capabilities region: off=%RGp cb=%u\n", off, cb));
1580	return VINF_IOM_MMIO_UNUSED_00;
1581	}
1582
1583	/**
1584	* @callback_method_impl{FNIOMMMIONEWREAD,
1585	* Memory mapped I/O Handler for PCI Capabilities write operations.}
1586	*
1587	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
1588	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is limited
1589	* to cb == 1, cb == 2, or cb==4 type writes.
1590	*/
1591	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioWrite(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void const pv, unsigned cb)
1592	{
1593	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1594	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1595
1596	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
1597
1598	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
1599
1600	uint32_t offIntra;
1601	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocDeviceCap))
1602	{
1603	#ifdef IN_RING3
1604	/*
1605	* Pass this MMIO write access back to the client to handle
1606	*/
1607	return pVirtioCC->pfnDevCapWrite(pDevIns, offIntra, pv, cb);
1608	#else
1609	return VINF_IOM_R3_MMIO_WRITE;
1610	#endif
1611	}
1612
1613	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocCommonCfgCap))
1614	return virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, true /* fWrite /, offIntra, cb, (void )pv);
1615
1616	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocIsrCap) && cb == sizeof(uint8_t))
1617	{
1618	pVirtio->uISR = (uint8_t )pv;
1619	Log6Func(("Setting uISR = 0x%02x (virtq interrupt: %d, dev confg interrupt: %d)\n",
1620	pVirtio->uISR & 0xff,
1621	pVirtio->uISR & VIRTIO_ISR_VIRTQ_INTERRUPT,
1622	RT_BOOL(pVirtio->uISR & VIRTIO_ISR_DEVICE_CONFIG)));
1623	return VINF_SUCCESS;
1624	}
1625
1626	/* This should be guest driver dropping index of a new descriptor in avail ring */
1627	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocNotifyCap) && cb == sizeof(uint16_t))
1628	{
1629	#ifdef IN_RING3
1630	virtioR3QueueNotified(pVirtio, pVirtioCC, offIntra / VIRTIO_NOTIFY_OFFSET_MULTIPLIER, (uint16_t )pv);
1631	return VINF_SUCCESS;
1632	#else
1633	return VINF_IOM_R3_MMIO_WRITE;
1634	#endif
1635	}
1636
1637	ASSERT_GUEST_MSG_FAILED(("Bad write access to mapped capabilities region: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb));
1638	return VINF_SUCCESS;
1639	}
1640
1641	#ifdef IN_RING3
1642
1643	/**
1644	* @callback_method_impl{FNPCICONFIGREAD}
1645	*/
1646	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigRead(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
1647	uint32_t uAddress, unsigned cb, uint32_t *pu32Value)
1648	{
1649	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1650	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1651	RT_NOREF(pPciDev);
1652
1653	Log7Func(("pDevIns=%p pPciDev=%p uAddress=%#x cb=%u pu32Value=%p\n",
1654	pDevIns, pPciDev, uAddress, cb, pu32Value));
1655	if (uAddress == pVirtio->uPciCfgDataOff)
1656	{
1657	/*
1658	* VirtIO 1.0 spec section 4.1.4.7 describes a required alternative access capability
1659	* whereby the guest driver can specify a bar, offset, and length via the PCI configuration space
1660	* (the virtio_pci_cfg_cap capability), and access data items.
1661	*/
1662	uint32_t uLength = pVirtioCC->pPciCfgCap->pciCap.uLength;
1663	uint32_t uOffset = pVirtioCC->pPciCfgCap->pciCap.uOffset;
1664	uint8_t uBar = pVirtioCC->pPciCfgCap->pciCap.uBar;
1665
1666	if ( (uLength != 1 && uLength != 2 && uLength != 4)
1667	\|\| cb != uLength
1668	\|\| uBar != VIRTIO_REGION_PCI_CAP)
1669	{
1670	ASSERT_GUEST_MSG_FAILED(("Guest read virtio_pci_cfg_cap.pci_cfg_data using mismatching config. Ignoring\n"));
1671	*pu32Value = UINT32_MAX;
1672	return VINF_SUCCESS;
1673	}
1674
1675	VBOXSTRICTRC rcStrict = virtioMmioRead(pDevIns, pVirtio, uOffset, pu32Value, cb);
1676	Log2Func(("virtio: Guest read virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%d, length=%d, result=%d -> %Rrc\n",
1677	uBar, uOffset, uLength, *pu32Value, VBOXSTRICTRC_VAL(rcStrict)));
1678	return rcStrict;
1679	}
1680	return VINF_PDM_PCI_DO_DEFAULT;
1681	}
1682
1683	/**
1684	* @callback_method_impl{FNPCICONFIGWRITE}
1685	*/
1686	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigWrite(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
1687	uint32_t uAddress, unsigned cb, uint32_t u32Value)
1688	{
1689	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1690	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1691	RT_NOREF(pPciDev);
1692
1693	Log7Func(("pDevIns=%p pPciDev=%p uAddress=%#x cb=%u u32Value=%#x\n", pDevIns, pPciDev, uAddress, cb, u32Value));
1694	if (uAddress == pVirtio->uPciCfgDataOff)
1695	{
1696	/* VirtIO 1.0 spec section 4.1.4.7 describes a required alternative access capability
1697	* whereby the guest driver can specify a bar, offset, and length via the PCI configuration space
1698	* (the virtio_pci_cfg_cap capability), and access data items. */
1699
1700	uint32_t uLength = pVirtioCC->pPciCfgCap->pciCap.uLength;
1701	uint32_t uOffset = pVirtioCC->pPciCfgCap->pciCap.uOffset;
1702	uint8_t uBar = pVirtioCC->pPciCfgCap->pciCap.uBar;
1703
1704	if ( (uLength != 1 && uLength != 2 && uLength != 4)
1705	\|\| cb != uLength
1706	\|\| uBar != VIRTIO_REGION_PCI_CAP)
1707	{
1708	ASSERT_GUEST_MSG_FAILED(("Guest write virtio_pci_cfg_cap.pci_cfg_data using mismatching config. Ignoring\n"));
1709	return VINF_SUCCESS;
1710	}
1711
1712	VBOXSTRICTRC rcStrict = virtioMmioWrite(pDevIns, pVirtio, uOffset, &u32Value, cb);
1713	Log2Func(("Guest wrote virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%x, length=%x, value=%d -> %Rrc\n",
1714	uBar, uOffset, uLength, u32Value, VBOXSTRICTRC_VAL(rcStrict)));
1715	return rcStrict;
1716	}
1717	return VINF_PDM_PCI_DO_DEFAULT;
1718	}
1719
1720
1721	/*********************************************************************************************************************************
1722	* Saved state. *
1723	*********************************************************************************************************************************/
1724
1725	/**
1726	* Called from the FNSSMDEVSAVEEXEC function of the device.
1727	*
1728	* @param pVirtio Pointer to the shared virtio state.
1729	* @param pHlp The ring-3 device helpers.
1730	* @param pSSM The saved state handle.
1731	* @returns VBox status code.
1732	*/
1733	int virtioCoreR3SaveExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM)
1734	{
1735	LogFunc(("\n"));
1736	pHlp->pfnSSMPutU64(pSSM, VIRTIO_SAVEDSTATE_MARKER);
1737	pHlp->pfnSSMPutU32(pSSM, VIRTIO_SAVEDSTATE_VERSION);
1738
1739	pHlp->pfnSSMPutBool(pSSM, pVirtio->fGenUpdatePending);
1740	pHlp->pfnSSMPutU8(pSSM, pVirtio->uDeviceStatus);
1741	pHlp->pfnSSMPutU8(pSSM, pVirtio->uConfigGeneration);
1742	pHlp->pfnSSMPutU8(pSSM, pVirtio->uPciCfgDataOff);
1743	pHlp->pfnSSMPutU8(pSSM, pVirtio->uISR);
1744	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueSelect);
1745	pHlp->pfnSSMPutU32(pSSM, pVirtio->uDeviceFeaturesSelect);
1746	pHlp->pfnSSMPutU32(pSSM, pVirtio->uDriverFeaturesSelect);
1747	pHlp->pfnSSMPutU64(pSSM, pVirtio->uDriverFeatures);
1748
1749	for (uint32_t i = 0; i < VIRTQ_MAX_CNT; i++)
1750	{
1751	pHlp->pfnSSMPutGCPhys64(pSSM, pVirtio->aGCPhysQueueDesc[i]);
1752	pHlp->pfnSSMPutGCPhys64(pSSM, pVirtio->aGCPhysQueueAvail[i]);
1753	pHlp->pfnSSMPutGCPhys64(pSSM, pVirtio->aGCPhysQueueUsed[i]);
1754	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueNotifyOff[i]);
1755	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueMsixVector[i]);
1756	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueEnable[i]);
1757	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueSize[i]);
1758	pHlp->pfnSSMPutU16(pSSM, pVirtio->virtqState[i].uAvailIdx);
1759	pHlp->pfnSSMPutU16(pSSM, pVirtio->virtqState[i].uUsedIdx);
1760	int rc = pHlp->pfnSSMPutMem(pSSM, pVirtio->virtqState[i].szVirtqName, 32);
1761	AssertRCReturn(rc, rc);
1762	}
1763
1764	return VINF_SUCCESS;
1765	}
1766
1767	/**
1768	* Called from the FNSSMDEVLOADEXEC function of the device.
1769	*
1770	* @param pVirtio Pointer to the shared virtio state.
1771	* @param pHlp The ring-3 device helpers.
1772	* @param pSSM The saved state handle.
1773	* @returns VBox status code.
1774	*/
1775	int virtioCoreR3LoadExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM)
1776	{
1777	LogFunc(("\n"));
1778	/*
1779	* Check the marker and (embedded) version number.
1780	*/
1781	uint64_t uMarker = 0;
1782	int rc = pHlp->pfnSSMGetU64(pSSM, &uMarker);
1783	AssertRCReturn(rc, rc);
1784	if (uMarker != VIRTIO_SAVEDSTATE_MARKER)
1785	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
1786	N_("Expected marker value %#RX64 found %#RX64 instead"),
1787	VIRTIO_SAVEDSTATE_MARKER, uMarker);
1788	uint32_t uVersion = 0;
1789	rc = pHlp->pfnSSMGetU32(pSSM, &uVersion);
1790	AssertRCReturn(rc, rc);
1791	if (uVersion != VIRTIO_SAVEDSTATE_VERSION)
1792	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
1793	N_("Unsupported virtio version: %u"), uVersion);
1794	/*
1795	* Load the state.
1796	*/
1797	pHlp->pfnSSMGetBool(pSSM, &pVirtio->fGenUpdatePending);
1798	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uDeviceStatus);
1799	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uConfigGeneration);
1800	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uPciCfgDataOff);
1801	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uISR);
1802	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueSelect);
1803	pHlp->pfnSSMGetU32(pSSM, &pVirtio->uDeviceFeaturesSelect);
1804	pHlp->pfnSSMGetU32(pSSM, &pVirtio->uDriverFeaturesSelect);
1805	pHlp->pfnSSMGetU64(pSSM, &pVirtio->uDriverFeatures);
1806
1807	for (uint32_t i = 0; i < VIRTQ_MAX_CNT; i++)
1808	{
1809	pHlp->pfnSSMGetGCPhys64(pSSM, &pVirtio->aGCPhysQueueDesc[i]);
1810	pHlp->pfnSSMGetGCPhys64(pSSM, &pVirtio->aGCPhysQueueAvail[i]);
1811	pHlp->pfnSSMGetGCPhys64(pSSM, &pVirtio->aGCPhysQueueUsed[i]);
1812	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueNotifyOff[i]);
1813	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueMsixVector[i]);
1814	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueEnable[i]);
1815	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueSize[i]);
1816	pHlp->pfnSSMGetU16(pSSM, &pVirtio->virtqState[i].uAvailIdx);
1817	pHlp->pfnSSMGetU16(pSSM, &pVirtio->virtqState[i].uUsedIdx);
1818	rc = pHlp->pfnSSMGetMem(pSSM, pVirtio->virtqState[i].szVirtqName,
1819	sizeof(pVirtio->virtqState[i].szVirtqName));
1820	AssertRCReturn(rc, rc);
1821	}
1822
1823	return VINF_SUCCESS;
1824	}
1825
1826
1827	/*********************************************************************************************************************************
1828	* Device Level *
1829	*********************************************************************************************************************************/
1830
1831	/**
1832	* This must be called by the client to handle VM state changes
1833	* after the client takes care of its device-specific tasks for the state change.
1834	* (i.e. Reset, suspend, power-off, resume)
1835	*
1836	* @param pDevIns The device instance.
1837	* @param pVirtio Pointer to the shared virtio state.
1838	*/
1839	void virtioCoreR3VmStateChanged(PVIRTIOCORE pVirtio, VIRTIOVMSTATECHANGED enmState)
1840	{
1841	LogFunc(("State changing to %s\n",
1842	virtioCoreGetStateChangeText(enmState)));
1843
1844	switch(enmState)
1845	{
1846	case kvirtIoVmStateChangedReset:
1847	virtioCoreResetAll(pVirtio);
1848	break;
1849	case kvirtIoVmStateChangedSuspend:
1850	break;
1851	case kvirtIoVmStateChangedPowerOff:
1852	break;
1853	case kvirtIoVmStateChangedResume:
1854	virtioNotifyGuestDriver(pVirtio->pDevIns, pVirtio, 0 /* idxQueue /, true / fForce */);
1855	break;
1856	default:
1857	LogRelFunc(("Bad enum value"));
1858	return;
1859	}
1860	}
1861
1862	/**
1863	* This should be called from PDMDEVREGR3::pfnDestruct.
1864	*
1865	* @param pDevIns The device instance.
1866	* @param pVirtio Pointer to the shared virtio state.
1867	* @param pVirtioCC Pointer to the ring-3 virtio state.
1868	*/
1869	void virtioCoreR3Term(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1870	{
1871	if (pVirtioCC->pbPrevDevSpecificCfg)
1872	{
1873	RTMemFree(pVirtioCC->pbPrevDevSpecificCfg);
1874	pVirtioCC->pbPrevDevSpecificCfg = NULL;
1875	}
1876	RT_NOREF(pDevIns, pVirtio);
1877	}
1878
1879
1880	/**
1881	* Setup PCI device controller and Virtio state
1882	*
1883	* This should be called from PDMDEVREGR3::pfnConstruct.
1884	*
1885	* @param pDevIns The device instance.
1886	* @param pVirtio Pointer to the shared virtio state. This
1887	* must be the first member in the shared
1888	* device instance data!
1889	* @param pVirtioCC Pointer to the ring-3 virtio state. This
1890	* must be the first member in the ring-3
1891	* device instance data!
1892	* @param pPciParams Values to populate industry standard PCI Configuration Space data structure
1893	* @param pcszInstance Device instance name (format-specifier)
1894	* @param fDevSpecificFeatures VirtIO device-specific features offered by
1895	* client
1896	* @param cbDevSpecificCfg Size of virtio_pci_device_cap device-specific struct
1897	* @param pvDevSpecificCfg Address of client's dev-specific
1898	* configuration struct.
1899	*/
1900	int virtioCoreR3Init(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, PVIRTIOPCIPARAMS pPciParams,
1901	const char pcszInstance, uint64_t fDevSpecificFeatures, void pvDevSpecificCfg, uint16_t cbDevSpecificCfg)
1902	{
1903	/*
1904	* The pVirtio state must be the first member of the shared device instance
1905	* data, otherwise we cannot get our bearings in the PCI configuration callbacks.
1906	*/
1907	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
1908	AssertLogRelReturn(pVirtioCC == PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC), VERR_STATE_CHANGED);
1909
1910	pVirtio->pDevIns = pDevIns;
1911
1912	/*
1913	* Caller must initialize these.
1914	*/
1915	AssertReturn(pVirtioCC->pfnStatusChanged, VERR_INVALID_POINTER);
1916	AssertReturn(pVirtioCC->pfnQueueNotified, VERR_INVALID_POINTER);
1917
1918	#if 0 /* Until pdmR3DvHlp_PCISetIrq() impl is fixed and Assert that limits vec to 0 is removed */
1919	# ifdef VBOX_WITH_MSI_DEVICES
1920	pVirtio->fMsiSupport = true;
1921	# endif
1922	#endif
1923
1924	/*
1925	* The host features offered include both device-specific features
1926	* and reserved feature bits (device independent)
1927	*/
1928	pVirtio->uDeviceFeatures = VIRTIO_F_VERSION_1
1929	\| VIRTIO_DEV_INDEPENDENT_FEATURES_OFFERED
1930	\| fDevSpecificFeatures;
1931
1932	RTStrCopy(pVirtio->szInstance, sizeof(pVirtio->szInstance), pcszInstance);
1933
1934	pVirtio->uDeviceStatus = 0;
1935	pVirtioCC->cbDevSpecificCfg = cbDevSpecificCfg;
1936	pVirtioCC->pbDevSpecificCfg = (uint8_t *)pvDevSpecificCfg;
1937	pVirtioCC->pbPrevDevSpecificCfg = (uint8_t *)RTMemDup(pvDevSpecificCfg, cbDevSpecificCfg);
1938	AssertLogRelReturn(pVirtioCC->pbPrevDevSpecificCfg, VERR_NO_MEMORY);
1939
1940	/* Set PCI config registers (assume 32-bit mode) */
1941	PPDMPCIDEV pPciDev = pDevIns->apPciDevs[0];
1942	PDMPCIDEV_ASSERT_VALID(pDevIns, pPciDev);
1943
1944	PDMPciDevSetRevisionId(pPciDev, DEVICE_PCI_REVISION_ID_VIRTIO);
1945	PDMPciDevSetVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
1946	PDMPciDevSetSubSystemVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
1947	PDMPciDevSetDeviceId(pPciDev, pPciParams->uDeviceId);
1948	PDMPciDevSetClassBase(pPciDev, pPciParams->uClassBase);
1949	PDMPciDevSetClassSub(pPciDev, pPciParams->uClassSub);
1950	PDMPciDevSetClassProg(pPciDev, pPciParams->uClassProg);
1951	PDMPciDevSetSubSystemId(pPciDev, pPciParams->uSubsystemId);
1952	PDMPciDevSetInterruptLine(pPciDev, pPciParams->uInterruptLine);
1953	PDMPciDevSetInterruptPin(pPciDev, pPciParams->uInterruptPin);
1954
1955	/* Register PCI device */
1956	int rc = PDMDevHlpPCIRegister(pDevIns, pPciDev);
1957	if (RT_FAILURE(rc))
1958	return PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Device")); /* can we put params in this error? */
1959
1960	rc = PDMDevHlpPCIInterceptConfigAccesses(pDevIns, pPciDev, virtioR3PciConfigRead, virtioR3PciConfigWrite);
1961	AssertRCReturn(rc, rc);
1962
1963
1964	/* Construct & map PCI vendor-specific capabilities for virtio host negotiation with guest driver */
1965
1966	/* The following capability mapped via VirtIO 1.0: struct virtio_pci_cfg_cap (VIRTIO_PCI_CFG_CAP_T)
1967	* as a mandatory but suboptimal alternative interface to host device capabilities, facilitating
1968	* access the memory of any BAR. If the guest uses it (the VirtIO driver on Linux doesn't),
1969	* Unlike Common, Notify, ISR and Device capabilities, it is accessed directly via PCI Config region.
1970	* therefore does not contribute to the capabilities region (BAR) the other capabilities use.
1971	*/
1972	#define CFG_ADDR_2_IDX(addr) ((uint8_t)(((uintptr_t)(addr) - (uintptr_t)&pPciDev->abConfig[0])))
1973	#define SET_PCI_CAP_LOC(a_pPciDev, a_pCfg, a_LocCap, a_uMmioLengthAlign) \
1974	do { \
1975	(a_LocCap).offMmio = (a_pCfg)->uOffset; \
1976	(a_LocCap).cbMmio = RT_ALIGN_T((a_pCfg)->uLength, a_uMmioLengthAlign, uint16_t); \
1977	(a_LocCap).offPci = (uint16_t)(uintptr_t)((uint8_t *)(a_pCfg) - &(a_pPciDev)->abConfig[0]); \
1978	(a_LocCap).cbPci = (a_pCfg)->uCapLen; \
1979	} while (0)
1980
1981	PVIRTIO_PCI_CAP_T pCfg;
1982	uint32_t cbRegion = 0;
1983
1984	/* Common capability (VirtIO 1.0 spec, section 4.1.4.3) */
1985	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[0x40];
1986	pCfg->uCfgType = VIRTIO_PCI_CAP_COMMON_CFG;
1987	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
1988	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
1989	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
1990	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
1991	pCfg->uOffset = RT_ALIGN_32(0, 4); /* reminder, in case someone changes offset */
1992	pCfg->uLength = sizeof(VIRTIO_PCI_COMMON_CFG_T);
1993	cbRegion += pCfg->uLength;
1994	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocCommonCfgCap, 2);
1995	pVirtioCC->pCommonCfgCap = pCfg;
1996
1997	/*
1998	* Notify capability (VirtIO 1.0 spec, section 4.1.4.4). Note: uLength is based the choice
1999	* of this implementation that each queue's uQueueNotifyOff is set equal to (QueueSelect) ordinal
2000	* value of the queue
2001	*/
2002	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2003	pCfg->uCfgType = VIRTIO_PCI_CAP_NOTIFY_CFG;
2004	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2005	pCfg->uCapLen = sizeof(VIRTIO_PCI_NOTIFY_CAP_T);
2006	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2007	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2008	pCfg->uOffset = pVirtioCC->pCommonCfgCap->uOffset + pVirtioCC->pCommonCfgCap->uLength;
2009	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2010
2011
2012	pCfg->uLength = VIRTQ_MAX_CNT * VIRTIO_NOTIFY_OFFSET_MULTIPLIER + 2; /* will change in VirtIO 1.1 */
2013	cbRegion += pCfg->uLength;
2014	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocNotifyCap, 1);
2015	pVirtioCC->pNotifyCap = (PVIRTIO_PCI_NOTIFY_CAP_T)pCfg;
2016	pVirtioCC->pNotifyCap->uNotifyOffMultiplier = VIRTIO_NOTIFY_OFFSET_MULTIPLIER;
2017
2018	/* ISR capability (VirtIO 1.0 spec, section 4.1.4.5)
2019	*
2020	* VirtIO 1.0 spec says 8-bit, unaligned in MMIO space. Example/diagram
2021	* of spec shows it as a 32-bit field with upper bits 'reserved'
2022	* Will take spec words more literally than the diagram for now.
2023	*/
2024	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2025	pCfg->uCfgType = VIRTIO_PCI_CAP_ISR_CFG;
2026	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2027	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2028	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2029	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2030	pCfg->uOffset = pVirtioCC->pNotifyCap->pciCap.uOffset + pVirtioCC->pNotifyCap->pciCap.uLength;
2031	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2032	pCfg->uLength = sizeof(uint8_t);
2033	cbRegion += pCfg->uLength;
2034	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocIsrCap, 4);
2035	pVirtioCC->pIsrCap = pCfg;
2036
2037	/* PCI Cfg capability (VirtIO 1.0 spec, section 4.1.4.7)
2038	* This capability doesn't get page-MMIO mapped. Instead uBar, uOffset and uLength are intercepted
2039	* by trapping PCI configuration I/O and get modulated by consumers to locate fetch and read/write
2040	* values from any region. NOTE: The linux driver not only doesn't use this feature, it will not
2041	* even list it as present if uLength isn't non-zero and also 4-byte-aligned as the linux driver is
2042	* initializing.
2043	*/
2044	pVirtio->uPciCfgDataOff = pCfg->uCapNext + RT_OFFSETOF(VIRTIO_PCI_CFG_CAP_T, uPciCfgData);
2045	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2046	pCfg->uCfgType = VIRTIO_PCI_CAP_PCI_CFG;
2047	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2048	pCfg->uCapLen = sizeof(VIRTIO_PCI_CFG_CAP_T);
2049	pCfg->uCapNext = (pVirtio->fMsiSupport \|\| pVirtioCC->pbDevSpecificCfg) ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2050	pCfg->uBar = 0;
2051	pCfg->uOffset = 0;
2052	pCfg->uLength = 0;
2053	cbRegion += pCfg->uLength;
2054	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocPciCfgCap, 1);
2055	pVirtioCC->pPciCfgCap = (PVIRTIO_PCI_CFG_CAP_T)pCfg;
2056
2057	if (pVirtioCC->pbDevSpecificCfg)
2058	{
2059	/* Following capability (via VirtIO 1.0, section 4.1.4.6). Client defines the
2060	* device-specific config fields struct and passes size to this constructor */
2061	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2062	pCfg->uCfgType = VIRTIO_PCI_CAP_DEVICE_CFG;
2063	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2064	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2065	pCfg->uCapNext = pVirtio->fMsiSupport ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2066	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2067	pCfg->uOffset = pVirtioCC->pIsrCap->uOffset + pVirtioCC->pIsrCap->uLength;
2068	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2069	pCfg->uLength = cbDevSpecificCfg;
2070	cbRegion += pCfg->uLength;
2071	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocDeviceCap, 4);
2072	pVirtioCC->pDeviceCap = pCfg;
2073	}
2074	else
2075	Assert(pVirtio->LocDeviceCap.cbMmio == 0 && pVirtio->LocDeviceCap.cbPci == 0);
2076
2077	if (pVirtio->fMsiSupport)
2078	{
2079	PDMMSIREG aMsiReg;
2080	RT_ZERO(aMsiReg);
2081	aMsiReg.iMsixCapOffset = pCfg->uCapNext;
2082	aMsiReg.iMsixNextOffset = 0;
2083	aMsiReg.iMsixBar = VIRTIO_REGION_MSIX_CAP;
2084	aMsiReg.cMsixVectors = VBOX_MSIX_MAX_ENTRIES;
2085	rc = PDMDevHlpPCIRegisterMsi(pDevIns, &aMsiReg); /* see MsixR3init() */
2086	if (RT_FAILURE(rc))
2087	{
2088	/* See PDMDevHlp.cpp:pdmR3DevHlp_PCIRegisterMsi */
2089	LogFunc(("Failed to configure MSI-X (%Rrc). Reverting to INTx\n", rc));
2090	pVirtio->fMsiSupport = false;
2091	}
2092	else
2093	Log2Func(("Using MSI-X for guest driver notification\n"));
2094	}
2095	else
2096	LogFunc(("MSI-X not available for VBox, using INTx notification\n"));
2097
2098	/* Set offset to first capability and enable PCI dev capabilities */
2099	PDMPciDevSetCapabilityList(pPciDev, 0x40);
2100	PDMPciDevSetStatus(pPciDev, VBOX_PCI_STATUS_CAP_LIST);
2101
2102	/* Linux drivers/virtio/virtio_pci_modern.c tries to map at least a page for the
2103	* 'unknown' device-specific capability without querying the capability to figure
2104	* out size, so pad with an extra page
2105	*/
2106	rc = PDMDevHlpPCIIORegionCreateMmio(pDevIns, VIRTIO_REGION_PCI_CAP, RT_ALIGN_32(cbRegion + PAGE_SIZE, PAGE_SIZE),
2107	PCI_ADDRESS_SPACE_MEM, virtioMmioWrite, virtioMmioRead, pVirtio,
2108	IOMMMIO_FLAGS_READ_PASSTHRU \| IOMMMIO_FLAGS_WRITE_PASSTHRU, "virtio-scsi MMIO",
2109	&pVirtio->hMmioPciCap);
2110	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Capabilities address space")));
2111
2112	/*
2113	* Statistics.
2114	*/
2115	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsAllocated, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2116	"Total number of allocated descriptor chains", "DescChainsAllocated");
2117	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsFreed, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2118	"Total number of freed descriptor chains", "DescChainsFreed");
2119	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsIn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2120	"Total number of inbound segments", "DescChainsSegsIn");
2121	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsOut, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2122	"Total number of outbound segments", "DescChainsSegsOut");
2123
2124	return VINF_SUCCESS;
2125	}
2126
2127	#else /* !IN_RING3 */
2128
2129	/**
2130	* Sets up the core ring-0/raw-mode virtio bits.
2131	*
2132	* @returns VBox status code.
2133	* @param pDevIns The device instance.
2134	* @param pVirtio Pointer to the shared virtio state. This must be the first
2135	* member in the shared device instance data!
2136	* @param pVirtioCC Pointer to the current context virtio state. This must be the
2137	* first member in the currenct context's device instance data!
2138	*/
2139	int virtioCoreRZInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
2140	{
2141	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
2142	AssertLogRelReturn(pVirtioCC == PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC), VERR_STATE_CHANGED);
2143
2144	int rc = PDMDevHlpMmioSetUpContext(pDevIns, pVirtio->hMmioPciCap, virtioMmioWrite, virtioMmioRead, pVirtio);
2145	AssertRCReturn(rc, rc);
2146	return rc;
2147	}
2148
2149	#endif /* !IN_RING3 */
2150

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Devices/VirtIO/Virtio_1_0.cpp@ 83748

Download in other formats: