Virtio_1_0.cpp@ 83575

Last change on this file since 83575 was 83575, checked in by vboxsync, 5 years ago
Virtio_1_0.cpp: Ditto uint64_t casting cleaning up in virtioCoreR3QueuePut. bugref:9440
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 85.4 KB

Line
1	/* $Id: Virtio_1_0.cpp 83575 2020-04-05 22:40:07Z vboxsync $ */
2	/** @file
3	* Virtio_1_0 - Virtio Common (PCI, feature & config mgt, queue mgt & proxy, notification mgt)
4	*/
5
6	/*
7	* Copyright (C) 2009-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_DEV_VIRTIO
23
24	#include <VBox/log.h>
25	#include <VBox/msi.h>
26	#include <VBox/AssertGuest.h>
27	#include <iprt/param.h>
28	#include <iprt/assert.h>
29	#include <iprt/uuid.h>
30	#include <iprt/mem.h>
31	#include <iprt/assert.h>
32	#include <iprt/sg.h>
33	#include <iprt/string.h>
34	#include <VBox/vmm/pdmdev.h>
35	#include "Virtio_1_0.h"
36
37
38	/*********************************************************************************************************************************
39	* Defined Constants And Macros *
40	*********************************************************************************************************************************/
41	#define INSTANCE(a_pVirtio) ((a_pVirtio)->szInstance)
42	#define VIRTQNAME(a_pVirtio, a_idxQueue) ((a_pVirtio)->virtqState[(a_idxQueue)].szVirtqName)
43	#define IS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
44
45	/**
46	* This macro returns true if the @a a_offAccess and access length (@a
47	* a_cbAccess) are within the range of the mapped capability struct described by
48	* @a a_LocCapData.
49	*
50	* @param[in] a_offAccess The offset into the MMIO bar of the access.
51	* @param[in] a_cbAccess The access size.
52	* @param[out] a_offIntraVar The variable to return the intra-capability
53	* offset into. ASSUMES this is uint32_t.
54	* @param[in] a_LocCapData The capability location info.
55	*/
56	#define MATCHES_VIRTIO_CAP_STRUCT(a_offAccess, a_cbAccess, a_offIntraVar, a_LocCapData) \
57	( ((a_offIntraVar) = (uint32_t)((a_offAccess) - (a_LocCapData).offMmio)) < (uint32_t)(a_LocCapData).cbMmio \
58	&& (a_offIntraVar) + (uint32_t)(a_cbAccess) <= (uint32_t)(a_LocCapData).cbMmio )
59
60
61	/** Marks the start of the virtio saved state (just for sanity). */
62	#define VIRTIO_SAVEDSTATE_MARKER UINT64_C(0x1133557799bbddff)
63	/** The current saved state version for the virtio core. */
64	#define VIRTIO_SAVEDSTATE_VERSION UINT32_C(1)
65
66
67	/*********************************************************************************************************************************
68	* Structures and Typedefs *
69	*********************************************************************************************************************************/
70	/**
71	* virtq related structs
72	* (struct names follow VirtIO 1.0 spec, typedef use VBox style)
73	*/
74	typedef struct virtq_desc
75	{
76	uint64_t GCPhysBuf; /*< addr GC Phys. address of buffer /
77	uint32_t cb; /*< len Buffer length /
78	uint16_t fFlags; /*< flags Buffer specific flags /
79	uint16_t uDescIdxNext; /*< next Idx set if VIRTIO_DESC_F_NEXT /
80	} VIRTQ_DESC_T, *PVIRTQ_DESC_T;
81
82	typedef struct virtq_avail
83	{
84	uint16_t fFlags; /*< flags avail ring drv to dev flags /
85	uint16_t uIdx; /*< idx Index of next free ring slot /
86	uint16_t auRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: avail drv to dev bufs /
87	/* uint16_t uUsedEventIdx; - used_event (if VIRTQ_USED_F_EVENT_IDX) */
88	} VIRTQ_AVAIL_T, *PVIRTQ_AVAIL_T;
89
90	typedef struct virtq_used_elem
91	{
92	uint32_t uDescIdx; /*< idx Start of used desc chain /
93	uint32_t cbElem; /*< len Total len of used desc chain /
94	} VIRTQ_USED_ELEM_T;
95
96	typedef struct virt_used
97	{
98	uint16_t fFlags; /*< flags used ring host-to-guest flags /
99	uint16_t uIdx; /*< idx Index of next ring slot /
100	VIRTQ_USED_ELEM_T aRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: used dev to drv bufs /
101	/* uint16_t uAvailEventIdx; - avail_event if (VIRTQ_USED_F_EVENT_IDX) */
102	} VIRTQ_USED_T, *PVIRTQ_USED_T;
103
104
105	const char *virtioCoreGetStateChangeText(VIRTIOVMSTATECHANGED enmState)
106	{
107	switch (enmState)
108	{
109	case kvirtIoVmStateChangedReset: return "VM RESET";
110	case kvirtIoVmStateChangedSuspend: return "VM SUSPEND";
111	case kvirtIoVmStateChangedPowerOff: return "VM POWER OFF";
112	case kvirtIoVmStateChangedResume: return "VM RESUME";
113	default: return "<BAD ENUM>";
114	}
115	}
116
117	/* Internal Functions */
118
119	static void virtioNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, bool fForce);
120	static int virtioKick(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uVec, bool fForce);
121
122	/** @name Internal queue operations
123	* @{ */
124
125	#if 0 /* unused */
126	DECLINLINE(int) virtqIsEventNeeded(uint16_t uEventIdx, uint16_t uDescIdxNew, uint16_t uDescIdxOld)
127	{
128	return (uint16_t)(uDescIdxNew - uEventIdx - 1) < (uint16_t)(uDescIdxNew - uDescIdxOld);
129	}
130	#endif
131
132	/**
133	* Accessor for virtq descriptor
134	*/
135	#ifdef IN_RING3
136	DECLINLINE(void) virtioReadDesc(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
137	uint32_t idxDesc, PVIRTQ_DESC_T pDesc)
138	{
139	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
140	uint16_t const cQueueItems = RT_MAX(pVirtio->uQueueSize[idxQueue], 1); /* Make sure to avoid div-by-zero. */
141	PDMDevHlpPCIPhysRead(pDevIns,
142	pVirtio->aGCPhysQueueDesc[idxQueue] + sizeof(VIRTQ_DESC_T) * (idxDesc % cQueueItems),
143	pDesc, sizeof(VIRTQ_DESC_T));
144	}
145	#endif
146
147	/**
148	* Accessors for virtq avail ring
149	*/
150	#ifdef IN_RING3
151	DECLINLINE(uint16_t) virtioReadAvailDescIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint32_t availIdx)
152	{
153	uint16_t uDescIdx;
154	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
155	uint16_t const cQueueItems = RT_MAX(pVirtio->uQueueSize[idxQueue], 1); /* Make sure to avoid div-by-zero. */
156	PDMDevHlpPCIPhysRead(pDevIns,
157	pVirtio->aGCPhysQueueAvail[idxQueue]
158	+ RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[availIdx % cQueueItems]),
159	&uDescIdx, sizeof(uDescIdx));
160	return uDescIdx;
161	}
162	#endif
163
164	DECLINLINE(uint16_t) virtioReadAvailRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
165	{
166	uint16_t uIdx = 0;
167	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
168	PDMDevHlpPCIPhysRead(pDevIns,
169	pVirtio->aGCPhysQueueAvail[idxQueue] + RT_UOFFSETOF(VIRTQ_AVAIL_T, uIdx),
170	&uIdx, sizeof(uIdx));
171	return uIdx;
172	}
173
174	DECLINLINE(bool) virtqIsEmpty(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
175	{
176	uint16_t uAvailGuest = virtioReadAvailRingIdx(pDevIns, pVirtio, idxQueue);
177	bool fEmpty = uAvailGuest == pVirtio->virtqState[idxQueue].uAvailIdx;
178
179	Log6Func(("%s: uAvailGuest=%u uAvailIdx=%u. (%s)\n",
180	VIRTQNAME(pVirtio, idxQueue), uAvailGuest, pVirtio->virtqState[idxQueue].uAvailIdx,
181	fEmpty ? "Queue empty" : "Queue has available descriptors"));
182	return fEmpty;
183	}
184
185	#if 0 /* unused - Will be used when VIRTIO_F_EVENT_IDX optional feature is implemented, VirtIO 1.0, 2.4.7 */
186	DECLINLINE(uint16_t) virtioReadAvailFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
187	{
188	uint16_t fFlags;
189	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
190	PDMDevHlpPCIPhysRead(pDevIns,
191	pVirtio->aGCPhysQueueAvail[idxQueue] + RT_UOFFSETOF(VIRTQ_AVAIL_T, fFlags),
192	&fFlags, sizeof(fFlags));
193	return fFlags;
194	}
195	#endif
196
197	#ifdef IN_RING3
198	DECLINLINE(uint16_t) virtioReadAvailUsedEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
199	{
200	uint16_t uUsedEventIdx;
201	/* VirtIO 1.0 uUsedEventIdx (used_event) immediately follows ring */
202	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
203	PDMDevHlpPCIPhysRead(pDevIns,
204	pVirtio->aGCPhysQueueAvail[idxQueue] + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtio->uQueueSize[idxQueue]]),
205	&uUsedEventIdx, sizeof(uUsedEventIdx));
206	return uUsedEventIdx;
207	}
208	#endif
209
210	/** @} */
211
212	/** @name Accessors for virtq used ring
213	* @{
214	*/
215
216	#ifdef IN_RING3
217	DECLINLINE(void) virtioWriteUsedElem(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
218	uint32_t usedIdx, uint32_t uDescIdx, uint32_t uLen)
219	{
220	VIRTQ_USED_ELEM_T elem = { uDescIdx, uLen };
221	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
222	uint16_t const cQueueItems = RT_MAX(pVirtio->uQueueSize[idxQueue], 1); /* Make sure to avoid div-by-zero. */
223	PDMDevHlpPCIPhysWrite(pDevIns,
224	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[usedIdx % cQueueItems]),
225	&elem, sizeof(elem));
226	}
227
228	DECLINLINE(void) virtioWriteUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint16_t fFlags)
229	{
230	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
231	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
232	PDMDevHlpPCIPhysWrite(pDevIns,
233	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
234	&fFlags, sizeof(fFlags));
235	}
236	#endif
237
238	DECLINLINE(void) virtioWriteUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint16_t uIdx)
239	{
240	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
241	PDMDevHlpPCIPhysWrite(pDevIns,
242	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
243	&uIdx, sizeof(uIdx));
244	}
245
246	#ifdef LOG_ENABLED
247	DECLINLINE(uint16_t) virtioReadUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
248	{
249	uint16_t uIdx = 0;
250	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
251	PDMDevHlpPCIPhysRead(pDevIns,
252	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
253	&uIdx, sizeof(uIdx));
254	return uIdx;
255	}
256	#endif
257
258	DECLINLINE(uint16_t) virtioReadUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
259	{
260	uint16_t fFlags = 0;
261	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
262	PDMDevHlpPCIPhysRead(pDevIns,
263	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
264	&fFlags, sizeof(fFlags));
265	return fFlags;
266	}
267
268	#if 0 /* unused - May be used when VIRTIO_F_EVENT_IDX optional feature is implemented VirtIO 1.0, 2.4.9.2*/
269	DECLINLINE(void) virtioWriteUsedAvailEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint32_t uAvailEventIdx)
270	{
271	/** VirtIO 1.0 uAvailEventIdx (avail_event) immediately follows ring */
272	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
273	PDMDevHlpPCIPhysWrite(pDevIns,
274	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[pVirtio->uQueueSize[idxQueue]]),
275	&uAvailEventIdx, sizeof(uAvailEventIdx));
276	}
277	#endif
278
279
280	/** @} */
281
282	void virtioCoreSgBufInit(PVIRTIOSGBUF pGcSgBuf, PVIRTIOSGSEG paSegs, size_t cSegs)
283	{
284	AssertPtr(pGcSgBuf);
285	Assert( (cSegs > 0 && VALID_PTR(paSegs)) \|\| (!cSegs && !paSegs));
286	Assert(cSegs < (~(unsigned)0 >> 1));
287
288	pGcSgBuf->paSegs = paSegs;
289	pGcSgBuf->cSegs = (unsigned)cSegs;
290	pGcSgBuf->idxSeg = 0;
291	if (cSegs && paSegs)
292	{
293	pGcSgBuf->gcPhysCur = paSegs[0].gcPhys;
294	pGcSgBuf->cbSegLeft = paSegs[0].cbSeg;
295	}
296	else
297	{
298	pGcSgBuf->gcPhysCur = 0;
299	pGcSgBuf->cbSegLeft = 0;
300	}
301	}
302
303	static RTGCPHYS virtioCoreSgBufGet(PVIRTIOSGBUF pGcSgBuf, size_t *pcbData)
304	{
305	size_t cbData;
306	RTGCPHYS pGcBuf;
307
308	/* Check that the S/G buffer has memory left. */
309	if (RT_LIKELY(pGcSgBuf->idxSeg < pGcSgBuf->cSegs && pGcSgBuf->cbSegLeft))
310	{ /* likely */ }
311	else
312	{
313	*pcbData = 0;
314	return 0;
315	}
316
317	AssertMsg( pGcSgBuf->cbSegLeft <= 128 * _1M
318	&& (RTGCPHYS)pGcSgBuf->gcPhysCur >= (RTGCPHYS)pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys
319	&& (RTGCPHYS)pGcSgBuf->gcPhysCur + pGcSgBuf->cbSegLeft <=
320	(RTGCPHYS)pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys + pGcSgBuf->paSegs[pGcSgBuf->idxSeg].cbSeg,
321	("pGcSgBuf->idxSeg=%d pGcSgBuf->cSegs=%d pGcSgBuf->gcPhysCur=%p pGcSgBuf->cbSegLeft=%zd "
322	"pGcSgBuf->paSegs[%d].gcPhys=%p pGcSgBuf->paSegs[%d].cbSeg=%zd\n",
323	pGcSgBuf->idxSeg, pGcSgBuf->cSegs, pGcSgBuf->gcPhysCur, pGcSgBuf->cbSegLeft,
324	pGcSgBuf->idxSeg, pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys, pGcSgBuf->idxSeg,
325	pGcSgBuf->paSegs[pGcSgBuf->idxSeg].cbSeg));
326
327	cbData = RT_MIN(*pcbData, pGcSgBuf->cbSegLeft);
328	pGcBuf = pGcSgBuf->gcPhysCur;
329	pGcSgBuf->cbSegLeft -= cbData;
330	if (!pGcSgBuf->cbSegLeft)
331	{
332	pGcSgBuf->idxSeg++;
333
334	if (pGcSgBuf->idxSeg < pGcSgBuf->cSegs)
335	{
336	pGcSgBuf->gcPhysCur = pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys;
337	pGcSgBuf->cbSegLeft = pGcSgBuf->paSegs[pGcSgBuf->idxSeg].cbSeg;
338	}
339	*pcbData = cbData;
340	}
341	else
342	pGcSgBuf->gcPhysCur = pGcSgBuf->gcPhysCur + cbData;
343
344	return pGcBuf;
345	}
346
347	void virtioCoreSgBufReset(PVIRTIOSGBUF pGcSgBuf)
348	{
349	AssertPtrReturnVoid(pGcSgBuf);
350
351	pGcSgBuf->idxSeg = 0;
352	if (pGcSgBuf->cSegs)
353	{
354	pGcSgBuf->gcPhysCur = pGcSgBuf->paSegs[0].gcPhys;
355	pGcSgBuf->cbSegLeft = pGcSgBuf->paSegs[0].cbSeg;
356	}
357	else
358	{
359	pGcSgBuf->gcPhysCur = 0;
360	pGcSgBuf->cbSegLeft = 0;
361	}
362	}
363
364	RTGCPHYS virtioCoreSgBufAdvance(PVIRTIOSGBUF pGcSgBuf, size_t cbAdvance)
365	{
366	AssertReturn(pGcSgBuf, 0);
367
368	size_t cbLeft = cbAdvance;
369	while (cbLeft)
370	{
371	size_t cbThisAdvance = cbLeft;
372	virtioCoreSgBufGet(pGcSgBuf, &cbThisAdvance);
373	if (!cbThisAdvance)
374	break;
375
376	cbLeft -= cbThisAdvance;
377	}
378	return cbAdvance - cbLeft;
379	}
380
381	RTGCPHYS virtioCoreSgBufGetNextSegment(PVIRTIOSGBUF pGcSgBuf, size_t *pcbSeg)
382	{
383	AssertReturn(pGcSgBuf, 0);
384	AssertPtrReturn(pcbSeg, 0);
385
386	if (!*pcbSeg)
387	*pcbSeg = pGcSgBuf->cbSegLeft;
388
389	return virtioCoreSgBufGet(pGcSgBuf, pcbSeg);
390	}
391
392	#ifdef LOG_ENABLED
393
394	/**
395	* Does a formatted hex dump using Log(()), recommend using VIRTIO_HEX_DUMP() macro to
396	* control enabling of logging efficiently.
397	*
398	* @param pv pointer to buffer to dump contents of
399	* @param cb count of characters to dump from buffer
400	* @param uBase base address of per-row address prefixing of hex output
401	* @param pszTitle Optional title. If present displays title that lists
402	* provided text with value of cb to indicate size next to it.
403	*/
404	void virtioCoreHexDump(uint8_t pv, uint32_t cb, uint32_t uBase, const char pszTitle)
405	{
406	if (pszTitle)
407	Log(("%s [%d bytes]:\n", pszTitle, cb));
408	for (uint32_t row = 0; row < RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
409	{
410	Log(("%04x: ", row * 16 + uBase)); /* line address */
411	for (uint8_t col = 0; col < 16; col++)
412	{
413	uint32_t idx = row * 16 + col;
414	if (idx >= cb)
415	Log(("-- %s", (col + 1) % 8 ? "" : " "));
416	else
417	Log(("%02x %s", pv[idx], (col + 1) % 8 ? "" : " "));
418	}
419	for (uint32_t idx = row * 16; idx < row * 16 + 16; idx++)
420	Log(("%c", (idx >= cb) ? ' ' : (pv[idx] >= 0x20 && pv[idx] <= 0x7e ? pv[idx] : '.')));
421	Log(("\n"));
422	}
423	Log(("\n"));
424	RT_NOREF2(uBase, pv);
425	}
426
427	/**
428	* Do a hex dump of memory in guest physical context
429	*
430	* @param gcPhys pointer to buffer to dump contents of
431	* @param cb count of characters to dump from buffer
432	* @param uBase base address of per-row address prefixing of hex output
433	* @param pszTitle Optional title. If present displays title that lists
434	* provided text with value of cb to indicate size next to it.
435	*/
436	void virtioCoreGcPhysHexDump(PPDMDEVINS pDevIns, RTGCPHYS gcPhys, uint32_t cb, uint32_t uBase, const char *pszTitle)
437	{
438	if (pszTitle)
439	Log(("%s [%d bytes]:\n", pszTitle, cb));
440	for (uint32_t row = 0; row < RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
441	{
442	uint8_t c;
443	Log(("%04x: ", row * 16 + uBase)); /* line address */
444	for (uint8_t col = 0; col < 16; col++)
445	{
446	uint32_t idx = row * 16 + col;
447	PDMDevHlpPCIPhysRead(pDevIns, gcPhys + idx, &c, 1);
448	if (idx >= cb)
449	Log(("-- %s", (col + 1) % 8 ? "" : " "));
450	else
451	Log(("%02x %s", c, (col + 1) % 8 ? "" : " "));
452	}
453	for (uint32_t idx = row * 16; idx < row * 16 + 16; idx++)
454	{
455	PDMDevHlpPCIPhysRead(pDevIns, gcPhys + idx, &c, 1);
456	Log(("%c", (idx >= cb) ? ' ' : (c >= 0x20 && c <= 0x7e ? c : '.')));
457	}
458	Log(("\n"));
459	}
460	Log(("\n"));
461	RT_NOREF(uBase);
462	}
463	#endif /* LOG_ENABLED */
464
465	/**
466	* Log memory-mapped I/O input or output value.
467	*
468	* This is designed to be invoked by macros that can make contextual assumptions
469	* (e.g. implicitly derive MACRO parameters from the invoking function). It is exposed
470	* for the VirtIO client doing the device-specific implementation in order to log in a
471	* similar fashion accesses to the device-specific MMIO configuration structure. Macros
472	* that leverage this function are found in virtioCommonCfgAccessed() and can be
473	* used as an example of how to use this effectively for the device-specific
474	* code.
475	*
476	* @param pszFunc To avoid displaying this function's name via __FUNCTION__ or LogFunc()
477	* @param pszMember Name of struct member
478	* @param pv pointer to value
479	* @param cb size of value
480	* @param uOffset offset into member where value starts
481	* @param fWrite True if write I/O
482	* @param fHasIndex True if the member is indexed
483	* @param idx The index if fHasIndex
484	*/
485	void virtioCoreLogMappedIoValue(const char pszFunc, const char pszMember, uint32_t uMemberSize,
486	const void *pv, uint32_t cb, uint32_t uOffset, int fWrite,
487	int fHasIndex, uint32_t idx)
488	{
489	if (!LogIs6Enabled())
490	return;
491
492	char szIdx[16];
493	if (fHasIndex)
494	RTStrPrintf(szIdx, sizeof(szIdx), "[%d]", idx);
495	else
496	szIdx[0] = '\0';
497
498	if (cb == 1 \|\| cb == 2 \|\| cb == 4 \|\| cb == 8)
499	{
500	char szDepiction[64];
501	size_t cchDepiction;
502	if (uOffset != 0 \|\| cb != uMemberSize) /* display bounds if partial member access */
503	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s[%d:%d]",
504	pszMember, szIdx, uOffset, uOffset + cb - 1);
505	else
506	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s", pszMember, szIdx);
507
508	/* padding */
509	if (cchDepiction < 30)
510	szDepiction[cchDepiction++] = ' ';
511	while (cchDepiction < 30)
512	szDepiction[cchDepiction++] = '.';
513	szDepiction[cchDepiction] = '\0';
514
515	RTUINT64U uValue;
516	uValue.u = 0;
517	memcpy(uValue.au8, pv, cb);
518	Log6(("%s: Guest %s %s %#0*RX64\n",
519	pszFunc, fWrite ? "wrote" : "read ", szDepiction, 2 + cb * 2, uValue.u));
520	}
521	else /* odd number or oversized access, ... log inline hex-dump style */
522	{
523	Log6(("%s: Guest %s %s%s[%d:%d]: %.*Rhxs\n",
524	pszFunc, fWrite ? "wrote" : "read ", pszMember,
525	szIdx, uOffset, uOffset + cb, cb, pv));
526	}
527	RT_NOREF2(fWrite, pszFunc);
528	}
529
530
531	/**
532	* Makes the MMIO-mapped Virtio uDeviceStatus registers non-cryptic
533	*/
534	DECLINLINE(void) virtioLogDeviceStatus(uint8_t bStatus)
535	{
536	if (bStatus == 0)
537	Log6(("RESET"));
538	else
539	{
540	int primed = 0;
541	if (bStatus & VIRTIO_STATUS_ACKNOWLEDGE)
542	Log6(("%sACKNOWLEDGE", primed++ ? "" : ""));
543	if (bStatus & VIRTIO_STATUS_DRIVER)
544	Log6(("%sDRIVER", primed++ ? " \| " : ""));
545	if (bStatus & VIRTIO_STATUS_FEATURES_OK)
546	Log6(("%sFEATURES_OK", primed++ ? " \| " : ""));
547	if (bStatus & VIRTIO_STATUS_DRIVER_OK)
548	Log6(("%sDRIVER_OK", primed++ ? " \| " : ""));
549	if (bStatus & VIRTIO_STATUS_FAILED)
550	Log6(("%sFAILED", primed++ ? " \| " : ""));
551	if (bStatus & VIRTIO_STATUS_DEVICE_NEEDS_RESET)
552	Log6(("%sNEEDS_RESET", primed++ ? " \| " : ""));
553	(void)primed;
554	}
555	}
556
557	#ifdef IN_RING3
558	/**
559	* Allocate client context for client to work with VirtIO-provided with queue
560	*
561	* @param pVirtio Pointer to the shared virtio state.
562	* @param idxQueue Queue number
563	* @param pcszName Name to give queue
564	*
565	* @returns VBox status code.
566	*/
567	int virtioCoreR3QueueAttach(PVIRTIOCORE pVirtio, uint16_t idxQueue, const char *pcszName)
568	{
569	LogFunc(("%s\n", pcszName));
570	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
571	pVirtq->uAvailIdx = 0;
572	pVirtq->uUsedIdx = 0;
573	pVirtq->fEventThresholdReached = false;
574	RTStrCopy(pVirtq->szVirtqName, sizeof(pVirtq->szVirtqName), pcszName);
575	return VINF_SUCCESS;
576	}
577	#endif /* IN_RING3 */
578
579
580	/**
581	* Check if the associated queue is empty
582	*
583	* @param pDevIns The device instance (for reading).
584	* @param pVirtio Pointer to the shared virtio state.
585	* @param idxQueue Queue number
586	*
587	* @retval true Queue is empty or unavailable.
588	* @retval false Queue is available and has entries
589	*/
590	bool virtioCoreQueueIsEmpty(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
591	{
592	if (pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
593	return virtqIsEmpty(pDevIns, pVirtio, idxQueue);
594	LogFunc(("VirtIO not ready: Returning 'true' for queue empty\n"));
595	return true;
596	}
597
598	#ifdef IN_RING3
599
600
601	int virtioCoreR3DescChainGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
602	uint16_t uHeadIdx, PPVIRTIO_DESC_CHAIN_T ppDescChain)
603	{
604	AssertReturn(ppDescChain, VERR_INVALID_PARAMETER);
605
606	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
607
608	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
609
610	PVIRTIOSGSEG paSegsIn = (PVIRTIOSGSEG)RTMemAlloc(VIRTQ_MAX_SIZE * sizeof(VIRTIOSGSEG));
611	AssertReturn(paSegsIn, VERR_NO_MEMORY);
612
613	PVIRTIOSGSEG paSegsOut = (PVIRTIOSGSEG)RTMemAlloc(VIRTQ_MAX_SIZE * sizeof(VIRTIOSGSEG));
614	AssertReturn(paSegsOut, VERR_NO_MEMORY);
615
616	AssertMsgReturn(IS_DRIVER_OK(pVirtio) && pVirtio->uQueueEnable[idxQueue],
617	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
618
619	uint16_t uDescIdx = uHeadIdx;
620
621	Log6Func(("%s DESC CHAIN: (head) desc_idx=%u\n", pVirtq->szVirtqName, uHeadIdx));
622	RT_NOREF(pVirtq);
623
624	VIRTQ_DESC_T desc;
625
626	uint32_t cbIn = 0, cbOut = 0, cSegsIn = 0, cSegsOut = 0;
627
628	do
629	{
630	PVIRTIOSGSEG pSeg;
631
632	/*
633	* Malicious guests may go beyond paSegsIn or paSegsOut boundaries by linking
634	* several descriptors into a loop. Since there is no legitimate way to get a sequences of
635	* linked descriptors exceeding the total number of descriptors in the ring (see @bugref{8620}),
636	* the following aborts I/O if breach and employs a simple log throttling algorithm to notify.
637	*/
638	if (cSegsIn + cSegsOut >= VIRTQ_MAX_SIZE)
639	{
640	static volatile uint32_t s_cMessages = 0;
641	static volatile uint32_t s_cThreshold = 1;
642	if (ASMAtomicIncU32(&s_cMessages) == ASMAtomicReadU32(&s_cThreshold))
643	{
644	LogRelMax(64, ("Too many linked descriptors; check if the guest arranges descriptors in a loop.\n"));
645	if (ASMAtomicReadU32(&s_cMessages) != 1)
646	LogRelMax(64, ("(the above error has occured %u times so far)\n", ASMAtomicReadU32(&s_cMessages)));
647	ASMAtomicWriteU32(&s_cThreshold, ASMAtomicReadU32(&s_cThreshold) * 10);
648	}
649	break;
650	}
651	RT_UNTRUSTED_VALIDATED_FENCE();
652
653	virtioReadDesc(pDevIns, pVirtio, idxQueue, uDescIdx, &desc);
654
655	if (desc.fFlags & VIRTQ_DESC_F_WRITE)
656	{
657	Log6Func(("%s IN desc_idx=%u seg=%u addr=%RGp cb=%u\n", VIRTQNAME(pVirtio, idxQueue), uDescIdx, cSegsIn, desc.GCPhysBuf, desc.cb));
658	cbIn += desc.cb;
659	pSeg = &(paSegsIn[cSegsIn++]);
660	}
661	else
662	{
663	Log6Func(("%s OUT desc_idx=%u seg=%u addr=%RGp cb=%u\n", VIRTQNAME(pVirtio, idxQueue), uDescIdx, cSegsOut, desc.GCPhysBuf, desc.cb));
664	cbOut += desc.cb;
665	pSeg = &(paSegsOut[cSegsOut++]);
666	}
667
668	pSeg->gcPhys = desc.GCPhysBuf;
669	pSeg->cbSeg = desc.cb;
670
671	uDescIdx = desc.uDescIdxNext;
672	} while (desc.fFlags & VIRTQ_DESC_F_NEXT);
673
674	PVIRTIO_DESC_CHAIN_T pDescChain = (PVIRTIO_DESC_CHAIN_T)RTMemAllocZ(sizeof(VIRTIO_DESC_CHAIN_T));
675	AssertReturn(pDescChain, VERR_NO_MEMORY);
676
677	pDescChain->uHeadIdx = uHeadIdx;
678	*ppDescChain = pDescChain;
679
680	if (cSegsIn)
681	{
682	PVIRTIOSGBUF pSgPhysIn = (PVIRTIOSGBUF)RTMemAllocZ(sizeof(VIRTIOSGBUF));
683	AssertReturn(pSgPhysIn, VERR_NO_MEMORY);
684
685	virtioCoreSgBufInit(pSgPhysIn, paSegsIn, cSegsIn);
686	pDescChain->pSgPhysReturn = pSgPhysIn;
687	pDescChain->cbPhysReturn = cbIn;
688	}
689
690	if (cSegsOut)
691	{
692	PVIRTIOSGBUF pSgPhysOut = (PVIRTIOSGBUF)RTMemAllocZ(sizeof(VIRTIOSGBUF));
693	AssertReturn(pSgPhysOut, VERR_NO_MEMORY);
694
695	virtioCoreSgBufInit(pSgPhysOut, paSegsOut, cSegsOut);
696	pDescChain->pSgPhysSend = pSgPhysOut;
697	pDescChain->cbPhysSend = cbOut;
698	}
699
700
701	Log6Func(("%s -- segs OUT: %u (%u bytes) IN: %u (%u bytes) --\n", pVirtq->szVirtqName, cSegsOut, cbOut, cSegsIn, cbIn));
702
703	return VINF_SUCCESS;
704	}
705
706	/*
707	* Notifies guest (via ISR or MSI-X) of device configuration change
708	*
709	* @param pVirtio Pointer to the shared virtio state.
710	*/
711	void virtioCoreNotifyConfigChanged(PVIRTIOCORE pVirtio)
712	{
713	virtioKick(pVirtio->pDevIns, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig, false);
714	}
715
716	/**
717	* Enable or Disable notification for the specified queue
718	*
719	* @param pVirtio Pointer to the shared virtio state.
720	* @param idxQueue Queue number
721	* @param fEnabled Selects notification mode (enabled or disabled)
722	*/
723	void virtioCoreQueueSetNotify(PVIRTIOCORE pVirtio, uint16_t idxQueue, bool fEnabled)
724	{
725	if (pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
726	{
727	uint16_t fFlags = virtioReadUsedRingFlags(pVirtio->pDevIns, pVirtio, idxQueue);
728
729	if (fEnabled)
730	fFlags &= ~ VIRTQ_USED_F_NO_NOTIFY;
731	else
732	fFlags \|= VIRTQ_USED_F_NO_NOTIFY;
733
734	virtioWriteUsedRingFlags(pVirtio->pDevIns, pVirtio, idxQueue, fFlags);
735	}
736	}
737
738	/**
739	* Initiate orderly reset procedure. This is an exposed API for clients that might need it.
740	* Invoked by client to reset the device and driver (see VirtIO 1.0 section 2.1.1/2.1.2)
741	*
742	* @param pVirtio Pointer to the virtio state.
743	*/
744	void virtioCoreResetAll(PVIRTIOCORE pVirtio)
745	{
746	LogFunc(("\n"));
747	pVirtio->uDeviceStatus \|= VIRTIO_STATUS_DEVICE_NEEDS_RESET;
748	if (pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
749	{
750	pVirtio->fGenUpdatePending = true;
751	virtioKick(pVirtio->pDevIns, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig, false /* fForce */);
752	}
753	}
754	/**
755	* Get count of new (e.g. pending) elements in available ring.
756	*
757	* @param pDevIns The device instance.
758	* @param pVirtio Pointer to the shared virtio state.
759	* @param idxQueue Queue number
760	*
761	* @returns how many entries have been added to ring as a delta of the consumer's
762	* avail index and the queue's guest-side current avail index.
763	*/
764	int virtioCoreR3QueuePendingCount(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
765	{
766	uint16_t uAvailRingIdx = virtioReadAvailRingIdx(pDevIns, pVirtio, idxQueue);
767	uint16_t uNextAvailIdx = pVirtio->virtqState[idxQueue].uAvailIdx;
768	uint16_t uDelta = uAvailRingIdx - uNextAvailIdx;
769	if (uAvailRingIdx > uNextAvailIdx)
770	return uDelta;
771	return VIRTQ_MAX_CNT + uDelta;
772	}
773	/**
774	* Fetches descriptor chain using avail ring of indicated queue and converts the descriptor
775	* chain into its OUT (to device) and IN to guest components, but does NOT remove it from
776	* the 'avail' queue. I.e. doesn't advance the index. This can be used with virtioQueueSkip(),
777	* which does advance the avail index. Together they facilitate a mechanism that allows
778	* work with a queue element (descriptor chain) to be aborted if necessary, by not advancing
779	* the pointer, or, upon success calling the skip function (above) to move to the next element.
780	*
781	* Additionally it converts the OUT desc chain data to a contiguous virtual
782	* memory buffer for easy consumption by the caller. The caller must return the
783	* descriptor chain pointer via virtioCoreR3QueuePut() and then call virtioCoreQueueSync()
784	* at some point to return the data to the guest and complete the transaction.
785	*
786	* @param pDevIns The device instance.
787	* @param pVirtio Pointer to the shared virtio state.
788	* @param idxQueue Queue number
789	* @param ppDescChain Address to store pointer to descriptor chain that contains the
790	* pre-processed transaction information pulled from the virtq.
791	*
792	* @returns VBox status code:
793	* @retval VINF_SUCCESS Success
794	* @retval VERR_INVALID_STATE VirtIO not in ready state (asserted).
795	* @retval VERR_NOT_AVAILABLE If the queue is empty.
796	*/
797
798	int virtioCoreR3QueuePeek(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
799	PPVIRTIO_DESC_CHAIN_T ppDescChain)
800	{
801	return virtioCoreR3QueueGet(pDevIns, pVirtio, idxQueue, ppDescChain, false);
802	}
803
804	/**
805	* Skip the next entry in the specified queue (typically used with virtioCoreR3QueuePeek())
806	*
807	* @param pVirtio Pointer to the virtio state.
808	* @param idxQueue Index of queue
809	*/
810	int virtioCoreR3QueueSkip(PVIRTIOCORE pVirtio, uint16_t idxQueue)
811	{
812	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
813	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
814
815	AssertMsgReturn(IS_DRIVER_OK(pVirtio) && pVirtio->uQueueEnable[idxQueue],
816	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
817
818	if (virtioCoreQueueIsEmpty(pVirtio->pDevIns, pVirtio, idxQueue))
819	return VERR_NOT_AVAILABLE;
820
821	Log2Func(("%s avail_idx=%u\n", pVirtq->szVirtqName, pVirtq->uAvailIdx));
822	pVirtq->uAvailIdx++;
823
824	return VINF_SUCCESS;
825	}
826
827	/**
828	* Fetches descriptor chain using avail ring of indicated queue and converts the descriptor
829	* chain into its OUT (to device) and IN to guest components.
830	*
831	* Additionally it converts the OUT desc chain data to a contiguous virtual
832	* memory buffer for easy consumption by the caller. The caller must return the
833	* descriptor chain pointer via virtioCoreR3QueuePut() and then call virtioCoreQueueSync()
834	* at some point to return the data to the guest and complete the transaction.
835	*
836	* @param pDevIns The device instance.
837	* @param pVirtio Pointer to the shared virtio state.
838	* @param idxQueue Queue number
839	* @param ppDescChain Address to store pointer to descriptor chain that contains the
840	* pre-processed transaction information pulled from the virtq.
841	* @param fRemove flags whether to remove desc chain from queue (false = peek)
842	*
843	* @returns VBox status code:
844	* @retval VINF_SUCCESS Success
845	* @retval VERR_INVALID_STATE VirtIO not in ready state (asserted).
846	* @retval VERR_NOT_AVAILABLE If the queue is empty.
847	*/
848	int virtioCoreR3QueueGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
849	PPVIRTIO_DESC_CHAIN_T ppDescChain, bool fRemove)
850	{
851	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
852
853	if (virtqIsEmpty(pDevIns, pVirtio, idxQueue))
854	return VERR_NOT_AVAILABLE;
855
856	uint16_t uHeadIdx = virtioReadAvailDescIdx(pDevIns, pVirtio, idxQueue, pVirtq->uAvailIdx);
857
858	if (fRemove)
859	pVirtq->uAvailIdx++;
860
861	int rc = virtioCoreR3DescChainGet(pDevIns, pVirtio, idxQueue, uHeadIdx, ppDescChain);
862	return rc;
863	}
864
865	/**
866	* Returns data to the guest to complete a transaction initiated by virtQueueGet().
867	*
868	* The caller passes in a pointer to a scatter-gather buffer of virtual memory segments
869	* and a pointer to the descriptor chain context originally derived from the pulled
870	* queue entry, and this function will write the virtual memory s/g buffer into the
871	* guest's physical memory free the descriptor chain. The caller handles the freeing
872	* (as needed) of the virtual memory buffer.
873	*
874	* @note This does a write-ahead to the used ring of the guest's queue. The data
875	* written won't be seen by the guest until the next call to virtioCoreQueueSync()
876	*
877	*
878	* @param pDevIns The device instance (for reading).
879	* @param pVirtio Pointer to the shared virtio state.
880	* @param idxQueue Queue number
881	*
882	* @param pSgVirtReturn Points to scatter-gather buffer of virtual memory
883	* segments the caller is returning to the guest.
884	*
885	* @param pDescChain This contains the context of the scatter-gather
886	* buffer originally pulled from the queue.
887	*
888	* @param fFence If true, put up copy fence (memory barrier) after
889	* copying to guest phys. mem.
890	*
891	* @returns VBox status code.
892	* @retval VINF_SUCCESS Success
893	* @retval VERR_INVALID_STATE VirtIO not in ready state
894	* @retval VERR_NOT_AVAILABLE Queue is empty
895	*/
896	int virtioCoreR3QueuePut(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, PRTSGBUF pSgVirtReturn,
897	PVIRTIO_DESC_CHAIN_T pDescChain, bool fFence)
898	{
899	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
900	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
901	PVIRTIOSGBUF pSgPhysReturn = pDescChain->pSgPhysReturn;
902
903	AssertMsgReturn(IS_DRIVER_OK(pVirtio) /&& pVirtio->uQueueEnable[idxQueue]/,
904	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
905
906	Log6Func(("Copying client data to %s, desc chain (head desc_idx %d)\n",
907	VIRTQNAME(pVirtio, idxQueue), virtioReadUsedRingIdx(pDevIns, pVirtio, idxQueue)));
908
909	/* Copy s/g buf (virtual memory) to guest phys mem (IN direction). */
910
911	size_t cbCopy = 0, cbTotal = 0, cbRemain = 0;
912
913	if (pSgVirtReturn)
914	{
915	cbRemain = cbTotal = RTSgBufCalcTotalLength(pSgVirtReturn);
916	virtioCoreSgBufReset(pSgPhysReturn); /* Reset ptr because req data may have already been written */
917	while (cbRemain)
918	{
919	PVIRTIOSGSEG paSeg = &pSgPhysReturn->paSegs[pSgPhysReturn->idxSeg];
920	/** @todo r=bird: Shouldn't this be: RT_MIN(pSgVirtReturn->cbSegLeft, pSgPhysReturn->cbSegLeft); */
921	cbCopy = RT_MIN(pSgVirtReturn->cbSegLeft, paSeg->cbSeg - (size_t)(pSgPhysReturn->gcPhysCur - paSeg->gcPhys));
922	Assert(cbCopy > 0); /** @todo r=bird: There is no check that there is sufficient space in the output
923	* buffer (pSgPhysReturn), so we might loop here forever if the caller is careless,
924	* right? I'm pretty sure virtioScsiR3SendEvent could do more checks. */
925	PDMDevHlpPhysWrite(pDevIns, (RTGCPHYS)pSgPhysReturn->gcPhysCur, pSgVirtReturn->pvSegCur, cbCopy);
926	RTSgBufAdvance(pSgVirtReturn, cbCopy);
927	virtioCoreSgBufAdvance(pSgPhysReturn, cbCopy);
928	cbRemain -= cbCopy;
929	}
930
931	if (fFence)
932	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); /* needed? */
933
934	Assert(!(cbCopy >> 32));
935	}
936
937	/* If this write-ahead crosses threshold where the driver wants to get an event flag it */
938	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
939	if (pVirtq->uUsedIdx == virtioReadAvailUsedEvent(pDevIns, pVirtio, idxQueue))
940	pVirtq->fEventThresholdReached = true;
941
942	/*
943	* Place used buffer's descriptor in used ring but don't update used ring's slot index.
944	* That will be done with a subsequent client call to virtioCoreQueueSync() */
945	virtioWriteUsedElem(pDevIns, pVirtio, idxQueue, pVirtq->uUsedIdx++, pDescChain->uHeadIdx, (uint32_t)cbTotal);
946
947	if (pSgVirtReturn)
948	Log6Func((".... Copied %zu bytes in %d segs to %u byte buffer, residual=%zu\n",
949	cbTotal - cbRemain, pSgVirtReturn->cSegs, pDescChain->cbPhysReturn, pDescChain->cbPhysReturn - cbTotal));
950
951	Log6Func(("Write ahead used_idx=%u, %s used_idx=%u\n",
952	pVirtq->uUsedIdx, VIRTQNAME(pVirtio, idxQueue), virtioReadUsedRingIdx(pDevIns, pVirtio, idxQueue)));
953
954	if (pDescChain->pSgPhysSend)
955	{
956	RTMemFree(pDescChain->pSgPhysSend->paSegs);
957	RTMemFree(pDescChain->pSgPhysSend);
958	}
959	if (pDescChain->pSgPhysReturn)
960	{
961	RTMemFree(pSgPhysReturn->paSegs);
962	RTMemFree(pSgPhysReturn);
963	}
964	RTMemFree(pDescChain);
965
966	return VINF_SUCCESS;
967	}
968
969	#endif /* IN_RING3 */
970
971	/**
972	* Updates the indicated virtq's "used ring" descriptor index to match the
973	* current write-head index, thus exposing the data added to the used ring by all
974	* virtioCoreR3QueuePut() calls since the last sync. This should be called after one or
975	* more virtioCoreR3QueuePut() calls to inform the guest driver there is data in the queue.
976	* Explicit notifications (e.g. interrupt or MSI-X) will be sent to the guest,
977	* depending on VirtIO features negotiated and conditions, otherwise the guest
978	* will detect the update by polling. (see VirtIO 1.0
979	* specification, Section 2.4 "Virtqueues").
980	*
981	* @param pDevIns The device instance.
982	* @param pVirtio Pointer to the shared virtio state.
983	* @param idxQueue Queue number
984	*
985	* @returns VBox status code.
986	* @retval VINF_SUCCESS Success
987	* @retval VERR_INVALID_STATE VirtIO not in ready state
988	*/
989	int virtioCoreQueueSync(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
990	{
991	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
992	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
993
994	AssertMsgReturn(IS_DRIVER_OK(pVirtio) && pVirtio->uQueueEnable[idxQueue],
995	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
996
997	Log6Func(("Updating %s used_idx from %u to %u\n",
998	VIRTQNAME(pVirtio, idxQueue), virtioReadUsedRingIdx(pDevIns, pVirtio, idxQueue), pVirtq->uUsedIdx));
999
1000	virtioWriteUsedRingIdx(pDevIns, pVirtio, idxQueue, pVirtq->uUsedIdx);
1001	virtioNotifyGuestDriver(pDevIns, pVirtio, idxQueue, false);
1002
1003	return VINF_SUCCESS;
1004	}
1005
1006	#ifdef IN_RING3
1007	/**
1008	*/
1009	static void virtioR3QueueNotified(PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, uint16_t idxQueue, uint16_t uNotifyIdx)
1010	{
1011	/* See VirtIO 1.0, section 4.1.5.2 It implies that idxQueue and uNotifyIdx should match.
1012	* Disregarding this notification may cause throughput to stop, however there's no way to know
1013	* which was queue was intended for wake-up if the two parameters disagree. */
1014
1015	AssertMsg(uNotifyIdx == idxQueue,
1016	("Guest kicked virtq %d's notify addr w/non-corresponding virtq idx %d\n",
1017	idxQueue, uNotifyIdx));
1018	RT_NOREF(uNotifyIdx);
1019
1020	AssertReturnVoid(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1021	Log6Func(("%s\n", pVirtio->virtqState[idxQueue].szVirtqName));
1022
1023	/* Inform client */
1024	pVirtioCC->pfnQueueNotified(pVirtio, pVirtioCC, idxQueue);
1025	}
1026	#endif /* IN_RING3 */
1027
1028	/**
1029	* Trigger MSI-X or INT# interrupt to notify guest of data added to used ring of
1030	* the specified virtq, depending on the interrupt configuration of the device
1031	* and depending on negotiated and realtime constraints flagged by the guest driver.
1032	*
1033	* See VirtIO 1.0 specification (section 2.4.7).
1034	*
1035	* @param pDevIns The device instance.
1036	* @param pVirtio Pointer to the shared virtio state.
1037	* @param idxQueue Queue to check for guest interrupt handling preference
1038	* @param fForce Overrides idxQueue, forcing notification regardless of driver's
1039	* notification preferences. This is a safeguard to prevent
1040	* stalls upon resuming the VM. VirtIO 1.0 specification Section 4.1.5.5
1041	* indicates spurious interrupts are harmless to guest driver's state,
1042	* as they only cause the guest driver to [re]scan queues for work to do.
1043	*/
1044	static void virtioNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, bool fForce)
1045	{
1046
1047	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1048	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
1049
1050	AssertMsgReturnVoid(IS_DRIVER_OK(pVirtio), ("Guest driver not in ready state.\n"));
1051	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1052	{
1053	if (pVirtq->fEventThresholdReached)
1054	{
1055	virtioKick(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtio->uQueueMsixVector[idxQueue], fForce);
1056	pVirtq->fEventThresholdReached = false;
1057	return;
1058	}
1059	Log6Func(("...skipping interrupt: VIRTIO_F_EVENT_IDX set but threshold not reached\n"));
1060	}
1061	else
1062	{
1063	/** If guest driver hasn't suppressed interrupts, interrupt */
1064	if (fForce \|\| !(virtioReadUsedRingFlags(pDevIns, pVirtio, idxQueue) & VIRTQ_AVAIL_F_NO_INTERRUPT))
1065	{
1066	virtioKick(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtio->uQueueMsixVector[idxQueue], fForce);
1067	return;
1068	}
1069
1070	Log6Func(("...skipping interrupt. Guest flagged VIRTQ_AVAIL_F_NO_INTERRUPT for queue\n"));
1071	}
1072	}
1073
1074	/**
1075	* Raise interrupt or MSI-X
1076	*
1077	* @param pDevIns The device instance.
1078	* @param pVirtio Pointer to the shared virtio state.
1079	* @param uCause Interrupt cause bit mask to set in PCI ISR port.
1080	* @param uVec MSI-X vector, if enabled
1081	* @param uForce True of out-of-band
1082	*/
1083	static int virtioKick(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uMsixVector, bool fForce)
1084	{
1085	if (fForce)
1086	Log6Func(("reason: resumed after suspend\n"));
1087	else
1088	if (uCause == VIRTIO_ISR_VIRTQ_INTERRUPT)
1089	Log6Func(("reason: buffer added to 'used' ring.\n"));
1090	else
1091	if (uCause == VIRTIO_ISR_DEVICE_CONFIG)
1092	Log6Func(("reason: device config change\n"));
1093
1094	if (!pVirtio->fMsiSupport)
1095	{
1096	pVirtio->uISR \|= uCause;
1097	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_HIGH);
1098	}
1099	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1100	PDMDevHlpPCISetIrq(pDevIns, uMsixVector, 1);
1101	return VINF_SUCCESS;
1102	}
1103
1104	/**
1105	* Lower interrupt (Called when guest reads ISR and when resetting)
1106	*
1107	* @param pDevIns The device instance.
1108	*/
1109	static void virtioLowerInterrupt(PPDMDEVINS pDevIns, uint16_t uMsixVector)
1110	{
1111	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1112	if (!pVirtio->fMsiSupport)
1113	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_LOW);
1114	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1115	PDMDevHlpPCISetIrq(pDevIns, pVirtio->uMsixConfig, PDM_IRQ_LEVEL_LOW);
1116	}
1117
1118	#ifdef IN_RING3
1119	static void virtioResetQueue(PVIRTIOCORE pVirtio, uint16_t idxQueue)
1120	{
1121	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1122	PVIRTQSTATE pVirtQ = &pVirtio->virtqState[idxQueue];
1123	pVirtQ->uAvailIdx = 0;
1124	pVirtQ->uUsedIdx = 0;
1125	pVirtio->uQueueEnable[idxQueue] = false;
1126	pVirtio->uQueueSize[idxQueue] = VIRTQ_MAX_SIZE;
1127	pVirtio->uQueueNotifyOff[idxQueue] = idxQueue;
1128	pVirtio->uQueueMsixVector[idxQueue] = idxQueue + 2;
1129
1130	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1131	pVirtio->uQueueMsixVector[idxQueue] = VIRTIO_MSI_NO_VECTOR;
1132
1133	virtioLowerInterrupt(pVirtio->pDevIns, pVirtio->uQueueMsixVector[idxQueue]);
1134	}
1135
1136	static void virtioResetDevice(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
1137	{
1138	Log2Func(("\n"));
1139	pVirtio->uDeviceFeaturesSelect = 0;
1140	pVirtio->uDriverFeaturesSelect = 0;
1141	pVirtio->uConfigGeneration = 0;
1142	pVirtio->uDeviceStatus = 0;
1143	pVirtio->uISR = 0;
1144
1145	if (!pVirtio->fMsiSupport)
1146	virtioLowerInterrupt(pDevIns, 0);
1147	else
1148	{
1149	virtioLowerInterrupt(pDevIns, pVirtio->uMsixConfig);
1150	for (int i = 0; i < VIRTQ_MAX_CNT; i++)
1151	{
1152	virtioLowerInterrupt(pDevIns, pVirtio->uQueueMsixVector[i]);
1153	pVirtio->uQueueMsixVector[i];
1154	}
1155	}
1156
1157	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1158	pVirtio->uMsixConfig = VIRTIO_MSI_NO_VECTOR;
1159
1160	for (uint16_t idxQueue = 0; idxQueue < VIRTQ_MAX_CNT; idxQueue++)
1161	virtioResetQueue(pVirtio, idxQueue);
1162	}
1163
1164	/**
1165	* Invoked by this implementation when guest driver resets the device.
1166	* The driver itself will not until the device has read the status change.
1167	*/
1168	static void virtioGuestR3WasReset(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1169	{
1170	LogFunc(("Guest reset the device\n"));
1171
1172	/* Let the client know */
1173	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 0);
1174	virtioResetDevice(pDevIns, pVirtio);
1175	}
1176	#endif /* IN_RING3 */
1177
1178	/**
1179	* Handle accesses to Common Configuration capability
1180	*
1181	* @returns VBox status code
1182	*
1183	* @param pDevIns The device instance.
1184	* @param pVirtio Pointer to the shared virtio state.
1185	* @param pVirtioCC Pointer to the current context virtio state.
1186	* @param fWrite Set if write access, clear if read access.
1187	* @param offCfg The common configuration capability offset.
1188	* @param cb Number of bytes to read or write
1189	* @param pv Pointer to location to write to or read from
1190	*/
1191	static int virtioCommonCfgAccessed(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1192	int fWrite, uint32_t offCfg, unsigned cb, void *pv)
1193	{
1194	/**
1195	* This macro resolves to boolean true if the implied parameters, offCfg and cb,
1196	* match the field offset and size of a field in the Common Cfg struct, (or if
1197	* it is a 64-bit field, if it accesses either 32-bit part as a 32-bit access)
1198	* This is mandated by section 4.1.3.1 of the VirtIO 1.0 specification)
1199	*
1200	* @param member Member of VIRTIO_PCI_COMMON_CFG_T
1201	* @param offCfg Implied parameter: Offset into VIRTIO_PCI_COMMON_CFG_T
1202	* @param cb Implied parameter: Number of bytes to access
1203	* @result true or false
1204	*/
1205	#define MATCH_COMMON_CFG(member) \
1206	( ( RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member) == 8 \
1207	&& ( offCfg == RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member) \
1208	\|\| offCfg == RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member) + sizeof(uint32_t)) \
1209	&& cb == sizeof(uint32_t)) \
1210	\|\| ( offCfg == RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member) \
1211	&& cb == RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member)) )
1212
1213	#ifdef LOG_ENABLED
1214	# define LOG_COMMON_CFG_ACCESS(member, a_offIntra) \
1215	if (LogIs7Enabled()) { \
1216	virtioCoreLogMappedIoValue(__FUNCTION__, #member, RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member), \
1217	pv, cb, a_offIntra, fWrite, false, 0); \
1218	}
1219	# define LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, a_offIntra) \
1220	if (LogIs7Enabled()) { \
1221	virtioCoreLogMappedIoValue(__FUNCTION__, #member, RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member), \
1222	pv, cb, a_offIntra, fWrite, true, idx); \
1223	}
1224	#else
1225	# define LOG_COMMON_CFG_ACCESS(member, a_offIntra) do { } while (0)
1226	# define LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, a_offIntra) do { } while (0)
1227	#endif
1228
1229	#define COMMON_CFG_ACCESSOR(member) \
1230	do \
1231	{ \
1232	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1233	if (fWrite) \
1234	memcpy((char )&pVirtio->member + offIntra, (const char )pv, cb); \
1235	else \
1236	memcpy(pv, (const char *)&pVirtio->member + offIntra, cb); \
1237	LOG_COMMON_CFG_ACCESS(member, offIntra); \
1238	} while(0)
1239
1240	#define COMMON_CFG_ACCESSOR_INDEXED(member, idx) \
1241	do \
1242	{ \
1243	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1244	if (fWrite) \
1245	memcpy((char *)&pVirtio->member[idx] + offIntra, pv, cb); \
1246	else \
1247	memcpy(pv, (const char *)&pVirtio->member[idx] + offIntra, cb); \
1248	LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, offIntra); \
1249	} while(0)
1250
1251	#define COMMON_CFG_ACCESSOR_READONLY(member) \
1252	do \
1253	{ \
1254	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1255	if (fWrite) \
1256	LogFunc(("Guest attempted to write readonly virtio_pci_common_cfg.%s\n", #member)); \
1257	else \
1258	{ \
1259	memcpy(pv, (const char *)&pVirtio->member + offIntra, cb); \
1260	LOG_COMMON_CFG_ACCESS(member, offIntra); \
1261	} \
1262	} while(0)
1263
1264	#define COMMON_CFG_ACCESSOR_INDEXED_READONLY(member, idx) \
1265	do \
1266	{ \
1267	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1268	if (fWrite) \
1269	LogFunc(("Guest attempted to write readonly virtio_pci_common_cfg.%s[%d]\n", #member, idx)); \
1270	else \
1271	{ \
1272	memcpy(pv, (char const *)&pVirtio->member[idx] + offIntra, cb); \
1273	LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, offIntra); \
1274	} \
1275	} while(0)
1276
1277
1278	int rc = VINF_SUCCESS;
1279	uint64_t val;
1280	if (MATCH_COMMON_CFG(uDeviceFeatures))
1281	{
1282	if (fWrite) /* Guest WRITE pCommonCfg>uDeviceFeatures */
1283	{
1284	LogFunc(("Guest attempted to write readonly virtio_pci_common_cfg.device_feature\n"));
1285	return VINF_SUCCESS;
1286	}
1287	else /* Guest READ pCommonCfg->uDeviceFeatures */
1288	{
1289	switch (pVirtio->uDeviceFeaturesSelect)
1290	{
1291	case 0:
1292	val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1293	memcpy(pv, &val, cb);
1294	LOG_COMMON_CFG_ACCESS(uDeviceFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDeviceFeatures));
1295	break;
1296	case 1:
1297	val = pVirtio->uDeviceFeatures >> 32;
1298	memcpy(pv, &val, cb);
1299	LOG_COMMON_CFG_ACCESS(uDeviceFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDeviceFeatures) + 4);
1300	break;
1301	default:
1302	LogFunc(("Guest read uDeviceFeatures with out of range selector (%#x), returning 0\n",
1303	pVirtio->uDeviceFeaturesSelect));
1304	return VINF_IOM_MMIO_UNUSED_00;
1305	}
1306	}
1307	}
1308	else if (MATCH_COMMON_CFG(uDriverFeatures))
1309	{
1310	if (fWrite) /* Guest WRITE pCommonCfg->udriverFeatures */
1311	{
1312	switch (pVirtio->uDriverFeaturesSelect)
1313	{
1314	case 0:
1315	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1316	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures));
1317	break;
1318	case 1:
1319	memcpy((char *)&pVirtio->uDriverFeatures + sizeof(uint32_t), pv, cb);
1320	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures) + 4);
1321	break;
1322	default:
1323	LogFunc(("Guest wrote uDriverFeatures with out of range selector (%#x), returning 0\n",
1324	pVirtio->uDriverFeaturesSelect));
1325	return VINF_SUCCESS;
1326	}
1327	}
1328	else /* Guest READ pCommonCfg->udriverFeatures */
1329	{
1330	switch (pVirtio->uDriverFeaturesSelect)
1331	{
1332	case 0:
1333	val = pVirtio->uDriverFeatures & 0xffffffff;
1334	memcpy(pv, &val, cb);
1335	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures));
1336	break;
1337	case 1:
1338	val = (pVirtio->uDriverFeatures >> 32) & 0xffffffff;
1339	memcpy(pv, &val, cb);
1340	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures) + 4);
1341	break;
1342	default:
1343	LogFunc(("Guest read uDriverFeatures with out of range selector (%#x), returning 0\n",
1344	pVirtio->uDriverFeaturesSelect));
1345	return VINF_IOM_MMIO_UNUSED_00;
1346	}
1347	}
1348	}
1349	else if (MATCH_COMMON_CFG(uNumQueues))
1350	{
1351	if (fWrite)
1352	{
1353	Log2Func(("Guest attempted to write readonly virtio_pci_common_cfg.num_queues\n"));
1354	return VINF_SUCCESS;
1355	}
1356	else
1357	{
1358	(uint16_t )pv = VIRTQ_MAX_CNT;
1359	LOG_COMMON_CFG_ACCESS(uNumQueues, 0);
1360	}
1361	}
1362	else if (MATCH_COMMON_CFG(uDeviceStatus))
1363	{
1364	if (fWrite) /* Guest WRITE pCommonCfg->uDeviceStatus */
1365	{
1366	uint8_t const fNewStatus = (uint8_t )pv;
1367	Log7Func(("Guest wrote uDeviceStatus (%#x, was %#x, change #%x) ................ (",
1368	fNewStatus, pVirtio->uDeviceStatus, fNewStatus ^ pVirtio->uDeviceStatus));
1369	if (LogIs7Enabled())
1370	virtioLogDeviceStatus(fNewStatus);
1371	Log7((")\n"));
1372
1373	/* If the status changed or we were reset, we need to go to ring-3 as
1374	it requires notifying the parent device. */
1375	bool const fStatusChanged = (fNewStatus & VIRTIO_STATUS_DRIVER_OK)
1376	!= (pVirtio->uPrevDeviceStatus & VIRTIO_STATUS_DRIVER_OK);
1377	#ifndef IN_RING3
1378	if (fStatusChanged \|\| fNewStatus == 0)
1379	{
1380	Log6Func(("=>ring3\n"));
1381	return VINF_IOM_R3_MMIO_WRITE;
1382	}
1383	#endif
1384	pVirtio->uDeviceStatus = fNewStatus;
1385
1386	#ifdef IN_RING3
1387	/*
1388	* Notify client only if status actually changed from last time and when we're reset.
1389	*/
1390	if (pVirtio->uDeviceStatus == 0)
1391	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1392	if (fStatusChanged)
1393	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, fNewStatus & VIRTIO_STATUS_DRIVER_OK);
1394	#endif
1395	/*
1396	* Save the current status for the next write so we can see what changed.
1397	*/
1398	pVirtio->uPrevDeviceStatus = pVirtio->uDeviceStatus;
1399	}
1400	else /* Guest READ pCommonCfg->uDeviceStatus */
1401	{
1402	Log7Func(("Guest read uDeviceStatus ................ ("));
1403	(uint8_t )pv = pVirtio->uDeviceStatus;
1404	if (LogIs7Enabled())
1405	virtioLogDeviceStatus(pVirtio->uDeviceStatus);
1406	Log7((")\n"));
1407	}
1408	}
1409	else
1410	if (MATCH_COMMON_CFG(uMsixConfig))
1411	COMMON_CFG_ACCESSOR(uMsixConfig);
1412	else
1413	if (MATCH_COMMON_CFG(uDeviceFeaturesSelect))
1414	COMMON_CFG_ACCESSOR(uDeviceFeaturesSelect);
1415	else
1416	if (MATCH_COMMON_CFG(uDriverFeaturesSelect))
1417	COMMON_CFG_ACCESSOR(uDriverFeaturesSelect);
1418	else
1419	if (MATCH_COMMON_CFG(uConfigGeneration))
1420	COMMON_CFG_ACCESSOR_READONLY(uConfigGeneration);
1421	else
1422	if (MATCH_COMMON_CFG(uQueueSelect))
1423	COMMON_CFG_ACCESSOR(uQueueSelect);
1424	else
1425	if (MATCH_COMMON_CFG(uQueueSize))
1426	COMMON_CFG_ACCESSOR_INDEXED(uQueueSize, pVirtio->uQueueSelect);
1427	else
1428	if (MATCH_COMMON_CFG(uQueueMsixVector))
1429	COMMON_CFG_ACCESSOR_INDEXED(uQueueMsixVector, pVirtio->uQueueSelect);
1430	else
1431	if (MATCH_COMMON_CFG(uQueueEnable))
1432	COMMON_CFG_ACCESSOR_INDEXED(uQueueEnable, pVirtio->uQueueSelect);
1433	else
1434	if (MATCH_COMMON_CFG(uQueueNotifyOff))
1435	COMMON_CFG_ACCESSOR_INDEXED_READONLY(uQueueNotifyOff, pVirtio->uQueueSelect);
1436	else
1437	if (MATCH_COMMON_CFG(aGCPhysQueueDesc))
1438	COMMON_CFG_ACCESSOR_INDEXED(aGCPhysQueueDesc, pVirtio->uQueueSelect);
1439	else
1440	if (MATCH_COMMON_CFG(aGCPhysQueueAvail))
1441	COMMON_CFG_ACCESSOR_INDEXED(aGCPhysQueueAvail, pVirtio->uQueueSelect);
1442	else
1443	if (MATCH_COMMON_CFG(aGCPhysQueueUsed))
1444	COMMON_CFG_ACCESSOR_INDEXED(aGCPhysQueueUsed, pVirtio->uQueueSelect);
1445	else
1446	{
1447	Log2Func(("Bad guest %s access to virtio_pci_common_cfg: offCfg=%#x (%d), cb=%d\n",
1448	fWrite ? "write" : "read ", offCfg, offCfg, cb));
1449	return fWrite ? VINF_SUCCESS : VINF_IOM_MMIO_UNUSED_00;
1450	}
1451
1452	#undef COMMON_CFG_ACCESSOR_READONLY
1453	#undef COMMON_CFG_ACCESSOR_INDEXED_READONLY
1454	#undef COMMON_CFG_ACCESSOR_INDEXED
1455	#undef COMMON_CFG_ACCESSOR
1456	#undef LOG_COMMON_CFG_ACCESS_INDEXED
1457	#undef LOG_COMMON_CFG_ACCESS
1458	#undef MATCH_COMMON_CFG
1459	#ifndef IN_RING3
1460	RT_NOREF(pDevIns, pVirtioCC);
1461	#endif
1462	return rc;
1463	}
1464
1465	/**
1466	* @callback_method_impl{FNIOMMMIONEWREAD,
1467	* Memory mapped I/O Handler for PCI Capabilities read operations.}
1468	*
1469	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
1470	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is limited to cb == 1, cb == 2, or cb==4 type reads.
1471	*
1472	*/
1473	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioRead(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void pv, unsigned cb)
1474	{
1475	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1476	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1477	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
1478	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
1479
1480	uint32_t offIntra;
1481	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocDeviceCap))
1482	{
1483	#ifdef IN_RING3
1484	/*
1485	* Callback to client to manage device-specific configuration.
1486	*/
1487	VBOXSTRICTRC rcStrict = pVirtioCC->pfnDevCapRead(pDevIns, offIntra, pv, cb);
1488
1489	/*
1490	* Additionally, anytime any part of the device-specific configuration (which our client maintains)
1491	* is READ it needs to be checked to see if it changed since the last time any part was read, in
1492	* order to maintain the config generation (see VirtIO 1.0 spec, section 4.1.4.3.1)
1493	*/
1494	bool fDevSpecificFieldChanged = !!memcmp(pVirtioCC->pbDevSpecificCfg + offIntra,
1495	pVirtioCC->pbPrevDevSpecificCfg + offIntra,
1496	RT_MIN(cb, pVirtioCC->cbDevSpecificCfg - offIntra));
1497
1498	memcpy(pVirtioCC->pbPrevDevSpecificCfg, pVirtioCC->pbDevSpecificCfg, pVirtioCC->cbDevSpecificCfg);
1499
1500	if (pVirtio->fGenUpdatePending \|\| fDevSpecificFieldChanged)
1501	{
1502	++pVirtio->uConfigGeneration;
1503	Log6Func(("Bumped cfg. generation to %d because %s%s\n",
1504	pVirtio->uConfigGeneration,
1505	fDevSpecificFieldChanged ? "<dev cfg changed> " : "",
1506	pVirtio->fGenUpdatePending ? "<update was pending>" : ""));
1507	pVirtio->fGenUpdatePending = false;
1508	}
1509
1510	virtioLowerInterrupt(pDevIns, 0);
1511	return rcStrict;
1512	#else
1513	return VINF_IOM_R3_MMIO_READ;
1514	#endif
1515	}
1516
1517	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocCommonCfgCap))
1518	return virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, false /* fWrite */, offIntra, cb, pv);
1519
1520	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocIsrCap) && cb == sizeof(uint8_t))
1521	{
1522	(uint8_t )pv = pVirtio->uISR;
1523	Log6Func(("Read and clear ISR\n"));
1524	pVirtio->uISR = 0; /* VirtIO specification requires reads of ISR to clear it */
1525	virtioLowerInterrupt(pDevIns, 0);
1526	return VINF_SUCCESS;
1527	}
1528
1529	ASSERT_GUEST_MSG_FAILED(("Bad read access to mapped capabilities region: off=%RGp cb=%u\n", off, cb));
1530	return VINF_IOM_MMIO_UNUSED_00;
1531	}
1532
1533	/**
1534	* @callback_method_impl{FNIOMMMIONEWREAD,
1535	* Memory mapped I/O Handler for PCI Capabilities write operations.}
1536	*
1537	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
1538	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is limited
1539	* to cb == 1, cb == 2, or cb==4 type writes.
1540	*/
1541	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioWrite(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void const pv, unsigned cb)
1542	{
1543	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1544	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1545
1546	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
1547
1548	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
1549
1550	uint32_t offIntra;
1551	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocDeviceCap))
1552	{
1553	#ifdef IN_RING3
1554	/*
1555	* Pass this MMIO write access back to the client to handle
1556	*/
1557	return pVirtioCC->pfnDevCapWrite(pDevIns, offIntra, pv, cb);
1558	#else
1559	return VINF_IOM_R3_MMIO_WRITE;
1560	#endif
1561	}
1562
1563	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocCommonCfgCap))
1564	return virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, true /* fWrite /, offIntra, cb, (void )pv);
1565
1566	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocIsrCap) && cb == sizeof(uint8_t))
1567	{
1568	pVirtio->uISR = (uint8_t )pv;
1569	Log6Func(("Setting uISR = 0x%02x (virtq interrupt: %d, dev confg interrupt: %d)\n",
1570	pVirtio->uISR & 0xff,
1571	pVirtio->uISR & VIRTIO_ISR_VIRTQ_INTERRUPT,
1572	RT_BOOL(pVirtio->uISR & VIRTIO_ISR_DEVICE_CONFIG)));
1573	return VINF_SUCCESS;
1574	}
1575
1576	/* This should be guest driver dropping index of a new descriptor in avail ring */
1577	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocNotifyCap) && cb == sizeof(uint16_t))
1578	{
1579	#ifdef IN_RING3
1580	virtioR3QueueNotified(pVirtio, pVirtioCC, offIntra / VIRTIO_NOTIFY_OFFSET_MULTIPLIER, (uint16_t )pv);
1581	return VINF_SUCCESS;
1582	#else
1583	return VINF_IOM_R3_MMIO_WRITE;
1584	#endif
1585	}
1586
1587	ASSERT_GUEST_MSG_FAILED(("Bad write access to mapped capabilities region: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb));
1588	return VINF_SUCCESS;
1589	}
1590
1591	#ifdef IN_RING3
1592
1593	/**
1594	* @callback_method_impl{FNPCICONFIGREAD}
1595	*/
1596	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigRead(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
1597	uint32_t uAddress, unsigned cb, uint32_t *pu32Value)
1598	{
1599	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1600	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1601	RT_NOREF(pPciDev);
1602
1603	Log7Func(("pDevIns=%p pPciDev=%p uAddress=%#x cb=%u pu32Value=%p\n",
1604	pDevIns, pPciDev, uAddress, cb, pu32Value));
1605	if (uAddress == pVirtio->uPciCfgDataOff)
1606	{
1607	/*
1608	* VirtIO 1.0 spec section 4.1.4.7 describes a required alternative access capability
1609	* whereby the guest driver can specify a bar, offset, and length via the PCI configuration space
1610	* (the virtio_pci_cfg_cap capability), and access data items.
1611	*/
1612	uint32_t uLength = pVirtioCC->pPciCfgCap->pciCap.uLength;
1613	uint32_t uOffset = pVirtioCC->pPciCfgCap->pciCap.uOffset;
1614	uint8_t uBar = pVirtioCC->pPciCfgCap->pciCap.uBar;
1615
1616	if ( (uLength != 1 && uLength != 2 && uLength != 4)
1617	\|\| cb != uLength
1618	\|\| uBar != VIRTIO_REGION_PCI_CAP)
1619	{
1620	ASSERT_GUEST_MSG_FAILED(("Guest read virtio_pci_cfg_cap.pci_cfg_data using mismatching config. Ignoring\n"));
1621	*pu32Value = UINT32_MAX;
1622	return VINF_SUCCESS;
1623	}
1624
1625	VBOXSTRICTRC rcStrict = virtioMmioRead(pDevIns, pVirtio, uOffset, pu32Value, cb);
1626	Log2Func(("virtio: Guest read virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%d, length=%d, result=%d -> %Rrc\n",
1627	uBar, uOffset, uLength, *pu32Value, VBOXSTRICTRC_VAL(rcStrict)));
1628	return rcStrict;
1629	}
1630	return VINF_PDM_PCI_DO_DEFAULT;
1631	}
1632
1633	/**
1634	* @callback_method_impl{FNPCICONFIGWRITE}
1635	*/
1636	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigWrite(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
1637	uint32_t uAddress, unsigned cb, uint32_t u32Value)
1638	{
1639	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1640	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1641	RT_NOREF(pPciDev);
1642
1643	Log7Func(("pDevIns=%p pPciDev=%p uAddress=%#x cb=%u u32Value=%#x\n", pDevIns, pPciDev, uAddress, cb, u32Value));
1644	if (uAddress == pVirtio->uPciCfgDataOff)
1645	{
1646	/* VirtIO 1.0 spec section 4.1.4.7 describes a required alternative access capability
1647	* whereby the guest driver can specify a bar, offset, and length via the PCI configuration space
1648	* (the virtio_pci_cfg_cap capability), and access data items. */
1649
1650	uint32_t uLength = pVirtioCC->pPciCfgCap->pciCap.uLength;
1651	uint32_t uOffset = pVirtioCC->pPciCfgCap->pciCap.uOffset;
1652	uint8_t uBar = pVirtioCC->pPciCfgCap->pciCap.uBar;
1653
1654	if ( (uLength != 1 && uLength != 2 && uLength != 4)
1655	\|\| cb != uLength
1656	\|\| uBar != VIRTIO_REGION_PCI_CAP)
1657	{
1658	ASSERT_GUEST_MSG_FAILED(("Guest write virtio_pci_cfg_cap.pci_cfg_data using mismatching config. Ignoring\n"));
1659	return VINF_SUCCESS;
1660	}
1661
1662	VBOXSTRICTRC rcStrict = virtioMmioWrite(pDevIns, pVirtio, uOffset, &u32Value, cb);
1663	Log2Func(("Guest wrote virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%x, length=%x, value=%d -> %Rrc\n",
1664	uBar, uOffset, uLength, u32Value, VBOXSTRICTRC_VAL(rcStrict)));
1665	return rcStrict;
1666	}
1667	return VINF_PDM_PCI_DO_DEFAULT;
1668	}
1669
1670
1671	/*********************************************************************************************************************************
1672	* Saved state. *
1673	*********************************************************************************************************************************/
1674
1675	/**
1676	* Called from the FNSSMDEVSAVEEXEC function of the device.
1677	*
1678	* @param pVirtio Pointer to the shared virtio state.
1679	* @param pHlp The ring-3 device helpers.
1680	* @param pSSM The saved state handle.
1681	* @returns VBox status code.
1682	*/
1683	int virtioCoreR3SaveExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM)
1684	{
1685	LogFunc(("\n"));
1686	pHlp->pfnSSMPutU64(pSSM, VIRTIO_SAVEDSTATE_MARKER);
1687	pHlp->pfnSSMPutU32(pSSM, VIRTIO_SAVEDSTATE_VERSION);
1688
1689	pHlp->pfnSSMPutBool(pSSM, pVirtio->fGenUpdatePending);
1690	pHlp->pfnSSMPutU8(pSSM, pVirtio->uDeviceStatus);
1691	pHlp->pfnSSMPutU8(pSSM, pVirtio->uConfigGeneration);
1692	pHlp->pfnSSMPutU8(pSSM, pVirtio->uPciCfgDataOff);
1693	pHlp->pfnSSMPutU8(pSSM, pVirtio->uISR);
1694	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueSelect);
1695	pHlp->pfnSSMPutU32(pSSM, pVirtio->uDeviceFeaturesSelect);
1696	pHlp->pfnSSMPutU32(pSSM, pVirtio->uDriverFeaturesSelect);
1697	pHlp->pfnSSMPutU64(pSSM, pVirtio->uDriverFeatures);
1698
1699	for (uint32_t i = 0; i < VIRTQ_MAX_CNT; i++)
1700	{
1701	pHlp->pfnSSMPutGCPhys64(pSSM, pVirtio->aGCPhysQueueDesc[i]);
1702	pHlp->pfnSSMPutGCPhys64(pSSM, pVirtio->aGCPhysQueueAvail[i]);
1703	pHlp->pfnSSMPutGCPhys64(pSSM, pVirtio->aGCPhysQueueUsed[i]);
1704	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueNotifyOff[i]);
1705	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueMsixVector[i]);
1706	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueEnable[i]);
1707	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueSize[i]);
1708	pHlp->pfnSSMPutU16(pSSM, pVirtio->virtqState[i].uAvailIdx);
1709	pHlp->pfnSSMPutU16(pSSM, pVirtio->virtqState[i].uUsedIdx);
1710	int rc = pHlp->pfnSSMPutMem(pSSM, pVirtio->virtqState[i].szVirtqName, 32);
1711	AssertRCReturn(rc, rc);
1712	}
1713
1714	return VINF_SUCCESS;
1715	}
1716
1717	/**
1718	* Called from the FNSSMDEVLOADEXEC function of the device.
1719	*
1720	* @param pVirtio Pointer to the shared virtio state.
1721	* @param pHlp The ring-3 device helpers.
1722	* @param pSSM The saved state handle.
1723	* @returns VBox status code.
1724	*/
1725	int virtioCoreR3LoadExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM)
1726	{
1727	LogFunc(("\n"));
1728	/*
1729	* Check the marker and (embedded) version number.
1730	*/
1731	uint64_t uMarker = 0;
1732	int rc = pHlp->pfnSSMGetU64(pSSM, &uMarker);
1733	AssertRCReturn(rc, rc);
1734	if (uMarker != VIRTIO_SAVEDSTATE_MARKER)
1735	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
1736	N_("Expected marker value %#RX64 found %#RX64 instead"),
1737	VIRTIO_SAVEDSTATE_MARKER, uMarker);
1738	uint32_t uVersion = 0;
1739	rc = pHlp->pfnSSMGetU32(pSSM, &uVersion);
1740	AssertRCReturn(rc, rc);
1741	if (uVersion != VIRTIO_SAVEDSTATE_VERSION)
1742	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
1743	N_("Unsupported virtio version: %u"), uVersion);
1744	/*
1745	* Load the state.
1746	*/
1747	pHlp->pfnSSMGetBool(pSSM, &pVirtio->fGenUpdatePending);
1748	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uDeviceStatus);
1749	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uConfigGeneration);
1750	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uPciCfgDataOff);
1751	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uISR);
1752	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueSelect);
1753	pHlp->pfnSSMGetU32(pSSM, &pVirtio->uDeviceFeaturesSelect);
1754	pHlp->pfnSSMGetU32(pSSM, &pVirtio->uDriverFeaturesSelect);
1755	pHlp->pfnSSMGetU64(pSSM, &pVirtio->uDriverFeatures);
1756
1757	for (uint32_t i = 0; i < VIRTQ_MAX_CNT; i++)
1758	{
1759	pHlp->pfnSSMGetGCPhys64(pSSM, &pVirtio->aGCPhysQueueDesc[i]);
1760	pHlp->pfnSSMGetGCPhys64(pSSM, &pVirtio->aGCPhysQueueAvail[i]);
1761	pHlp->pfnSSMGetGCPhys64(pSSM, &pVirtio->aGCPhysQueueUsed[i]);
1762	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueNotifyOff[i]);
1763	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueMsixVector[i]);
1764	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueEnable[i]);
1765	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueSize[i]);
1766	pHlp->pfnSSMGetU16(pSSM, &pVirtio->virtqState[i].uAvailIdx);
1767	pHlp->pfnSSMGetU16(pSSM, &pVirtio->virtqState[i].uUsedIdx);
1768	rc = pHlp->pfnSSMGetMem(pSSM, pVirtio->virtqState[i].szVirtqName,
1769	sizeof(pVirtio->virtqState[i].szVirtqName));
1770	AssertRCReturn(rc, rc);
1771	}
1772
1773	return VINF_SUCCESS;
1774	}
1775
1776
1777	/*********************************************************************************************************************************
1778	* Device Level *
1779	*********************************************************************************************************************************/
1780
1781	/**
1782	* This must be called by the client to handle VM state changes
1783	* after the client takes care of its device-specific tasks for the state change.
1784	* (i.e. Reset, suspend, power-off, resume)
1785	*
1786	* @param pDevIns The device instance.
1787	* @param pVirtio Pointer to the shared virtio state.
1788	*/
1789	void virtioCoreR3VmStateChanged(PVIRTIOCORE pVirtio, VIRTIOVMSTATECHANGED enmState)
1790	{
1791	LogFunc(("State changing to %s\n",
1792	virtioCoreGetStateChangeText(enmState)));
1793
1794	switch(enmState)
1795	{
1796	case kvirtIoVmStateChangedReset:
1797	virtioCoreResetAll(pVirtio);
1798	break;
1799	case kvirtIoVmStateChangedSuspend:
1800	break;
1801	case kvirtIoVmStateChangedPowerOff:
1802	break;
1803	case kvirtIoVmStateChangedResume:
1804	virtioNotifyGuestDriver(pVirtio->pDevIns, pVirtio, 0 /* idxQueue /, true / fForce */);
1805	break;
1806	default:
1807	LogRelFunc(("Bad enum value"));
1808	return;
1809	}
1810	}
1811
1812	/**
1813	* This should be called from PDMDEVREGR3::pfnDestruct.
1814	*
1815	* @param pDevIns The device instance.
1816	* @param pVirtio Pointer to the shared virtio state.
1817	* @param pVirtioCC Pointer to the ring-3 virtio state.
1818	*/
1819	void virtioCoreR3Term(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1820	{
1821	if (pVirtioCC->pbPrevDevSpecificCfg)
1822	{
1823	RTMemFree(pVirtioCC->pbPrevDevSpecificCfg);
1824	pVirtioCC->pbPrevDevSpecificCfg = NULL;
1825	}
1826	RT_NOREF(pDevIns, pVirtio);
1827	}
1828
1829
1830	/**
1831	* Setup PCI device controller and Virtio state
1832	*
1833	* This should be called from PDMDEVREGR3::pfnConstruct.
1834	*
1835	* @param pDevIns The device instance.
1836	* @param pVirtio Pointer to the shared virtio state. This
1837	* must be the first member in the shared
1838	* device instance data!
1839	* @param pVirtioCC Pointer to the ring-3 virtio state. This
1840	* must be the first member in the ring-3
1841	* device instance data!
1842	* @param pPciParams Values to populate industry standard PCI Configuration Space data structure
1843	* @param pcszInstance Device instance name (format-specifier)
1844	* @param fDevSpecificFeatures VirtIO device-specific features offered by
1845	* client
1846	* @param cbDevSpecificCfg Size of virtio_pci_device_cap device-specific struct
1847	* @param pvDevSpecificCfg Address of client's dev-specific
1848	* configuration struct.
1849	*/
1850	int virtioCoreR3Init(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, PVIRTIOPCIPARAMS pPciParams,
1851	const char pcszInstance, uint64_t fDevSpecificFeatures, void pvDevSpecificCfg, uint16_t cbDevSpecificCfg)
1852	{
1853	/*
1854	* The pVirtio state must be the first member of the shared device instance
1855	* data, otherwise we cannot get our bearings in the PCI configuration callbacks.
1856	*/
1857	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
1858	AssertLogRelReturn(pVirtioCC == PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC), VERR_STATE_CHANGED);
1859
1860	pVirtio->pDevIns = pDevIns;
1861
1862	/*
1863	* Caller must initialize these.
1864	*/
1865	AssertReturn(pVirtioCC->pfnStatusChanged, VERR_INVALID_POINTER);
1866	AssertReturn(pVirtioCC->pfnQueueNotified, VERR_INVALID_POINTER);
1867
1868	#if 0 /* Until pdmR3DvHlp_PCISetIrq() impl is fixed and Assert that limits vec to 0 is removed */
1869	# ifdef VBOX_WITH_MSI_DEVICES
1870	pVirtio->fMsiSupport = true;
1871	# endif
1872	#endif
1873
1874	/*
1875	* The host features offered include both device-specific features
1876	* and reserved feature bits (device independent)
1877	*/
1878	pVirtio->uDeviceFeatures = VIRTIO_F_VERSION_1
1879	\| VIRTIO_DEV_INDEPENDENT_FEATURES_OFFERED
1880	\| fDevSpecificFeatures;
1881
1882	RTStrCopy(pVirtio->szInstance, sizeof(pVirtio->szInstance), pcszInstance);
1883
1884	pVirtio->uDeviceStatus = 0;
1885	pVirtioCC->cbDevSpecificCfg = cbDevSpecificCfg;
1886	pVirtioCC->pbDevSpecificCfg = (uint8_t *)pvDevSpecificCfg;
1887	pVirtioCC->pbPrevDevSpecificCfg = (uint8_t *)RTMemDup(pvDevSpecificCfg, cbDevSpecificCfg);
1888	AssertLogRelReturn(pVirtioCC->pbPrevDevSpecificCfg, VERR_NO_MEMORY);
1889
1890	/* Set PCI config registers (assume 32-bit mode) */
1891	PPDMPCIDEV pPciDev = pDevIns->apPciDevs[0];
1892	PDMPCIDEV_ASSERT_VALID(pDevIns, pPciDev);
1893
1894	PDMPciDevSetRevisionId(pPciDev, DEVICE_PCI_REVISION_ID_VIRTIO);
1895	PDMPciDevSetVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
1896	PDMPciDevSetSubSystemVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
1897	PDMPciDevSetDeviceId(pPciDev, pPciParams->uDeviceId);
1898	PDMPciDevSetClassBase(pPciDev, pPciParams->uClassBase);
1899	PDMPciDevSetClassSub(pPciDev, pPciParams->uClassSub);
1900	PDMPciDevSetClassProg(pPciDev, pPciParams->uClassProg);
1901	PDMPciDevSetSubSystemId(pPciDev, pPciParams->uSubsystemId);
1902	PDMPciDevSetInterruptLine(pPciDev, pPciParams->uInterruptLine);
1903	PDMPciDevSetInterruptPin(pPciDev, pPciParams->uInterruptPin);
1904
1905	/* Register PCI device */
1906	int rc = PDMDevHlpPCIRegister(pDevIns, pPciDev);
1907	if (RT_FAILURE(rc))
1908	return PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Device")); /* can we put params in this error? */
1909
1910	rc = PDMDevHlpPCIInterceptConfigAccesses(pDevIns, pPciDev, virtioR3PciConfigRead, virtioR3PciConfigWrite);
1911	AssertRCReturn(rc, rc);
1912
1913
1914	/* Construct & map PCI vendor-specific capabilities for virtio host negotiation with guest driver */
1915
1916	/* The following capability mapped via VirtIO 1.0: struct virtio_pci_cfg_cap (VIRTIO_PCI_CFG_CAP_T)
1917	* as a mandatory but suboptimal alternative interface to host device capabilities, facilitating
1918	* access the memory of any BAR. If the guest uses it (the VirtIO driver on Linux doesn't),
1919	* Unlike Common, Notify, ISR and Device capabilities, it is accessed directly via PCI Config region.
1920	* therefore does not contribute to the capabilities region (BAR) the other capabilities use.
1921	*/
1922	#define CFG_ADDR_2_IDX(addr) ((uint8_t)(((uintptr_t)(addr) - (uintptr_t)&pPciDev->abConfig[0])))
1923	#define SET_PCI_CAP_LOC(a_pPciDev, a_pCfg, a_LocCap, a_uMmioLengthAlign) \
1924	do { \
1925	(a_LocCap).offMmio = (a_pCfg)->uOffset; \
1926	(a_LocCap).cbMmio = RT_ALIGN_T((a_pCfg)->uLength, a_uMmioLengthAlign, uint16_t); \
1927	(a_LocCap).offPci = (uint16_t)(uintptr_t)((uint8_t *)(a_pCfg) - &(a_pPciDev)->abConfig[0]); \
1928	(a_LocCap).cbPci = (a_pCfg)->uCapLen; \
1929	} while (0)
1930
1931	PVIRTIO_PCI_CAP_T pCfg;
1932	uint32_t cbRegion = 0;
1933
1934	/* Common capability (VirtIO 1.0 spec, section 4.1.4.3) */
1935	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[0x40];
1936	pCfg->uCfgType = VIRTIO_PCI_CAP_COMMON_CFG;
1937	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
1938	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
1939	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
1940	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
1941	pCfg->uOffset = RT_ALIGN_32(0, 4); /* reminder, in case someone changes offset */
1942	pCfg->uLength = sizeof(VIRTIO_PCI_COMMON_CFG_T);
1943	cbRegion += pCfg->uLength;
1944	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocCommonCfgCap, 2);
1945	pVirtioCC->pCommonCfgCap = pCfg;
1946
1947	/*
1948	* Notify capability (VirtIO 1.0 spec, section 4.1.4.4). Note: uLength is based the choice
1949	* of this implementation that each queue's uQueueNotifyOff is set equal to (QueueSelect) ordinal
1950	* value of the queue
1951	*/
1952	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
1953	pCfg->uCfgType = VIRTIO_PCI_CAP_NOTIFY_CFG;
1954	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
1955	pCfg->uCapLen = sizeof(VIRTIO_PCI_NOTIFY_CAP_T);
1956	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
1957	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
1958	pCfg->uOffset = pVirtioCC->pCommonCfgCap->uOffset + pVirtioCC->pCommonCfgCap->uLength;
1959	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
1960
1961
1962	pCfg->uLength = VIRTQ_MAX_CNT * VIRTIO_NOTIFY_OFFSET_MULTIPLIER + 2; /* will change in VirtIO 1.1 */
1963	cbRegion += pCfg->uLength;
1964	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocNotifyCap, 1);
1965	pVirtioCC->pNotifyCap = (PVIRTIO_PCI_NOTIFY_CAP_T)pCfg;
1966	pVirtioCC->pNotifyCap->uNotifyOffMultiplier = VIRTIO_NOTIFY_OFFSET_MULTIPLIER;
1967
1968	/* ISR capability (VirtIO 1.0 spec, section 4.1.4.5)
1969	*
1970	* VirtIO 1.0 spec says 8-bit, unaligned in MMIO space. Example/diagram
1971	* of spec shows it as a 32-bit field with upper bits 'reserved'
1972	* Will take spec words more literally than the diagram for now.
1973	*/
1974	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
1975	pCfg->uCfgType = VIRTIO_PCI_CAP_ISR_CFG;
1976	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
1977	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
1978	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
1979	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
1980	pCfg->uOffset = pVirtioCC->pNotifyCap->pciCap.uOffset + pVirtioCC->pNotifyCap->pciCap.uLength;
1981	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
1982	pCfg->uLength = sizeof(uint8_t);
1983	cbRegion += pCfg->uLength;
1984	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocIsrCap, 4);
1985	pVirtioCC->pIsrCap = pCfg;
1986
1987	/* PCI Cfg capability (VirtIO 1.0 spec, section 4.1.4.7)
1988	* This capability doesn't get page-MMIO mapped. Instead uBar, uOffset and uLength are intercepted
1989	* by trapping PCI configuration I/O and get modulated by consumers to locate fetch and read/write
1990	* values from any region. NOTE: The linux driver not only doesn't use this feature, it will not
1991	* even list it as present if uLength isn't non-zero and also 4-byte-aligned as the linux driver is
1992	* initializing.
1993	*/
1994	pVirtio->uPciCfgDataOff = pCfg->uCapNext + RT_OFFSETOF(VIRTIO_PCI_CFG_CAP_T, uPciCfgData);
1995	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
1996	pCfg->uCfgType = VIRTIO_PCI_CAP_PCI_CFG;
1997	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
1998	pCfg->uCapLen = sizeof(VIRTIO_PCI_CFG_CAP_T);
1999	pCfg->uCapNext = (pVirtio->fMsiSupport \|\| pVirtioCC->pbDevSpecificCfg) ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2000	pCfg->uBar = 0;
2001	pCfg->uOffset = 0;
2002	pCfg->uLength = 0;
2003	cbRegion += pCfg->uLength;
2004	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocPciCfgCap, 1);
2005	pVirtioCC->pPciCfgCap = (PVIRTIO_PCI_CFG_CAP_T)pCfg;
2006
2007	if (pVirtioCC->pbDevSpecificCfg)
2008	{
2009	/* Following capability (via VirtIO 1.0, section 4.1.4.6). Client defines the
2010	* device-specific config fields struct and passes size to this constructor */
2011	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2012	pCfg->uCfgType = VIRTIO_PCI_CAP_DEVICE_CFG;
2013	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2014	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2015	pCfg->uCapNext = pVirtio->fMsiSupport ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2016	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2017	pCfg->uOffset = pVirtioCC->pIsrCap->uOffset + pVirtioCC->pIsrCap->uLength;
2018	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2019	pCfg->uLength = cbDevSpecificCfg;
2020	cbRegion += pCfg->uLength;
2021	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocDeviceCap, 4);
2022	pVirtioCC->pDeviceCap = pCfg;
2023	}
2024	else
2025	Assert(pVirtio->LocDeviceCap.cbMmio == 0 && pVirtio->LocDeviceCap.cbPci == 0);
2026
2027	if (pVirtio->fMsiSupport)
2028	{
2029	PDMMSIREG aMsiReg;
2030	RT_ZERO(aMsiReg);
2031	aMsiReg.iMsixCapOffset = pCfg->uCapNext;
2032	aMsiReg.iMsixNextOffset = 0;
2033	aMsiReg.iMsixBar = VIRTIO_REGION_MSIX_CAP;
2034	aMsiReg.cMsixVectors = VBOX_MSIX_MAX_ENTRIES;
2035	rc = PDMDevHlpPCIRegisterMsi(pDevIns, &aMsiReg); /* see MsixR3init() */
2036	if (RT_FAILURE(rc))
2037	{
2038	/* See PDMDevHlp.cpp:pdmR3DevHlp_PCIRegisterMsi */
2039	LogFunc(("Failed to configure MSI-X (%Rrc). Reverting to INTx\n", rc));
2040	pVirtio->fMsiSupport = false;
2041	}
2042	else
2043	Log2Func(("Using MSI-X for guest driver notification\n"));
2044	}
2045	else
2046	LogFunc(("MSI-X not available for VBox, using INTx notification\n"));
2047
2048	/* Set offset to first capability and enable PCI dev capabilities */
2049	PDMPciDevSetCapabilityList(pPciDev, 0x40);
2050	PDMPciDevSetStatus(pPciDev, VBOX_PCI_STATUS_CAP_LIST);
2051
2052	/* Linux drivers/virtio/virtio_pci_modern.c tries to map at least a page for the
2053	* 'unknown' device-specific capability without querying the capability to figure
2054	* out size, so pad with an extra page
2055	*/
2056	rc = PDMDevHlpPCIIORegionCreateMmio(pDevIns, VIRTIO_REGION_PCI_CAP, RT_ALIGN_32(cbRegion + PAGE_SIZE, PAGE_SIZE),
2057	PCI_ADDRESS_SPACE_MEM, virtioMmioWrite, virtioMmioRead, pVirtio,
2058	IOMMMIO_FLAGS_READ_PASSTHRU \| IOMMMIO_FLAGS_WRITE_PASSTHRU, "virtio-scsi MMIO",
2059	&pVirtio->hMmioPciCap);
2060	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Capabilities address space")));
2061
2062	return rc;
2063	}
2064
2065	#else /* !IN_RING3 */
2066
2067	/**
2068	* Sets up the core ring-0/raw-mode virtio bits.
2069	*
2070	* @returns VBox status code.
2071	* @param pDevIns The device instance.
2072	* @param pVirtio Pointer to the shared virtio state. This must be the first
2073	* member in the shared device instance data!
2074	* @param pVirtioCC Pointer to the current context virtio state. This must be the
2075	* first member in the currenct context's device instance data!
2076	*/
2077	int virtioCoreRZInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
2078	{
2079	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
2080	AssertLogRelReturn(pVirtioCC == PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC), VERR_STATE_CHANGED);
2081
2082	int rc = PDMDevHlpMmioSetUpContext(pDevIns, pVirtio->hMmioPciCap, virtioMmioWrite, virtioMmioRead, pVirtio);
2083	AssertRCReturn(rc, rc);
2084	return rc;
2085	}
2086
2087	#endif /* !IN_RING3 */
2088

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Devices/VirtIO/Virtio_1_0.cpp@ 83575

Download in other formats: