Virtio_1_0.cpp@ 84468

Last change on this file since 84468 was 84468, checked in by vboxsync, 5 years ago
Network/DevVirtioNet_1.0.cpp: Removed force interrupt flag from API and calls since it is no longer a necessary workardound option due to fixing how avail ring flags are read.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 89.4 KB

Line
1	/* $Id: Virtio_1_0.cpp 84468 2020-05-23 07:17:02Z vboxsync $ */
2	/** @file
3	* Virtio_1_0 - Virtio Common (PCI, feature & config mgt, queue mgt & proxy, notification mgt)
4	*/
5
6	/*
7	* Copyright (C) 2009-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_DEV_VIRTIO
23
24	#include <VBox/log.h>
25	#include <VBox/msi.h>
26	#include <VBox/AssertGuest.h>
27	#include <iprt/param.h>
28	#include <iprt/assert.h>
29	#include <iprt/uuid.h>
30	#include <iprt/mem.h>
31	#include <iprt/assert.h>
32	#include <iprt/sg.h>
33	#include <iprt/string.h>
34	#include <VBox/vmm/pdmdev.h>
35	#include "Virtio_1_0.h"
36
37
38	/*********************************************************************************************************************************
39	* Defined Constants And Macros *
40	*********************************************************************************************************************************/
41	#define INSTANCE(a_pVirtio) ((a_pVirtio)->szInstance)
42	#define VIRTQNAME(a_pVirtio, a_idxQueue) ((a_pVirtio)->virtqState[(a_idxQueue)].szVirtqName)
43	#define IS_DRIVER_OK(a_pVirtio) ((a_pVirtio)->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
44
45	/**
46	* This macro returns true if the @a a_offAccess and access length (@a
47	* a_cbAccess) are within the range of the mapped capability struct described by
48	* @a a_LocCapData.
49	*
50	* @param[in] a_offAccess The offset into the MMIO bar of the access.
51	* @param[in] a_cbAccess The access size.
52	* @param[out] a_offIntraVar The variable to return the intra-capability
53	* offset into. ASSUMES this is uint32_t.
54	* @param[in] a_LocCapData The capability location info.
55	*/
56	#define MATCHES_VIRTIO_CAP_STRUCT(a_offAccess, a_cbAccess, a_offIntraVar, a_LocCapData) \
57	( ((a_offIntraVar) = (uint32_t)((a_offAccess) - (a_LocCapData).offMmio)) < (uint32_t)(a_LocCapData).cbMmio \
58	&& (a_offIntraVar) + (uint32_t)(a_cbAccess) <= (uint32_t)(a_LocCapData).cbMmio )
59
60
61	/** Marks the start of the virtio saved state (just for sanity). */
62	#define VIRTIO_SAVEDSTATE_MARKER UINT64_C(0x1133557799bbddff)
63	/** The current saved state version for the virtio core. */
64	#define VIRTIO_SAVEDSTATE_VERSION UINT32_C(1)
65
66
67	/*********************************************************************************************************************************
68	* Structures and Typedefs *
69	*********************************************************************************************************************************/
70
71
72	/** @name virtq related flags
73	* @{ */
74	#define VIRTQ_DESC_F_NEXT 1 /*< Indicates this descriptor chains to next /
75	#define VIRTQ_DESC_F_WRITE 2 /*< Marks buffer as write-only (default ro) /
76	#define VIRTQ_DESC_F_INDIRECT 4 /*< Buffer is list of buffer descriptors /
77
78	#define VIRTQ_USED_F_NO_NOTIFY 1 /*< Dev to Drv: Don't notify when buf added /
79	#define VIRTQ_AVAIL_F_NO_INTERRUPT 1 /*< Drv to Dev: Don't notify when buf eaten /
80	/** @} */
81
82	/**
83	* virtq related structs
84	* (struct names follow VirtIO 1.0 spec, typedef use VBox style)
85	*/
86	typedef struct virtq_desc
87	{
88	uint64_t GCPhysBuf; /*< addr GC Phys. address of buffer /
89	uint32_t cb; /*< len Buffer length /
90	uint16_t fFlags; /*< flags Buffer specific flags /
91	uint16_t uDescIdxNext; /*< next Idx set if VIRTIO_DESC_F_NEXT /
92	} VIRTQ_DESC_T, *PVIRTQ_DESC_T;
93
94	typedef struct virtq_avail
95	{
96	uint16_t fFlags; /*< flags avail ring guest-to-host flags /
97	uint16_t uIdx; /*< idx Index of next free ring slot /
98	uint16_t auRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: avail drv to dev bufs /
99	/* uint16_t uUsedEventIdx; - used_event (if VIRTQ_USED_F_EVENT_IDX) */
100	} VIRTQ_AVAIL_T, *PVIRTQ_AVAIL_T;
101
102	typedef struct virtq_used_elem
103	{
104	uint32_t uDescIdx; /*< idx Start of used desc chain /
105	uint32_t cbElem; /*< len Total len of used desc chain /
106	} VIRTQ_USED_ELEM_T;
107
108	typedef struct virt_used
109	{
110	uint16_t fFlags; /*< flags used ring host-to-guest flags /
111	uint16_t uIdx; /*< idx Index of next ring slot /
112	VIRTQ_USED_ELEM_T aRing[RT_FLEXIBLE_ARRAY]; /*< ring Ring: used dev to drv bufs /
113	/* uint16_t uAvailEventIdx; - avail_event if (VIRTQ_USED_F_EVENT_IDX) */
114	} VIRTQ_USED_T, *PVIRTQ_USED_T;
115
116
117	const char *virtioCoreGetStateChangeText(VIRTIOVMSTATECHANGED enmState)
118	{
119	switch (enmState)
120	{
121	case kvirtIoVmStateChangedReset: return "VM RESET";
122	case kvirtIoVmStateChangedSuspend: return "VM SUSPEND";
123	case kvirtIoVmStateChangedPowerOff: return "VM POWER OFF";
124	case kvirtIoVmStateChangedResume: return "VM RESUME";
125	default: return "<BAD ENUM>";
126	}
127	}
128
129	/* Internal Functions */
130
131	static void virtioNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue);
132	static int virtioKick(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uVec);
133
134	/** @name Internal queue operations
135	* @{ */
136
137	/**
138	* Accessor for virtq descriptor
139	*/
140	#ifdef IN_RING3
141	DECLINLINE(void) virtioReadDesc(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
142	uint32_t idxDesc, PVIRTQ_DESC_T pDesc)
143	{
144	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
145	uint16_t const cQueueItems = RT_MAX(pVirtio->uQueueSize[idxQueue], 1); /* Make sure to avoid div-by-zero. */
146	PDMDevHlpPCIPhysRead(pDevIns,
147	pVirtio->aGCPhysQueueDesc[idxQueue] + sizeof(VIRTQ_DESC_T) * (idxDesc % cQueueItems),
148	pDesc, sizeof(VIRTQ_DESC_T));
149	}
150	#endif
151
152	/**
153	* Accessors for virtq avail ring
154	*/
155	#ifdef IN_RING3
156	DECLINLINE(uint16_t) virtioReadAvailDescIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint32_t availIdx)
157	{
158	uint16_t uDescIdx;
159	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
160	uint16_t const cQueueItems = RT_MAX(pVirtio->uQueueSize[idxQueue], 1); /* Make sure to avoid div-by-zero. */
161	PDMDevHlpPCIPhysRead(pDevIns,
162	pVirtio->aGCPhysQueueAvail[idxQueue]
163	+ RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[availIdx % cQueueItems]),
164	&uDescIdx, sizeof(uDescIdx));
165	return uDescIdx;
166	}
167
168	DECLINLINE(uint16_t) virtioReadAvailUsedEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
169	{
170	uint16_t uUsedEventIdx;
171	/* VirtIO 1.0 uUsedEventIdx (used_event) immediately follows ring */
172	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
173	PDMDevHlpPCIPhysRead(pDevIns,
174	pVirtio->aGCPhysQueueAvail[idxQueue] + RT_UOFFSETOF_DYN(VIRTQ_AVAIL_T, auRing[pVirtio->uQueueSize[idxQueue]]),
175	&uUsedEventIdx, sizeof(uUsedEventIdx));
176	return uUsedEventIdx;
177	}
178	#endif
179
180	DECLINLINE(uint16_t) virtioReadAvailRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
181	{
182	uint16_t uIdx = 0;
183	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
184	PDMDevHlpPCIPhysRead(pDevIns,
185	pVirtio->aGCPhysQueueAvail[idxQueue] + RT_UOFFSETOF(VIRTQ_AVAIL_T, uIdx),
186	&uIdx, sizeof(uIdx));
187	return uIdx;
188	}
189
190	DECLINLINE(bool) virtqIsEmpty(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
191	{
192	uint16_t uAvailGuestIdx = virtioReadAvailRingIdx(pDevIns, pVirtio, idxQueue);
193	bool fEmpty = uAvailGuestIdx == pVirtio->virtqState[idxQueue].uAvailIdx;
194
195	Log6Func(("%s idx=%u, shadow idx=%u (%s)\n",
196	VIRTQNAME(pVirtio, idxQueue), uAvailGuestIdx, pVirtio->virtqState[idxQueue].uAvailIdx,
197	fEmpty ? "Queue empty" : "Queue has available descriptors"));
198	return fEmpty;
199	}
200
201	DECLINLINE(uint16_t) virtioReadAvailRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
202	{
203	uint16_t fFlags = 0;
204	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
205	PDMDevHlpPCIPhysRead(pDevIns,
206	pVirtio->aGCPhysQueueAvail[idxQueue] + RT_UOFFSETOF(VIRTQ_AVAIL_T, fFlags),
207	&fFlags, sizeof(fFlags));
208	return fFlags;
209	}
210
211	/** @} */
212
213	/** @name Accessors for virtq used ring
214	* @{
215	*/
216
217	#ifdef IN_RING3
218	DECLINLINE(void) virtioWriteUsedElem(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
219	uint32_t usedIdx, uint32_t uDescIdx, uint32_t uLen)
220	{
221	VIRTQ_USED_ELEM_T elem = { uDescIdx, uLen };
222	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
223	uint16_t const cQueueItems = RT_MAX(pVirtio->uQueueSize[idxQueue], 1); /* Make sure to avoid div-by-zero. */
224	PDMDevHlpPCIPhysWrite(pDevIns,
225	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[usedIdx % cQueueItems]),
226	&elem, sizeof(elem));
227	}
228
229	DECLINLINE(void) virtioWriteUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint16_t fFlags)
230	{
231	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
232	RT_UNTRUSTED_VALIDATED_FENCE(); /* VirtIO 1.0, Section 3.2.1.4.1 */
233	PDMDevHlpPCIPhysWrite(pDevIns,
234	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
235	&fFlags, sizeof(fFlags));
236	}
237	#endif
238
239	DECLINLINE(void) virtioWriteUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint16_t uIdx)
240	{
241	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
242	PDMDevHlpPCIPhysWrite(pDevIns,
243	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
244	&uIdx, sizeof(uIdx));
245	}
246
247	#ifdef LOG_ENABLED
248	DECLINLINE(uint16_t) virtioReadUsedRingIdx(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
249	{
250	uint16_t uIdx = 0;
251	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
252	PDMDevHlpPCIPhysRead(pDevIns,
253	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, uIdx),
254	&uIdx, sizeof(uIdx));
255	return uIdx;
256	}
257	#endif
258
259
260	#ifdef IN_RING3
261	DECLINLINE(uint16_t) virtioReadUsedRingFlags(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
262	{
263	uint16_t fFlags = 0;
264	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
265	PDMDevHlpPCIPhysRead(pDevIns,
266	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF(VIRTQ_USED_T, fFlags),
267	&fFlags, sizeof(fFlags));
268	return fFlags;
269	}
270
271	DECLINLINE(void) virtioWriteUsedAvailEvent(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint32_t uAvailEventIdx)
272	{
273	/** VirtIO 1.0 uAvailEventIdx (avail_event) immediately follows ring */
274	AssertMsg(pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK, ("Called with guest driver not ready\n"));
275	PDMDevHlpPCIPhysWrite(pDevIns,
276	pVirtio->aGCPhysQueueUsed[idxQueue] + RT_UOFFSETOF_DYN(VIRTQ_USED_T, aRing[pVirtio->uQueueSize[idxQueue]]),
277	&uAvailEventIdx, sizeof(uAvailEventIdx));
278	}
279	#endif
280
281	/** @} */
282
283	void virtioCoreSgBufInit(PVIRTIOSGBUF pGcSgBuf, PVIRTIOSGSEG paSegs, size_t cSegs)
284	{
285	AssertPtr(pGcSgBuf);
286	Assert( (cSegs > 0 && VALID_PTR(paSegs)) \|\| (!cSegs && !paSegs));
287	Assert(cSegs < (~(unsigned)0 >> 1));
288
289	pGcSgBuf->paSegs = paSegs;
290	pGcSgBuf->cSegs = (unsigned)cSegs;
291	pGcSgBuf->idxSeg = 0;
292	if (cSegs && paSegs)
293	{
294	pGcSgBuf->gcPhysCur = paSegs[0].gcPhys;
295	pGcSgBuf->cbSegLeft = paSegs[0].cbSeg;
296	}
297	else
298	{
299	pGcSgBuf->gcPhysCur = 0;
300	pGcSgBuf->cbSegLeft = 0;
301	}
302	}
303
304	static RTGCPHYS virtioCoreSgBufGet(PVIRTIOSGBUF pGcSgBuf, size_t *pcbData)
305	{
306	size_t cbData;
307	RTGCPHYS pGcBuf;
308
309	/* Check that the S/G buffer has memory left. */
310	if (RT_LIKELY(pGcSgBuf->idxSeg < pGcSgBuf->cSegs && pGcSgBuf->cbSegLeft))
311	{ /* likely */ }
312	else
313	{
314	*pcbData = 0;
315	return 0;
316	}
317
318	AssertMsg( pGcSgBuf->cbSegLeft <= 128 * _1M
319	&& (RTGCPHYS)pGcSgBuf->gcPhysCur >= (RTGCPHYS)pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys
320	&& (RTGCPHYS)pGcSgBuf->gcPhysCur + pGcSgBuf->cbSegLeft <=
321	(RTGCPHYS)pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys + pGcSgBuf->paSegs[pGcSgBuf->idxSeg].cbSeg,
322	("pGcSgBuf->idxSeg=%d pGcSgBuf->cSegs=%d pGcSgBuf->gcPhysCur=%p pGcSgBuf->cbSegLeft=%zd "
323	"pGcSgBuf->paSegs[%d].gcPhys=%p pGcSgBuf->paSegs[%d].cbSeg=%zd\n",
324	pGcSgBuf->idxSeg, pGcSgBuf->cSegs, pGcSgBuf->gcPhysCur, pGcSgBuf->cbSegLeft,
325	pGcSgBuf->idxSeg, pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys, pGcSgBuf->idxSeg,
326	pGcSgBuf->paSegs[pGcSgBuf->idxSeg].cbSeg));
327
328	cbData = RT_MIN(*pcbData, pGcSgBuf->cbSegLeft);
329	pGcBuf = pGcSgBuf->gcPhysCur;
330	pGcSgBuf->cbSegLeft -= cbData;
331	if (!pGcSgBuf->cbSegLeft)
332	{
333	pGcSgBuf->idxSeg++;
334
335	if (pGcSgBuf->idxSeg < pGcSgBuf->cSegs)
336	{
337	pGcSgBuf->gcPhysCur = pGcSgBuf->paSegs[pGcSgBuf->idxSeg].gcPhys;
338	pGcSgBuf->cbSegLeft = pGcSgBuf->paSegs[pGcSgBuf->idxSeg].cbSeg;
339	}
340	*pcbData = cbData;
341	}
342	else
343	pGcSgBuf->gcPhysCur = pGcSgBuf->gcPhysCur + cbData;
344
345	return pGcBuf;
346	}
347
348	void virtioCoreSgBufReset(PVIRTIOSGBUF pGcSgBuf)
349	{
350	AssertPtrReturnVoid(pGcSgBuf);
351
352	pGcSgBuf->idxSeg = 0;
353	if (pGcSgBuf->cSegs)
354	{
355	pGcSgBuf->gcPhysCur = pGcSgBuf->paSegs[0].gcPhys;
356	pGcSgBuf->cbSegLeft = pGcSgBuf->paSegs[0].cbSeg;
357	}
358	else
359	{
360	pGcSgBuf->gcPhysCur = 0;
361	pGcSgBuf->cbSegLeft = 0;
362	}
363	}
364
365	RTGCPHYS virtioCoreSgBufAdvance(PVIRTIOSGBUF pGcSgBuf, size_t cbAdvance)
366	{
367	AssertReturn(pGcSgBuf, 0);
368
369	size_t cbLeft = cbAdvance;
370	while (cbLeft)
371	{
372	size_t cbThisAdvance = cbLeft;
373	virtioCoreSgBufGet(pGcSgBuf, &cbThisAdvance);
374	if (!cbThisAdvance)
375	break;
376
377	cbLeft -= cbThisAdvance;
378	}
379	return cbAdvance - cbLeft;
380	}
381
382	RTGCPHYS virtioCoreSgBufGetNextSegment(PVIRTIOSGBUF pGcSgBuf, size_t *pcbSeg)
383	{
384	AssertReturn(pGcSgBuf, 0);
385	AssertPtrReturn(pcbSeg, 0);
386
387	if (!*pcbSeg)
388	*pcbSeg = pGcSgBuf->cbSegLeft;
389
390	return virtioCoreSgBufGet(pGcSgBuf, pcbSeg);
391	}
392
393	size_t virtioCoreSgBufCalcTotalLength(PVIRTIOSGBUF pGcSgBuf)
394	{
395	size_t cb = 0;
396	unsigned i = pGcSgBuf->cSegs;
397	while (i-- > 0)
398	cb += pGcSgBuf->paSegs[i].cbSeg;
399	return cb;
400	}
401
402	#ifdef LOG_ENABLED
403
404	void virtioPrintFeatures(VIRTIOCORE *pVirtio)
405	{
406	#ifdef LOG_ENABLED
407	static struct
408	{
409	uint64_t fFeatureBit;
410	const char *pcszDesc;
411	} const s_aFeatures[] =
412	{
413	{ VIRTIO_F_RING_INDIRECT_DESC, " RING_INDIRECT_DESC Driver can use descriptors with VIRTQ_DESC_F_INDIRECT flag set\n" },
414	{ VIRTIO_F_RING_EVENT_IDX, " RING_EVENT_IDX Enables use_event and avail_event fields described in 2.4.7, 2.4.8\n" },
415	{ VIRTIO_F_VERSION_1, " VERSION Used to detect legacy drivers.\n" },
416	};
417
418	#define MAXLINE 80
419	/* Display as a single buf to prevent interceding log messages */
420	uint16_t cbBuf = RT_ELEMENTS(s_aFeatures) * 132;
421	char pszBuf = (char )RTMemAllocZ(cbBuf);
422	Assert(pszBuf);
423	char *cp = pszBuf;
424	for (unsigned i = 0; i < RT_ELEMENTS(s_aFeatures); ++i)
425	{
426	bool isOffered = RT_BOOL(pVirtio->uDeviceFeatures & s_aFeatures[i].fFeatureBit);
427	bool isNegotiated = RT_BOOL(pVirtio->uDriverFeatures & s_aFeatures[i].fFeatureBit);
428	cp += RTStrPrintf(cp, cbBuf - (cp - pszBuf), " %s %s %s",
429	isOffered ? "+" : "-", isNegotiated ? "x" : " ", s_aFeatures[i].pcszDesc);
430	}
431	Log3(("VirtIO Features Configuration\n\n"
432	" Offered Accepted Feature Description\n"
433	" ------- -------- ------- -----------\n"
434	"%s\n", pszBuf));
435	RTMemFree(pszBuf);
436
437	#else /* !LOG_ENABLED */
438	RT_NOREF3(pThis, fFeatures, pcszText);
439	#endif /* !LOG_ENABLED */
440	}
441
442
443	/**
444	* Does a formatted hex dump using Log(()), recommend using VIRTIO_HEX_DUMP() macro to
445	* control enabling of logging efficiently.
446	*
447	* @param pv pointer to buffer to dump contents of
448	* @param cb count of characters to dump from buffer
449	* @param uBase base address of per-row address prefixing of hex output
450	* @param pszTitle Optional title. If present displays title that lists
451	* provided text with value of cb to indicate size next to it.
452	*/
453	void virtioCoreHexDump(uint8_t pv, uint32_t cb, uint32_t uBase, const char pszTitle)
454	{
455	if (pszTitle)
456	Log(("%s [%d bytes]:\n", pszTitle, cb));
457	for (uint32_t row = 0; row < RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
458	{
459	Log(("%04x: ", row * 16 + uBase)); /* line address */
460	for (uint8_t col = 0; col < 16; col++)
461	{
462	uint32_t idx = row * 16 + col;
463	if (idx >= cb)
464	Log(("-- %s", (col + 1) % 8 ? "" : " "));
465	else
466	Log(("%02x %s", pv[idx], (col + 1) % 8 ? "" : " "));
467	}
468	for (uint32_t idx = row * 16; idx < row * 16 + 16; idx++)
469	Log(("%c", (idx >= cb) ? ' ' : (pv[idx] >= 0x20 && pv[idx] <= 0x7e ? pv[idx] : '.')));
470	Log(("\n"));
471	}
472	Log(("\n"));
473	RT_NOREF2(uBase, pv);
474	}
475
476	/**
477	* Do a hex dump of memory in guest physical context
478	*
479	* @param gcPhys pointer to buffer to dump contents of
480	* @param cb count of characters to dump from buffer
481	* @param uBase base address of per-row address prefixing of hex output
482	* @param pszTitle Optional title. If present displays title that lists
483	* provided text with value of cb to indicate size next to it.
484	*/
485	void virtioCoreGcPhysHexDump(PPDMDEVINS pDevIns, RTGCPHYS gcPhys, uint32_t cb, uint32_t uBase, const char *pszTitle)
486	{
487	if (pszTitle)
488	Log(("%s [%d bytes]:\n", pszTitle, cb));
489	for (uint32_t row = 0; row < RT_MAX(1, (cb / 16) + 1) && row * 16 < cb; row++)
490	{
491	uint8_t c;
492	Log(("%04x: ", row * 16 + uBase)); /* line address */
493	for (uint8_t col = 0; col < 16; col++)
494	{
495	uint32_t idx = row * 16 + col;
496	PDMDevHlpPCIPhysRead(pDevIns, gcPhys + idx, &c, 1);
497	if (idx >= cb)
498	Log(("-- %s", (col + 1) % 8 ? "" : " "));
499	else
500	Log(("%02x %s", c, (col + 1) % 8 ? "" : " "));
501	}
502	for (uint32_t idx = row * 16; idx < row * 16 + 16; idx++)
503	{
504	PDMDevHlpPCIPhysRead(pDevIns, gcPhys + idx, &c, 1);
505	Log(("%c", (idx >= cb) ? ' ' : (c >= 0x20 && c <= 0x7e ? c : '.')));
506	}
507	Log(("\n"));
508	}
509	Log(("\n"));
510	RT_NOREF(uBase);
511	}
512	#endif /* LOG_ENABLED */
513
514	/**
515	* Log memory-mapped I/O input or output value.
516	*
517	* This is designed to be invoked by macros that can make contextual assumptions
518	* (e.g. implicitly derive MACRO parameters from the invoking function). It is exposed
519	* for the VirtIO client doing the device-specific implementation in order to log in a
520	* similar fashion accesses to the device-specific MMIO configuration structure. Macros
521	* that leverage this function are found in virtioCommonCfgAccessed() and can be
522	* used as an example of how to use this effectively for the device-specific
523	* code.
524	*
525	* @param pszFunc To avoid displaying this function's name via __FUNCTION__ or LogFunc()
526	* @param pszMember Name of struct member
527	* @param pv pointer to value
528	* @param cb size of value
529	* @param uOffset offset into member where value starts
530	* @param fWrite True if write I/O
531	* @param fHasIndex True if the member is indexed
532	* @param idx The index if fHasIndex
533	*/
534	void virtioCoreLogMappedIoValue(const char pszFunc, const char pszMember, uint32_t uMemberSize,
535	const void *pv, uint32_t cb, uint32_t uOffset, int fWrite,
536	int fHasIndex, uint32_t idx)
537	{
538	if (!LogIs6Enabled())
539	return;
540
541	char szIdx[16];
542	if (fHasIndex)
543	RTStrPrintf(szIdx, sizeof(szIdx), "[%d]", idx);
544	else
545	szIdx[0] = '\0';
546
547	if (cb == 1 \|\| cb == 2 \|\| cb == 4 \|\| cb == 8)
548	{
549	char szDepiction[64];
550	size_t cchDepiction;
551	if (uOffset != 0 \|\| cb != uMemberSize) /* display bounds if partial member access */
552	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s[%d:%d]",
553	pszMember, szIdx, uOffset, uOffset + cb - 1);
554	else
555	cchDepiction = RTStrPrintf(szDepiction, sizeof(szDepiction), "%s%s", pszMember, szIdx);
556
557	/* padding */
558	if (cchDepiction < 30)
559	szDepiction[cchDepiction++] = ' ';
560	while (cchDepiction < 30)
561	szDepiction[cchDepiction++] = '.';
562	szDepiction[cchDepiction] = '\0';
563
564	RTUINT64U uValue;
565	uValue.u = 0;
566	memcpy(uValue.au8, pv, cb);
567	Log6(("%s: Guest %s %s %#0*RX64\n",
568	pszFunc, fWrite ? "wrote" : "read ", szDepiction, 2 + cb * 2, uValue.u));
569	}
570	else /* odd number or oversized access, ... log inline hex-dump style */
571	{
572	Log6(("%s: Guest %s %s%s[%d:%d]: %.*Rhxs\n",
573	pszFunc, fWrite ? "wrote" : "read ", pszMember,
574	szIdx, uOffset, uOffset + cb, cb, pv));
575	}
576	RT_NOREF2(fWrite, pszFunc);
577	}
578
579
580	/**
581	* Makes the MMIO-mapped Virtio uDeviceStatus registers non-cryptic
582	*/
583	DECLINLINE(void) virtioLogDeviceStatus(uint8_t bStatus)
584	{
585	if (bStatus == 0)
586	Log6(("RESET"));
587	else
588	{
589	int primed = 0;
590	if (bStatus & VIRTIO_STATUS_ACKNOWLEDGE)
591	Log6(("%sACKNOWLEDGE", primed++ ? "" : ""));
592	if (bStatus & VIRTIO_STATUS_DRIVER)
593	Log6(("%sDRIVER", primed++ ? " \| " : ""));
594	if (bStatus & VIRTIO_STATUS_FEATURES_OK)
595	Log6(("%sFEATURES_OK", primed++ ? " \| " : ""));
596	if (bStatus & VIRTIO_STATUS_DRIVER_OK)
597	Log6(("%sDRIVER_OK", primed++ ? " \| " : ""));
598	if (bStatus & VIRTIO_STATUS_FAILED)
599	Log6(("%sFAILED", primed++ ? " \| " : ""));
600	if (bStatus & VIRTIO_STATUS_DEVICE_NEEDS_RESET)
601	Log6(("%sNEEDS_RESET", primed++ ? " \| " : ""));
602	(void)primed;
603	}
604	}
605
606	#ifdef IN_RING3
607	/**
608	* Allocate client context for client to work with VirtIO-provided with queue
609	*
610	* @param pVirtio Pointer to the shared virtio state.
611	* @param idxQueue Queue number
612	* @param pcszName Name to give queue
613	*
614	* @returns VBox status code.
615	*/
616	int virtioCoreR3QueueAttach(PVIRTIOCORE pVirtio, uint16_t idxQueue, const char *pcszName)
617	{
618	LogFunc(("%s\n", pcszName));
619	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
620	pVirtq->uAvailIdx = 0;
621	pVirtq->uUsedIdx = 0;
622	pVirtq->fEventThresholdReached = false;
623	RTStrCopy(pVirtq->szVirtqName, sizeof(pVirtq->szVirtqName), pcszName);
624	return VINF_SUCCESS;
625	}
626	#endif /* IN_RING3 */
627
628
629	/**
630	* Check if the associated queue is empty
631	*
632	* @param pDevIns The device instance (for reading).
633	* @param pVirtio Pointer to the shared virtio state.
634	* @param idxQueue Queue number
635	*
636	* @retval true Queue is empty or unavailable.
637	* @retval false Queue is available and has entries
638	*/
639	bool virtioCoreQueueIsEmpty(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
640	{
641	if (pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
642	return virtqIsEmpty(pDevIns, pVirtio, idxQueue);
643	LogFunc(("VirtIO not ready: Returning 'true' for queue empty\n"));
644	return true;
645	}
646
647	#ifdef IN_RING3
648
649
650	int virtioCoreR3DescChainGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
651	uint16_t uHeadIdx, PPVIRTIO_DESC_CHAIN_T ppDescChain)
652	{
653	AssertReturn(ppDescChain, VERR_INVALID_POINTER);
654	*ppDescChain = NULL;
655
656	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
657
658	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
659
660	AssertMsgReturn(IS_DRIVER_OK(pVirtio) && pVirtio->uQueueEnable[idxQueue],
661	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
662
663	uint16_t uDescIdx = uHeadIdx;
664
665	Log6Func(("%s DESC CHAIN: (head) desc_idx=%u\n", pVirtq->szVirtqName, uHeadIdx));
666	RT_NOREF(pVirtq);
667
668	/*
669	* Allocate and initialize the descriptor chain structure.
670	*/
671	PVIRTIO_DESC_CHAIN_T pDescChain = (PVIRTIO_DESC_CHAIN_T)RTMemAllocZ(sizeof(VIRTIO_DESC_CHAIN_T));
672	AssertReturn(pDescChain, VERR_NO_MEMORY);
673	pDescChain->u32Magic = VIRTIO_DESC_CHAIN_MAGIC;
674	pDescChain->cRefs = 1;
675	pDescChain->uHeadIdx = uHeadIdx;
676	*ppDescChain = pDescChain;
677
678	/*
679	* Gather segments.
680	*/
681	VIRTQ_DESC_T desc;
682
683	uint32_t cbIn = 0;
684	uint32_t cbOut = 0;
685	uint32_t cSegsIn = 0;
686	uint32_t cSegsOut = 0;
687	PVIRTIOSGSEG paSegsIn = pDescChain->aSegsIn;
688	PVIRTIOSGSEG paSegsOut = pDescChain->aSegsOut;
689
690	do
691	{
692	PVIRTIOSGSEG pSeg;
693
694	/*
695	* Malicious guests may go beyond paSegsIn or paSegsOut boundaries by linking
696	* several descriptors into a loop. Since there is no legitimate way to get a sequences of
697	* linked descriptors exceeding the total number of descriptors in the ring (see @bugref{8620}),
698	* the following aborts I/O if breach and employs a simple log throttling algorithm to notify.
699	*/
700	if (cSegsIn + cSegsOut >= VIRTQ_MAX_SIZE)
701	{
702	static volatile uint32_t s_cMessages = 0;
703	static volatile uint32_t s_cThreshold = 1;
704	if (ASMAtomicIncU32(&s_cMessages) == ASMAtomicReadU32(&s_cThreshold))
705	{
706	LogRelMax(64, ("Too many linked descriptors; check if the guest arranges descriptors in a loop.\n"));
707	if (ASMAtomicReadU32(&s_cMessages) != 1)
708	LogRelMax(64, ("(the above error has occured %u times so far)\n", ASMAtomicReadU32(&s_cMessages)));
709	ASMAtomicWriteU32(&s_cThreshold, ASMAtomicReadU32(&s_cThreshold) * 10);
710	}
711	break;
712	}
713	RT_UNTRUSTED_VALIDATED_FENCE();
714
715	virtioReadDesc(pDevIns, pVirtio, idxQueue, uDescIdx, &desc);
716
717	if (desc.fFlags & VIRTQ_DESC_F_WRITE)
718	{
719	Log6Func(("%s IN desc_idx=%u seg=%u addr=%RGp cb=%u\n", VIRTQNAME(pVirtio, idxQueue), uDescIdx, cSegsIn, desc.GCPhysBuf, desc.cb));
720	cbIn += desc.cb;
721	pSeg = &paSegsIn[cSegsIn++];
722	}
723	else
724	{
725	Log6Func(("%s OUT desc_idx=%u seg=%u addr=%RGp cb=%u\n", VIRTQNAME(pVirtio, idxQueue), uDescIdx, cSegsOut, desc.GCPhysBuf, desc.cb));
726	cbOut += desc.cb;
727	pSeg = &paSegsOut[cSegsOut++];
728	}
729
730	pSeg->gcPhys = desc.GCPhysBuf;
731	pSeg->cbSeg = desc.cb;
732
733	uDescIdx = desc.uDescIdxNext;
734	} while (desc.fFlags & VIRTQ_DESC_F_NEXT);
735
736	/*
737	* Add segments to the descriptor chain structure.
738	*/
739	if (cSegsIn)
740	{
741	virtioCoreSgBufInit(&pDescChain->SgBufIn, paSegsIn, cSegsIn);
742	pDescChain->pSgPhysReturn = &pDescChain->SgBufIn;
743	pDescChain->cbPhysReturn = cbIn;
744	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsIn, cSegsIn);
745	}
746
747	if (cSegsOut)
748	{
749	virtioCoreSgBufInit(&pDescChain->SgBufOut, paSegsOut, cSegsOut);
750	pDescChain->pSgPhysSend = &pDescChain->SgBufOut;
751	pDescChain->cbPhysSend = cbOut;
752	STAM_REL_COUNTER_ADD(&pVirtio->StatDescChainsSegsOut, cSegsOut);
753	}
754
755	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsAllocated);
756	Log6Func(("%s -- segs OUT: %u (%u bytes) IN: %u (%u bytes) --\n", pVirtq->szVirtqName, cSegsOut, cbOut, cSegsIn, cbIn));
757
758	return VINF_SUCCESS;
759	}
760
761
762	/**
763	* Retains a reference to the given descriptor chain.
764	*
765	* @returns New reference count.
766	* @retval UINT32_MAX on invalid parameter.
767	* @param pDescChain The descriptor chain to reference.
768	*/
769	uint32_t virtioCoreR3DescChainRetain(PVIRTIO_DESC_CHAIN_T pDescChain)
770	{
771	AssertReturn(pDescChain, UINT32_MAX);
772	AssertReturn(pDescChain->u32Magic == VIRTIO_DESC_CHAIN_MAGIC, UINT32_MAX);
773	uint32_t cRefs = ASMAtomicIncU32(&pDescChain->cRefs);
774	Assert(cRefs > 1);
775	Assert(cRefs < 16);
776	return cRefs;
777	}
778
779
780	/**
781	* Releases a reference to the given descriptor chain.
782	*
783	* @returns New reference count.
784	* @retval 0 if freed or invalid parameter.
785	* @param pVirtio Pointer to the shared virtio state.
786	* @param pDescChain The descriptor chain to reference. NULL is quietly
787	* ignored (returns 0).
788	*/
789	uint32_t virtioCoreR3DescChainRelease(PVIRTIOCORE pVirtio, PVIRTIO_DESC_CHAIN_T pDescChain)
790	{
791	if (!pDescChain)
792	return 0;
793	AssertReturn(pDescChain, 0);
794	AssertReturn(pDescChain->u32Magic == VIRTIO_DESC_CHAIN_MAGIC, 0);
795	uint32_t cRefs = ASMAtomicDecU32(&pDescChain->cRefs);
796	Assert(cRefs < 16);
797	if (cRefs == 0)
798	{
799	pDescChain->u32Magic = ~VIRTIO_DESC_CHAIN_MAGIC;
800	RTMemFree(pDescChain);
801	STAM_REL_COUNTER_INC(&pVirtio->StatDescChainsFreed);
802	}
803	return cRefs;
804	}
805
806
807	/*
808	* Notifies guest (via ISR or MSI-X) of device configuration change
809	*
810	* @param pVirtio Pointer to the shared virtio state.
811	*/
812	void virtioCoreNotifyConfigChanged(PVIRTIOCORE pVirtio)
813	{
814	virtioKick(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig);
815	}
816
817	/**
818	* Enable or Disable notification for the specified queue
819	*
820	* @param pVirtio Pointer to the shared virtio state.
821	* @param idxQueue Queue number
822	* @param fEnabled Selects notification mode (enabled or disabled)
823	*/
824	void virtioCoreQueueSetNotify(PVIRTIOCORE pVirtio, uint16_t idxQueue, bool fEnabled)
825	{
826	if (pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
827	{
828	uint16_t fFlags = virtioReadUsedRingFlags(pVirtio->pDevInsR3, pVirtio, idxQueue);
829
830	if (fEnabled)
831	fFlags &= ~ VIRTQ_USED_F_NO_NOTIFY;
832	else
833	fFlags \|= VIRTQ_USED_F_NO_NOTIFY;
834
835	virtioWriteUsedRingFlags(pVirtio->pDevInsR3, pVirtio, idxQueue, fFlags);
836	}
837	}
838
839	/**
840	* Initiate orderly reset procedure. This is an exposed API for clients that might need it.
841	* Invoked by client to reset the device and driver (see VirtIO 1.0 section 2.1.1/2.1.2)
842	*
843	* @param pVirtio Pointer to the virtio state.
844	*/
845	void virtioCoreResetAll(PVIRTIOCORE pVirtio)
846	{
847	LogFunc(("\n"));
848	pVirtio->uDeviceStatus \|= VIRTIO_STATUS_DEVICE_NEEDS_RESET;
849	if (pVirtio->uDeviceStatus & VIRTIO_STATUS_DRIVER_OK)
850	{
851	pVirtio->fGenUpdatePending = true;
852	virtioKick(pVirtio->pDevInsR3, pVirtio, VIRTIO_ISR_DEVICE_CONFIG, pVirtio->uMsixConfig);
853	}
854	}
855	/**
856	* Get count of new (e.g. pending) elements in available ring.
857	*
858	* @param pDevIns The device instance.
859	* @param pVirtio Pointer to the shared virtio state.
860	* @param idxQueue Queue number
861	*
862	* @returns how many entries have been added to ring as a delta of the consumer's
863	* avail index and the queue's guest-side current avail index.
864	*/
865	int virtioCoreR3QueuePendingCount(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
866	{
867	uint16_t uAvailRingIdx = virtioReadAvailRingIdx(pDevIns, pVirtio, idxQueue);
868	uint16_t uNextAvailIdx = pVirtio->virtqState[idxQueue].uAvailIdx;
869	uint16_t uDelta = uAvailRingIdx - uNextAvailIdx;
870	if (uAvailRingIdx > uNextAvailIdx)
871	return uDelta;
872	return VIRTQ_MAX_CNT + uDelta;
873	}
874	/**
875	* Fetches descriptor chain using avail ring of indicated queue and converts the descriptor
876	* chain into its OUT (to device) and IN to guest components, but does NOT remove it from
877	* the 'avail' queue. I.e. doesn't advance the index. This can be used with virtioQueueSkip(),
878	* which does advance the avail index. Together they facilitate a mechanism that allows
879	* work with a queue element (descriptor chain) to be aborted if necessary, by not advancing
880	* the pointer, or, upon success calling the skip function (above) to move to the next element.
881	*
882	* Additionally it converts the OUT desc chain data to a contiguous virtual
883	* memory buffer for easy consumption by the caller. The caller must return the
884	* descriptor chain pointer via virtioCoreR3QueuePut() and then call virtioCoreQueueSync()
885	* at some point to return the data to the guest and complete the transaction.
886	*
887	* @param pDevIns The device instance.
888	* @param pVirtio Pointer to the shared virtio state.
889	* @param idxQueue Queue number
890	* @param ppDescChain Address to store pointer to descriptor chain that contains the
891	* pre-processed transaction information pulled from the virtq.
892	*
893	* @returns VBox status code:
894	* @retval VINF_SUCCESS Success
895	* @retval VERR_INVALID_STATE VirtIO not in ready state (asserted).
896	* @retval VERR_NOT_AVAILABLE If the queue is empty.
897	*/
898
899	int virtioCoreR3QueuePeek(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
900	PPVIRTIO_DESC_CHAIN_T ppDescChain)
901	{
902	return virtioCoreR3QueueGet(pDevIns, pVirtio, idxQueue, ppDescChain, false);
903	}
904
905	/**
906	* Skip the next entry in the specified queue (typically used with virtioCoreR3QueuePeek())
907	*
908	* @param pVirtio Pointer to the virtio state.
909	* @param idxQueue Index of queue
910	*/
911	int virtioCoreR3QueueSkip(PVIRTIOCORE pVirtio, uint16_t idxQueue)
912	{
913	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
914	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
915
916	AssertMsgReturn(IS_DRIVER_OK(pVirtio) && pVirtio->uQueueEnable[idxQueue],
917	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
918
919	if (virtioCoreQueueIsEmpty(pVirtio->pDevInsR3, pVirtio, idxQueue))
920	return VERR_NOT_AVAILABLE;
921
922	Log2Func(("%s avail_idx=%u\n", pVirtq->szVirtqName, pVirtq->uAvailIdx));
923	pVirtq->uAvailIdx++;
924
925	return VINF_SUCCESS;
926	}
927
928	/**
929	* Fetches descriptor chain using avail ring of indicated queue and converts the descriptor
930	* chain into its OUT (to device) and IN to guest components.
931	*
932	* Additionally it converts the OUT desc chain data to a contiguous virtual
933	* memory buffer for easy consumption by the caller. The caller must return the
934	* descriptor chain pointer via virtioCoreR3QueuePut() and then call virtioCoreQueueSync()
935	* at some point to return the data to the guest and complete the transaction.
936	*
937	* @param pDevIns The device instance.
938	* @param pVirtio Pointer to the shared virtio state.
939	* @param idxQueue Queue number
940	* @param ppDescChain Address to store pointer to descriptor chain that contains the
941	* pre-processed transaction information pulled from the virtq.
942	* Returned reference must be released by calling
943	* virtioCoreR3DescChainRelease().
944	* @param fRemove flags whether to remove desc chain from queue (false = peek)
945	*
946	* @returns VBox status code:
947	* @retval VINF_SUCCESS Success
948	* @retval VERR_INVALID_STATE VirtIO not in ready state (asserted).
949	* @retval VERR_NOT_AVAILABLE If the queue is empty.
950	*/
951	int virtioCoreR3QueueGet(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue,
952	PPVIRTIO_DESC_CHAIN_T ppDescChain, bool fRemove)
953	{
954	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
955
956	if (virtqIsEmpty(pDevIns, pVirtio, idxQueue))
957	return VERR_NOT_AVAILABLE;
958
959	uint16_t uHeadIdx = virtioReadAvailDescIdx(pDevIns, pVirtio, idxQueue, pVirtq->uAvailIdx);
960
961	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
962	virtioWriteUsedAvailEvent(pDevIns,pVirtio, idxQueue, pVirtq->uAvailIdx + 1);
963
964	if (fRemove)
965	pVirtq->uAvailIdx++;
966
967	int rc = virtioCoreR3DescChainGet(pDevIns, pVirtio, idxQueue, uHeadIdx, ppDescChain);
968	return rc;
969	}
970
971	/**
972	* Returns data to the guest to complete a transaction initiated by virtQueueGet().
973	*
974	* The caller passes in a pointer to a scatter-gather buffer of virtual memory segments
975	* and a pointer to the descriptor chain context originally derived from the pulled
976	* queue entry, and this function will write the virtual memory s/g buffer into the
977	* guest's physical memory free the descriptor chain. The caller handles the freeing
978	* (as needed) of the virtual memory buffer.
979	*
980	* @note This does a write-ahead to the used ring of the guest's queue. The data
981	* written won't be seen by the guest until the next call to virtioCoreQueueSync()
982	*
983	*
984	* @param pDevIns The device instance (for reading).
985	* @param pVirtio Pointer to the shared virtio state.
986	* @param idxQueue Queue number
987	*
988	* @param pSgVirtReturn Points to scatter-gather buffer of virtual memory
989	* segments the caller is returning to the guest.
990	*
991	* @param pDescChain This contains the context of the scatter-gather
992	* buffer originally pulled from the queue.
993	*
994	* @param fFence If true, put up copy fence (memory barrier) after
995	* copying to guest phys. mem.
996	*
997	* @returns VBox status code.
998	* @retval VINF_SUCCESS Success
999	* @retval VERR_INVALID_STATE VirtIO not in ready state
1000	* @retval VERR_NOT_AVAILABLE Queue is empty
1001	*
1002	* @note This function will not release any reference to pDescChain. The
1003	* caller must take care of that.
1004	*/
1005	int virtioCoreR3QueuePut(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, PRTSGBUF pSgVirtReturn,
1006	PVIRTIO_DESC_CHAIN_T pDescChain, bool fFence)
1007	{
1008	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1009	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
1010	PVIRTIOSGBUF pSgPhysReturn = pDescChain->pSgPhysReturn;
1011
1012	Assert(pDescChain->u32Magic == VIRTIO_DESC_CHAIN_MAGIC);
1013	Assert(pDescChain->cRefs > 0);
1014
1015	AssertMsgReturn(IS_DRIVER_OK(pVirtio), ("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1016
1017	Log6Func(("Copying client data to %s, desc chain (head desc_idx %d)\n",
1018	VIRTQNAME(pVirtio, idxQueue), virtioReadUsedRingIdx(pDevIns, pVirtio, idxQueue)));
1019
1020	/* Copy s/g buf (virtual memory) to guest phys mem (IN direction). */
1021
1022	size_t cbCopy = 0, cbTotal = 0, cbRemain = 0;
1023
1024	if (pSgVirtReturn)
1025	{
1026	size_t cbTarget = virtioCoreSgBufCalcTotalLength(pSgPhysReturn);
1027	cbRemain = cbTotal = RTSgBufCalcTotalLength(pSgVirtReturn);
1028	AssertMsgReturn(cbTarget >= cbRemain, ("No space to write data to phys memory"), VERR_BUFFER_OVERFLOW);
1029	virtioCoreSgBufReset(pSgPhysReturn); /* Reset ptr because req data may have already been written */
1030	while (cbRemain)
1031	{
1032	cbCopy = RT_MIN(pSgVirtReturn->cbSegLeft, pSgPhysReturn->cbSegLeft);
1033	Assert(cbCopy > 0);
1034	PDMDevHlpPhysWrite(pDevIns, (RTGCPHYS)pSgPhysReturn->gcPhysCur, pSgVirtReturn->pvSegCur, cbCopy);
1035	RTSgBufAdvance(pSgVirtReturn, cbCopy);
1036	virtioCoreSgBufAdvance(pSgPhysReturn, cbCopy);
1037	cbRemain -= cbCopy;
1038	}
1039
1040	if (fFence)
1041	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE(); /* needed? */
1042
1043	Assert(!(cbCopy >> 32));
1044	}
1045
1046	/* If this write-ahead crosses threshold where the driver wants to get an event flag it */
1047	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1048	if (pVirtq->uUsedIdx == virtioReadAvailUsedEvent(pDevIns, pVirtio, idxQueue))
1049	pVirtq->fEventThresholdReached = true;
1050
1051	/*
1052	* Place used buffer's descriptor in used ring but don't update used ring's slot index.
1053	* That will be done with a subsequent client call to virtioCoreQueueSync() */
1054	virtioWriteUsedElem(pDevIns, pVirtio, idxQueue, pVirtq->uUsedIdx++, pDescChain->uHeadIdx, (uint32_t)cbTotal);
1055
1056	if (pSgVirtReturn)
1057	Log6Func((".... Copied %zu bytes in %d segs to %u byte buffer, residual=%zu\n",
1058	cbTotal - cbRemain, pSgVirtReturn->cSegs, pDescChain->cbPhysReturn, pDescChain->cbPhysReturn - cbTotal));
1059
1060	Log6Func(("Write ahead used_idx=%u, %s used_idx=%u\n",
1061	pVirtq->uUsedIdx, VIRTQNAME(pVirtio, idxQueue), virtioReadUsedRingIdx(pDevIns, pVirtio, idxQueue)));
1062
1063	return VINF_SUCCESS;
1064	}
1065
1066	#endif /* IN_RING3 */
1067
1068	/**
1069	* Updates the indicated virtq's "used ring" descriptor index to match the
1070	* current write-head index, thus exposing the data added to the used ring by all
1071	* virtioCoreR3QueuePut() calls since the last sync. This should be called after one or
1072	* more virtioCoreR3QueuePut() calls to inform the guest driver there is data in the queue.
1073	* Explicit notifications (e.g. interrupt or MSI-X) will be sent to the guest,
1074	* depending on VirtIO features negotiated and conditions, otherwise the guest
1075	* will detect the update by polling. (see VirtIO 1.0 specification, Section 2.4 "Virtqueues").
1076	*
1077	* @param pDevIns The device instance.
1078	* @param pVirtio Pointer to the shared virtio state.
1079	* @param idxQueue Queue number
1080	*
1081	* @returns VBox status code.
1082	* @retval VINF_SUCCESS Success
1083	* @retval VERR_INVALID_STATE VirtIO not in ready state
1084	*/
1085	int virtioCoreQueueSync(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
1086	{
1087	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1088	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
1089
1090	AssertMsgReturn(IS_DRIVER_OK(pVirtio) && pVirtio->uQueueEnable[idxQueue],
1091	("Guest driver not in ready state.\n"), VERR_INVALID_STATE);
1092
1093	Log6Func(("Updating %s used_idx from %u to %u\n",
1094	VIRTQNAME(pVirtio, idxQueue), virtioReadUsedRingIdx(pDevIns, pVirtio, idxQueue), pVirtq->uUsedIdx));
1095
1096	virtioWriteUsedRingIdx(pDevIns, pVirtio, idxQueue, pVirtq->uUsedIdx);
1097	virtioNotifyGuestDriver(pDevIns, pVirtio, idxQueue);
1098
1099	return VINF_SUCCESS;
1100	}
1101
1102	/**
1103	*/
1104	static void virtioQueueNotified(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue, uint16_t uNotifyIdx)
1105	{
1106
1107	PVIRTIOCORECC pVirtioCC = PDMDEVINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1108
1109	/* See VirtIO 1.0, section 4.1.5.2 It implies that idxQueue and uNotifyIdx should match.
1110	* Disregarding this notification may cause throughput to stop, however there's no way to know
1111	* which was queue was intended for wake-up if the two parameters disagree. */
1112
1113	AssertMsg(uNotifyIdx == idxQueue,
1114	("Guest kicked virtq %d's notify addr w/non-corresponding virtq idx %d\n",
1115	idxQueue, uNotifyIdx));
1116	RT_NOREF(uNotifyIdx);
1117
1118	AssertReturnVoid(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1119	Log6Func(("%s\n", pVirtio->virtqState[idxQueue].szVirtqName));
1120
1121	/* Inform client */
1122	pVirtioCC->pfnQueueNotified(pDevIns, pVirtio, idxQueue);
1123	}
1124
1125	/**
1126	* Trigger MSI-X or INT# interrupt to notify guest of data added to used ring of
1127	* the specified virtq, depending on the interrupt configuration of the device
1128	* and depending on negotiated and realtime constraints flagged by the guest driver.
1129	*
1130	* See VirtIO 1.0 specification (section 2.4.7).
1131	*
1132	* @param pDevIns The device instance.
1133	* @param pVirtio Pointer to the shared virtio state.
1134	* @param idxQueue Queue to check for guest interrupt handling preference
1135	*/
1136	static void virtioNotifyGuestDriver(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint16_t idxQueue)
1137	{
1138
1139	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1140	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
1141
1142	if (!IS_DRIVER_OK(pVirtio))
1143	{
1144	LogFunc(("Guest driver not in ready state.\n"));
1145	return;
1146	}
1147
1148	if (pVirtio->uDriverFeatures & VIRTIO_F_EVENT_IDX)
1149	{
1150	if (pVirtq->fEventThresholdReached)
1151	{
1152	#ifdef IN_RING3
1153	Log6Func(("...kicking guest %s, VIRTIO_F_EVENT_IDX set and threshold (%d) reached\n",
1154	VIRTQNAME(pVirtio, idxQueue), (uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, idxQueue)));
1155	#endif
1156	virtioKick(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtio->uQueueMsixVector[idxQueue]);
1157	pVirtq->fEventThresholdReached = false;
1158	return;
1159	}
1160	#ifdef IN_RING3
1161	Log6Func(("...skipping interrupt %s, VIRTIO_F_EVENT_IDX set but threshold (%d) not reached (%d)\n",
1162	VIRTQNAME(pVirtio, idxQueue),(uint16_t)virtioReadAvailUsedEvent(pDevIns, pVirtio, idxQueue), pVirtq->uUsedIdx));
1163	#endif
1164	}
1165	else
1166	{
1167	/** If guest driver hasn't suppressed interrupts, interrupt */
1168	if (!(virtioReadAvailRingFlags(pDevIns, pVirtio, idxQueue) & VIRTQ_AVAIL_F_NO_INTERRUPT))
1169	{
1170	virtioKick(pDevIns, pVirtio, VIRTIO_ISR_VIRTQ_INTERRUPT, pVirtio->uQueueMsixVector[idxQueue]);
1171	return;
1172	}
1173	Log6Func(("...skipping interrupt, queue %s, Guest flagged VIRTQ_AVAIL_F_NO_INTERRUPT for queue\n",
1174	VIRTQNAME(pVirtio, idxQueue)));
1175	}
1176	}
1177
1178	/**
1179	* Raise interrupt or MSI-X
1180	*
1181	* @param pDevIns The device instance.
1182	* @param pVirtio Pointer to the shared virtio state.
1183	* @param uCause Interrupt cause bit mask to set in PCI ISR port.
1184	* @param uVec MSI-X vector, if enabled
1185	*/
1186	static int virtioKick(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, uint8_t uCause, uint16_t uMsixVector)
1187	{
1188	if (uCause == VIRTIO_ISR_VIRTQ_INTERRUPT)
1189	Log6Func(("reason: buffer added to 'used' ring.\n"));
1190	else
1191	if (uCause == VIRTIO_ISR_DEVICE_CONFIG)
1192	Log6Func(("reason: device config change\n"));
1193
1194	if (!pVirtio->fMsiSupport)
1195	{
1196	pVirtio->uISR \|= uCause;
1197	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_HIGH);
1198	}
1199	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1200	PDMDevHlpPCISetIrq(pDevIns, uMsixVector, 1);
1201	return VINF_SUCCESS;
1202	}
1203
1204	/**
1205	* Lower interrupt (Called when guest reads ISR and when resetting)
1206	*
1207	* @param pDevIns The device instance.
1208	*/
1209	static void virtioLowerInterrupt(PPDMDEVINS pDevIns, uint16_t uMsixVector)
1210	{
1211	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1212	if (!pVirtio->fMsiSupport)
1213	PDMDevHlpPCISetIrq(pDevIns, 0, PDM_IRQ_LEVEL_LOW);
1214	else if (uMsixVector != VIRTIO_MSI_NO_VECTOR)
1215	PDMDevHlpPCISetIrq(pDevIns, pVirtio->uMsixConfig, PDM_IRQ_LEVEL_LOW);
1216	}
1217
1218	#ifdef IN_RING3
1219	static void virtioResetQueue(PVIRTIOCORE pVirtio, uint16_t idxQueue)
1220	{
1221	Assert(idxQueue < RT_ELEMENTS(pVirtio->virtqState));
1222	PVIRTQSTATE pVirtq = &pVirtio->virtqState[idxQueue];
1223	pVirtq->uAvailIdx = 0;
1224	pVirtq->uUsedIdx = 0;
1225	pVirtq->fEventThresholdReached = false;
1226	pVirtio->uQueueEnable[idxQueue] = false;
1227	pVirtio->uQueueSize[idxQueue] = VIRTQ_MAX_SIZE;
1228	pVirtio->uQueueNotifyOff[idxQueue] = idxQueue;
1229	pVirtio->uQueueMsixVector[idxQueue] = idxQueue + 2;
1230	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1231	pVirtio->uQueueMsixVector[idxQueue] = VIRTIO_MSI_NO_VECTOR;
1232
1233	virtioLowerInterrupt(pVirtio->pDevInsR3, pVirtio->uQueueMsixVector[idxQueue]);
1234	}
1235
1236	static void virtioResetDevice(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
1237	{
1238	Log2Func(("\n"));
1239	pVirtio->uDeviceFeaturesSelect = 0;
1240	pVirtio->uDriverFeaturesSelect = 0;
1241	pVirtio->uConfigGeneration = 0;
1242	pVirtio->uDeviceStatus = 0;
1243	pVirtio->uISR = 0;
1244
1245	if (!pVirtio->fMsiSupport)
1246	virtioLowerInterrupt(pDevIns, 0);
1247	else
1248	{
1249	virtioLowerInterrupt(pDevIns, pVirtio->uMsixConfig);
1250	for (int i = 0; i < VIRTQ_MAX_CNT; i++)
1251	{
1252	virtioLowerInterrupt(pDevIns, pVirtio->uQueueMsixVector[i]);
1253	pVirtio->uQueueMsixVector[i];
1254	}
1255	}
1256
1257	if (!pVirtio->fMsiSupport) /* VirtIO 1.0, 4.1.4.3 and 4.1.5.1.2 */
1258	pVirtio->uMsixConfig = VIRTIO_MSI_NO_VECTOR;
1259
1260	for (uint16_t idxQueue = 0; idxQueue < VIRTQ_MAX_CNT; idxQueue++)
1261	virtioResetQueue(pVirtio, idxQueue);
1262	}
1263
1264	/**
1265	* Invoked by this implementation when guest driver resets the device.
1266	* The driver itself will not until the device has read the status change.
1267	*/
1268	static void virtioGuestR3WasReset(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1269	{
1270	LogFunc(("Guest reset the device\n"));
1271
1272	/* Let the client know */
1273	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, 0);
1274	virtioResetDevice(pDevIns, pVirtio);
1275	}
1276	#endif /* IN_RING3 */
1277
1278	/**
1279	* Handle accesses to Common Configuration capability
1280	*
1281	* @returns VBox status code
1282	*
1283	* @param pDevIns The device instance.
1284	* @param pVirtio Pointer to the shared virtio state.
1285	* @param pVirtioCC Pointer to the current context virtio state.
1286	* @param fWrite Set if write access, clear if read access.
1287	* @param offCfg The common configuration capability offset.
1288	* @param cb Number of bytes to read or write
1289	* @param pv Pointer to location to write to or read from
1290	*/
1291	static int virtioCommonCfgAccessed(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC,
1292	int fWrite, uint32_t offCfg, unsigned cb, void *pv)
1293	{
1294	/**
1295	* This macro resolves to boolean true if the implied parameters, offCfg and cb,
1296	* match the field offset and size of a field in the Common Cfg struct, (or if
1297	* it is a 64-bit field, if it accesses either 32-bit part as a 32-bit access)
1298	* This is mandated by section 4.1.3.1 of the VirtIO 1.0 specification)
1299	*
1300	* @param member Member of VIRTIO_PCI_COMMON_CFG_T
1301	* @param offCfg Implied parameter: Offset into VIRTIO_PCI_COMMON_CFG_T
1302	* @param cb Implied parameter: Number of bytes to access
1303	* @result true or false
1304	*/
1305	#define MATCH_COMMON_CFG(member) \
1306	( ( RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member) == 8 \
1307	&& ( offCfg == RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member) \
1308	\|\| offCfg == RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member) + sizeof(uint32_t)) \
1309	&& cb == sizeof(uint32_t)) \
1310	\|\| ( offCfg == RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member) \
1311	&& cb == RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member)) )
1312
1313	#ifdef LOG_ENABLED
1314	# define LOG_COMMON_CFG_ACCESS(member, a_offIntra) \
1315	if (LogIs7Enabled()) { \
1316	virtioCoreLogMappedIoValue(__FUNCTION__, #member, RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member), \
1317	pv, cb, a_offIntra, fWrite, false, 0); \
1318	}
1319	# define LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, a_offIntra) \
1320	if (LogIs7Enabled()) { \
1321	virtioCoreLogMappedIoValue(__FUNCTION__, #member, RT_SIZEOFMEMB(VIRTIO_PCI_COMMON_CFG_T, member), \
1322	pv, cb, a_offIntra, fWrite, true, idx); \
1323	}
1324	#else
1325	# define LOG_COMMON_CFG_ACCESS(member, a_offIntra) do { } while (0)
1326	# define LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, a_offIntra) do { } while (0)
1327	#endif
1328
1329	#define COMMON_CFG_ACCESSOR(member) \
1330	do \
1331	{ \
1332	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1333	if (fWrite) \
1334	memcpy((char )&pVirtio->member + offIntra, (const char )pv, cb); \
1335	else \
1336	memcpy(pv, (const char *)&pVirtio->member + offIntra, cb); \
1337	LOG_COMMON_CFG_ACCESS(member, offIntra); \
1338	} while(0)
1339
1340	#define COMMON_CFG_ACCESSOR_INDEXED(member, idx) \
1341	do \
1342	{ \
1343	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1344	if (fWrite) \
1345	memcpy((char *)&pVirtio->member[idx] + offIntra, pv, cb); \
1346	else \
1347	memcpy(pv, (const char *)&pVirtio->member[idx] + offIntra, cb); \
1348	LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, offIntra); \
1349	} while(0)
1350
1351	#define COMMON_CFG_ACCESSOR_READONLY(member) \
1352	do \
1353	{ \
1354	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1355	if (fWrite) \
1356	LogFunc(("Guest attempted to write readonly virtio_pci_common_cfg.%s\n", #member)); \
1357	else \
1358	{ \
1359	memcpy(pv, (const char *)&pVirtio->member + offIntra, cb); \
1360	LOG_COMMON_CFG_ACCESS(member, offIntra); \
1361	} \
1362	} while(0)
1363
1364	#define COMMON_CFG_ACCESSOR_INDEXED_READONLY(member, idx) \
1365	do \
1366	{ \
1367	uint32_t offIntra = offCfg - RT_OFFSETOF(VIRTIO_PCI_COMMON_CFG_T, member); \
1368	if (fWrite) \
1369	LogFunc(("Guest attempted to write readonly virtio_pci_common_cfg.%s[%d]\n", #member, idx)); \
1370	else \
1371	{ \
1372	memcpy(pv, (char const *)&pVirtio->member[idx] + offIntra, cb); \
1373	LOG_COMMON_CFG_ACCESS_INDEXED(member, idx, offIntra); \
1374	} \
1375	} while(0)
1376
1377
1378	int rc = VINF_SUCCESS;
1379	uint64_t val;
1380	if (MATCH_COMMON_CFG(uDeviceFeatures))
1381	{
1382	if (fWrite) /* Guest WRITE pCommonCfg>uDeviceFeatures */
1383	{
1384	LogFunc(("Guest attempted to write readonly virtio_pci_common_cfg.device_feature\n"));
1385	return VINF_SUCCESS;
1386	}
1387	else /* Guest READ pCommonCfg->uDeviceFeatures */
1388	{
1389	switch (pVirtio->uDeviceFeaturesSelect)
1390	{
1391	case 0:
1392	val = pVirtio->uDeviceFeatures & UINT32_C(0xffffffff);
1393	memcpy(pv, &val, cb);
1394	LOG_COMMON_CFG_ACCESS(uDeviceFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDeviceFeatures));
1395	break;
1396	case 1:
1397	val = pVirtio->uDeviceFeatures >> 32;
1398	memcpy(pv, &val, cb);
1399	LOG_COMMON_CFG_ACCESS(uDeviceFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDeviceFeatures) + 4);
1400	break;
1401	default:
1402	LogFunc(("Guest read uDeviceFeatures with out of range selector (%#x), returning 0\n",
1403	pVirtio->uDeviceFeaturesSelect));
1404	return VINF_IOM_MMIO_UNUSED_00;
1405	}
1406	}
1407	}
1408	else if (MATCH_COMMON_CFG(uDriverFeatures))
1409	{
1410	if (fWrite) /* Guest WRITE pCommonCfg->udriverFeatures */
1411	{
1412	switch (pVirtio->uDriverFeaturesSelect)
1413	{
1414	case 0:
1415	memcpy(&pVirtio->uDriverFeatures, pv, cb);
1416	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures));
1417	break;
1418	case 1:
1419	memcpy((char *)&pVirtio->uDriverFeatures + sizeof(uint32_t), pv, cb);
1420	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures) + 4);
1421	break;
1422	default:
1423	LogFunc(("Guest wrote uDriverFeatures with out of range selector (%#x), returning 0\n",
1424	pVirtio->uDriverFeaturesSelect));
1425	return VINF_SUCCESS;
1426	}
1427	}
1428	else /* Guest READ pCommonCfg->udriverFeatures */
1429	{
1430	switch (pVirtio->uDriverFeaturesSelect)
1431	{
1432	case 0:
1433	val = pVirtio->uDriverFeatures & 0xffffffff;
1434	memcpy(pv, &val, cb);
1435	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures));
1436	break;
1437	case 1:
1438	val = (pVirtio->uDriverFeatures >> 32) & 0xffffffff;
1439	memcpy(pv, &val, cb);
1440	LOG_COMMON_CFG_ACCESS(uDriverFeatures, offCfg - RT_UOFFSETOF(VIRTIO_PCI_COMMON_CFG_T, uDriverFeatures) + 4);
1441	break;
1442	default:
1443	LogFunc(("Guest read uDriverFeatures with out of range selector (%#x), returning 0\n",
1444	pVirtio->uDriverFeaturesSelect));
1445	return VINF_IOM_MMIO_UNUSED_00;
1446	}
1447	}
1448	}
1449	else if (MATCH_COMMON_CFG(uNumQueues))
1450	{
1451	if (fWrite)
1452	{
1453	Log2Func(("Guest attempted to write readonly virtio_pci_common_cfg.num_queues\n"));
1454	return VINF_SUCCESS;
1455	}
1456	else
1457	{
1458	(uint16_t )pv = VIRTQ_MAX_CNT;
1459	LOG_COMMON_CFG_ACCESS(uNumQueues, 0);
1460	}
1461	}
1462	else if (MATCH_COMMON_CFG(uDeviceStatus))
1463	{
1464	if (fWrite) /* Guest WRITE pCommonCfg->uDeviceStatus */
1465	{
1466	uint8_t const fNewStatus = (uint8_t )pv;
1467	Log7Func(("Guest wrote uDeviceStatus ................ ("));
1468	if (LogIs7Enabled())
1469	virtioLogDeviceStatus(fNewStatus ^ pVirtio->uDeviceStatus);
1470	Log7((")\n"));
1471
1472	/* If the status changed or we were reset, we need to go to ring-3 as
1473	it requires notifying the parent device. */
1474	bool const fStatusChanged = (fNewStatus & VIRTIO_STATUS_DRIVER_OK)
1475	!= (pVirtio->uPrevDeviceStatus & VIRTIO_STATUS_DRIVER_OK);
1476	#ifndef IN_RING3
1477	if (fStatusChanged \|\| fNewStatus == 0)
1478	{
1479	Log6Func(("=>ring3\n"));
1480	return VINF_IOM_R3_MMIO_WRITE;
1481	}
1482	#endif
1483	pVirtio->uDeviceStatus = fNewStatus;
1484
1485	#ifdef IN_RING3
1486	/*
1487	* Notify client only if status actually changed from last time and when we're reset.
1488	*/
1489	if (pVirtio->uDeviceStatus == 0)
1490	virtioGuestR3WasReset(pDevIns, pVirtio, pVirtioCC);
1491	if (fStatusChanged)
1492	pVirtioCC->pfnStatusChanged(pVirtio, pVirtioCC, fNewStatus & VIRTIO_STATUS_DRIVER_OK);
1493	#endif
1494	/*
1495	* Save the current status for the next write so we can see what changed.
1496	*/
1497	pVirtio->uPrevDeviceStatus = pVirtio->uDeviceStatus;
1498	}
1499	else /* Guest READ pCommonCfg->uDeviceStatus */
1500	{
1501	Log7Func(("Guest read uDeviceStatus ................ ("));
1502	(uint8_t )pv = pVirtio->uDeviceStatus;
1503	if (LogIs7Enabled())
1504	virtioLogDeviceStatus(pVirtio->uDeviceStatus);
1505	Log7((")\n"));
1506	}
1507	}
1508	else
1509	if (MATCH_COMMON_CFG(uMsixConfig))
1510	COMMON_CFG_ACCESSOR(uMsixConfig);
1511	else
1512	if (MATCH_COMMON_CFG(uDeviceFeaturesSelect))
1513	COMMON_CFG_ACCESSOR(uDeviceFeaturesSelect);
1514	else
1515	if (MATCH_COMMON_CFG(uDriverFeaturesSelect))
1516	COMMON_CFG_ACCESSOR(uDriverFeaturesSelect);
1517	else
1518	if (MATCH_COMMON_CFG(uConfigGeneration))
1519	COMMON_CFG_ACCESSOR_READONLY(uConfigGeneration);
1520	else
1521	if (MATCH_COMMON_CFG(uQueueSelect))
1522	COMMON_CFG_ACCESSOR(uQueueSelect);
1523	else
1524	if (MATCH_COMMON_CFG(uQueueSize))
1525	COMMON_CFG_ACCESSOR_INDEXED(uQueueSize, pVirtio->uQueueSelect);
1526	else
1527	if (MATCH_COMMON_CFG(uQueueMsixVector))
1528	COMMON_CFG_ACCESSOR_INDEXED(uQueueMsixVector, pVirtio->uQueueSelect);
1529	else
1530	if (MATCH_COMMON_CFG(uQueueEnable))
1531	COMMON_CFG_ACCESSOR_INDEXED(uQueueEnable, pVirtio->uQueueSelect);
1532	else
1533	if (MATCH_COMMON_CFG(uQueueNotifyOff))
1534	COMMON_CFG_ACCESSOR_INDEXED_READONLY(uQueueNotifyOff, pVirtio->uQueueSelect);
1535	else
1536	if (MATCH_COMMON_CFG(aGCPhysQueueDesc))
1537	COMMON_CFG_ACCESSOR_INDEXED(aGCPhysQueueDesc, pVirtio->uQueueSelect);
1538	else
1539	if (MATCH_COMMON_CFG(aGCPhysQueueAvail))
1540	COMMON_CFG_ACCESSOR_INDEXED(aGCPhysQueueAvail, pVirtio->uQueueSelect);
1541	else
1542	if (MATCH_COMMON_CFG(aGCPhysQueueUsed))
1543	COMMON_CFG_ACCESSOR_INDEXED(aGCPhysQueueUsed, pVirtio->uQueueSelect);
1544	else
1545	{
1546	Log2Func(("Bad guest %s access to virtio_pci_common_cfg: offCfg=%#x (%d), cb=%d\n",
1547	fWrite ? "write" : "read ", offCfg, offCfg, cb));
1548	return fWrite ? VINF_SUCCESS : VINF_IOM_MMIO_UNUSED_00;
1549	}
1550
1551	#undef COMMON_CFG_ACCESSOR_READONLY
1552	#undef COMMON_CFG_ACCESSOR_INDEXED_READONLY
1553	#undef COMMON_CFG_ACCESSOR_INDEXED
1554	#undef COMMON_CFG_ACCESSOR
1555	#undef LOG_COMMON_CFG_ACCESS_INDEXED
1556	#undef LOG_COMMON_CFG_ACCESS
1557	#undef MATCH_COMMON_CFG
1558	#ifndef IN_RING3
1559	RT_NOREF(pDevIns, pVirtioCC);
1560	#endif
1561	return rc;
1562	}
1563
1564	/**
1565	* @callback_method_impl{FNIOMMMIONEWREAD,
1566	* Memory mapped I/O Handler for PCI Capabilities read operations.}
1567	*
1568	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
1569	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to reads
1570	* of 1, 2 or 4 bytes, only.
1571	*
1572	*/
1573	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioRead(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void pv, unsigned cb)
1574	{
1575	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1576	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1577	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
1578	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
1579
1580
1581	uint32_t offIntra;
1582	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocDeviceCap))
1583	{
1584	#ifdef IN_RING3
1585	/*
1586	* Callback to client to manage device-specific configuration.
1587	*/
1588	VBOXSTRICTRC rcStrict = pVirtioCC->pfnDevCapRead(pDevIns, offIntra, pv, cb);
1589
1590	/*
1591	* Additionally, anytime any part of the device-specific configuration (which our client maintains)
1592	* is READ it needs to be checked to see if it changed since the last time any part was read, in
1593	* order to maintain the config generation (see VirtIO 1.0 spec, section 4.1.4.3.1)
1594	*/
1595	bool fDevSpecificFieldChanged = RT_BOOL(memcmp(pVirtioCC->pbDevSpecificCfg + offIntra,
1596	pVirtioCC->pbPrevDevSpecificCfg + offIntra,
1597	RT_MIN(cb, pVirtioCC->cbDevSpecificCfg - offIntra)));
1598
1599	memcpy(pVirtioCC->pbPrevDevSpecificCfg, pVirtioCC->pbDevSpecificCfg, pVirtioCC->cbDevSpecificCfg);
1600
1601	if (pVirtio->fGenUpdatePending \|\| fDevSpecificFieldChanged)
1602	{
1603	++pVirtio->uConfigGeneration;
1604	Log6Func(("Bumped cfg. generation to %d because %s%s\n",
1605	pVirtio->uConfigGeneration,
1606	fDevSpecificFieldChanged ? "<dev cfg changed> " : "",
1607	pVirtio->fGenUpdatePending ? "<update was pending>" : ""));
1608	pVirtio->fGenUpdatePending = false;
1609	}
1610
1611	virtioLowerInterrupt(pDevIns, 0);
1612	return rcStrict;
1613	#else
1614	return VINF_IOM_R3_MMIO_READ;
1615	#endif
1616	}
1617
1618	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocCommonCfgCap))
1619	return virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, false /* fWrite */, offIntra, cb, pv);
1620
1621	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocIsrCap) && cb == sizeof(uint8_t))
1622	{
1623	(uint8_t )pv = pVirtio->uISR;
1624	Log6Func(("Read and clear ISR\n"));
1625	pVirtio->uISR = 0; /* VirtIO specification requires reads of ISR to clear it */
1626	virtioLowerInterrupt(pDevIns, 0);
1627	return VINF_SUCCESS;
1628	}
1629
1630	ASSERT_GUEST_MSG_FAILED(("Bad read access to mapped capabilities region: off=%RGp cb=%u\n", off, cb));
1631	return VINF_IOM_MMIO_UNUSED_00;
1632	}
1633
1634	/**
1635	* @callback_method_impl{FNIOMMMIONEWREAD,
1636	* Memory mapped I/O Handler for PCI Capabilities write operations.}
1637	*
1638	* This MMIO handler specifically supports the VIRTIO_PCI_CAP_PCI_CFG capability defined
1639	* in the VirtIO 1.0 specification, section 4.1.4.7, and as such is restricted to writes
1640	* of 1, 2 or 4 bytes, only.
1641	*/
1642	static DECLCALLBACK(VBOXSTRICTRC) virtioMmioWrite(PPDMDEVINS pDevIns, void pvUser, RTGCPHYS off, void const pv, unsigned cb)
1643	{
1644	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1645	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1646
1647	AssertReturn(cb == 1 \|\| cb == 2 \|\| cb == 4, VERR_INVALID_PARAMETER);
1648
1649	Assert(pVirtio == (PVIRTIOCORE)pvUser); RT_NOREF(pvUser);
1650	uint32_t offIntra;
1651	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocDeviceCap))
1652	{
1653	#ifdef IN_RING3
1654	/*
1655	* Pass this MMIO write access back to the client to handle
1656	*/
1657	return pVirtioCC->pfnDevCapWrite(pDevIns, offIntra, pv, cb);
1658	#else
1659	return VINF_IOM_R3_MMIO_WRITE;
1660	#endif
1661	}
1662
1663	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocCommonCfgCap))
1664	return virtioCommonCfgAccessed(pDevIns, pVirtio, pVirtioCC, true /* fWrite /, offIntra, cb, (void )pv);
1665
1666	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocIsrCap) && cb == sizeof(uint8_t))
1667	{
1668	pVirtio->uISR = (uint8_t )pv;
1669	Log6Func(("Setting uISR = 0x%02x (virtq interrupt: %d, dev confg interrupt: %d)\n",
1670	pVirtio->uISR & 0xff,
1671	pVirtio->uISR & VIRTIO_ISR_VIRTQ_INTERRUPT,
1672	RT_BOOL(pVirtio->uISR & VIRTIO_ISR_DEVICE_CONFIG)));
1673	return VINF_SUCCESS;
1674	}
1675
1676	/* This should be guest driver dropping index of a new descriptor in avail ring */
1677	if (MATCHES_VIRTIO_CAP_STRUCT(off, cb, offIntra, pVirtio->LocNotifyCap) && cb == sizeof(uint16_t))
1678	{
1679	virtioQueueNotified(pDevIns, pVirtio, offIntra / VIRTIO_NOTIFY_OFFSET_MULTIPLIER, (uint16_t )pv);
1680	return VINF_SUCCESS;
1681	}
1682
1683	ASSERT_GUEST_MSG_FAILED(("Bad write access to mapped capabilities region: off=%RGp pv=%#p{%.*Rhxs} cb=%u\n", off, pv, cb, pv, cb));
1684	return VINF_SUCCESS;
1685	}
1686
1687	#ifdef IN_RING3
1688
1689	/**
1690	* @callback_method_impl{FNPCICONFIGREAD}
1691	*/
1692	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigRead(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
1693	uint32_t uAddress, unsigned cb, uint32_t *pu32Value)
1694	{
1695	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1696	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1697	RT_NOREF(pPciDev);
1698
1699	Log7Func(("pDevIns=%p pPciDev=%p uAddress=%#x cb=%u pu32Value=%p\n",
1700	pDevIns, pPciDev, uAddress, cb, pu32Value));
1701	if (uAddress == pVirtio->uPciCfgDataOff)
1702	{
1703	/*
1704	* VirtIO 1.0 spec section 4.1.4.7 describes a required alternative access capability
1705	* whereby the guest driver can specify a bar, offset, and length via the PCI configuration space
1706	* (the virtio_pci_cfg_cap capability), and access data items.
1707	*/
1708	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
1709	uint32_t uLength = pPciCap->uLength;
1710
1711	if ( (uLength != 1 && uLength != 2 && uLength != 4)
1712	\|\| cb != uLength
1713	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
1714	{
1715	ASSERT_GUEST_MSG_FAILED(("Guest read virtio_pci_cfg_cap.pci_cfg_data using mismatching config. Ignoring\n"));
1716	*pu32Value = UINT32_MAX;
1717	return VINF_SUCCESS;
1718	}
1719
1720	VBOXSTRICTRC rcStrict = virtioMmioRead(pDevIns, pVirtio, pPciCap->uOffset, pu32Value, cb);
1721	Log2Func(("virtio: Guest read virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%d, length=%d, result=%d -> %Rrc\n",
1722	pPciCap->uBar, pPciCap->uOffset, uLength, *pu32Value, VBOXSTRICTRC_VAL(rcStrict)));
1723	return rcStrict;
1724	}
1725	return VINF_PDM_PCI_DO_DEFAULT;
1726	}
1727
1728	/**
1729	* @callback_method_impl{FNPCICONFIGWRITE}
1730	*/
1731	static DECLCALLBACK(VBOXSTRICTRC) virtioR3PciConfigWrite(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev,
1732	uint32_t uAddress, unsigned cb, uint32_t u32Value)
1733	{
1734	PVIRTIOCORE pVirtio = PDMINS_2_DATA(pDevIns, PVIRTIOCORE);
1735	PVIRTIOCORECC pVirtioCC = PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC);
1736	RT_NOREF(pPciDev);
1737
1738	Log7Func(("pDevIns=%p pPciDev=%p uAddress=%#x cb=%u u32Value=%#x\n", pDevIns, pPciDev, uAddress, cb, u32Value));
1739	if (uAddress == pVirtio->uPciCfgDataOff)
1740	{
1741	/* VirtIO 1.0 spec section 4.1.4.7 describes a required alternative access capability
1742	* whereby the guest driver can specify a bar, offset, and length via the PCI configuration space
1743	* (the virtio_pci_cfg_cap capability), and access data items. */
1744
1745	struct virtio_pci_cap *pPciCap = &pVirtioCC->pPciCfgCap->pciCap;
1746	uint32_t uLength = pPciCap->uLength;
1747
1748	if ( (uLength != 1 && uLength != 2 && uLength != 4)
1749	\|\| cb != uLength
1750	\|\| pPciCap->uBar != VIRTIO_REGION_PCI_CAP)
1751	{
1752	ASSERT_GUEST_MSG_FAILED(("Guest write virtio_pci_cfg_cap.pci_cfg_data using mismatching config. Ignoring\n"));
1753	return VINF_SUCCESS;
1754	}
1755
1756	VBOXSTRICTRC rcStrict = virtioMmioWrite(pDevIns, pVirtio, pPciCap->uOffset, &u32Value, cb);
1757	Log2Func(("Guest wrote virtio_pci_cfg_cap.pci_cfg_data, bar=%d, offset=%x, length=%x, value=%d -> %Rrc\n",
1758	pPciCap->uBar, pPciCap->uOffset, uLength, u32Value, VBOXSTRICTRC_VAL(rcStrict)));
1759	return rcStrict;
1760	}
1761	return VINF_PDM_PCI_DO_DEFAULT;
1762	}
1763
1764
1765	/*********************************************************************************************************************************
1766	* Saved state. *
1767	*********************************************************************************************************************************/
1768
1769	/**
1770	* Called from the FNSSMDEVSAVEEXEC function of the device.
1771	*
1772	* @param pVirtio Pointer to the shared virtio state.
1773	* @param pHlp The ring-3 device helpers.
1774	* @param pSSM The saved state handle.
1775	* @returns VBox status code.
1776	*/
1777	int virtioCoreR3SaveExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM)
1778	{
1779	LogFunc(("\n"));
1780	pHlp->pfnSSMPutU64(pSSM, VIRTIO_SAVEDSTATE_MARKER);
1781	pHlp->pfnSSMPutU32(pSSM, VIRTIO_SAVEDSTATE_VERSION);
1782
1783	pHlp->pfnSSMPutBool(pSSM, pVirtio->fGenUpdatePending);
1784	pHlp->pfnSSMPutU8(pSSM, pVirtio->uDeviceStatus);
1785	pHlp->pfnSSMPutU8(pSSM, pVirtio->uConfigGeneration);
1786	pHlp->pfnSSMPutU8(pSSM, pVirtio->uPciCfgDataOff);
1787	pHlp->pfnSSMPutU8(pSSM, pVirtio->uISR);
1788	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueSelect);
1789	pHlp->pfnSSMPutU32(pSSM, pVirtio->uDeviceFeaturesSelect);
1790	pHlp->pfnSSMPutU32(pSSM, pVirtio->uDriverFeaturesSelect);
1791	pHlp->pfnSSMPutU64(pSSM, pVirtio->uDriverFeatures);
1792
1793	for (uint32_t i = 0; i < VIRTQ_MAX_CNT; i++)
1794	{
1795	pHlp->pfnSSMPutGCPhys64(pSSM, pVirtio->aGCPhysQueueDesc[i]);
1796	pHlp->pfnSSMPutGCPhys64(pSSM, pVirtio->aGCPhysQueueAvail[i]);
1797	pHlp->pfnSSMPutGCPhys64(pSSM, pVirtio->aGCPhysQueueUsed[i]);
1798	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueNotifyOff[i]);
1799	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueMsixVector[i]);
1800	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueEnable[i]);
1801	pHlp->pfnSSMPutU16(pSSM, pVirtio->uQueueSize[i]);
1802	pHlp->pfnSSMPutU16(pSSM, pVirtio->virtqState[i].uAvailIdx);
1803	pHlp->pfnSSMPutU16(pSSM, pVirtio->virtqState[i].uUsedIdx);
1804	int rc = pHlp->pfnSSMPutMem(pSSM, pVirtio->virtqState[i].szVirtqName, 32);
1805	AssertRCReturn(rc, rc);
1806	}
1807
1808	return VINF_SUCCESS;
1809	}
1810
1811	/**
1812	* Called from the FNSSMDEVLOADEXEC function of the device.
1813	*
1814	* @param pVirtio Pointer to the shared virtio state.
1815	* @param pHlp The ring-3 device helpers.
1816	* @param pSSM The saved state handle.
1817	* @returns VBox status code.
1818	*/
1819	int virtioCoreR3LoadExec(PVIRTIOCORE pVirtio, PCPDMDEVHLPR3 pHlp, PSSMHANDLE pSSM)
1820	{
1821	LogFunc(("\n"));
1822	/*
1823	* Check the marker and (embedded) version number.
1824	*/
1825	uint64_t uMarker = 0;
1826	int rc = pHlp->pfnSSMGetU64(pSSM, &uMarker);
1827	AssertRCReturn(rc, rc);
1828	if (uMarker != VIRTIO_SAVEDSTATE_MARKER)
1829	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
1830	N_("Expected marker value %#RX64 found %#RX64 instead"),
1831	VIRTIO_SAVEDSTATE_MARKER, uMarker);
1832	uint32_t uVersion = 0;
1833	rc = pHlp->pfnSSMGetU32(pSSM, &uVersion);
1834	AssertRCReturn(rc, rc);
1835	if (uVersion != VIRTIO_SAVEDSTATE_VERSION)
1836	return pHlp->pfnSSMSetLoadError(pSSM, VERR_SSM_DATA_UNIT_FORMAT_CHANGED, RT_SRC_POS,
1837	N_("Unsupported virtio version: %u"), uVersion);
1838	/*
1839	* Load the state.
1840	*/
1841	pHlp->pfnSSMGetBool(pSSM, &pVirtio->fGenUpdatePending);
1842	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uDeviceStatus);
1843	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uConfigGeneration);
1844	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uPciCfgDataOff);
1845	pHlp->pfnSSMGetU8(pSSM, &pVirtio->uISR);
1846	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueSelect);
1847	pHlp->pfnSSMGetU32(pSSM, &pVirtio->uDeviceFeaturesSelect);
1848	pHlp->pfnSSMGetU32(pSSM, &pVirtio->uDriverFeaturesSelect);
1849	pHlp->pfnSSMGetU64(pSSM, &pVirtio->uDriverFeatures);
1850
1851	for (uint32_t i = 0; i < VIRTQ_MAX_CNT; i++)
1852	{
1853	pHlp->pfnSSMGetGCPhys64(pSSM, &pVirtio->aGCPhysQueueDesc[i]);
1854	pHlp->pfnSSMGetGCPhys64(pSSM, &pVirtio->aGCPhysQueueAvail[i]);
1855	pHlp->pfnSSMGetGCPhys64(pSSM, &pVirtio->aGCPhysQueueUsed[i]);
1856	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueNotifyOff[i]);
1857	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueMsixVector[i]);
1858	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueEnable[i]);
1859	pHlp->pfnSSMGetU16(pSSM, &pVirtio->uQueueSize[i]);
1860	pHlp->pfnSSMGetU16(pSSM, &pVirtio->virtqState[i].uAvailIdx);
1861	pHlp->pfnSSMGetU16(pSSM, &pVirtio->virtqState[i].uUsedIdx);
1862	rc = pHlp->pfnSSMGetMem(pSSM, pVirtio->virtqState[i].szVirtqName,
1863	sizeof(pVirtio->virtqState[i].szVirtqName));
1864	AssertRCReturn(rc, rc);
1865	}
1866
1867	return VINF_SUCCESS;
1868	}
1869
1870
1871	/*********************************************************************************************************************************
1872	* Device Level *
1873	*********************************************************************************************************************************/
1874
1875	/**
1876	* This must be called by the client to handle VM state changes
1877	* after the client takes care of its device-specific tasks for the state change.
1878	* (i.e. Reset, suspend, power-off, resume)
1879	*
1880	* @param pDevIns The device instance.
1881	* @param pVirtio Pointer to the shared virtio state.
1882	*/
1883	void virtioCoreR3VmStateChanged(PVIRTIOCORE pVirtio, VIRTIOVMSTATECHANGED enmState)
1884	{
1885	LogFunc(("State changing to %s\n",
1886	virtioCoreGetStateChangeText(enmState)));
1887
1888	switch(enmState)
1889	{
1890	case kvirtIoVmStateChangedReset:
1891	virtioCoreResetAll(pVirtio);
1892	break;
1893	case kvirtIoVmStateChangedSuspend:
1894	break;
1895	case kvirtIoVmStateChangedPowerOff:
1896	break;
1897	case kvirtIoVmStateChangedResume:
1898	virtioNotifyGuestDriver(pVirtio->pDevInsR3, pVirtio, 0 /* idxQueue */);
1899	break;
1900	default:
1901	LogRelFunc(("Bad enum value"));
1902	return;
1903	}
1904	}
1905
1906	/**
1907	* This should be called from PDMDEVREGR3::pfnDestruct.
1908	*
1909	* @param pDevIns The device instance.
1910	* @param pVirtio Pointer to the shared virtio state.
1911	* @param pVirtioCC Pointer to the ring-3 virtio state.
1912	*/
1913	void virtioCoreR3Term(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC)
1914	{
1915	if (pVirtioCC->pbPrevDevSpecificCfg)
1916	{
1917	RTMemFree(pVirtioCC->pbPrevDevSpecificCfg);
1918	pVirtioCC->pbPrevDevSpecificCfg = NULL;
1919	}
1920	RT_NOREF(pDevIns, pVirtio);
1921	}
1922
1923
1924	/**rr
1925	* Setup PCI device controller and Virtio state
1926	*
1927	* This should be called from PDMDEVREGR3::pfnConstruct.
1928	*
1929	* @param pDevIns The device instance.
1930	* @param pVirtio Pointer to the shared virtio state. This
1931	* must be the first member in the shared
1932	* device instance data!
1933	* @param pVirtioCC Pointer to the ring-3 virtio state. This
1934	* must be the first member in the ring-3
1935	* device instance data!
1936	* @param pPciParams Values to populate industry standard PCI Configuration Space data structure
1937	* @param pcszInstance Device instance name (format-specifier)
1938	* @param fDevSpecificFeatures VirtIO device-specific features offered by
1939	* client
1940	* @param cbDevSpecificCfg Size of virtio_pci_device_cap device-specific struct
1941	* @param pvDevSpecificCfg Address of client's dev-specific
1942	* configuration struct.
1943	*/
1944	int virtioCoreR3Init(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio, PVIRTIOCORECC pVirtioCC, PVIRTIOPCIPARAMS pPciParams,
1945	const char pcszInstance, uint64_t fDevSpecificFeatures, void pvDevSpecificCfg, uint16_t cbDevSpecificCfg)
1946	{
1947	/*
1948	* The pVirtio state must be the first member of the shared device instance
1949	* data, otherwise we cannot get our bearings in the PCI configuration callbacks.
1950	*/
1951	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
1952	AssertLogRelReturn(pVirtioCC == PDMINS_2_DATA_CC(pDevIns, PVIRTIOCORECC), VERR_STATE_CHANGED);
1953
1954	pVirtio->pDevInsR3 = pDevIns;
1955
1956	/*
1957	* Caller must initialize these.
1958	*/
1959	AssertReturn(pVirtioCC->pfnStatusChanged, VERR_INVALID_POINTER);
1960	AssertReturn(pVirtioCC->pfnQueueNotified, VERR_INVALID_POINTER);
1961
1962	#if 0 /* Until pdmR3DvHlp_PCISetIrq() impl is fixed and Assert that limits vec to 0 is removed */
1963	# ifdef VBOX_WITH_MSI_DEVICES
1964	pVirtio->fMsiSupport = true;
1965	# endif
1966	#endif
1967
1968	/*
1969	* The host features offered include both device-specific features
1970	* and reserved feature bits (device independent)
1971	*/
1972	pVirtio->uDeviceFeatures = VIRTIO_F_VERSION_1
1973	\| VIRTIO_DEV_INDEPENDENT_FEATURES_OFFERED
1974	\| fDevSpecificFeatures;
1975
1976	RTStrCopy(pVirtio->szInstance, sizeof(pVirtio->szInstance), pcszInstance);
1977
1978	pVirtio->uDeviceStatus = 0;
1979	pVirtioCC->cbDevSpecificCfg = cbDevSpecificCfg;
1980	pVirtioCC->pbDevSpecificCfg = (uint8_t *)pvDevSpecificCfg;
1981	pVirtioCC->pbPrevDevSpecificCfg = (uint8_t *)RTMemDup(pvDevSpecificCfg, cbDevSpecificCfg);
1982	AssertLogRelReturn(pVirtioCC->pbPrevDevSpecificCfg, VERR_NO_MEMORY);
1983
1984	/* Set PCI config registers (assume 32-bit mode) */
1985	PPDMPCIDEV pPciDev = pDevIns->apPciDevs[0];
1986	PDMPCIDEV_ASSERT_VALID(pDevIns, pPciDev);
1987
1988	PDMPciDevSetRevisionId(pPciDev, DEVICE_PCI_REVISION_ID_VIRTIO);
1989	PDMPciDevSetVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
1990	PDMPciDevSetSubSystemVendorId(pPciDev, DEVICE_PCI_VENDOR_ID_VIRTIO);
1991	PDMPciDevSetDeviceId(pPciDev, pPciParams->uDeviceId);
1992	PDMPciDevSetClassBase(pPciDev, pPciParams->uClassBase);
1993	PDMPciDevSetClassSub(pPciDev, pPciParams->uClassSub);
1994	PDMPciDevSetClassProg(pPciDev, pPciParams->uClassProg);
1995	PDMPciDevSetSubSystemId(pPciDev, pPciParams->uSubsystemId);
1996	PDMPciDevSetInterruptLine(pPciDev, pPciParams->uInterruptLine);
1997	PDMPciDevSetInterruptPin(pPciDev, pPciParams->uInterruptPin);
1998
1999	/* Register PCI device */
2000	int rc = PDMDevHlpPCIRegister(pDevIns, pPciDev);
2001	if (RT_FAILURE(rc))
2002	return PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Device")); /* can we put params in this error? */
2003
2004	rc = PDMDevHlpPCIInterceptConfigAccesses(pDevIns, pPciDev, virtioR3PciConfigRead, virtioR3PciConfigWrite);
2005	AssertRCReturn(rc, rc);
2006
2007
2008	/* Construct & map PCI vendor-specific capabilities for virtio host negotiation with guest driver */
2009
2010	/* The following capability mapped via VirtIO 1.0: struct virtio_pci_cfg_cap (VIRTIO_PCI_CFG_CAP_T)
2011	* as a mandatory but suboptimal alternative interface to host device capabilities, facilitating
2012	* access the memory of any BAR. If the guest uses it (the VirtIO driver on Linux doesn't),
2013	* Unlike Common, Notify, ISR and Device capabilities, it is accessed directly via PCI Config region.
2014	* therefore does not contribute to the capabilities region (BAR) the other capabilities use.
2015	*/
2016	#define CFG_ADDR_2_IDX(addr) ((uint8_t)(((uintptr_t)(addr) - (uintptr_t)&pPciDev->abConfig[0])))
2017	#define SET_PCI_CAP_LOC(a_pPciDev, a_pCfg, a_LocCap, a_uMmioLengthAlign) \
2018	do { \
2019	(a_LocCap).offMmio = (a_pCfg)->uOffset; \
2020	(a_LocCap).cbMmio = RT_ALIGN_T((a_pCfg)->uLength, a_uMmioLengthAlign, uint16_t); \
2021	(a_LocCap).offPci = (uint16_t)(uintptr_t)((uint8_t *)(a_pCfg) - &(a_pPciDev)->abConfig[0]); \
2022	(a_LocCap).cbPci = (a_pCfg)->uCapLen; \
2023	} while (0)
2024
2025	PVIRTIO_PCI_CAP_T pCfg;
2026	uint32_t cbRegion = 0;
2027
2028	/* Common capability (VirtIO 1.0 spec, section 4.1.4.3) */
2029	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[0x40];
2030	pCfg->uCfgType = VIRTIO_PCI_CAP_COMMON_CFG;
2031	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2032	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2033	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2034	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2035	pCfg->uOffset = RT_ALIGN_32(0, 4); /* reminder, in case someone changes offset */
2036	pCfg->uLength = sizeof(VIRTIO_PCI_COMMON_CFG_T);
2037	cbRegion += pCfg->uLength;
2038	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocCommonCfgCap, 2);
2039	pVirtioCC->pCommonCfgCap = pCfg;
2040
2041	/*
2042	* Notify capability (VirtIO 1.0 spec, section 4.1.4.4). Note: uLength is based on the choice
2043	* of this implementation to make each queue's uQueueNotifyOff equal to (QueueSelect) ordinal
2044	* value of the queue (different strategies are possible according to spec).
2045	*/
2046	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2047	pCfg->uCfgType = VIRTIO_PCI_CAP_NOTIFY_CFG;
2048	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2049	pCfg->uCapLen = sizeof(VIRTIO_PCI_NOTIFY_CAP_T);
2050	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2051	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2052	pCfg->uOffset = pVirtioCC->pCommonCfgCap->uOffset + pVirtioCC->pCommonCfgCap->uLength;
2053	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2054
2055
2056	pCfg->uLength = VIRTQ_MAX_CNT * VIRTIO_NOTIFY_OFFSET_MULTIPLIER + 2; /* will change in VirtIO 1.1 */
2057	cbRegion += pCfg->uLength;
2058	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocNotifyCap, 1);
2059	pVirtioCC->pNotifyCap = (PVIRTIO_PCI_NOTIFY_CAP_T)pCfg;
2060	pVirtioCC->pNotifyCap->uNotifyOffMultiplier = VIRTIO_NOTIFY_OFFSET_MULTIPLIER;
2061
2062	/* ISR capability (VirtIO 1.0 spec, section 4.1.4.5)
2063	*
2064	* VirtIO 1.0 spec says 8-bit, unaligned in MMIO space. Example/diagram
2065	* of spec shows it as a 32-bit field with upper bits 'reserved'
2066	* Will take spec's words more literally than the diagram for now.
2067	*/
2068	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2069	pCfg->uCfgType = VIRTIO_PCI_CAP_ISR_CFG;
2070	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2071	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2072	pCfg->uCapNext = CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen;
2073	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2074	pCfg->uOffset = pVirtioCC->pNotifyCap->pciCap.uOffset + pVirtioCC->pNotifyCap->pciCap.uLength;
2075	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2076	pCfg->uLength = sizeof(uint8_t);
2077	cbRegion += pCfg->uLength;
2078	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocIsrCap, 4);
2079	pVirtioCC->pIsrCap = pCfg;
2080
2081	/* PCI Cfg capability (VirtIO 1.0 spec, section 4.1.4.7)
2082	* This capability doesn't get page-MMIO mapped. Instead uBar, uOffset and uLength are intercepted
2083	* by trapping PCI configuration I/O and get modulated by consumers to locate fetch and read/write
2084	* values from any region. NOTE: The linux driver not only doesn't use this feature, it will not
2085	* even list it as present if uLength isn't non-zero and also 4-byte-aligned as the linux driver is
2086	* initializing.
2087	*/
2088	pVirtio->uPciCfgDataOff = pCfg->uCapNext + RT_OFFSETOF(VIRTIO_PCI_CFG_CAP_T, uPciCfgData);
2089	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2090	pCfg->uCfgType = VIRTIO_PCI_CAP_PCI_CFG;
2091	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2092	pCfg->uCapLen = sizeof(VIRTIO_PCI_CFG_CAP_T);
2093	pCfg->uCapNext = (pVirtio->fMsiSupport \|\| pVirtioCC->pbDevSpecificCfg) ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2094	pCfg->uBar = 0;
2095	pCfg->uOffset = 0;
2096	pCfg->uLength = 0;
2097	cbRegion += pCfg->uLength;
2098	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocPciCfgCap, 1);
2099	pVirtioCC->pPciCfgCap = (PVIRTIO_PCI_CFG_CAP_T)pCfg;
2100
2101	if (pVirtioCC->pbDevSpecificCfg)
2102	{
2103	/* Following capability (via VirtIO 1.0, section 4.1.4.6). Client defines the
2104	* device-specific config fields struct and passes size to this constructor */
2105	pCfg = (PVIRTIO_PCI_CAP_T)&pPciDev->abConfig[pCfg->uCapNext];
2106	pCfg->uCfgType = VIRTIO_PCI_CAP_DEVICE_CFG;
2107	pCfg->uCapVndr = VIRTIO_PCI_CAP_ID_VENDOR;
2108	pCfg->uCapLen = sizeof(VIRTIO_PCI_CAP_T);
2109	pCfg->uCapNext = pVirtio->fMsiSupport ? CFG_ADDR_2_IDX(pCfg) + pCfg->uCapLen : 0;
2110	pCfg->uBar = VIRTIO_REGION_PCI_CAP;
2111	pCfg->uOffset = pVirtioCC->pIsrCap->uOffset + pVirtioCC->pIsrCap->uLength;
2112	pCfg->uOffset = RT_ALIGN_32(pCfg->uOffset, 4);
2113	pCfg->uLength = cbDevSpecificCfg;
2114	cbRegion += pCfg->uLength;
2115	SET_PCI_CAP_LOC(pPciDev, pCfg, pVirtio->LocDeviceCap, 4);
2116	pVirtioCC->pDeviceCap = pCfg;
2117	}
2118	else
2119	Assert(pVirtio->LocDeviceCap.cbMmio == 0 && pVirtio->LocDeviceCap.cbPci == 0);
2120
2121	if (pVirtio->fMsiSupport)
2122	{
2123	PDMMSIREG aMsiReg;
2124	RT_ZERO(aMsiReg);
2125	aMsiReg.iMsixCapOffset = pCfg->uCapNext;
2126	aMsiReg.iMsixNextOffset = 0;
2127	aMsiReg.iMsixBar = VIRTIO_REGION_MSIX_CAP;
2128	aMsiReg.cMsixVectors = VBOX_MSIX_MAX_ENTRIES;
2129	rc = PDMDevHlpPCIRegisterMsi(pDevIns, &aMsiReg); /* see MsixR3init() */
2130	if (RT_FAILURE(rc))
2131	{
2132	/* See PDMDevHlp.cpp:pdmR3DevHlp_PCIRegisterMsi */
2133	LogFunc(("Failed to configure MSI-X (%Rrc). Reverting to INTx\n", rc));
2134	pVirtio->fMsiSupport = false;
2135	}
2136	else
2137	Log2Func(("Using MSI-X for guest driver notification\n"));
2138	}
2139	else
2140	LogFunc(("MSI-X not available for VBox, using INTx notification\n"));
2141
2142	/* Set offset to first capability and enable PCI dev capabilities */
2143	PDMPciDevSetCapabilityList(pPciDev, 0x40);
2144	PDMPciDevSetStatus(pPciDev, VBOX_PCI_STATUS_CAP_LIST);
2145
2146	/* Linux drivers/virtio/virtio_pci_modern.c tries to map at least a page for the
2147	* 'unknown' device-specific capability without querying the capability to figure
2148	* out size, so pad with an extra page
2149	*/
2150	size_t cbSize = RTStrPrintf(pVirtioCC->pcszMmioName, sizeof(pVirtioCC->pcszMmioName), "%s MMIO", pcszInstance);
2151	if (cbSize <= 0)
2152	return PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: out of memory allocating string")); /* can we put params in this error? */
2153
2154	rc = PDMDevHlpPCIIORegionCreateMmio(pDevIns, VIRTIO_REGION_PCI_CAP, RT_ALIGN_32(cbRegion + PAGE_SIZE, PAGE_SIZE),
2155	PCI_ADDRESS_SPACE_MEM, virtioMmioWrite, virtioMmioRead, pVirtio,
2156	IOMMMIO_FLAGS_READ_PASSTHRU \| IOMMMIO_FLAGS_WRITE_PASSTHRU,
2157	pVirtioCC->pcszMmioName,
2158	&pVirtio->hMmioPciCap);
2159	AssertLogRelRCReturn(rc, PDMDEV_SET_ERROR(pDevIns, rc, N_("virtio: cannot register PCI Capabilities address space")));
2160	/*
2161	* Statistics.
2162	*/
2163	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsAllocated, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2164	"Total number of allocated descriptor chains", "DescChainsAllocated");
2165	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsFreed, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2166	"Total number of freed descriptor chains", "DescChainsFreed");
2167	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsIn, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2168	"Total number of inbound segments", "DescChainsSegsIn");
2169	PDMDevHlpSTAMRegisterF(pDevIns, &pVirtio->StatDescChainsSegsOut, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,
2170	"Total number of outbound segments", "DescChainsSegsOut");
2171
2172	return VINF_SUCCESS;
2173	}
2174
2175	#else /* !IN_RING3 */
2176
2177	/**
2178	* Sets up the core ring-0/raw-mode virtio bits.
2179	*
2180	* @returns VBox status code.
2181	* @param pDevIns The device instance.
2182	* @param pVirtio Pointer to the shared virtio state. This must be the first
2183	* member in the shared device instance data!
2184	*/
2185	int virtioCoreRZInit(PPDMDEVINS pDevIns, PVIRTIOCORE pVirtio)
2186	{
2187	AssertLogRelReturn(pVirtio == PDMINS_2_DATA(pDevIns, PVIRTIOCORE), VERR_STATE_CHANGED);
2188
2189	#ifdef FUTURE_OPTIMIZATION
2190	int rc = PDMDevHlpSetDeviceCritSect(pDevIns, PDMDevHlpCritSectGetNop(pDevIns));
2191	AssertRCReturn(rc, rc);
2192	#endif
2193	int rc = PDMDevHlpMmioSetUpContext(pDevIns, pVirtio->hMmioPciCap, virtioMmioWrite, virtioMmioRead, pVirtio);
2194	AssertRCReturn(rc, rc);
2195	return rc;
2196	}
2197
2198	#endif /* !IN_RING3 */
2199

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Devices/VirtIO/Virtio_1_0.cpp@ 84468

Download in other formats: