SUPDrvGip.cpp@ 62679

Last change on this file since 62679 was 62664, checked in by vboxsync, 8 years ago
SUPDrv: warnings
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 178.4 KB

Line
1	/* $Id: SUPDrvGip.cpp 62664 2016-07-28 23:30:23Z vboxsync $ */
2	/** @file
3	* VBoxDrv - The VirtualBox Support Driver - Common code for GIP.
4	*/
5
6	/*
7	* Copyright (C) 2006-2016 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*********************************************************************************************************************************
29	* Header Files *
30	*********************************************************************************************************************************/
31	#define LOG_GROUP LOG_GROUP_SUP_DRV
32	#define SUPDRV_AGNOSTIC
33	#include "SUPDrvInternal.h"
34	#ifndef PAGE_SHIFT
35	# include <iprt/param.h>
36	#endif
37	#include <iprt/asm.h>
38	#include <iprt/asm-amd64-x86.h>
39	#include <iprt/asm-math.h>
40	#include <iprt/cpuset.h>
41	#include <iprt/handletable.h>
42	#include <iprt/mem.h>
43	#include <iprt/mp.h>
44	#include <iprt/power.h>
45	#include <iprt/process.h>
46	#include <iprt/semaphore.h>
47	#include <iprt/spinlock.h>
48	#include <iprt/thread.h>
49	#include <iprt/uuid.h>
50	#include <iprt/net.h>
51	#include <iprt/crc.h>
52	#include <iprt/string.h>
53	#include <iprt/timer.h>
54	#if defined(RT_OS_DARWIN) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_FREEBSD)
55	# include <iprt/rand.h>
56	# include <iprt/path.h>
57	#endif
58	#include <iprt/uint128.h>
59	#include <iprt/x86.h>
60
61	#include <VBox/param.h>
62	#include <VBox/log.h>
63	#include <VBox/err.h>
64
65	#if defined(RT_OS_SOLARIS) \|\| defined(RT_OS_DARWIN)
66	# include "dtrace/SUPDrv.h"
67	#else
68	/* ... */
69	#endif
70
71
72	/*********************************************************************************************************************************
73	* Defined Constants And Macros *
74	*********************************************************************************************************************************/
75	/** The frequency by which we recalculate the u32UpdateHz and
76	* u32UpdateIntervalNS GIP members. The value must be a power of 2.
77	*
78	* Warning: Bumping this too high might overflow u32UpdateIntervalNS.
79	*/
80	#define GIP_UPDATEHZ_RECALC_FREQ 0x800
81
82	/** A reserved TSC value used for synchronization as well as measurement of
83	* TSC deltas. */
84	#define GIP_TSC_DELTA_RSVD UINT64_MAX
85	/** The number of TSC delta measurement loops in total (includes primer and
86	* read-time loops). */
87	#define GIP_TSC_DELTA_LOOPS 96
88	/** The number of cache primer loops. */
89	#define GIP_TSC_DELTA_PRIMER_LOOPS 4
90	/** The number of loops until we keep computing the minumum read time. */
91	#define GIP_TSC_DELTA_READ_TIME_LOOPS 24
92
93	/** The TSC frequency refinement period in seconds.
94	* The timer fires after 200ms, then every second, this value just says when
95	* to stop it after that. */
96	#define GIP_TSC_REFINE_PERIOD_IN_SECS 12
97	/** The TSC-delta threshold for the SUPGIPUSETSCDELTA_PRACTICALLY_ZERO rating */
98	#define GIP_TSC_DELTA_THRESHOLD_PRACTICALLY_ZERO 32
99	/** The TSC-delta threshold for the SUPGIPUSETSCDELTA_ROUGHLY_ZERO rating */
100	#define GIP_TSC_DELTA_THRESHOLD_ROUGHLY_ZERO 448
101	/** The TSC delta value for the initial GIP master - 0 in regular builds.
102	* To test the delta code this can be set to a non-zero value. */
103	#if 0
104	# define GIP_TSC_DELTA_INITIAL_MASTER_VALUE INT64_C(170139095182512) /* 0x00009abd9854acb0 */
105	#else
106	# define GIP_TSC_DELTA_INITIAL_MASTER_VALUE INT64_C(0)
107	#endif
108
109	AssertCompile(GIP_TSC_DELTA_PRIMER_LOOPS < GIP_TSC_DELTA_READ_TIME_LOOPS);
110	AssertCompile(GIP_TSC_DELTA_PRIMER_LOOPS + GIP_TSC_DELTA_READ_TIME_LOOPS < GIP_TSC_DELTA_LOOPS);
111
112	/** @def VBOX_SVN_REV
113	* The makefile should define this if it can. */
114	#ifndef VBOX_SVN_REV
115	# define VBOX_SVN_REV 0
116	#endif
117
118	#if 0 /* Don't start the GIP timers. Useful when debugging the IPRT timer code. */
119	# define DO_NOT_START_GIP
120	#endif
121
122
123	/*********************************************************************************************************************************
124	* Internal Functions *
125	*********************************************************************************************************************************/
126	static DECLCALLBACK(void) supdrvGipSyncAndInvariantTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
127	static DECLCALLBACK(void) supdrvGipAsyncTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
128	static int supdrvGipSetFlags(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, uint32_t fOrMask, uint32_t fAndMask);
129	static void supdrvGipInitCpu(PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pCpu, uint64_t u64NanoTS, uint64_t uCpuHz);
130	static void supdrvTscResetSamples(PSUPDRVDEVEXT pDevExt, bool fClearDeltas);
131	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
132	static int supdrvTscDeltaThreadInit(PSUPDRVDEVEXT pDevExt);
133	static void supdrvTscDeltaTerm(PSUPDRVDEVEXT pDevExt);
134	static void supdrvTscDeltaThreadStartMeasurement(PSUPDRVDEVEXT pDevExt, bool fForceAll);
135	#else
136	static int supdrvMeasureInitialTscDeltas(PSUPDRVDEVEXT pDevExt);
137	static int supdrvMeasureTscDeltaOne(PSUPDRVDEVEXT pDevExt, uint32_t idxWorker);
138	#endif
139
140
141	/*********************************************************************************************************************************
142	* Global Variables *
143	*********************************************************************************************************************************/
144	DECLEXPORT(PSUPGLOBALINFOPAGE) g_pSUPGlobalInfoPage = NULL;
145
146
147
148	/*
149	*
150	* Misc Common GIP Code
151	* Misc Common GIP Code
152	* Misc Common GIP Code
153	*
154	*
155	*/
156
157
158	/**
159	* Finds the GIP CPU index corresponding to @a idCpu.
160	*
161	* @returns GIP CPU array index, UINT32_MAX if not found.
162	* @param pGip The GIP.
163	* @param idCpu The CPU ID.
164	*/
165	static uint32_t supdrvGipFindCpuIndexForCpuId(PSUPGLOBALINFOPAGE pGip, RTCPUID idCpu)
166	{
167	uint32_t i;
168	for (i = 0; i < pGip->cCpus; i++)
169	if (pGip->aCPUs[i].idCpu == idCpu)
170	return i;
171	return UINT32_MAX;
172	}
173
174
175
176	/*
177	*
178	* GIP Mapping and Unmapping Related Code.
179	* GIP Mapping and Unmapping Related Code.
180	* GIP Mapping and Unmapping Related Code.
181	*
182	*
183	*/
184
185
186	/**
187	* (Re-)initializes the per-cpu structure prior to starting or resuming the GIP
188	* updating.
189	*
190	* @param pGipCpu The per CPU structure for this CPU.
191	* @param u64NanoTS The current time.
192	*/
193	static void supdrvGipReInitCpu(PSUPGIPCPU pGipCpu, uint64_t u64NanoTS)
194	{
195	/*
196	* Here we don't really care about applying the TSC delta. The re-initialization of this
197	* value is not relevant especially while (re)starting the GIP as the first few ones will
198	* be ignored anyway, see supdrvGipDoUpdateCpu().
199	*/
200	pGipCpu->u64TSC = ASMReadTSC() - pGipCpu->u32UpdateIntervalTSC;
201	pGipCpu->u64NanoTS = u64NanoTS;
202	}
203
204
205	/**
206	* Set the current TSC and NanoTS value for the CPU.
207	*
208	* @param idCpu The CPU ID. Unused - we have to use the APIC ID.
209	* @param pvUser1 Pointer to the ring-0 GIP mapping.
210	* @param pvUser2 Pointer to the variable holding the current time.
211	*/
212	static DECLCALLBACK(void) supdrvGipReInitCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
213	{
214	PSUPGLOBALINFOPAGE pGip = (PSUPGLOBALINFOPAGE)pvUser1;
215	unsigned iCpu = pGip->aiCpuFromApicId[ASMGetApicId()];
216
217	if (RT_LIKELY(iCpu < pGip->cCpus && pGip->aCPUs[iCpu].idCpu == idCpu))
218	supdrvGipReInitCpu(&pGip->aCPUs[iCpu], (uint64_t )pvUser2);
219
220	NOREF(pvUser2);
221	NOREF(idCpu);
222	}
223
224
225	/**
226	* State structure for supdrvGipDetectGetGipCpuCallback.
227	*/
228	typedef struct SUPDRVGIPDETECTGETCPU
229	{
230	/** Bitmap of APIC IDs that has been seen (initialized to zero).
231	* Used to detect duplicate APIC IDs (paranoia). */
232	uint8_t volatile bmApicId[256 / 8];
233	/** Mask of supported GIP CPU getter methods (SUPGIPGETCPU_XXX) (all bits set
234	* initially). The callback clears the methods not detected. */
235	uint32_t volatile fSupported;
236	/** The first callback detecting any kind of range issues (initialized to
237	* NIL_RTCPUID). */
238	RTCPUID volatile idCpuProblem;
239	} SUPDRVGIPDETECTGETCPU;
240	/** Pointer to state structure for supdrvGipDetectGetGipCpuCallback. */
241	typedef SUPDRVGIPDETECTGETCPU *PSUPDRVGIPDETECTGETCPU;
242
243
244	/**
245	* Checks for alternative ways of getting the CPU ID.
246	*
247	* This also checks the APIC ID, CPU ID and CPU set index values against the
248	* GIP tables.
249	*
250	* @param idCpu The CPU ID. Unused - we have to use the APIC ID.
251	* @param pvUser1 Pointer to the state structure.
252	* @param pvUser2 Pointer to the GIP.
253	*/
254	static DECLCALLBACK(void) supdrvGipDetectGetGipCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
255	{
256	PSUPDRVGIPDETECTGETCPU pState = (PSUPDRVGIPDETECTGETCPU)pvUser1;
257	PSUPGLOBALINFOPAGE pGip = (PSUPGLOBALINFOPAGE)pvUser2;
258	uint32_t fSupported = 0;
259	uint16_t idApic;
260	int iCpuSet;
261	NOREF(pGip);
262
263	AssertMsg(idCpu == RTMpCpuId(), ("idCpu=%#x RTMpCpuId()=%#x\n", idCpu, RTMpCpuId())); /* paranoia^3 */
264
265	/*
266	* Check that the CPU ID and CPU set index are interchangable.
267	*/
268	iCpuSet = RTMpCpuIdToSetIndex(idCpu);
269	if ((RTCPUID)iCpuSet == idCpu)
270	{
271	AssertCompile(RT_IS_POWER_OF_TWO(RTCPUSET_MAX_CPUS));
272	if ( iCpuSet >= 0
273	&& iCpuSet < RTCPUSET_MAX_CPUS
274	&& RT_IS_POWER_OF_TWO(RTCPUSET_MAX_CPUS))
275	{
276	/*
277	* Check whether the IDTR.LIMIT contains a CPU number.
278	*/
279	#ifdef RT_ARCH_X86
280	uint16_t const cbIdt = sizeof(X86DESC64SYSTEM) * 256;
281	#else
282	uint16_t const cbIdt = sizeof(X86DESCGATE) * 256;
283	#endif
284	RTIDTR Idtr;
285	ASMGetIDTR(&Idtr);
286	if (Idtr.cbIdt >= cbIdt)
287	{
288	uint32_t uTmp = Idtr.cbIdt - cbIdt;
289	uTmp &= RTCPUSET_MAX_CPUS - 1;
290	if (uTmp == idCpu)
291	{
292	RTIDTR Idtr2;
293	ASMGetIDTR(&Idtr2);
294	if (Idtr2.cbIdt == Idtr.cbIdt)
295	fSupported \|= SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS;
296	}
297	}
298
299	/*
300	* Check whether RDTSCP is an option.
301	*/
302	if (ASMHasCpuId())
303	{
304	if ( ASMIsValidExtRange(ASMCpuId_EAX(UINT32_C(0x80000000)))
305	&& (ASMCpuId_EDX(UINT32_C(0x80000001)) & X86_CPUID_EXT_FEATURE_EDX_RDTSCP) )
306	{
307	uint32_t uAux;
308	ASMReadTscWithAux(&uAux);
309	if ((uAux & (RTCPUSET_MAX_CPUS - 1)) == idCpu)
310	{
311	ASMNopPause();
312	ASMReadTscWithAux(&uAux);
313	if ((uAux & (RTCPUSET_MAX_CPUS - 1)) == idCpu)
314	fSupported \|= SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS;
315	}
316	}
317	}
318	}
319	}
320
321	/*
322	* Check that the APIC ID is unique.
323	*/
324	idApic = ASMGetApicId();
325	if (RT_LIKELY( idApic < RT_ELEMENTS(pGip->aiCpuFromApicId)
326	&& !ASMAtomicBitTestAndSet(pState->bmApicId, idApic)))
327	fSupported \|= SUPGIPGETCPU_APIC_ID;
328	else
329	{
330	AssertCompile(sizeof(pState->bmApicId) * 8 == RT_ELEMENTS(pGip->aiCpuFromApicId));
331	ASMAtomicCmpXchgU32(&pState->idCpuProblem, idCpu, NIL_RTCPUID);
332	LogRel(("supdrvGipDetectGetGipCpuCallback: idCpu=%#x iCpuSet=%d idApic=%#x - duplicate APIC ID.\n",
333	idCpu, iCpuSet, idApic));
334	}
335
336	/*
337	* Check that the iCpuSet is within the expected range.
338	*/
339	if (RT_UNLIKELY( iCpuSet < 0
340	\|\| (unsigned)iCpuSet >= RTCPUSET_MAX_CPUS
341	\|\| (unsigned)iCpuSet >= RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)))
342	{
343	ASMAtomicCmpXchgU32(&pState->idCpuProblem, idCpu, NIL_RTCPUID);
344	LogRel(("supdrvGipDetectGetGipCpuCallback: idCpu=%#x iCpuSet=%d idApic=%#x - CPU set index is out of range.\n",
345	idCpu, iCpuSet, idApic));
346	}
347	else
348	{
349	RTCPUID idCpu2 = RTMpCpuIdFromSetIndex(iCpuSet);
350	if (RT_UNLIKELY(idCpu2 != idCpu))
351	{
352	ASMAtomicCmpXchgU32(&pState->idCpuProblem, idCpu, NIL_RTCPUID);
353	LogRel(("supdrvGipDetectGetGipCpuCallback: idCpu=%#x iCpuSet=%d idApic=%#x - CPU id/index roundtrip problem: %#x\n",
354	idCpu, iCpuSet, idApic, idCpu2));
355	}
356	}
357
358	/*
359	* Update the supported feature mask before we return.
360	*/
361	ASMAtomicAndU32(&pState->fSupported, fSupported);
362
363	NOREF(pvUser2);
364	}
365
366
367	/**
368	* Increase the timer freqency on hosts where this is possible (NT).
369	*
370	* The idea is that more interrupts is better for us... Also, it's better than
371	* we increase the timer frequence, because we might end up getting inaccurate
372	* callbacks if someone else does it.
373	*
374	* @param pDevExt Sets u32SystemTimerGranularityGrant if increased.
375	*/
376	static void supdrvGipRequestHigherTimerFrequencyFromSystem(PSUPDRVDEVEXT pDevExt)
377	{
378	if (pDevExt->u32SystemTimerGranularityGrant == 0)
379	{
380	uint32_t u32SystemResolution;
381	if ( RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 976563 /* 1024 HZ */, &u32SystemResolution))
382	\|\| RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 1000000 /* 1000 HZ */, &u32SystemResolution))
383	\|\| RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 1953125 /* 512 HZ */, &u32SystemResolution))
384	\|\| RT_SUCCESS_NP(RTTimerRequestSystemGranularity( 2000000 /* 500 HZ */, &u32SystemResolution))
385	)
386	{
387	#if 0 /* def VBOX_STRICT - this is somehow triggers bogus assertions on windows 10 */
388	uint32_t u32After = RTTimerGetSystemGranularity();
389	AssertMsg(u32After <= u32SystemResolution, ("u32After=%u u32SystemResolution=%u\n", u32After, u32SystemResolution));
390	#endif
391	pDevExt->u32SystemTimerGranularityGrant = u32SystemResolution;
392	}
393	}
394	}
395
396
397	/**
398	* Undoes supdrvGipRequestHigherTimerFrequencyFromSystem.
399	*
400	* @param pDevExt Clears u32SystemTimerGranularityGrant.
401	*/
402	static void supdrvGipReleaseHigherTimerFrequencyFromSystem(PSUPDRVDEVEXT pDevExt)
403	{
404	if (pDevExt->u32SystemTimerGranularityGrant)
405	{
406	int rc2 = RTTimerReleaseSystemGranularity(pDevExt->u32SystemTimerGranularityGrant);
407	AssertRC(rc2);
408	pDevExt->u32SystemTimerGranularityGrant = 0;
409	}
410	}
411
412
413	/**
414	* Maps the GIP into userspace and/or get the physical address of the GIP.
415	*
416	* @returns IPRT status code.
417	* @param pSession Session to which the GIP mapping should belong.
418	* @param ppGipR3 Where to store the address of the ring-3 mapping. (optional)
419	* @param pHCPhysGip Where to store the physical address. (optional)
420	*
421	* @remark There is no reference counting on the mapping, so one call to this function
422	* count globally as one reference. One call to SUPR0GipUnmap() is will unmap GIP
423	* and remove the session as a GIP user.
424	*/
425	SUPR0DECL(int) SUPR0GipMap(PSUPDRVSESSION pSession, PRTR3PTR ppGipR3, PRTHCPHYS pHCPhysGip)
426	{
427	int rc;
428	PSUPDRVDEVEXT pDevExt = pSession->pDevExt;
429	RTR3PTR pGipR3 = NIL_RTR3PTR;
430	RTHCPHYS HCPhys = NIL_RTHCPHYS;
431	LogFlow(("SUPR0GipMap: pSession=%p ppGipR3=%p pHCPhysGip=%p\n", pSession, ppGipR3, pHCPhysGip));
432
433	/*
434	* Validate
435	*/
436	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
437	AssertPtrNullReturn(ppGipR3, VERR_INVALID_POINTER);
438	AssertPtrNullReturn(pHCPhysGip, VERR_INVALID_POINTER);
439
440	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
441	RTSemMutexRequest(pDevExt->mtxGip, RT_INDEFINITE_WAIT);
442	#else
443	RTSemFastMutexRequest(pDevExt->mtxGip);
444	#endif
445	if (pDevExt->pGip)
446	{
447	/*
448	* Map it?
449	*/
450	rc = VINF_SUCCESS;
451	if (ppGipR3)
452	{
453	if (pSession->GipMapObjR3 == NIL_RTR0MEMOBJ)
454	rc = RTR0MemObjMapUser(&pSession->GipMapObjR3, pDevExt->GipMemObj, (RTR3PTR)-1, 0,
455	RTMEM_PROT_READ, NIL_RTR0PROCESS);
456	if (RT_SUCCESS(rc))
457	pGipR3 = RTR0MemObjAddressR3(pSession->GipMapObjR3);
458	}
459
460	/*
461	* Get physical address.
462	*/
463	if (pHCPhysGip && RT_SUCCESS(rc))
464	HCPhys = pDevExt->HCPhysGip;
465
466	/*
467	* Reference globally.
468	*/
469	if (!pSession->fGipReferenced && RT_SUCCESS(rc))
470	{
471	pSession->fGipReferenced = 1;
472	pDevExt->cGipUsers++;
473	if (pDevExt->cGipUsers == 1)
474	{
475	PSUPGLOBALINFOPAGE pGipR0 = pDevExt->pGip;
476	uint64_t u64NanoTS;
477
478	/*
479	* GIP starts/resumes updating again. On windows we bump the
480	* host timer frequency to make sure we don't get stuck in guest
481	* mode and to get better timer (and possibly clock) accuracy.
482	*/
483	LogFlow(("SUPR0GipMap: Resumes GIP updating\n"));
484
485	supdrvGipRequestHigherTimerFrequencyFromSystem(pDevExt);
486
487	/*
488	* document me
489	*/
490	if (pGipR0->aCPUs[0].u32TransactionId != 2 /* not the first time */)
491	{
492	unsigned i;
493	for (i = 0; i < pGipR0->cCpus; i++)
494	ASMAtomicUoWriteU32(&pGipR0->aCPUs[i].u32TransactionId,
495	(pGipR0->aCPUs[i].u32TransactionId + GIP_UPDATEHZ_RECALC_FREQ * 2)
496	& ~(GIP_UPDATEHZ_RECALC_FREQ * 2 - 1));
497	ASMAtomicWriteU64(&pGipR0->u64NanoTSLastUpdateHz, 0);
498	}
499
500	/*
501	* document me
502	*/
503	u64NanoTS = RTTimeSystemNanoTS() - pGipR0->u32UpdateIntervalNS;
504	if ( pGipR0->u32Mode == SUPGIPMODE_INVARIANT_TSC
505	\|\| pGipR0->u32Mode == SUPGIPMODE_SYNC_TSC
506	\|\| RTMpGetOnlineCount() == 1)
507	supdrvGipReInitCpu(&pGipR0->aCPUs[0], u64NanoTS);
508	else
509	RTMpOnAll(supdrvGipReInitCpuCallback, pGipR0, &u64NanoTS);
510
511	/*
512	* Detect alternative ways to figure the CPU ID in ring-3 and
513	* raw-mode context. Check the sanity of the APIC IDs, CPU IDs,
514	* and CPU set indexes while we're at it.
515	*/
516	if (RT_SUCCESS(rc))
517	{
518	SUPDRVGIPDETECTGETCPU DetectState;
519	RT_BZERO((void *)&DetectState.bmApicId, sizeof(DetectState.bmApicId));
520	DetectState.fSupported = UINT32_MAX;
521	DetectState.idCpuProblem = NIL_RTCPUID;
522	rc = RTMpOnAll(supdrvGipDetectGetGipCpuCallback, &DetectState, pGipR0);
523	if (DetectState.idCpuProblem == NIL_RTCPUID)
524	{
525	if ( DetectState.fSupported != UINT32_MAX
526	&& DetectState.fSupported != 0)
527	{
528	if (pGipR0->fGetGipCpu != DetectState.fSupported)
529	{
530	pGipR0->fGetGipCpu = DetectState.fSupported;
531	LogRel(("SUPR0GipMap: fGetGipCpu=%#x\n", DetectState.fSupported));
532	}
533	}
534	else
535	{
536	LogRel(("SUPR0GipMap: No supported ways of getting the APIC ID or CPU number in ring-3! (%#x)\n",
537	DetectState.fSupported));
538	rc = VERR_UNSUPPORTED_CPU;
539	}
540	}
541	else
542	{
543	LogRel(("SUPR0GipMap: APIC ID, CPU ID or CPU set index problem detected on CPU #%u (%#x)!\n",
544	DetectState.idCpuProblem, DetectState.idCpuProblem));
545	rc = VERR_INVALID_CPU_ID;
546	}
547	}
548
549	/*
550	* Start the GIP timer if all is well..
551	*/
552	if (RT_SUCCESS(rc))
553	{
554	#ifndef DO_NOT_START_GIP
555	rc = RTTimerStart(pDevExt->pGipTimer, 0 /* fire ASAP */); AssertRC(rc);
556	#endif
557	rc = VINF_SUCCESS;
558	}
559
560	/*
561	* Bail out on error.
562	*/
563	if (RT_FAILURE(rc))
564	{
565	LogRel(("SUPR0GipMap: failed rc=%Rrc\n", rc));
566	pDevExt->cGipUsers = 0;
567	pSession->fGipReferenced = 0;
568	if (pSession->GipMapObjR3 != NIL_RTR0MEMOBJ)
569	{
570	int rc2 = RTR0MemObjFree(pSession->GipMapObjR3, false); AssertRC(rc2);
571	if (RT_SUCCESS(rc2))
572	pSession->GipMapObjR3 = NIL_RTR0MEMOBJ;
573	}
574	HCPhys = NIL_RTHCPHYS;
575	pGipR3 = NIL_RTR3PTR;
576	}
577	}
578	}
579	}
580	else
581	{
582	rc = VERR_GENERAL_FAILURE;
583	Log(("SUPR0GipMap: GIP is not available!\n"));
584	}
585	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
586	RTSemMutexRelease(pDevExt->mtxGip);
587	#else
588	RTSemFastMutexRelease(pDevExt->mtxGip);
589	#endif
590
591	/*
592	* Write returns.
593	*/
594	if (pHCPhysGip)
595	*pHCPhysGip = HCPhys;
596	if (ppGipR3)
597	*ppGipR3 = pGipR3;
598
599	#ifdef DEBUG_DARWIN_GIP
600	OSDBGPRINT(("SUPR0GipMap: returns %d pHCPhysGip=%lx pGipR3=%p\n", rc, (unsigned long)HCPhys, (void )pGipR3));
601	#else
602	LogFlow(( "SUPR0GipMap: returns %d pHCPhysGip=%lx pGipR3=%p\n", rc, (unsigned long)HCPhys, (void )pGipR3));
603	#endif
604	return rc;
605	}
606
607
608	/**
609	* Unmaps any user mapping of the GIP and terminates all GIP access
610	* from this session.
611	*
612	* @returns IPRT status code.
613	* @param pSession Session to which the GIP mapping should belong.
614	*/
615	SUPR0DECL(int) SUPR0GipUnmap(PSUPDRVSESSION pSession)
616	{
617	int rc = VINF_SUCCESS;
618	PSUPDRVDEVEXT pDevExt = pSession->pDevExt;
619	#ifdef DEBUG_DARWIN_GIP
620	OSDBGPRINT(("SUPR0GipUnmap: pSession=%p pGip=%p GipMapObjR3=%p\n",
621	pSession,
622	pSession->GipMapObjR3 != NIL_RTR0MEMOBJ ? RTR0MemObjAddress(pSession->GipMapObjR3) : NULL,
623	pSession->GipMapObjR3));
624	#else
625	LogFlow(("SUPR0GipUnmap: pSession=%p\n", pSession));
626	#endif
627	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
628
629	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
630	RTSemMutexRequest(pDevExt->mtxGip, RT_INDEFINITE_WAIT);
631	#else
632	RTSemFastMutexRequest(pDevExt->mtxGip);
633	#endif
634
635	/*
636	* GIP test-mode session?
637	*/
638	if ( pSession->fGipTestMode
639	&& pDevExt->pGip)
640	{
641	supdrvGipSetFlags(pDevExt, pSession, 0, ~SUPGIP_FLAGS_TESTING_ENABLE);
642	Assert(!pSession->fGipTestMode);
643	}
644
645	/*
646	* Unmap anything?
647	*/
648	if (pSession->GipMapObjR3 != NIL_RTR0MEMOBJ)
649	{
650	rc = RTR0MemObjFree(pSession->GipMapObjR3, false);
651	AssertRC(rc);
652	if (RT_SUCCESS(rc))
653	pSession->GipMapObjR3 = NIL_RTR0MEMOBJ;
654	}
655
656	/*
657	* Dereference global GIP.
658	*/
659	if (pSession->fGipReferenced && !rc)
660	{
661	pSession->fGipReferenced = 0;
662	if ( pDevExt->cGipUsers > 0
663	&& !--pDevExt->cGipUsers)
664	{
665	LogFlow(("SUPR0GipUnmap: Suspends GIP updating\n"));
666	#ifndef DO_NOT_START_GIP
667	rc = RTTimerStop(pDevExt->pGipTimer); AssertRC(rc); rc = VINF_SUCCESS;
668	#endif
669	supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt);
670	}
671	}
672
673	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
674	RTSemMutexRelease(pDevExt->mtxGip);
675	#else
676	RTSemFastMutexRelease(pDevExt->mtxGip);
677	#endif
678
679	return rc;
680	}
681
682
683	/**
684	* Gets the GIP pointer.
685	*
686	* @returns Pointer to the GIP or NULL.
687	*/
688	SUPDECL(PSUPGLOBALINFOPAGE) SUPGetGIP(void)
689	{
690	return g_pSUPGlobalInfoPage;
691	}
692
693
694
695
696
697	/*
698	*
699	*
700	* GIP Initialization, Termination and CPU Offline / Online Related Code.
701	* GIP Initialization, Termination and CPU Offline / Online Related Code.
702	* GIP Initialization, Termination and CPU Offline / Online Related Code.
703	*
704	*
705	*/
706
707	/**
708	* Used by supdrvInitRefineInvariantTscFreqTimer and supdrvGipInitMeasureTscFreq
709	* to update the TSC frequency related GIP variables.
710	*
711	* @param pGip The GIP.
712	* @param nsElapsed The number of nanoseconds elapsed.
713	* @param cElapsedTscTicks The corresponding number of TSC ticks.
714	* @param iTick The tick number for debugging.
715	*/
716	static void supdrvGipInitSetCpuFreq(PSUPGLOBALINFOPAGE pGip, uint64_t nsElapsed, uint64_t cElapsedTscTicks, uint32_t iTick)
717	{
718	/*
719	* Calculate the frequency.
720	*/
721	uint64_t uCpuHz;
722	if ( cElapsedTscTicks < UINT64_MAX / RT_NS_1SEC
723	&& nsElapsed < UINT32_MAX)
724	uCpuHz = ASMMultU64ByU32DivByU32(cElapsedTscTicks, RT_NS_1SEC, (uint32_t)nsElapsed);
725	else
726	{
727	RTUINT128U CpuHz, Tmp, Divisor;
728	CpuHz.s.Lo = CpuHz.s.Hi = 0;
729	RTUInt128MulU64ByU64(&Tmp, cElapsedTscTicks, RT_NS_1SEC_64);
730	RTUInt128Div(&CpuHz, &Tmp, RTUInt128AssignU64(&Divisor, nsElapsed));
731	uCpuHz = CpuHz.s.Lo;
732	}
733
734	/*
735	* Update the GIP.
736	*/
737	ASMAtomicWriteU64(&pGip->u64CpuHz, uCpuHz);
738	if (pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)
739	{
740	ASMAtomicWriteU64(&pGip->aCPUs[0].u64CpuHz, uCpuHz);
741
742	/* For inspecting the frequency calcs using tstGIP-2, debugger or similar. */
743	if (iTick + 1 < pGip->cCpus)
744	ASMAtomicWriteU64(&pGip->aCPUs[iTick + 1].u64CpuHz, uCpuHz);
745	}
746	}
747
748
749	/**
750	* Timer callback function for TSC frequency refinement in invariant GIP mode.
751	*
752	* This is started during driver init and fires once
753	* GIP_TSC_REFINE_PERIOD_IN_SECS seconds later.
754	*
755	* @param pTimer The timer.
756	* @param pvUser Opaque pointer to the device instance data.
757	* @param iTick The timer tick.
758	*/
759	static DECLCALLBACK(void) supdrvInitRefineInvariantTscFreqTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
760	{
761	PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
762	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
763	RTCPUID idCpu;
764	uint64_t cNsElapsed;
765	uint64_t cTscTicksElapsed;
766	uint64_t nsNow;
767	uint64_t uTsc;
768	RTCCUINTREG fEFlags;
769
770	/* Paranoia. */
771	AssertReturnVoid(pGip);
772	AssertReturnVoid(pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC);
773
774	/*
775	* If we got a power event, stop the refinement process.
776	*/
777	if (pDevExt->fInvTscRefinePowerEvent)
778	{
779	int rc = RTTimerStop(pTimer); AssertRC(rc);
780	return;
781	}
782
783	/*
784	* Read the TSC and time, noting which CPU we are on.
785	*
786	* Don't bother spinning until RTTimeSystemNanoTS changes, since on
787	* systems where it matters we're in a context where we cannot waste that
788	* much time (DPC watchdog, called from clock interrupt).
789	*/
790	fEFlags = ASMIntDisableFlags();
791	uTsc = ASMReadTSC();
792	nsNow = RTTimeSystemNanoTS();
793	idCpu = RTMpCpuId();
794	ASMSetFlags(fEFlags);
795
796	cNsElapsed = nsNow - pDevExt->nsStartInvarTscRefine;
797	cTscTicksElapsed = uTsc - pDevExt->uTscStartInvarTscRefine;
798
799	/*
800	* If the above measurement was taken on a different CPU than the one we
801	* started the process on, cTscTicksElapsed will need to be adjusted with
802	* the TSC deltas of both the CPUs.
803	*
804	* We ASSUME that the delta calculation process takes less time than the
805	* TSC frequency refinement timer. If it doesn't, we'll complain and
806	* drop the frequency refinement.
807	*
808	* Note! We cannot entirely trust enmUseTscDelta here because it's
809	* downgraded after each delta calculation.
810	*/
811	if ( idCpu != pDevExt->idCpuInvarTscRefine
812	&& pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
813	{
814	uint32_t iStartCpuSet = RTMpCpuIdToSetIndex(pDevExt->idCpuInvarTscRefine);
815	uint32_t iStopCpuSet = RTMpCpuIdToSetIndex(idCpu);
816	uint16_t iStartGipCpu = iStartCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
817	? pGip->aiCpuFromCpuSetIdx[iStartCpuSet] : UINT16_MAX;
818	uint16_t iStopGipCpu = iStopCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
819	? pGip->aiCpuFromCpuSetIdx[iStopCpuSet] : UINT16_MAX;
820	int64_t iStartTscDelta = iStartGipCpu < pGip->cCpus ? pGip->aCPUs[iStartGipCpu].i64TSCDelta : INT64_MAX;
821	int64_t iStopTscDelta = iStopGipCpu < pGip->cCpus ? pGip->aCPUs[iStopGipCpu].i64TSCDelta : INT64_MAX;
822	if (RT_LIKELY(iStartTscDelta != INT64_MAX && iStopTscDelta != INT64_MAX))
823	{
824	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
825	{
826	/* cTscTicksElapsed = (uTsc - iStopTscDelta) - (pDevExt->uTscStartInvarTscRefine - iStartTscDelta); */
827	cTscTicksElapsed += iStartTscDelta - iStopTscDelta;
828	}
829	}
830	/*
831	* Allow 5 times the refinement period to elapse before we give up on the TSC delta
832	* calculations.
833	*/
834	else if (cNsElapsed > GIP_TSC_REFINE_PERIOD_IN_SECS * 5 * RT_NS_1SEC_64)
835	{
836	SUPR0Printf("vboxdrv: Failed to refine invariant TSC frequency because deltas are unavailable after %u (%u) seconds\n",
837	(uint32_t)(cNsElapsed / RT_NS_1SEC), GIP_TSC_REFINE_PERIOD_IN_SECS);
838	SUPR0Printf("vboxdrv: start: %u, %u, %#llx stop: %u, %u, %#llx\n",
839	iStartCpuSet, iStartGipCpu, iStartTscDelta, iStopCpuSet, iStopGipCpu, iStopTscDelta);
840	int rc = RTTimerStop(pTimer); AssertRC(rc);
841	return;
842	}
843	}
844
845	/*
846	* Calculate and update the CPU frequency variables in GIP.
847	*
848	* If there is a GIP user already and we've already refined the frequency
849	* a couple of times, don't update it as we want a stable frequency value
850	* for all VMs.
851	*/
852	if ( pDevExt->cGipUsers == 0
853	\|\| cNsElapsed < RT_NS_1SEC * 2)
854	{
855	supdrvGipInitSetCpuFreq(pGip, cNsElapsed, cTscTicksElapsed, (uint32_t)iTick);
856
857	/*
858	* Stop the timer once we've reached the defined refinement period.
859	*/
860	if (cNsElapsed > GIP_TSC_REFINE_PERIOD_IN_SECS * RT_NS_1SEC_64)
861	{
862	int rc = RTTimerStop(pTimer);
863	AssertRC(rc);
864	}
865	}
866	else
867	{
868	int rc = RTTimerStop(pTimer);
869	AssertRC(rc);
870	}
871	}
872
873
874	/**
875	* @callback_method_impl{FNRTPOWERNOTIFICATION}
876	*/
877	static DECLCALLBACK(void) supdrvGipPowerNotificationCallback(RTPOWEREVENT enmEvent, void *pvUser)
878	{
879	PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
880	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
881
882	/*
883	* If the TSC frequency refinement timer is running, we need to cancel it so it
884	* doesn't screw up the frequency after a long suspend.
885	*
886	* Recalculate all TSC-deltas on host resume as it may have changed, seen
887	* on Windows 7 running on the Dell Optiplex Intel Core i5-3570.
888	*/
889	if (enmEvent == RTPOWEREVENT_RESUME)
890	{
891	ASMAtomicWriteBool(&pDevExt->fInvTscRefinePowerEvent, true);
892	if ( RT_LIKELY(pGip)
893	&& pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED
894	&& !supdrvOSAreCpusOfflinedOnSuspend())
895	{
896	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
897	supdrvTscDeltaThreadStartMeasurement(pDevExt, true /* fForceAll */);
898	#else
899	RTCpuSetCopy(&pDevExt->TscDeltaCpuSet, &pGip->OnlineCpuSet);
900	supdrvMeasureInitialTscDeltas(pDevExt);
901	#endif
902	}
903	}
904	else if (enmEvent == RTPOWEREVENT_SUSPEND)
905	ASMAtomicWriteBool(&pDevExt->fInvTscRefinePowerEvent, true);
906	}
907
908
909	/**
910	* Start the TSC-frequency refinment timer for the invariant TSC GIP mode.
911	*
912	* We cannot use this in the synchronous and asynchronous tsc GIP modes because
913	* the CPU may change the TSC frequence between now and when the timer fires
914	* (supdrvInitAsyncRefineTscTimer).
915	*
916	* @param pDevExt Pointer to the device instance data.
917	*/
918	static void supdrvGipInitStartTimerForRefiningInvariantTscFreq(PSUPDRVDEVEXT pDevExt)
919	{
920	uint64_t u64NanoTS;
921	RTCCUINTREG fEFlags;
922	int rc;
923
924	/*
925	* Register a power management callback.
926	*/
927	pDevExt->fInvTscRefinePowerEvent = false;
928	rc = RTPowerNotificationRegister(supdrvGipPowerNotificationCallback, pDevExt);
929	AssertRC(rc); /* ignore */
930
931	/*
932	* Record the TSC and NanoTS as the starting anchor point for refinement
933	* of the TSC. We try get as close to a clock tick as possible on systems
934	* which does not provide high resolution time.
935	*/
936	u64NanoTS = RTTimeSystemNanoTS();
937	while (RTTimeSystemNanoTS() == u64NanoTS)
938	ASMNopPause();
939
940	fEFlags = ASMIntDisableFlags();
941	pDevExt->uTscStartInvarTscRefine = ASMReadTSC();
942	pDevExt->nsStartInvarTscRefine = RTTimeSystemNanoTS();
943	pDevExt->idCpuInvarTscRefine = RTMpCpuId();
944	ASMSetFlags(fEFlags);
945
946	/*
947	* Create a timer that runs on the same CPU so we won't have a depencency
948	* on the TSC-delta and can run in parallel to it. On systems that does not
949	* implement CPU specific timers we'll apply deltas in the timer callback,
950	* just like we do for CPUs going offline.
951	*
952	* The longer the refinement interval the better the accuracy, at least in
953	* theory. If it's too long though, ring-3 may already be starting its
954	* first VMs before we're done. On most systems we will be loading the
955	* support driver during boot and VMs won't be started for a while yet,
956	* it is really only a problem during development (especially with
957	* on-demand driver starting on windows).
958	*
959	* To avoid wasting time doing a long supdrvGipInitMeasureTscFreq() call
960	* to calculate the frequency during driver loading, the timer is set
961	* to fire after 200 ms the first time. It will then reschedule itself
962	* to fire every second until GIP_TSC_REFINE_PERIOD_IN_SECS has been
963	* reached or it notices that there is a user land client with GIP
964	* mapped (we want a stable frequency for all VMs).
965	*/
966	rc = RTTimerCreateEx(&pDevExt->pInvarTscRefineTimer, RT_NS_1SEC,
967	RTTIMER_FLAGS_CPU(RTMpCpuIdToSetIndex(pDevExt->idCpuInvarTscRefine)),
968	supdrvInitRefineInvariantTscFreqTimer, pDevExt);
969	if (RT_SUCCESS(rc))
970	{
971	rc = RTTimerStart(pDevExt->pInvarTscRefineTimer, 2*RT_NS_100MS);
972	if (RT_SUCCESS(rc))
973	return;
974	RTTimerDestroy(pDevExt->pInvarTscRefineTimer);
975	}
976
977	if (rc == VERR_CPU_OFFLINE \|\| rc == VERR_NOT_SUPPORTED)
978	{
979	rc = RTTimerCreateEx(&pDevExt->pInvarTscRefineTimer, RT_NS_1SEC, RTTIMER_FLAGS_CPU_ANY,
980	supdrvInitRefineInvariantTscFreqTimer, pDevExt);
981	if (RT_SUCCESS(rc))
982	{
983	rc = RTTimerStart(pDevExt->pInvarTscRefineTimer, 2*RT_NS_100MS);
984	if (RT_SUCCESS(rc))
985	return;
986	RTTimerDestroy(pDevExt->pInvarTscRefineTimer);
987	}
988	}
989
990	pDevExt->pInvarTscRefineTimer = NULL;
991	OSDBGPRINT(("vboxdrv: Failed to create or start TSC frequency refinement timer: rc=%Rrc\n", rc));
992	}
993
994
995	/**
996	* @callback_method_impl{PFNRTMPWORKER,
997	* RTMpOnSpecific callback for reading TSC and time on the CPU we started
998	* the measurements on.}
999	*/
1000	DECLCALLBACK(void) supdrvGipInitReadTscAndNanoTsOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
1001	{
1002	RTCCUINTREG fEFlags = ASMIntDisableFlags();
1003	uint64_t puTscStop = (uint64_t )pvUser1;
1004	uint64_t pnsStop = (uint64_t )pvUser2;
1005	RT_NOREF1(idCpu);
1006
1007	*puTscStop = ASMReadTSC();
1008	*pnsStop = RTTimeSystemNanoTS();
1009
1010	ASMSetFlags(fEFlags);
1011	}
1012
1013
1014	/**
1015	* Measures the TSC frequency of the system.
1016	*
1017	* The TSC frequency can vary on systems which are not reported as invariant.
1018	* On such systems the object of this function is to find out what the nominal,
1019	* maximum TSC frequency under 'normal' CPU operation.
1020	*
1021	* @returns VBox status code.
1022	* @param pGip Pointer to the GIP.
1023	* @param fRough Set if we're doing the rough calculation that the
1024	* TSC measuring code needs, where accuracy isn't all
1025	* that important (too high is better than too low).
1026	* When clear we try for best accuracy that we can
1027	* achieve in reasonably short time.
1028	*/
1029	static int supdrvGipInitMeasureTscFreq(PSUPGLOBALINFOPAGE pGip, bool fRough)
1030	{
1031	uint32_t nsTimerIncr = RTTimerGetSystemGranularity();
1032	int cTriesLeft = fRough ? 4 : 2;
1033	while (cTriesLeft-- > 0)
1034	{
1035	RTCCUINTREG fEFlags;
1036	uint64_t nsStart;
1037	uint64_t nsStop;
1038	uint64_t uTscStart;
1039	uint64_t uTscStop;
1040	RTCPUID idCpuStart;
1041	RTCPUID idCpuStop;
1042
1043	/*
1044	* Synchronize with the host OS clock tick on systems without high
1045	* resolution time API (older Windows version for example).
1046	*/
1047	nsStart = RTTimeSystemNanoTS();
1048	while (RTTimeSystemNanoTS() == nsStart)
1049	ASMNopPause();
1050
1051	/*
1052	* Read the TSC and current time, noting which CPU we're on.
1053	*/
1054	fEFlags = ASMIntDisableFlags();
1055	uTscStart = ASMReadTSC();
1056	nsStart = RTTimeSystemNanoTS();
1057	idCpuStart = RTMpCpuId();
1058	ASMSetFlags(fEFlags);
1059
1060	/*
1061	* Delay for a while.
1062	*/
1063	if (pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC)
1064	{
1065	/*
1066	* Sleep-wait since the TSC frequency is constant, it eases host load.
1067	* Shorter interval produces more variance in the frequency (esp. Windows).
1068	*/
1069	uint64_t msElapsed = 0;
1070	uint64_t msDelay = ( ((fRough ? 16 : 200) * RT_NS_1MS + nsTimerIncr - 1) / nsTimerIncr * nsTimerIncr - RT_NS_100US )
1071	/ RT_NS_1MS;
1072	do
1073	{
1074	RTThreadSleep((RTMSINTERVAL)(msDelay - msElapsed));
1075	nsStop = RTTimeSystemNanoTS();
1076	msElapsed = (nsStop - nsStart) / RT_NS_1MS;
1077	} while (msElapsed < msDelay);
1078
1079	while (RTTimeSystemNanoTS() == nsStop)
1080	ASMNopPause();
1081	}
1082	else
1083	{
1084	/*
1085	* Busy-wait keeping the frequency up.
1086	*/
1087	do
1088	{
1089	ASMNopPause();
1090	nsStop = RTTimeSystemNanoTS();
1091	} while (nsStop - nsStart < RT_NS_100MS);
1092	}
1093
1094	/*
1095	* Read the TSC and time again.
1096	*/
1097	fEFlags = ASMIntDisableFlags();
1098	uTscStop = ASMReadTSC();
1099	nsStop = RTTimeSystemNanoTS();
1100	idCpuStop = RTMpCpuId();
1101	ASMSetFlags(fEFlags);
1102
1103	/*
1104	* If the CPU changes, things get a bit complicated and what we
1105	* can get away with depends on the GIP mode / TSC reliability.
1106	*/
1107	if (idCpuStop != idCpuStart)
1108	{
1109	bool fDoXCall = false;
1110
1111	/*
1112	* Synchronous TSC mode: we're probably fine as it's unlikely
1113	* that we were rescheduled because of TSC throttling or power
1114	* management reasons, so just go ahead.
1115	*/
1116	if (pGip->u32Mode == SUPGIPMODE_SYNC_TSC)
1117	{
1118	/* Probably ok, maybe we should retry once?. */
1119	Assert(pGip->enmUseTscDelta == SUPGIPUSETSCDELTA_NOT_APPLICABLE);
1120	}
1121	/*
1122	* If we're just doing the rough measurement, do the cross call and
1123	* get on with things (we don't have deltas!).
1124	*/
1125	else if (fRough)
1126	fDoXCall = true;
1127	/*
1128	* Invariant TSC mode: It doesn't matter if we have delta available
1129	* for both CPUs. That is not something we can assume at this point.
1130	*
1131	* Note! We cannot necessarily trust enmUseTscDelta here because it's
1132	* downgraded after each delta calculation and the delta
1133	* calculations may not be complete yet.
1134	*/
1135	else if (pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC)
1136	{
1137	/** @todo This section of code is never reached atm, consider dropping it later on... */
1138	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
1139	{
1140	uint32_t iStartCpuSet = RTMpCpuIdToSetIndex(idCpuStart);
1141	uint32_t iStopCpuSet = RTMpCpuIdToSetIndex(idCpuStop);
1142	uint16_t iStartGipCpu = iStartCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
1143	? pGip->aiCpuFromCpuSetIdx[iStartCpuSet] : UINT16_MAX;
1144	uint16_t iStopGipCpu = iStopCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
1145	? pGip->aiCpuFromCpuSetIdx[iStopCpuSet] : UINT16_MAX;
1146	int64_t iStartTscDelta = iStartGipCpu < pGip->cCpus ? pGip->aCPUs[iStartGipCpu].i64TSCDelta : INT64_MAX;
1147	int64_t iStopTscDelta = iStopGipCpu < pGip->cCpus ? pGip->aCPUs[iStopGipCpu].i64TSCDelta : INT64_MAX;
1148	if (RT_LIKELY(iStartTscDelta != INT64_MAX && iStopTscDelta != INT64_MAX))
1149	{
1150	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
1151	{
1152	uTscStart -= iStartTscDelta;
1153	uTscStop -= iStopTscDelta;
1154	}
1155	}
1156	/*
1157	* Invalid CPU indexes are not caused by online/offline races, so
1158	* we have to trigger driver load failure if that happens as GIP
1159	* and IPRT assumptions are busted on this system.
1160	*/
1161	else if (iStopGipCpu >= pGip->cCpus \|\| iStartGipCpu >= pGip->cCpus)
1162	{
1163	SUPR0Printf("vboxdrv: Unexpected CPU index in supdrvGipInitMeasureTscFreq.\n");
1164	SUPR0Printf("vboxdrv: start: %u, %u, %#llx stop: %u, %u, %#llx\n",
1165	iStartCpuSet, iStartGipCpu, iStartTscDelta, iStopCpuSet, iStopGipCpu, iStopTscDelta);
1166	return VERR_INVALID_CPU_INDEX;
1167	}
1168	/*
1169	* No valid deltas. We retry, if we're on our last retry
1170	* we do the cross call instead just to get a result. The
1171	* frequency will be refined in a few seconds anyway.
1172	*/
1173	else if (cTriesLeft > 0)
1174	continue;
1175	else
1176	fDoXCall = true;
1177	}
1178	}
1179	/*
1180	* Asynchronous TSC mode: This is bad, as the reason we usually
1181	* use this mode is to deal with variable TSC frequencies and
1182	* deltas. So, we need to get the TSC from the same CPU as
1183	* started it, we also need to keep that CPU busy. So, retry
1184	* and fall back to the cross call on the last attempt.
1185	*/
1186	else
1187	{
1188	Assert(pGip->u32Mode == SUPGIPMODE_ASYNC_TSC);
1189	if (cTriesLeft > 0)
1190	continue;
1191	fDoXCall = true;
1192	}
1193
1194	if (fDoXCall)
1195	{
1196	/*
1197	* Try read the TSC and timestamp on the start CPU.
1198	*/
1199	int rc = RTMpOnSpecific(idCpuStart, supdrvGipInitReadTscAndNanoTsOnCpu, &uTscStop, &nsStop);
1200	if (RT_FAILURE(rc) && (!fRough \|\| cTriesLeft > 0))
1201	continue;
1202	}
1203	}
1204
1205	/*
1206	* Calculate the TSC frequency and update it (shared with the refinement timer).
1207	*/
1208	supdrvGipInitSetCpuFreq(pGip, nsStop - nsStart, uTscStop - uTscStart, 0);
1209	return VINF_SUCCESS;
1210	}
1211
1212	Assert(!fRough);
1213	return VERR_SUPDRV_TSC_FREQ_MEASUREMENT_FAILED;
1214	}
1215
1216
1217	/**
1218	* Finds our (@a idCpu) entry, or allocates a new one if not found.
1219	*
1220	* @returns Index of the CPU in the cache set.
1221	* @param pGip The GIP.
1222	* @param idCpu The CPU ID.
1223	*/
1224	static uint32_t supdrvGipFindOrAllocCpuIndexForCpuId(PSUPGLOBALINFOPAGE pGip, RTCPUID idCpu)
1225	{
1226	uint32_t i, cTries;
1227
1228	/*
1229	* ASSUMES that CPU IDs are constant.
1230	*/
1231	for (i = 0; i < pGip->cCpus; i++)
1232	if (pGip->aCPUs[i].idCpu == idCpu)
1233	return i;
1234
1235	cTries = 0;
1236	do
1237	{
1238	for (i = 0; i < pGip->cCpus; i++)
1239	{
1240	bool fRc;
1241	ASMAtomicCmpXchgSize(&pGip->aCPUs[i].idCpu, idCpu, NIL_RTCPUID, fRc);
1242	if (fRc)
1243	return i;
1244	}
1245	} while (cTries++ < 32);
1246	AssertReleaseFailed();
1247	return i - 1;
1248	}
1249
1250
1251	/**
1252	* The calling CPU should be accounted as online, update GIP accordingly.
1253	*
1254	* This is used by supdrvGipCreate() as well as supdrvGipMpEvent().
1255	*
1256	* @param pDevExt The device extension.
1257	* @param idCpu The CPU ID.
1258	*/
1259	static void supdrvGipMpEventOnlineOrInitOnCpu(PSUPDRVDEVEXT pDevExt, RTCPUID idCpu)
1260	{
1261	int iCpuSet = 0;
1262	uint16_t idApic = UINT16_MAX;
1263	uint32_t i = 0;
1264	uint64_t u64NanoTS = 0;
1265	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
1266
1267	AssertPtrReturnVoid(pGip);
1268	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1269	AssertRelease(idCpu == RTMpCpuId());
1270	Assert(pGip->cPossibleCpus == RTMpGetCount());
1271
1272	/*
1273	* Do this behind a spinlock with interrupts disabled as this can fire
1274	* on all CPUs simultaneously, see @bugref{6110}.
1275	*/
1276	RTSpinlockAcquire(pDevExt->hGipSpinlock);
1277
1278	/*
1279	* Update the globals.
1280	*/
1281	ASMAtomicWriteU16(&pGip->cPresentCpus, RTMpGetPresentCount());
1282	ASMAtomicWriteU16(&pGip->cOnlineCpus, RTMpGetOnlineCount());
1283	iCpuSet = RTMpCpuIdToSetIndex(idCpu);
1284	if (iCpuSet >= 0)
1285	{
1286	Assert(RTCpuSetIsMemberByIndex(&pGip->PossibleCpuSet, iCpuSet));
1287	RTCpuSetAddByIndex(&pGip->OnlineCpuSet, iCpuSet);
1288	RTCpuSetAddByIndex(&pGip->PresentCpuSet, iCpuSet);
1289	}
1290
1291	/*
1292	* Update the entry.
1293	*/
1294	u64NanoTS = RTTimeSystemNanoTS() - pGip->u32UpdateIntervalNS;
1295	i = supdrvGipFindOrAllocCpuIndexForCpuId(pGip, idCpu);
1296
1297	supdrvGipInitCpu(pGip, &pGip->aCPUs[i], u64NanoTS, pGip->u64CpuHz);
1298
1299	idApic = ASMGetApicId();
1300	ASMAtomicWriteU16(&pGip->aCPUs[i].idApic, idApic);
1301	ASMAtomicWriteS16(&pGip->aCPUs[i].iCpuSet, (int16_t)iCpuSet);
1302	ASMAtomicWriteSize(&pGip->aCPUs[i].idCpu, idCpu);
1303
1304	/*
1305	* Update the APIC ID and CPU set index mappings.
1306	*/
1307	ASMAtomicWriteU16(&pGip->aiCpuFromApicId[idApic], i);
1308	ASMAtomicWriteU16(&pGip->aiCpuFromCpuSetIdx[iCpuSet], i);
1309
1310	/* Add this CPU to this set of CPUs we need to calculate the TSC-delta for. */
1311	RTCpuSetAddByIndex(&pDevExt->TscDeltaCpuSet, RTMpCpuIdToSetIndex(idCpu));
1312
1313	/* Update the Mp online/offline counter. */
1314	ASMAtomicIncU32(&pDevExt->cMpOnOffEvents);
1315
1316	/* Commit it. */
1317	ASMAtomicWriteSize(&pGip->aCPUs[i].enmState, SUPGIPCPUSTATE_ONLINE);
1318
1319	RTSpinlockRelease(pDevExt->hGipSpinlock);
1320	}
1321
1322
1323	/**
1324	* RTMpOnSpecific callback wrapper for supdrvGipMpEventOnlineOrInitOnCpu().
1325	*
1326	* @param idCpu The CPU ID we are running on.
1327	* @param pvUser1 Opaque pointer to the device instance data.
1328	* @param pvUser2 Not used.
1329	*/
1330	static DECLCALLBACK(void) supdrvGipMpEventOnlineCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
1331	{
1332	PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser1;
1333	NOREF(pvUser2);
1334	supdrvGipMpEventOnlineOrInitOnCpu(pDevExt, idCpu);
1335	}
1336
1337
1338	/**
1339	* The CPU should be accounted as offline, update the GIP accordingly.
1340	*
1341	* This is used by supdrvGipMpEvent.
1342	*
1343	* @param pDevExt The device extension.
1344	* @param idCpu The CPU ID.
1345	*/
1346	static void supdrvGipMpEventOffline(PSUPDRVDEVEXT pDevExt, RTCPUID idCpu)
1347	{
1348	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
1349	int iCpuSet;
1350	unsigned i;
1351
1352	AssertPtrReturnVoid(pGip);
1353	RTSpinlockAcquire(pDevExt->hGipSpinlock);
1354
1355	iCpuSet = RTMpCpuIdToSetIndex(idCpu);
1356	AssertReturnVoid(iCpuSet >= 0);
1357
1358	i = pGip->aiCpuFromCpuSetIdx[iCpuSet];
1359	AssertReturnVoid(i < pGip->cCpus);
1360	AssertReturnVoid(pGip->aCPUs[i].idCpu == idCpu);
1361
1362	Assert(RTCpuSetIsMemberByIndex(&pGip->PossibleCpuSet, iCpuSet));
1363	RTCpuSetDelByIndex(&pGip->OnlineCpuSet, iCpuSet);
1364
1365	/* Update the Mp online/offline counter. */
1366	ASMAtomicIncU32(&pDevExt->cMpOnOffEvents);
1367
1368	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
1369	{
1370	/* Reset the TSC delta, we will recalculate it lazily. */
1371	ASMAtomicWriteS64(&pGip->aCPUs[i].i64TSCDelta, INT64_MAX);
1372	/* Remove this CPU from the set of CPUs that we have obtained the TSC deltas. */
1373	RTCpuSetDelByIndex(&pDevExt->TscDeltaObtainedCpuSet, iCpuSet);
1374	}
1375
1376	/* Commit it. */
1377	ASMAtomicWriteSize(&pGip->aCPUs[i].enmState, SUPGIPCPUSTATE_OFFLINE);
1378
1379	RTSpinlockRelease(pDevExt->hGipSpinlock);
1380	}
1381
1382
1383	/**
1384	* Multiprocessor event notification callback.
1385	*
1386	* This is used to make sure that the GIP master gets passed on to
1387	* another CPU. It also updates the associated CPU data.
1388	*
1389	* @param enmEvent The event.
1390	* @param idCpu The cpu it applies to.
1391	* @param pvUser Pointer to the device extension.
1392	*/
1393	static DECLCALLBACK(void) supdrvGipMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
1394	{
1395	PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
1396	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
1397
1398	if (pGip)
1399	{
1400	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1401	switch (enmEvent)
1402	{
1403	case RTMPEVENT_ONLINE:
1404	{
1405	RTThreadPreemptDisable(&PreemptState);
1406	if (idCpu == RTMpCpuId())
1407	{
1408	supdrvGipMpEventOnlineOrInitOnCpu(pDevExt, idCpu);
1409	RTThreadPreemptRestore(&PreemptState);
1410	}
1411	else
1412	{
1413	RTThreadPreemptRestore(&PreemptState);
1414	RTMpOnSpecific(idCpu, supdrvGipMpEventOnlineCallback, pDevExt, NULL /* pvUser2 */);
1415	}
1416
1417	/*
1418	* Recompute TSC-delta for the newly online'd CPU.
1419	*/
1420	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
1421	{
1422	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
1423	supdrvTscDeltaThreadStartMeasurement(pDevExt, false /* fForceAll */);
1424	#else
1425	uint32_t iCpu = supdrvGipFindOrAllocCpuIndexForCpuId(pGip, idCpu);
1426	supdrvMeasureTscDeltaOne(pDevExt, iCpu);
1427	#endif
1428	}
1429	break;
1430	}
1431
1432	case RTMPEVENT_OFFLINE:
1433	supdrvGipMpEventOffline(pDevExt, idCpu);
1434	break;
1435	}
1436	}
1437
1438	/*
1439	* Make sure there is a master GIP.
1440	*/
1441	if (enmEvent == RTMPEVENT_OFFLINE)
1442	{
1443	RTCPUID idGipMaster = ASMAtomicReadU32(&pDevExt->idGipMaster);
1444	if (idGipMaster == idCpu)
1445	{
1446	/*
1447	* The GIP master is going offline, find a new one.
1448	*/
1449	bool fIgnored;
1450	unsigned i;
1451	RTCPUID idNewGipMaster = NIL_RTCPUID;
1452	RTCPUSET OnlineCpus;
1453	RTMpGetOnlineSet(&OnlineCpus);
1454
1455	for (i = 0; i < RTCPUSET_MAX_CPUS; i++)
1456	if (RTCpuSetIsMemberByIndex(&OnlineCpus, i))
1457	{
1458	RTCPUID idCurCpu = RTMpCpuIdFromSetIndex(i);
1459	if (idCurCpu != idGipMaster)
1460	{
1461	idNewGipMaster = idCurCpu;
1462	break;
1463	}
1464	}
1465
1466	Log(("supdrvGipMpEvent: Gip master %#lx -> %#lx\n", (long)idGipMaster, (long)idNewGipMaster));
1467	ASMAtomicCmpXchgSize(&pDevExt->idGipMaster, idNewGipMaster, idGipMaster, fIgnored);
1468	NOREF(fIgnored);
1469	}
1470	}
1471	}
1472
1473
1474	/**
1475	* On CPU initialization callback for RTMpOnAll.
1476	*
1477	* @param idCpu The CPU ID.
1478	* @param pvUser1 The device extension.
1479	* @param pvUser2 The GIP.
1480	*/
1481	static DECLCALLBACK(void) supdrvGipInitOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
1482	{
1483	/* This is good enough, even though it will update some of the globals a
1484	bit to much. */
1485	supdrvGipMpEventOnlineOrInitOnCpu((PSUPDRVDEVEXT)pvUser1, idCpu);
1486	NOREF(pvUser2);
1487	}
1488
1489
1490	/**
1491	* Callback used by supdrvDetermineAsyncTSC to read the TSC on a CPU.
1492	*
1493	* @param idCpu Ignored.
1494	* @param pvUser1 Where to put the TSC.
1495	* @param pvUser2 Ignored.
1496	*/
1497	static DECLCALLBACK(void) supdrvGipInitDetermineAsyncTscWorker(RTCPUID idCpu, void pvUser1, void pvUser2)
1498	{
1499	Assert(RTMpCpuIdToSetIndex(idCpu) == (intptr_t)pvUser2);
1500	ASMAtomicWriteU64((uint64_t volatile *)pvUser1, ASMReadTSC());
1501	RT_NOREF2(idCpu, pvUser2);
1502	}
1503
1504
1505	/**
1506	* Determine if Async GIP mode is required because of TSC drift.
1507	*
1508	* When using the default/normal timer code it is essential that the time stamp counter
1509	* (TSC) runs never backwards, that is, a read operation to the counter should return
1510	* a bigger value than any previous read operation. This is guaranteed by the latest
1511	* AMD CPUs and by newer Intel CPUs which never enter the C2 state (P4). In any other
1512	* case we have to choose the asynchronous timer mode.
1513	*
1514	* @param poffMin Pointer to the determined difference between different
1515	* cores (optional, can be NULL).
1516	* @return false if the time stamp counters appear to be synchronized, true otherwise.
1517	*/
1518	static bool supdrvGipInitDetermineAsyncTsc(uint64_t *poffMin)
1519	{
1520	/*
1521	* Just iterate all the cpus 8 times and make sure that the TSC is
1522	* ever increasing. We don't bother taking TSC rollover into account.
1523	*/
1524	int iEndCpu = RTMpGetArraySize();
1525	int iCpu;
1526	int cLoops = 8;
1527	bool fAsync = false;
1528	int rc = VINF_SUCCESS;
1529	uint64_t offMax = 0;
1530	uint64_t offMin = ~(uint64_t)0;
1531	uint64_t PrevTsc = ASMReadTSC();
1532
1533	while (cLoops-- > 0)
1534	{
1535	for (iCpu = 0; iCpu < iEndCpu; iCpu++)
1536	{
1537	uint64_t CurTsc;
1538	rc = RTMpOnSpecific(RTMpCpuIdFromSetIndex(iCpu), supdrvGipInitDetermineAsyncTscWorker,
1539	&CurTsc, (void *)(uintptr_t)iCpu);
1540	if (RT_SUCCESS(rc))
1541	{
1542	if (CurTsc <= PrevTsc)
1543	{
1544	fAsync = true;
1545	offMin = offMax = PrevTsc - CurTsc;
1546	Log(("supdrvGipInitDetermineAsyncTsc: iCpu=%d cLoops=%d CurTsc=%llx PrevTsc=%llx\n",
1547	iCpu, cLoops, CurTsc, PrevTsc));
1548	break;
1549	}
1550
1551	/* Gather statistics (except the first time). */
1552	if (iCpu != 0 \|\| cLoops != 7)
1553	{
1554	uint64_t off = CurTsc - PrevTsc;
1555	if (off < offMin)
1556	offMin = off;
1557	if (off > offMax)
1558	offMax = off;
1559	Log2(("%d/%d: off=%llx\n", cLoops, iCpu, off));
1560	}
1561
1562	/* Next */
1563	PrevTsc = CurTsc;
1564	}
1565	else if (rc == VERR_NOT_SUPPORTED)
1566	break;
1567	else
1568	AssertMsg(rc == VERR_CPU_NOT_FOUND \|\| rc == VERR_CPU_OFFLINE, ("%d\n", rc));
1569	}
1570
1571	/* broke out of the loop. */
1572	if (iCpu < iEndCpu)
1573	break;
1574	}
1575
1576	if (poffMin)
1577	poffMin = offMin; / Almost RTMpOnSpecific profiling. */
1578	Log(("supdrvGipInitDetermineAsyncTsc: returns %d; iEndCpu=%d rc=%d offMin=%llx offMax=%llx\n",
1579	fAsync, iEndCpu, rc, offMin, offMax));
1580	#if !defined(RT_OS_SOLARIS) && !defined(RT_OS_OS2) && !defined(RT_OS_WINDOWS)
1581	OSDBGPRINT(("vboxdrv: fAsync=%d offMin=%#lx offMax=%#lx\n", fAsync, (long)offMin, (long)offMax));
1582	#endif
1583	return fAsync;
1584	}
1585
1586
1587	/**
1588	* supdrvGipInit() worker that determines the GIP TSC mode.
1589	*
1590	* @returns The most suitable TSC mode.
1591	* @param pDevExt Pointer to the device instance data.
1592	*/
1593	static SUPGIPMODE supdrvGipInitDetermineTscMode(PSUPDRVDEVEXT pDevExt)
1594	{
1595	uint64_t u64DiffCoresIgnored;
1596	uint32_t uEAX, uEBX, uECX, uEDX;
1597
1598	/*
1599	* Establish whether the CPU advertises TSC as invariant, we need that in
1600	* a couple of places below.
1601	*/
1602	bool fInvariantTsc = false;
1603	if (ASMHasCpuId())
1604	{
1605	uEAX = ASMCpuId_EAX(0x80000000);
1606	if (ASMIsValidExtRange(uEAX) && uEAX >= 0x80000007)
1607	{
1608	uEDX = ASMCpuId_EDX(0x80000007);
1609	if (uEDX & X86_CPUID_AMD_ADVPOWER_EDX_TSCINVAR)
1610	fInvariantTsc = true;
1611	}
1612	}
1613
1614	/*
1615	* On single CPU systems, we don't need to consider ASYNC mode.
1616	*/
1617	if (RTMpGetCount() <= 1)
1618	return fInvariantTsc ? SUPGIPMODE_INVARIANT_TSC : SUPGIPMODE_SYNC_TSC;
1619
1620	/*
1621	* Allow the user and/or OS specific bits to force async mode.
1622	*/
1623	if (supdrvOSGetForcedAsyncTscMode(pDevExt))
1624	return SUPGIPMODE_ASYNC_TSC;
1625
1626	/*
1627	* Use invariant mode if the CPU says TSC is invariant.
1628	*/
1629	if (fInvariantTsc)
1630	return SUPGIPMODE_INVARIANT_TSC;
1631
1632	/*
1633	* TSC is not invariant and we're on SMP, this presents two problems:
1634	*
1635	* (1) There might be a skew between the CPU, so that cpu0
1636	* returns a TSC that is slightly different from cpu1.
1637	* This screw may be due to (2), bad TSC initialization
1638	* or slightly different TSC rates.
1639	*
1640	* (2) Power management (and other things) may cause the TSC
1641	* to run at a non-constant speed, and cause the speed
1642	* to be different on the cpus. This will result in (1).
1643	*
1644	* If any of the above is detected, we will have to use ASYNC mode.
1645	*/
1646	/* (1). Try check for current differences between the cpus. */
1647	if (supdrvGipInitDetermineAsyncTsc(&u64DiffCoresIgnored))
1648	return SUPGIPMODE_ASYNC_TSC;
1649
1650	/* (2) If it's an AMD CPU with power management, we won't trust its TSC. */
1651	ASMCpuId(0, &uEAX, &uEBX, &uECX, &uEDX);
1652	if ( ASMIsValidStdRange(uEAX)
1653	&& ASMIsAmdCpuEx(uEBX, uECX, uEDX))
1654	{
1655	/* Check for APM support. */
1656	uEAX = ASMCpuId_EAX(0x80000000);
1657	if (ASMIsValidExtRange(uEAX) && uEAX >= 0x80000007)
1658	{
1659	uEDX = ASMCpuId_EDX(0x80000007);
1660	if (uEDX & 0x3e) /* STC\|TM\|THERMTRIP\|VID\|FID. Ignore TS. */
1661	return SUPGIPMODE_ASYNC_TSC;
1662	}
1663	}
1664
1665	return SUPGIPMODE_SYNC_TSC;
1666	}
1667
1668
1669	/**
1670	* Initializes per-CPU GIP information.
1671	*
1672	* @param pGip Pointer to the GIP.
1673	* @param pCpu Pointer to which GIP CPU to initialize.
1674	* @param u64NanoTS The current nanosecond timestamp.
1675	* @param uCpuHz The CPU frequency to set, 0 if the caller doesn't know.
1676	*/
1677	static void supdrvGipInitCpu(PSUPGLOBALINFOPAGE pGip, PSUPGIPCPU pCpu, uint64_t u64NanoTS, uint64_t uCpuHz)
1678	{
1679	pCpu->u32TransactionId = 2;
1680	pCpu->u64NanoTS = u64NanoTS;
1681	pCpu->u64TSC = ASMReadTSC();
1682	pCpu->u64TSCSample = GIP_TSC_DELTA_RSVD;
1683	pCpu->i64TSCDelta = pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED ? INT64_MAX : 0;
1684
1685	ASMAtomicWriteSize(&pCpu->enmState, SUPGIPCPUSTATE_INVALID);
1686	ASMAtomicWriteU32(&pCpu->idCpu, NIL_RTCPUID);
1687	ASMAtomicWriteS16(&pCpu->iCpuSet, -1);
1688	ASMAtomicWriteU16(&pCpu->idApic, UINT16_MAX);
1689
1690	/*
1691	* The first time we're called, we don't have a CPU frequency handy,
1692	* so pretend it's a 4 GHz CPU. On CPUs that are online, we'll get
1693	* called again and at that point we have a more plausible CPU frequency
1694	* value handy. The frequency history will also be adjusted again on
1695	* the 2nd timer callout (maybe we can skip that now?).
1696	*/
1697	if (!uCpuHz)
1698	{
1699	pCpu->u64CpuHz = _4G - 1;
1700	pCpu->u32UpdateIntervalTSC = (uint32_t)((_4G - 1) / pGip->u32UpdateHz);
1701	}
1702	else
1703	{
1704	pCpu->u64CpuHz = uCpuHz;
1705	pCpu->u32UpdateIntervalTSC = (uint32_t)(uCpuHz / pGip->u32UpdateHz);
1706	}
1707	pCpu->au32TSCHistory[0]
1708	= pCpu->au32TSCHistory[1]
1709	= pCpu->au32TSCHistory[2]
1710	= pCpu->au32TSCHistory[3]
1711	= pCpu->au32TSCHistory[4]
1712	= pCpu->au32TSCHistory[5]
1713	= pCpu->au32TSCHistory[6]
1714	= pCpu->au32TSCHistory[7]
1715	= pCpu->u32UpdateIntervalTSC;
1716	}
1717
1718
1719	/**
1720	* Initializes the GIP data.
1721	*
1722	* @param pDevExt Pointer to the device instance data.
1723	* @param pGip Pointer to the read-write kernel mapping of the GIP.
1724	* @param HCPhys The physical address of the GIP.
1725	* @param u64NanoTS The current nanosecond timestamp.
1726	* @param uUpdateHz The update frequency.
1727	* @param uUpdateIntervalNS The update interval in nanoseconds.
1728	* @param cCpus The CPU count.
1729	*/
1730	static void supdrvGipInit(PSUPDRVDEVEXT pDevExt, PSUPGLOBALINFOPAGE pGip, RTHCPHYS HCPhys,
1731	uint64_t u64NanoTS, unsigned uUpdateHz, unsigned uUpdateIntervalNS, unsigned cCpus)
1732	{
1733	size_t const cbGip = RT_ALIGN_Z(RT_OFFSETOF(SUPGLOBALINFOPAGE, aCPUs[cCpus]), PAGE_SIZE);
1734	unsigned i;
1735	#ifdef DEBUG_DARWIN_GIP
1736	OSDBGPRINT(("supdrvGipInit: pGip=%p HCPhys=%lx u64NanoTS=%llu uUpdateHz=%d cCpus=%u\n", pGip, (long)HCPhys, u64NanoTS, uUpdateHz, cCpus));
1737	#else
1738	LogFlow(("supdrvGipInit: pGip=%p HCPhys=%lx u64NanoTS=%llu uUpdateHz=%d cCpus=%u\n", pGip, (long)HCPhys, u64NanoTS, uUpdateHz, cCpus));
1739	#endif
1740
1741	/*
1742	* Initialize the structure.
1743	*/
1744	memset(pGip, 0, cbGip);
1745
1746	pGip->u32Magic = SUPGLOBALINFOPAGE_MAGIC;
1747	pGip->u32Version = SUPGLOBALINFOPAGE_VERSION;
1748	pGip->u32Mode = supdrvGipInitDetermineTscMode(pDevExt);
1749	if ( pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC
1750	/\|\| pGip->u32Mode == SUPGIPMODE_SYNC_TSC /)
1751	pGip->enmUseTscDelta = supdrvOSAreTscDeltasInSync() /* Allow OS override (windows). */
1752	? SUPGIPUSETSCDELTA_ZERO_CLAIMED : SUPGIPUSETSCDELTA_PRACTICALLY_ZERO /* downgrade later */;
1753	else
1754	pGip->enmUseTscDelta = SUPGIPUSETSCDELTA_NOT_APPLICABLE;
1755	pGip->cCpus = (uint16_t)cCpus;
1756	pGip->cPages = (uint16_t)(cbGip / PAGE_SIZE);
1757	pGip->u32UpdateHz = uUpdateHz;
1758	pGip->u32UpdateIntervalNS = uUpdateIntervalNS;
1759	pGip->fGetGipCpu = SUPGIPGETCPU_APIC_ID;
1760	RTCpuSetEmpty(&pGip->OnlineCpuSet);
1761	RTCpuSetEmpty(&pGip->PresentCpuSet);
1762	RTMpGetSet(&pGip->PossibleCpuSet);
1763	pGip->cOnlineCpus = RTMpGetOnlineCount();
1764	pGip->cPresentCpus = RTMpGetPresentCount();
1765	pGip->cPossibleCpus = RTMpGetCount();
1766	pGip->idCpuMax = RTMpGetMaxCpuId();
1767	for (i = 0; i < RT_ELEMENTS(pGip->aiCpuFromApicId); i++)
1768	pGip->aiCpuFromApicId[i] = UINT16_MAX;
1769	for (i = 0; i < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx); i++)
1770	pGip->aiCpuFromCpuSetIdx[i] = UINT16_MAX;
1771	for (i = 0; i < cCpus; i++)
1772	supdrvGipInitCpu(pGip, &pGip->aCPUs[i], u64NanoTS, 0 /uCpuHz/);
1773
1774	/*
1775	* Link it to the device extension.
1776	*/
1777	pDevExt->pGip = pGip;
1778	pDevExt->HCPhysGip = HCPhys;
1779	pDevExt->cGipUsers = 0;
1780	}
1781
1782
1783	/**
1784	* Creates the GIP.
1785	*
1786	* @returns VBox status code.
1787	* @param pDevExt Instance data. GIP stuff may be updated.
1788	*/
1789	int VBOXCALL supdrvGipCreate(PSUPDRVDEVEXT pDevExt)
1790	{
1791	PSUPGLOBALINFOPAGE pGip;
1792	RTHCPHYS HCPhysGip;
1793	uint32_t u32SystemResolution;
1794	uint32_t u32Interval;
1795	uint32_t u32MinInterval;
1796	uint32_t uMod;
1797	unsigned cCpus;
1798	int rc;
1799
1800	LogFlow(("supdrvGipCreate:\n"));
1801
1802	/*
1803	* Assert order.
1804	*/
1805	Assert(pDevExt->u32SystemTimerGranularityGrant == 0);
1806	Assert(pDevExt->GipMemObj == NIL_RTR0MEMOBJ);
1807	Assert(!pDevExt->pGipTimer);
1808	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
1809	Assert(pDevExt->mtxGip != NIL_RTSEMMUTEX);
1810	Assert(pDevExt->mtxTscDelta != NIL_RTSEMMUTEX);
1811	#else
1812	Assert(pDevExt->mtxGip != NIL_RTSEMFASTMUTEX);
1813	Assert(pDevExt->mtxTscDelta != NIL_RTSEMFASTMUTEX);
1814	#endif
1815
1816	/*
1817	* Check the CPU count.
1818	*/
1819	cCpus = RTMpGetArraySize();
1820	if ( cCpus > RTCPUSET_MAX_CPUS
1821	\|\| cCpus > 256 /* ApicId is used for the mappings */)
1822	{
1823	SUPR0Printf("VBoxDrv: Too many CPUs (%u) for the GIP (max %u)\n", cCpus, RT_MIN(RTCPUSET_MAX_CPUS, 256));
1824	return VERR_TOO_MANY_CPUS;
1825	}
1826
1827	/*
1828	* Allocate a contiguous set of pages with a default kernel mapping.
1829	*/
1830	rc = RTR0MemObjAllocCont(&pDevExt->GipMemObj, RT_UOFFSETOF(SUPGLOBALINFOPAGE, aCPUs[cCpus]), false /fExecutable/);
1831	if (RT_FAILURE(rc))
1832	{
1833	OSDBGPRINT(("supdrvGipCreate: failed to allocate the GIP page. rc=%d\n", rc));
1834	return rc;
1835	}
1836	pGip = (PSUPGLOBALINFOPAGE)RTR0MemObjAddress(pDevExt->GipMemObj); AssertPtr(pGip);
1837	HCPhysGip = RTR0MemObjGetPagePhysAddr(pDevExt->GipMemObj, 0); Assert(HCPhysGip != NIL_RTHCPHYS);
1838
1839	/*
1840	* Find a reasonable update interval and initialize the structure.
1841	*/
1842	supdrvGipRequestHigherTimerFrequencyFromSystem(pDevExt);
1843	/** @todo figure out why using a 100Ms interval upsets timekeeping in VMs.
1844	* See @bugref{6710}. */
1845	u32MinInterval = RT_NS_10MS;
1846	u32SystemResolution = RTTimerGetSystemGranularity();
1847	u32Interval = u32MinInterval;
1848	uMod = u32MinInterval % u32SystemResolution;
1849	if (uMod)
1850	u32Interval += u32SystemResolution - uMod;
1851
1852	supdrvGipInit(pDevExt, pGip, HCPhysGip, RTTimeSystemNanoTS(), RT_NS_1SEC / u32Interval /=Hz/, u32Interval, cCpus);
1853
1854	/*
1855	* Important sanity check...
1856	*/
1857	if (RT_UNLIKELY( pGip->enmUseTscDelta == SUPGIPUSETSCDELTA_ZERO_CLAIMED
1858	&& pGip->u32Mode == SUPGIPMODE_ASYNC_TSC
1859	&& !supdrvOSGetForcedAsyncTscMode(pDevExt)))
1860	{
1861	OSDBGPRINT(("supdrvGipCreate: Host-OS/user claims the TSC-deltas are zero but we detected async. TSC! Bad.\n"));
1862	return VERR_INTERNAL_ERROR_2;
1863	}
1864
1865	/* It doesn't make sense to do TSC-delta detection on systems we detect as async. */
1866	AssertReturn( pGip->u32Mode != SUPGIPMODE_ASYNC_TSC
1867	\|\| pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ZERO_CLAIMED, VERR_INTERNAL_ERROR_3);
1868
1869	/*
1870	* Do the TSC frequency measurements.
1871	*
1872	* If we're in invariant TSC mode, just to a quick preliminary measurement
1873	* that the TSC-delta measurement code can use to yield cross calls.
1874	*
1875	* If we're in any of the other two modes, neither which require MP init,
1876	* notifications or deltas for the job, do the full measurement now so
1877	* that supdrvGipInitOnCpu() can populate the TSC interval and history
1878	* array with more reasonable values.
1879	*/
1880	if (pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC)
1881	{
1882	rc = supdrvGipInitMeasureTscFreq(pGip, true /fRough/); /* cannot fail */
1883	supdrvGipInitStartTimerForRefiningInvariantTscFreq(pDevExt);
1884	}
1885	else
1886	rc = supdrvGipInitMeasureTscFreq(pGip, false /fRough/);
1887	if (RT_SUCCESS(rc))
1888	{
1889	/*
1890	* Start TSC-delta measurement thread before we start getting MP
1891	* events that will try kick it into action (includes the
1892	* RTMpOnAll/supdrvGipInitOnCpu call below).
1893	*/
1894	RTCpuSetEmpty(&pDevExt->TscDeltaCpuSet);
1895	RTCpuSetEmpty(&pDevExt->TscDeltaObtainedCpuSet);
1896	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
1897	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
1898	rc = supdrvTscDeltaThreadInit(pDevExt);
1899	#endif
1900	if (RT_SUCCESS(rc))
1901	{
1902	rc = RTMpNotificationRegister(supdrvGipMpEvent, pDevExt);
1903	if (RT_SUCCESS(rc))
1904	{
1905	/*
1906	* Do GIP initialization on all online CPUs. Wake up the
1907	* TSC-delta thread afterwards.
1908	*/
1909	rc = RTMpOnAll(supdrvGipInitOnCpu, pDevExt, pGip);
1910	if (RT_SUCCESS(rc))
1911	{
1912	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
1913	supdrvTscDeltaThreadStartMeasurement(pDevExt, true /* fForceAll */);
1914	#else
1915	uint16_t iCpu;
1916	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
1917	{
1918	/*
1919	* Measure the TSC deltas now that we have MP notifications.
1920	*/
1921	int cTries = 5;
1922	do
1923	{
1924	rc = supdrvMeasureInitialTscDeltas(pDevExt);
1925	if ( rc != VERR_TRY_AGAIN
1926	&& rc != VERR_CPU_OFFLINE)
1927	break;
1928	} while (--cTries > 0);
1929	for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
1930	Log(("supdrvTscDeltaInit: cpu[%u] delta %lld\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta));
1931	}
1932	else
1933	{
1934	for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
1935	AssertMsg(!pGip->aCPUs[iCpu].i64TSCDelta, ("iCpu=%u %lld mode=%d\n", iCpu, pGip->aCPUs[iCpu].i64TSCDelta, pGip->u32Mode));
1936	}
1937	if (RT_SUCCESS(rc))
1938	#endif
1939	{
1940	/*
1941	* Create the timer.
1942	* If CPU_ALL isn't supported we'll have to fall back to synchronous mode.
1943	*/
1944	if (pGip->u32Mode == SUPGIPMODE_ASYNC_TSC)
1945	{
1946	rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, RTTIMER_FLAGS_CPU_ALL,
1947	supdrvGipAsyncTimer, pDevExt);
1948	if (rc == VERR_NOT_SUPPORTED)
1949	{
1950	OSDBGPRINT(("supdrvGipCreate: omni timer not supported, falling back to synchronous mode\n"));
1951	pGip->u32Mode = SUPGIPMODE_SYNC_TSC;
1952	}
1953	}
1954	if (pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)
1955	rc = RTTimerCreateEx(&pDevExt->pGipTimer, u32Interval, 0 /* fFlags */,
1956	supdrvGipSyncAndInvariantTimer, pDevExt);
1957	if (RT_SUCCESS(rc))
1958	{
1959	/*
1960	* We're good.
1961	*/
1962	Log(("supdrvGipCreate: %u ns interval.\n", u32Interval));
1963	supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt);
1964
1965	g_pSUPGlobalInfoPage = pGip;
1966	return VINF_SUCCESS;
1967	}
1968
1969	OSDBGPRINT(("supdrvGipCreate: failed create GIP timer at %u ns interval. rc=%Rrc\n", u32Interval, rc));
1970	Assert(!pDevExt->pGipTimer);
1971	}
1972	}
1973	else
1974	OSDBGPRINT(("supdrvGipCreate: RTMpOnAll failed. rc=%Rrc\n", rc));
1975	}
1976	else
1977	OSDBGPRINT(("supdrvGipCreate: failed to register MP event notfication. rc=%Rrc\n", rc));
1978	}
1979	else
1980	OSDBGPRINT(("supdrvGipCreate: supdrvTscDeltaInit failed. rc=%Rrc\n", rc));
1981	}
1982	else
1983	OSDBGPRINT(("supdrvGipCreate: supdrvMeasureInitialTscDeltas failed. rc=%Rrc\n", rc));
1984
1985	/* Releases timer frequency increase too. */
1986	supdrvGipDestroy(pDevExt);
1987	return rc;
1988	}
1989
1990
1991	/**
1992	* Invalidates the GIP data upon termination.
1993	*
1994	* @param pGip Pointer to the read-write kernel mapping of the GIP.
1995	*/
1996	static void supdrvGipTerm(PSUPGLOBALINFOPAGE pGip)
1997	{
1998	unsigned i;
1999	pGip->u32Magic = 0;
2000	for (i = 0; i < pGip->cCpus; i++)
2001	{
2002	pGip->aCPUs[i].u64NanoTS = 0;
2003	pGip->aCPUs[i].u64TSC = 0;
2004	pGip->aCPUs[i].iTSCHistoryHead = 0;
2005	pGip->aCPUs[i].u64TSCSample = 0;
2006	pGip->aCPUs[i].i64TSCDelta = INT64_MAX;
2007	}
2008	}
2009
2010
2011	/**
2012	* Terminates the GIP.
2013	*
2014	* @param pDevExt Instance data. GIP stuff may be updated.
2015	*/
2016	void VBOXCALL supdrvGipDestroy(PSUPDRVDEVEXT pDevExt)
2017	{
2018	int rc;
2019	#ifdef DEBUG_DARWIN_GIP
2020	OSDBGPRINT(("supdrvGipDestroy: pDevExt=%p pGip=%p pGipTimer=%p GipMemObj=%p\n", pDevExt,
2021	pDevExt->GipMemObj != NIL_RTR0MEMOBJ ? RTR0MemObjAddress(pDevExt->GipMemObj) : NULL,
2022	pDevExt->pGipTimer, pDevExt->GipMemObj));
2023	#endif
2024
2025	/*
2026	* Stop receiving MP notifications before tearing anything else down.
2027	*/
2028	RTMpNotificationDeregister(supdrvGipMpEvent, pDevExt);
2029
2030	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
2031	/*
2032	* Terminate the TSC-delta measurement thread and resources.
2033	*/
2034	supdrvTscDeltaTerm(pDevExt);
2035	#endif
2036
2037	/*
2038	* Destroy the TSC-refinement timer.
2039	*/
2040	if (pDevExt->pInvarTscRefineTimer)
2041	{
2042	RTTimerDestroy(pDevExt->pInvarTscRefineTimer);
2043	pDevExt->pInvarTscRefineTimer = NULL;
2044	}
2045
2046	/*
2047	* Invalid the GIP data.
2048	*/
2049	if (pDevExt->pGip)
2050	{
2051	supdrvGipTerm(pDevExt->pGip);
2052	pDevExt->pGip = NULL;
2053	}
2054	g_pSUPGlobalInfoPage = NULL;
2055
2056	/*
2057	* Destroy the timer and free the GIP memory object.
2058	*/
2059	if (pDevExt->pGipTimer)
2060	{
2061	rc = RTTimerDestroy(pDevExt->pGipTimer); AssertRC(rc);
2062	pDevExt->pGipTimer = NULL;
2063	}
2064
2065	if (pDevExt->GipMemObj != NIL_RTR0MEMOBJ)
2066	{
2067	rc = RTR0MemObjFree(pDevExt->GipMemObj, true /* free mappings */); AssertRC(rc);
2068	pDevExt->GipMemObj = NIL_RTR0MEMOBJ;
2069	}
2070
2071	/*
2072	* Finally, make sure we've release the system timer resolution request
2073	* if one actually succeeded and is still pending.
2074	*/
2075	supdrvGipReleaseHigherTimerFrequencyFromSystem(pDevExt);
2076	}
2077
2078
2079
2080
2081	/*
2082	*
2083	*
2084	* GIP Update Timer Related Code
2085	* GIP Update Timer Related Code
2086	* GIP Update Timer Related Code
2087	*
2088	*
2089	*/
2090
2091
2092	/**
2093	* Worker routine for supdrvGipUpdate() and supdrvGipUpdatePerCpu() that
2094	* updates all the per cpu data except the transaction id.
2095	*
2096	* @param pDevExt The device extension.
2097	* @param pGipCpu Pointer to the per cpu data.
2098	* @param u64NanoTS The current time stamp.
2099	* @param u64TSC The current TSC.
2100	* @param iTick The current timer tick.
2101	*
2102	* @remarks Can be called with interrupts disabled!
2103	*/
2104	static void supdrvGipDoUpdateCpu(PSUPDRVDEVEXT pDevExt, PSUPGIPCPU pGipCpu, uint64_t u64NanoTS, uint64_t u64TSC, uint64_t iTick)
2105	{
2106	uint64_t u64TSCDelta;
2107	bool fUpdateCpuHz;
2108	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
2109	AssertPtrReturnVoid(pGip);
2110
2111	/* Delta between this and the previous update. */
2112	ASMAtomicUoWriteU32(&pGipCpu->u32PrevUpdateIntervalNS, (uint32_t)(u64NanoTS - pGipCpu->u64NanoTS));
2113
2114	/*
2115	* Update the NanoTS.
2116	*/
2117	ASMAtomicWriteU64(&pGipCpu->u64NanoTS, u64NanoTS);
2118
2119	/*
2120	* Calc TSC delta.
2121	*/
2122	u64TSCDelta = u64TSC - pGipCpu->u64TSC;
2123	ASMAtomicWriteU64(&pGipCpu->u64TSC, u64TSC);
2124
2125	/*
2126	* Determine if we need to update the CPU (TSC) frequency calculation.
2127	*
2128	* We don't need to keep recalculating the frequency when it's invariant,
2129	* unless the special tstGIP-2 testing mode is enabled.
2130	*/
2131	fUpdateCpuHz = pGip->u32Mode != SUPGIPMODE_INVARIANT_TSC;
2132	if (!(pGip->fFlags & SUPGIP_FLAGS_TESTING))
2133	{ /* likely*/ }
2134	else
2135	{
2136	uint32_t fGipFlags = pGip->fFlags;
2137	if (fGipFlags & (SUPGIP_FLAGS_TESTING_ENABLE \| SUPGIP_FLAGS_TESTING_START))
2138	{
2139	if (fGipFlags & SUPGIP_FLAGS_TESTING_START)
2140	{
2141	/* Cache the TSC frequency before forcing updates due to test mode. */
2142	if (!fUpdateCpuHz)
2143	pDevExt->uGipTestModeInvariantCpuHz = pGip->aCPUs[0].u64CpuHz;
2144	ASMAtomicAndU32(&pGip->fFlags, ~SUPGIP_FLAGS_TESTING_START);
2145	}
2146	fUpdateCpuHz = true;
2147	}
2148	else if (fGipFlags & SUPGIP_FLAGS_TESTING_STOP)
2149	{
2150	/* Restore the cached TSC frequency if any. */
2151	if (!fUpdateCpuHz)
2152	{
2153	Assert(pDevExt->uGipTestModeInvariantCpuHz);
2154	ASMAtomicWriteU64(&pGip->aCPUs[0].u64CpuHz, pDevExt->uGipTestModeInvariantCpuHz);
2155	}
2156	ASMAtomicAndU32(&pGip->fFlags, ~(SUPGIP_FLAGS_TESTING_STOP \| SUPGIP_FLAGS_TESTING));
2157	}
2158	}
2159
2160	/*
2161	* Calculate the CPU (TSC) frequency if necessary.
2162	*/
2163	if (fUpdateCpuHz)
2164	{
2165	uint64_t u64CpuHz;
2166	uint32_t u32UpdateIntervalTSC;
2167	uint32_t u32UpdateIntervalTSCSlack;
2168	uint32_t u32TransactionId;
2169	unsigned iTSCHistoryHead;
2170
2171	if (u64TSCDelta >> 32)
2172	{
2173	u64TSCDelta = pGipCpu->u32UpdateIntervalTSC;
2174	pGipCpu->cErrors++;
2175	}
2176
2177	/*
2178	* On the 2nd and 3rd callout, reset the history with the current TSC
2179	* interval since the values entered by supdrvGipInit are totally off.
2180	* The interval on the 1st callout completely unreliable, the 2nd is a bit
2181	* better, while the 3rd should be most reliable.
2182	*/
2183	/** @todo Could we drop this now that we initializes the history
2184	* with nominal TSC frequency values? */
2185	u32TransactionId = pGipCpu->u32TransactionId;
2186	if (RT_UNLIKELY( ( u32TransactionId == 5
2187	\|\| u32TransactionId == 7)
2188	&& ( iTick == 2
2189	\|\| iTick == 3) ))
2190	{
2191	unsigned i;
2192	for (i = 0; i < RT_ELEMENTS(pGipCpu->au32TSCHistory); i++)
2193	ASMAtomicUoWriteU32(&pGipCpu->au32TSCHistory[i], (uint32_t)u64TSCDelta);
2194	}
2195
2196	/*
2197	* Validate the NanoTS deltas between timer fires with an arbitrary threshold of 0.5%.
2198	* Wait until we have at least one full history since the above history reset. The
2199	* assumption is that the majority of the previous history values will be tolerable.
2200	* See @bugref{6710#c67}.
2201	*/
2202	/** @todo Could we drop the fudging there now that we initializes the history
2203	* with nominal TSC frequency values? */
2204	if ( u32TransactionId > 23 /* 7 + (8 * 2) */
2205	&& pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)
2206	{
2207	uint32_t uNanoTsThreshold = pGip->u32UpdateIntervalNS / 200;
2208	if ( pGipCpu->u32PrevUpdateIntervalNS > pGip->u32UpdateIntervalNS + uNanoTsThreshold
2209	\|\| pGipCpu->u32PrevUpdateIntervalNS < pGip->u32UpdateIntervalNS - uNanoTsThreshold)
2210	{
2211	uint32_t u32;
2212	u32 = pGipCpu->au32TSCHistory[0];
2213	u32 += pGipCpu->au32TSCHistory[1];
2214	u32 += pGipCpu->au32TSCHistory[2];
2215	u32 += pGipCpu->au32TSCHistory[3];
2216	u32 >>= 2;
2217	u64TSCDelta = pGipCpu->au32TSCHistory[4];
2218	u64TSCDelta += pGipCpu->au32TSCHistory[5];
2219	u64TSCDelta += pGipCpu->au32TSCHistory[6];
2220	u64TSCDelta += pGipCpu->au32TSCHistory[7];
2221	u64TSCDelta >>= 2;
2222	u64TSCDelta += u32;
2223	u64TSCDelta >>= 1;
2224	}
2225	}
2226
2227	/*
2228	* TSC History.
2229	*/
2230	Assert(RT_ELEMENTS(pGipCpu->au32TSCHistory) == 8);
2231	iTSCHistoryHead = (pGipCpu->iTSCHistoryHead + 1) & 7;
2232	ASMAtomicWriteU32(&pGipCpu->iTSCHistoryHead, iTSCHistoryHead);
2233	ASMAtomicWriteU32(&pGipCpu->au32TSCHistory[iTSCHistoryHead], (uint32_t)u64TSCDelta);
2234
2235	/*
2236	* UpdateIntervalTSC = average of last 8,2,1 intervals depending on update HZ.
2237	*
2238	* On Windows, we have an occasional (but recurring) sour value that messed up
2239	* the history but taking only 1 interval reduces the precision overall.
2240	*/
2241	if ( pGip->u32Mode == SUPGIPMODE_INVARIANT_TSC
2242	\|\| pGip->u32UpdateHz >= 1000)
2243	{
2244	uint32_t u32;
2245	u32 = pGipCpu->au32TSCHistory[0];
2246	u32 += pGipCpu->au32TSCHistory[1];
2247	u32 += pGipCpu->au32TSCHistory[2];
2248	u32 += pGipCpu->au32TSCHistory[3];
2249	u32 >>= 2;
2250	u32UpdateIntervalTSC = pGipCpu->au32TSCHistory[4];
2251	u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[5];
2252	u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[6];
2253	u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[7];
2254	u32UpdateIntervalTSC >>= 2;
2255	u32UpdateIntervalTSC += u32;
2256	u32UpdateIntervalTSC >>= 1;
2257
2258	/* Value chosen for a 2GHz Athlon64 running linux 2.6.10/11. */
2259	u32UpdateIntervalTSCSlack = u32UpdateIntervalTSC >> 14;
2260	}
2261	else if (pGip->u32UpdateHz >= 90)
2262	{
2263	u32UpdateIntervalTSC = (uint32_t)u64TSCDelta;
2264	u32UpdateIntervalTSC += pGipCpu->au32TSCHistory[(iTSCHistoryHead - 1) & 7];
2265	u32UpdateIntervalTSC >>= 1;
2266
2267	/* value chosen on a 2GHz thinkpad running windows */
2268	u32UpdateIntervalTSCSlack = u32UpdateIntervalTSC >> 7;
2269	}
2270	else
2271	{
2272	u32UpdateIntervalTSC = (uint32_t)u64TSCDelta;
2273
2274	/* This value hasn't be checked yet.. waiting for OS/2 and 33Hz timers.. :-) */
2275	u32UpdateIntervalTSCSlack = u32UpdateIntervalTSC >> 6;
2276	}
2277	ASMAtomicWriteU32(&pGipCpu->u32UpdateIntervalTSC, u32UpdateIntervalTSC + u32UpdateIntervalTSCSlack);
2278
2279	/*
2280	* CpuHz.
2281	*/
2282	u64CpuHz = ASMMult2xU32RetU64(u32UpdateIntervalTSC, RT_NS_1SEC);
2283	u64CpuHz /= pGip->u32UpdateIntervalNS;
2284	ASMAtomicWriteU64(&pGipCpu->u64CpuHz, u64CpuHz);
2285	}
2286	}
2287
2288
2289	/**
2290	* Updates the GIP.
2291	*
2292	* @param pDevExt The device extension.
2293	* @param u64NanoTS The current nanosecond timestamp.
2294	* @param u64TSC The current TSC timestamp.
2295	* @param idCpu The CPU ID.
2296	* @param iTick The current timer tick.
2297	*
2298	* @remarks Can be called with interrupts disabled!
2299	*/
2300	static void supdrvGipUpdate(PSUPDRVDEVEXT pDevExt, uint64_t u64NanoTS, uint64_t u64TSC, RTCPUID idCpu, uint64_t iTick)
2301	{
2302	/*
2303	* Determine the relevant CPU data.
2304	*/
2305	PSUPGIPCPU pGipCpu;
2306	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
2307	AssertPtrReturnVoid(pGip);
2308
2309	if (pGip->u32Mode != SUPGIPMODE_ASYNC_TSC)
2310	pGipCpu = &pGip->aCPUs[0];
2311	else
2312	{
2313	unsigned iCpu = pGip->aiCpuFromApicId[ASMGetApicId()];
2314	if (RT_UNLIKELY(iCpu >= pGip->cCpus))
2315	return;
2316	pGipCpu = &pGip->aCPUs[iCpu];
2317	if (RT_UNLIKELY(pGipCpu->idCpu != idCpu))
2318	return;
2319	}
2320
2321	/*
2322	* Start update transaction.
2323	*/
2324	if (!(ASMAtomicIncU32(&pGipCpu->u32TransactionId) & 1))
2325	{
2326	/* this can happen on win32 if we're taking to long and there are more CPUs around. shouldn't happen though. */
2327	AssertMsgFailed(("Invalid transaction id, %#x, not odd!\n", pGipCpu->u32TransactionId));
2328	ASMAtomicIncU32(&pGipCpu->u32TransactionId);
2329	pGipCpu->cErrors++;
2330	return;
2331	}
2332
2333	/*
2334	* Recalc the update frequency every 0x800th time.
2335	*/
2336	if ( pGip->u32Mode != SUPGIPMODE_INVARIANT_TSC /* cuz we're not recalculating the frequency on invariant hosts. */
2337	&& !(pGipCpu->u32TransactionId & (GIP_UPDATEHZ_RECALC_FREQ * 2 - 2)))
2338	{
2339	if (pGip->u64NanoTSLastUpdateHz)
2340	{
2341	#ifdef RT_ARCH_AMD64 /** @todo fix 64-bit div here to work on x86 linux. */
2342	uint64_t u64Delta = u64NanoTS - pGip->u64NanoTSLastUpdateHz;
2343	uint32_t u32UpdateHz = (uint32_t)((RT_NS_1SEC_64 * GIP_UPDATEHZ_RECALC_FREQ) / u64Delta);
2344	if (u32UpdateHz <= 2000 && u32UpdateHz >= 30)
2345	{
2346	/** @todo r=ramshankar: Changing u32UpdateHz might screw up TSC frequency
2347	* calculation on non-invariant hosts if it changes the history decision
2348	* taken in supdrvGipDoUpdateCpu(). */
2349	uint64_t u64Interval = u64Delta / GIP_UPDATEHZ_RECALC_FREQ;
2350	ASMAtomicWriteU32(&pGip->u32UpdateHz, u32UpdateHz);
2351	ASMAtomicWriteU32(&pGip->u32UpdateIntervalNS, (uint32_t)u64Interval);
2352	}
2353	#endif
2354	}
2355	ASMAtomicWriteU64(&pGip->u64NanoTSLastUpdateHz, u64NanoTS \| 1);
2356	}
2357
2358	/*
2359	* Update the data.
2360	*/
2361	supdrvGipDoUpdateCpu(pDevExt, pGipCpu, u64NanoTS, u64TSC, iTick);
2362
2363	/*
2364	* Complete transaction.
2365	*/
2366	ASMAtomicIncU32(&pGipCpu->u32TransactionId);
2367	}
2368
2369
2370	/**
2371	* Updates the per cpu GIP data for the calling cpu.
2372	*
2373	* @param pDevExt The device extension.
2374	* @param u64NanoTS The current nanosecond timestamp.
2375	* @param u64TSC The current TSC timesaver.
2376	* @param idCpu The CPU ID.
2377	* @param idApic The APIC id for the CPU index.
2378	* @param iTick The current timer tick.
2379	*
2380	* @remarks Can be called with interrupts disabled!
2381	*/
2382	static void supdrvGipUpdatePerCpu(PSUPDRVDEVEXT pDevExt, uint64_t u64NanoTS, uint64_t u64TSC,
2383	RTCPUID idCpu, uint8_t idApic, uint64_t iTick)
2384	{
2385	uint32_t iCpu;
2386	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
2387
2388	/*
2389	* Avoid a potential race when a CPU online notification doesn't fire on
2390	* the onlined CPU but the tick creeps in before the event notification is
2391	* run.
2392	*/
2393	if (RT_UNLIKELY(iTick == 1))
2394	{
2395	iCpu = supdrvGipFindOrAllocCpuIndexForCpuId(pGip, idCpu);
2396	if (pGip->aCPUs[iCpu].enmState == SUPGIPCPUSTATE_OFFLINE)
2397	supdrvGipMpEventOnlineOrInitOnCpu(pDevExt, idCpu);
2398	}
2399
2400	iCpu = pGip->aiCpuFromApicId[idApic];
2401	if (RT_LIKELY(iCpu < pGip->cCpus))
2402	{
2403	PSUPGIPCPU pGipCpu = &pGip->aCPUs[iCpu];
2404	if (pGipCpu->idCpu == idCpu)
2405	{
2406	/*
2407	* Start update transaction.
2408	*/
2409	if (!(ASMAtomicIncU32(&pGipCpu->u32TransactionId) & 1))
2410	{
2411	AssertMsgFailed(("Invalid transaction id, %#x, not odd!\n", pGipCpu->u32TransactionId));
2412	ASMAtomicIncU32(&pGipCpu->u32TransactionId);
2413	pGipCpu->cErrors++;
2414	return;
2415	}
2416
2417	/*
2418	* Update the data.
2419	*/
2420	supdrvGipDoUpdateCpu(pDevExt, pGipCpu, u64NanoTS, u64TSC, iTick);
2421
2422	/*
2423	* Complete transaction.
2424	*/
2425	ASMAtomicIncU32(&pGipCpu->u32TransactionId);
2426	}
2427	}
2428	}
2429
2430
2431	/**
2432	* Timer callback function for the sync and invariant GIP modes.
2433	*
2434	* @param pTimer The timer.
2435	* @param pvUser Opaque pointer to the device extension.
2436	* @param iTick The timer tick.
2437	*/
2438	static DECLCALLBACK(void) supdrvGipSyncAndInvariantTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
2439	{
2440	PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
2441	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
2442	RTCCUINTREG fEFlags = ASMIntDisableFlags(); /* No interruptions please (real problem on S10). */
2443	uint64_t u64TSC = ASMReadTSC();
2444	uint64_t u64NanoTS = RTTimeSystemNanoTS();
2445	RT_NOREF1(pTimer);
2446
2447	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
2448	{
2449	/*
2450	* The calculations in supdrvGipUpdate() is somewhat timing sensitive,
2451	* missing timer ticks is not an option for GIP because the GIP users
2452	* will end up incrementing the time in 1ns per time getter call until
2453	* there is a complete timer update. So, if the delta has yet to be
2454	* calculated, we just pretend it is zero for now (the GIP users
2455	* probably won't have it for a wee while either and will do the same).
2456	*
2457	* We could maybe on some platforms try cross calling a CPU with a
2458	* working delta here, but it's not worth the hassle since the
2459	* likelihood of this happening is really low. On Windows, Linux, and
2460	* Solaris timers fire on the CPU they were registered/started on.
2461	* Darwin timers doesn't necessarily (they are high priority threads).
2462	*/
2463	uint32_t iCpuSet = RTMpCpuIdToSetIndex(RTMpCpuId());
2464	uint16_t iGipCpu = RT_LIKELY(iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx))
2465	? pGip->aiCpuFromCpuSetIdx[iCpuSet] : UINT16_MAX;
2466	Assert(!ASMIntAreEnabled());
2467	if (RT_LIKELY(iGipCpu < pGip->cCpus))
2468	{
2469	int64_t iTscDelta = pGip->aCPUs[iGipCpu].i64TSCDelta;
2470	if (iTscDelta != INT64_MAX)
2471	u64TSC -= iTscDelta;
2472	}
2473	}
2474
2475	supdrvGipUpdate(pDevExt, u64NanoTS, u64TSC, NIL_RTCPUID, iTick);
2476
2477	ASMSetFlags(fEFlags);
2478	}
2479
2480
2481	/**
2482	* Timer callback function for async GIP mode.
2483	* @param pTimer The timer.
2484	* @param pvUser Opaque pointer to the device extension.
2485	* @param iTick The timer tick.
2486	*/
2487	static DECLCALLBACK(void) supdrvGipAsyncTimer(PRTTIMER pTimer, void *pvUser, uint64_t iTick)
2488	{
2489	PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
2490	RTCCUINTREG fEFlags = ASMIntDisableFlags(); /* No interruptions please (real problem on S10). */
2491	RTCPUID idCpu = RTMpCpuId();
2492	uint64_t u64TSC = ASMReadTSC();
2493	uint64_t NanoTS = RTTimeSystemNanoTS();
2494	RT_NOREF1(pTimer);
2495
2496	/** @todo reset the transaction number and whatnot when iTick == 1. */
2497	if (pDevExt->idGipMaster == idCpu)
2498	supdrvGipUpdate(pDevExt, NanoTS, u64TSC, idCpu, iTick);
2499	else
2500	supdrvGipUpdatePerCpu(pDevExt, NanoTS, u64TSC, idCpu, ASMGetApicId(), iTick);
2501
2502	ASMSetFlags(fEFlags);
2503	}
2504
2505
2506
2507
2508	/*
2509	*
2510	*
2511	* TSC Delta Measurements And Related Code
2512	* TSC Delta Measurements And Related Code
2513	* TSC Delta Measurements And Related Code
2514	*
2515	*
2516	*/
2517
2518
2519	/*
2520	* Select TSC delta measurement algorithm.
2521	*/
2522	#if 0
2523	# define GIP_TSC_DELTA_METHOD_1
2524	#else
2525	# define GIP_TSC_DELTA_METHOD_2
2526	#endif
2527
2528	/** For padding variables to keep them away from other cache lines. Better too
2529	* large than too small!
2530	* @remarks Current AMD64 and x86 CPUs seems to use 64 bytes. There are claims
2531	* that NetBurst had 128 byte cache lines while the 486 thru Pentium
2532	* III had 32 bytes cache lines. */
2533	#define GIP_TSC_DELTA_CACHE_LINE_SIZE 128
2534
2535
2536	/**
2537	* TSC delta measurement algorithm \#2 result entry.
2538	*/
2539	typedef struct SUPDRVTSCDELTAMETHOD2ENTRY
2540	{
2541	uint32_t iSeqMine;
2542	uint32_t iSeqOther;
2543	uint64_t uTsc;
2544	} SUPDRVTSCDELTAMETHOD2ENTRY;
2545
2546	/**
2547	* TSC delta measurement algorithm \#2 Data.
2548	*/
2549	typedef struct SUPDRVTSCDELTAMETHOD2
2550	{
2551	/** Padding to make sure the iCurSeqNo is in its own cache line. */
2552	uint64_t au64CacheLinePaddingBefore[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t)];
2553	/** The current sequence number of this worker. */
2554	uint32_t volatile iCurSeqNo;
2555	/** Padding to make sure the iCurSeqNo is in its own cache line. */
2556	uint32_t au64CacheLinePaddingAfter[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint32_t) - 1];
2557	/** Result table. */
2558	SUPDRVTSCDELTAMETHOD2ENTRY aResults[64];
2559	} SUPDRVTSCDELTAMETHOD2;
2560	/** Pointer to the data for TSC delta measurement algorithm \#2 .*/
2561	typedef SUPDRVTSCDELTAMETHOD2 *PSUPDRVTSCDELTAMETHOD2;
2562
2563
2564	/**
2565	* The TSC delta synchronization struct, version 2.
2566	*
2567	* The synchronization variable is completely isolated in its own cache line
2568	* (provided our max cache line size estimate is correct).
2569	*/
2570	typedef struct SUPTSCDELTASYNC2
2571	{
2572	/** Padding to make sure the uVar1 is in its own cache line. */
2573	uint64_t au64CacheLinePaddingBefore[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t)];
2574
2575	/** The synchronization variable, holds values GIP_TSC_DELTA_SYNC_. /
2576	volatile uint32_t uSyncVar;
2577	/** Sequence synchronizing variable used for post 'GO' synchronization. */
2578	volatile uint32_t uSyncSeq;
2579
2580	/** Padding to make sure the uVar1 is in its own cache line. */
2581	uint64_t au64CacheLinePaddingAfter[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t) - 2];
2582
2583	/** Start RDTSC value. Put here mainly to save stack space. */
2584	uint64_t uTscStart;
2585	/** Copy of SUPDRVGIPTSCDELTARGS::cMaxTscTicks. */
2586	uint64_t cMaxTscTicks;
2587	} SUPTSCDELTASYNC2;
2588	AssertCompileSize(SUPTSCDELTASYNC2, GIP_TSC_DELTA_CACHE_LINE_SIZE * 2 + sizeof(uint64_t));
2589	typedef SUPTSCDELTASYNC2 *PSUPTSCDELTASYNC2;
2590
2591	/** Prestart wait. */
2592	#define GIP_TSC_DELTA_SYNC2_PRESTART_WAIT UINT32_C(0x0ffe)
2593	/** Prestart aborted. */
2594	#define GIP_TSC_DELTA_SYNC2_PRESTART_ABORT UINT32_C(0x0fff)
2595	/** Ready (on your mark). */
2596	#define GIP_TSC_DELTA_SYNC2_READY UINT32_C(0x1000)
2597	/** Steady (get set). */
2598	#define GIP_TSC_DELTA_SYNC2_STEADY UINT32_C(0x1001)
2599	/** Go! */
2600	#define GIP_TSC_DELTA_SYNC2_GO UINT32_C(0x1002)
2601	/** Used by the verification test. */
2602	#define GIP_TSC_DELTA_SYNC2_GO_GO UINT32_C(0x1003)
2603
2604	/** We reached the time limit. */
2605	#define GIP_TSC_DELTA_SYNC2_TIMEOUT UINT32_C(0x1ffe)
2606	/** The other party won't touch the sync struct ever again. */
2607	#define GIP_TSC_DELTA_SYNC2_FINAL UINT32_C(0x1fff)
2608
2609
2610	/**
2611	* Argument package/state passed by supdrvMeasureTscDeltaOne() to the RTMpOn
2612	* callback worker.
2613	* @todo add
2614	*/
2615	typedef struct SUPDRVGIPTSCDELTARGS
2616	{
2617	/** The device extension. */
2618	PSUPDRVDEVEXT pDevExt;
2619	/** Pointer to the GIP CPU array entry for the worker. */
2620	PSUPGIPCPU pWorker;
2621	/** Pointer to the GIP CPU array entry for the master. */
2622	PSUPGIPCPU pMaster;
2623	/** The maximum number of ticks to spend in supdrvMeasureTscDeltaCallback.
2624	* (This is what we need a rough TSC frequency for.) */
2625	uint64_t cMaxTscTicks;
2626	/** Used to abort synchronization setup. */
2627	bool volatile fAbortSetup;
2628
2629	/** Padding to make sure the master variables live in its own cache lines. */
2630	uint64_t au64CacheLinePaddingBefore[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t)];
2631
2632	/** @name Master
2633	* @{ */
2634	/** The time the master spent in the MP worker. */
2635	uint64_t cElapsedMasterTscTicks;
2636	/** The iTry value when stopped at. */
2637	uint32_t iTry;
2638	/** Set if the run timed out. */
2639	bool volatile fTimedOut;
2640	/** Pointer to the master's synchronization struct (on stack). */
2641	PSUPTSCDELTASYNC2 volatile pSyncMaster;
2642	/** Master data union. */
2643	union
2644	{
2645	/** Data (master) for delta verification. */
2646	struct
2647	{
2648	/** Verification test TSC values for the master. */
2649	uint64_t volatile auTscs[32];
2650	} Verify;
2651	/** Data (master) for measurement method \#2. */
2652	struct
2653	{
2654	/** Data and sequence number. */
2655	SUPDRVTSCDELTAMETHOD2 Data;
2656	/** The lag setting for the next run. */
2657	bool fLag;
2658	/** Number of hits. */
2659	uint32_t cHits;
2660	} M2;
2661	} uMaster;
2662	/** The verifier verdict, VINF_SUCCESS if ok, VERR_OUT_OF_RANGE if not,
2663	* VERR_TRY_AGAIN on timeout. */
2664	int32_t rcVerify;
2665	#ifdef TSCDELTA_VERIFY_WITH_STATS
2666	/** The maximum difference between TSC read during delta verification. */
2667	int64_t cMaxVerifyTscTicks;
2668	/** The minimum difference between two TSC reads during verification. */
2669	int64_t cMinVerifyTscTicks;
2670	/** The bad TSC diff, worker relative to master (= worker - master).
2671	* Negative value means the worker is behind the master. */
2672	int64_t iVerifyBadTscDiff;
2673	#endif
2674	/** @} */
2675
2676	/** Padding to make sure the worker variables live is in its own cache line. */
2677	uint64_t au64CacheLinePaddingBetween[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t)];
2678
2679	/** @name Proletarian
2680	* @{ */
2681	/** Pointer to the worker's synchronization struct (on stack). */
2682	PSUPTSCDELTASYNC2 volatile pSyncWorker;
2683	/** The time the worker spent in the MP worker. */
2684	uint64_t cElapsedWorkerTscTicks;
2685	/** Worker data union. */
2686	union
2687	{
2688	/** Data (worker) for delta verification. */
2689	struct
2690	{
2691	/** Verification test TSC values for the worker. */
2692	uint64_t volatile auTscs[32];
2693	} Verify;
2694	/** Data (worker) for measurement method \#2. */
2695	struct
2696	{
2697	/** Data and sequence number. */
2698	SUPDRVTSCDELTAMETHOD2 Data;
2699	/** The lag setting for the next run (set by master). */
2700	bool fLag;
2701	} M2;
2702	} uWorker;
2703	/** @} */
2704
2705	/** Padding to make sure the above is in its own cache line. */
2706	uint64_t au64CacheLinePaddingAfter[GIP_TSC_DELTA_CACHE_LINE_SIZE / sizeof(uint64_t)];
2707	} SUPDRVGIPTSCDELTARGS;
2708	typedef SUPDRVGIPTSCDELTARGS *PSUPDRVGIPTSCDELTARGS;
2709
2710
2711	/** @name Macros that implements the basic synchronization steps common to
2712	* the algorithms.
2713	*
2714	* Must be used from loop as the timeouts are implemented via 'break' statements
2715	* at the moment.
2716	*
2717	* @{
2718	*/
2719	#if defined(DEBUG_bird) /* \|\| defined(VBOX_STRICT) */
2720	# define TSCDELTA_DBG_VARS() uint32_t iDbgCounter
2721	# define TSCDELTA_DBG_START_LOOP() do { iDbgCounter = 0; } while (0)
2722	# define TSCDELTA_DBG_CHECK_LOOP() \
2723	do { iDbgCounter++; if ((iDbgCounter & UINT32_C(0x01ffffff)) == 0) RT_BREAKPOINT(); } while (0)
2724	#else
2725	# define TSCDELTA_DBG_VARS() ((void)0)
2726	# define TSCDELTA_DBG_START_LOOP() ((void)0)
2727	# define TSCDELTA_DBG_CHECK_LOOP() ((void)0)
2728	#endif
2729	#if 0
2730	# define TSCDELTA_DBG_SYNC_MSG(a_Args) SUPR0Printf a_Args
2731	#else
2732	# define TSCDELTA_DBG_SYNC_MSG(a_Args) ((void)0)
2733	#endif
2734	#if 0
2735	# define TSCDELTA_DBG_SYNC_MSG2(a_Args) SUPR0Printf a_Args
2736	#else
2737	# define TSCDELTA_DBG_SYNC_MSG2(a_Args) ((void)0)
2738	#endif
2739	#if 0
2740	# define TSCDELTA_DBG_SYNC_MSG9(a_Args) SUPR0Printf a_Args
2741	#else
2742	# define TSCDELTA_DBG_SYNC_MSG9(a_Args) ((void)0)
2743	#endif
2744
2745
2746	static bool supdrvTscDeltaSync2_Before(PSUPTSCDELTASYNC2 pMySync, PSUPTSCDELTASYNC2 pOtherSync,
2747	bool fIsMaster, PRTCCUINTREG pfEFlags, PSUPDRVGIPTSCDELTARGS pArgs)
2748	{
2749	uint32_t iMySeq = fIsMaster ? 0 : 256;
2750	uint32_t const iMaxSeq = iMySeq + 16; /* For the last loop, darn linux/freebsd C-ishness. */
2751	uint32_t u32Tmp;
2752	uint32_t iSync2Loops = 0;
2753	RTCCUINTREG fEFlags;
2754	TSCDELTA_DBG_VARS();
2755
2756	pfEFlags = X86_EFL_IF \| X86_EFL_1; / should shut up most nagging compilers. */
2757
2758	/*
2759	* The master tells the worker to get on it's mark.
2760	*/
2761	if (fIsMaster)
2762	{
2763	if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_STEADY, GIP_TSC_DELTA_SYNC2_READY)))
2764	{ /* likely*/ }
2765	else
2766	{
2767	TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #1 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
2768	return false;
2769	}
2770	}
2771
2772	/*
2773	* Wait for the on your mark signal (ack in the master case). We process timeouts here.
2774	*/
2775	ASMAtomicWriteU32(&(pMySync)->uSyncSeq, 0);
2776	for (;;)
2777	{
2778	fEFlags = ASMIntDisableFlags();
2779	u32Tmp = ASMAtomicReadU32(&pMySync->uSyncVar);
2780	if (u32Tmp == GIP_TSC_DELTA_SYNC2_STEADY)
2781	break;
2782	ASMSetFlags(fEFlags);
2783	ASMNopPause();
2784
2785	/* Abort? */
2786	if (u32Tmp != GIP_TSC_DELTA_SYNC2_READY)
2787	{
2788	TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #2 u32Tmp=%#x\n", fIsMaster ? "master" : "worker", u32Tmp));
2789	return false;
2790	}
2791
2792	/* Check for timeouts every so often (not every loop in case RDTSC is
2793	trapping or something). Must check the first time around. */
2794	#if 0 /* For debugging the timeout paths. */
2795	static uint32_t volatile xxx;
2796	#endif
2797	if ( ( (iSync2Loops & 0x3ff) == 0
2798	&& ASMReadTSC() - pMySync->uTscStart > pMySync->cMaxTscTicks)
2799	#if 0 /* This is crazy, I know, but enable this code and the results are markedly better when enabled on the 1.4GHz AMD (debug). */
2800	\|\| (!fIsMaster && (++xxx & 0xf) == 0)
2801	#endif
2802	)
2803	{
2804	/* Try switch our own state into timeout mode so the master cannot tell us to 'GO',
2805	ignore the timeout if we've got the go ahead already (simpler). */
2806	if (ASMAtomicCmpXchgU32(&pMySync->uSyncVar, GIP_TSC_DELTA_SYNC2_TIMEOUT, GIP_TSC_DELTA_SYNC2_READY))
2807	{
2808	TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: timeout\n", fIsMaster ? "master" : "worker"));
2809	ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_TIMEOUT, GIP_TSC_DELTA_SYNC2_STEADY);
2810	ASMAtomicWriteBool(&pArgs->fTimedOut, true);
2811	return false;
2812	}
2813	}
2814	iSync2Loops++;
2815	}
2816
2817	/*
2818	* Interrupts are now disabled and will remain disabled until we do
2819	* TSCDELTA_MASTER_SYNC_AFTER / TSCDELTA_OTHER_SYNC_AFTER.
2820	*/
2821	*pfEFlags = fEFlags;
2822
2823	/*
2824	* The worker tells the master that it is on its mark and that the master
2825	* need to get into position as well.
2826	*/
2827	if (!fIsMaster)
2828	{
2829	if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_STEADY, GIP_TSC_DELTA_SYNC2_READY)))
2830	{ /* likely */ }
2831	else
2832	{
2833	ASMSetFlags(fEFlags);
2834	TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #3 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
2835	return false;
2836	}
2837	}
2838
2839	/*
2840	* The master sends the 'go' to the worker and wait for ACK.
2841	*/
2842	if (fIsMaster)
2843	{
2844	if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO, GIP_TSC_DELTA_SYNC2_STEADY)))
2845	{ /* likely */ }
2846	else
2847	{
2848	ASMSetFlags(fEFlags);
2849	TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #4 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
2850	return false;
2851	}
2852	}
2853
2854	/*
2855	* Wait for the 'go' signal (ack in the master case).
2856	*/
2857	TSCDELTA_DBG_START_LOOP();
2858	for (;;)
2859	{
2860	u32Tmp = ASMAtomicReadU32(&pMySync->uSyncVar);
2861	if (u32Tmp == GIP_TSC_DELTA_SYNC2_GO)
2862	break;
2863	if (RT_LIKELY(u32Tmp == GIP_TSC_DELTA_SYNC2_STEADY))
2864	{ /* likely */ }
2865	else
2866	{
2867	ASMSetFlags(fEFlags);
2868	TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #5 u32Tmp=%#x\n", fIsMaster ? "master" : "worker", u32Tmp));
2869	return false;
2870	}
2871
2872	TSCDELTA_DBG_CHECK_LOOP();
2873	ASMNopPause();
2874	}
2875
2876	/*
2877	* The worker acks the 'go' (shouldn't fail).
2878	*/
2879	if (!fIsMaster)
2880	{
2881	if (RT_LIKELY(ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO, GIP_TSC_DELTA_SYNC2_STEADY)))
2882	{ /* likely */ }
2883	else
2884	{
2885	ASMSetFlags(fEFlags);
2886	TSCDELTA_DBG_SYNC_MSG(("sync/before/%s: #6 uSyncVar=%#x\n", fIsMaster ? "master" : "worker", pOtherSync->uSyncVar));
2887	return false;
2888	}
2889	}
2890
2891	/*
2892	* Try enter mostly lockstep execution with it.
2893	*/
2894	for (;;)
2895	{
2896	uint32_t iOtherSeq1, iOtherSeq2;
2897	ASMCompilerBarrier();
2898	ASMSerializeInstruction();
2899
2900	ASMAtomicWriteU32(&pMySync->uSyncSeq, iMySeq);
2901	ASMNopPause();
2902	iOtherSeq1 = ASMAtomicXchgU32(&pOtherSync->uSyncSeq, iMySeq);
2903	ASMNopPause();
2904	iOtherSeq2 = ASMAtomicReadU32(&pMySync->uSyncSeq);
2905
2906	ASMCompilerBarrier();
2907	if (iOtherSeq1 == iOtherSeq2)
2908	return true;
2909
2910	/* Did the other guy give up? Should we give up? */
2911	if ( iOtherSeq1 == UINT32_MAX
2912	\|\| iOtherSeq2 == UINT32_MAX)
2913	return true;
2914	if (++iMySeq >= iMaxSeq)
2915	{
2916	ASMAtomicWriteU32(&pMySync->uSyncSeq, UINT32_MAX);
2917	return true;
2918	}
2919	ASMNopPause();
2920	}
2921	}
2922
2923	#define TSCDELTA_MASTER_SYNC_BEFORE(a_pMySync, a_pOtherSync, a_pfEFlags, a_pArgs) \
2924	if (RT_LIKELY(supdrvTscDeltaSync2_Before(a_pMySync, a_pOtherSync, true /fIsMaster/, a_pfEFlags, a_pArgs))) \
2925	{ /likely/ } \
2926	else if (true) \
2927	{ \
2928	TSCDELTA_DBG_SYNC_MSG9(("sync/before/master: #89\n")); \
2929	break; \
2930	} else do {} while (0)
2931	#define TSCDELTA_OTHER_SYNC_BEFORE(a_pMySync, a_pOtherSync, a_pfEFlags, a_pArgs) \
2932	if (RT_LIKELY(supdrvTscDeltaSync2_Before(a_pMySync, a_pOtherSync, false /fIsMaster/, a_pfEFlags, a_pArgs))) \
2933	{ /likely/ } \
2934	else if (true) \
2935	{ \
2936	TSCDELTA_DBG_SYNC_MSG9(("sync/before/other: #89\n")); \
2937	break; \
2938	} else do {} while (0)
2939
2940
2941	static bool supdrvTscDeltaSync2_After(PSUPTSCDELTASYNC2 pMySync, PSUPTSCDELTASYNC2 pOtherSync,
2942	bool fIsMaster, RTCCUINTREG fEFlags)
2943	{
2944	TSCDELTA_DBG_VARS();
2945	RT_NOREF1(pOtherSync);
2946
2947	/*
2948	* Wait for the 'ready' signal. In the master's case, this means the
2949	* worker has completed its data collection, while in the worker's case it
2950	* means the master is done processing the data and it's time for the next
2951	* loop iteration (or whatever).
2952	*/
2953	ASMSetFlags(fEFlags);
2954	TSCDELTA_DBG_START_LOOP();
2955	for (;;)
2956	{
2957	uint32_t u32Tmp = ASMAtomicReadU32(&pMySync->uSyncVar);
2958	if ( u32Tmp == GIP_TSC_DELTA_SYNC2_READY
2959	\|\| (u32Tmp == GIP_TSC_DELTA_SYNC2_STEADY && !fIsMaster) /* kicked twice => race */ )
2960	return true;
2961	ASMNopPause();
2962	if (RT_LIKELY(u32Tmp == GIP_TSC_DELTA_SYNC2_GO))
2963	{ /* likely */}
2964	else
2965	{
2966	TSCDELTA_DBG_SYNC_MSG(("sync/after/other: #1 u32Tmp=%#x\n", u32Tmp));
2967	return false; /* shouldn't ever happen! */
2968	}
2969	TSCDELTA_DBG_CHECK_LOOP();
2970	ASMNopPause();
2971	}
2972	}
2973
2974	#define TSCDELTA_MASTER_SYNC_AFTER(a_pMySync, a_pOtherSync, a_fEFlags) \
2975	if (RT_LIKELY(supdrvTscDeltaSync2_After(a_pMySync, a_pOtherSync, true /fIsMaster/, a_fEFlags))) \
2976	{ /* likely */ } \
2977	else if (true) \
2978	{ \
2979	TSCDELTA_DBG_SYNC_MSG9(("sync/after/master: #97\n")); \
2980	break; \
2981	} else do {} while (0)
2982
2983	#define TSCDELTA_MASTER_KICK_OTHER_OUT_OF_AFTER(a_pMySync, a_pOtherSync) \
2984	/* \
2985	* Tell the worker that we're done processing the data and ready for the next round. \
2986	*/ \
2987	if (RT_LIKELY(ASMAtomicCmpXchgU32(&(a_pOtherSync)->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_GO))) \
2988	{ /* likely */ } \
2989	else if (true)\
2990	{ \
2991	TSCDELTA_DBG_SYNC_MSG(("sync/after/master: #99 uSyncVar=%#x\n", (a_pOtherSync)->uSyncVar)); \
2992	break; \
2993	} else do {} while (0)
2994
2995	#define TSCDELTA_OTHER_SYNC_AFTER(a_pMySync, a_pOtherSync, a_fEFlags) \
2996	if (true) { \
2997	/* \
2998	* Tell the master that we're done collecting data and wait for the next round to start. \
2999	*/ \
3000	if (RT_LIKELY(ASMAtomicCmpXchgU32(&(a_pOtherSync)->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_GO))) \
3001	{ /* likely */ } \
3002	else \
3003	{ \
3004	ASMSetFlags(a_fEFlags); \
3005	TSCDELTA_DBG_SYNC_MSG(("sync/after/other: #0 uSyncVar=%#x\n", (a_pOtherSync)->uSyncVar)); \
3006	break; \
3007	} \
3008	if (RT_LIKELY(supdrvTscDeltaSync2_After(a_pMySync, a_pOtherSync, false /fIsMaster/, a_fEFlags))) \
3009	{ /* likely */ } \
3010	else \
3011	{ \
3012	TSCDELTA_DBG_SYNC_MSG9(("sync/after/other: #98\n")); \
3013	break; \
3014	} \
3015	} else do {} while (0)
3016	/** @} */
3017
3018
3019	#ifdef GIP_TSC_DELTA_METHOD_1
3020	/**
3021	* TSC delta measurement algorithm \#1 (GIP_TSC_DELTA_METHOD_1).
3022	*
3023	*
3024	* We ignore the first few runs of the loop in order to prime the
3025	* cache. Also, we need to be careful about using 'pause' instruction
3026	* in critical busy-wait loops in this code - it can cause undesired
3027	* behaviour with hyperthreading.
3028	*
3029	* We try to minimize the measurement error by computing the minimum
3030	* read time of the compare statement in the worker by taking TSC
3031	* measurements across it.
3032	*
3033	* It must be noted that the computed minimum read time is mostly to
3034	* eliminate huge deltas when the worker is too early and doesn't by
3035	* itself help produce more accurate deltas. We allow two times the
3036	* computed minimum as an arbitrary acceptable threshold. Therefore,
3037	* it is still possible to get negative deltas where there are none
3038	* when the worker is earlier. As long as these occasional negative
3039	* deltas are lower than the time it takes to exit guest-context and
3040	* the OS to reschedule EMT on a different CPU, we won't expose a TSC
3041	* that jumped backwards. It is due to the existence of the negative
3042	* deltas that we don't recompute the delta with the master and
3043	* worker interchanged to eliminate the remaining measurement error.
3044	*
3045	*
3046	* @param pArgs The argument/state data.
3047	* @param pMySync My synchronization structure.
3048	* @param pOtherSync My partner's synchronization structure.
3049	* @param fIsMaster Set if master, clear if worker.
3050	* @param iTry The attempt number.
3051	*/
3052	static void supdrvTscDeltaMethod1Loop(PSUPDRVGIPTSCDELTARGS pArgs, PSUPTSCDELTASYNC2 pMySync, PSUPTSCDELTASYNC2 pOtherSync,
3053	bool fIsMaster, uint32_t iTry)
3054	{
3055	PSUPGIPCPU pGipCpuWorker = pArgs->pWorker;
3056	PSUPGIPCPU pGipCpuMaster = pArgs->pMaster;
3057	uint64_t uMinCmpReadTime = UINT64_MAX;
3058	unsigned iLoop;
3059	NOREF(iTry);
3060
3061	for (iLoop = 0; iLoop < GIP_TSC_DELTA_LOOPS; iLoop++)
3062	{
3063	RTCCUINTREG fEFlags;
3064	if (fIsMaster)
3065	{
3066	/*
3067	* The master.
3068	*/
3069	AssertMsg(pGipCpuMaster->u64TSCSample == GIP_TSC_DELTA_RSVD,
3070	("%#llx idMaster=%#x idWorker=%#x (idGipMaster=%#x)\n",
3071	pGipCpuMaster->u64TSCSample, pGipCpuMaster->idCpu, pGipCpuWorker->idCpu, pArgs->pDevExt->idGipMaster));
3072	TSCDELTA_MASTER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
3073
3074	do
3075	{
3076	ASMSerializeInstruction();
3077	ASMAtomicWriteU64(&pGipCpuMaster->u64TSCSample, ASMReadTSC());
3078	} while (pGipCpuMaster->u64TSCSample == GIP_TSC_DELTA_RSVD);
3079
3080	TSCDELTA_MASTER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
3081
3082	/* Process the data. */
3083	if (iLoop > GIP_TSC_DELTA_PRIMER_LOOPS + GIP_TSC_DELTA_READ_TIME_LOOPS)
3084	{
3085	if (pGipCpuWorker->u64TSCSample != GIP_TSC_DELTA_RSVD)
3086	{
3087	int64_t iDelta = pGipCpuWorker->u64TSCSample
3088	- (pGipCpuMaster->u64TSCSample - pGipCpuMaster->i64TSCDelta);
3089	if ( iDelta >= GIP_TSC_DELTA_INITIAL_MASTER_VALUE
3090	? iDelta < pGipCpuWorker->i64TSCDelta
3091	: iDelta > pGipCpuWorker->i64TSCDelta \|\| pGipCpuWorker->i64TSCDelta == INT64_MAX)
3092	pGipCpuWorker->i64TSCDelta = iDelta;
3093	}
3094	}
3095
3096	/* Reset our TSC sample and tell the worker to move on. */
3097	ASMAtomicWriteU64(&pGipCpuMaster->u64TSCSample, GIP_TSC_DELTA_RSVD);
3098	TSCDELTA_MASTER_KICK_OTHER_OUT_OF_AFTER(pMySync, pOtherSync);
3099	}
3100	else
3101	{
3102	/*
3103	* The worker.
3104	*/
3105	uint64_t uTscWorker;
3106	uint64_t uTscWorkerFlushed;
3107	uint64_t uCmpReadTime;
3108
3109	ASMAtomicReadU64(&pGipCpuMaster->u64TSCSample); /* Warm the cache line. */
3110	TSCDELTA_OTHER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
3111
3112	/*
3113	* Keep reading the TSC until we notice that the master has read his. Reading
3114	* the TSC -after- the master has updated the memory is way too late. We thus
3115	* compensate by trying to measure how long it took for the worker to notice
3116	* the memory flushed from the master.
3117	*/
3118	do
3119	{
3120	ASMSerializeInstruction();
3121	uTscWorker = ASMReadTSC();
3122	} while (pGipCpuMaster->u64TSCSample == GIP_TSC_DELTA_RSVD);
3123	ASMSerializeInstruction();
3124	uTscWorkerFlushed = ASMReadTSC();
3125
3126	uCmpReadTime = uTscWorkerFlushed - uTscWorker;
3127	if (iLoop > GIP_TSC_DELTA_PRIMER_LOOPS + GIP_TSC_DELTA_READ_TIME_LOOPS)
3128	{
3129	/* This is totally arbitrary a.k.a I don't like it but I have no better ideas for now. */
3130	if (uCmpReadTime < (uMinCmpReadTime << 1))
3131	{
3132	ASMAtomicWriteU64(&pGipCpuWorker->u64TSCSample, uTscWorker);
3133	if (uCmpReadTime < uMinCmpReadTime)
3134	uMinCmpReadTime = uCmpReadTime;
3135	}
3136	else
3137	ASMAtomicWriteU64(&pGipCpuWorker->u64TSCSample, GIP_TSC_DELTA_RSVD);
3138	}
3139	else if (iLoop > GIP_TSC_DELTA_PRIMER_LOOPS)
3140	{
3141	if (uCmpReadTime < uMinCmpReadTime)
3142	uMinCmpReadTime = uCmpReadTime;
3143	}
3144
3145	TSCDELTA_OTHER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
3146	}
3147	}
3148
3149	TSCDELTA_DBG_SYNC_MSG9(("sync/method1loop/%s: #92 iLoop=%u MyState=%#x\n", fIsMaster ? "master" : "worker", iLoop,
3150	pMySync->uSyncVar));
3151
3152	/*
3153	* We must reset the worker TSC sample value in case it gets picked as a
3154	* GIP master later on (it's trashed above, naturally).
3155	*/
3156	if (!fIsMaster)
3157	ASMAtomicWriteU64(&pGipCpuWorker->u64TSCSample, GIP_TSC_DELTA_RSVD);
3158	}
3159	#endif /* GIP_TSC_DELTA_METHOD_1 */
3160
3161
3162	#ifdef GIP_TSC_DELTA_METHOD_2
3163	/*
3164	* TSC delta measurement algorithm \#2 configuration and code - Experimental!!
3165	*/
3166
3167	# define GIP_TSC_DELTA_M2_LOOPS (7 + GIP_TSC_DELTA_M2_PRIMER_LOOPS)
3168	# define GIP_TSC_DELTA_M2_PRIMER_LOOPS 0
3169
3170
3171	static void supdrvTscDeltaMethod2ProcessDataOnMaster(PSUPDRVGIPTSCDELTARGS pArgs)
3172	{
3173	int64_t iMasterTscDelta = pArgs->pMaster->i64TSCDelta;
3174	int64_t iBestDelta = pArgs->pWorker->i64TSCDelta;
3175	uint32_t idxResult;
3176	uint32_t cHits = 0;
3177
3178	/*
3179	* Look for matching entries in the master and worker tables.
3180	*/
3181	for (idxResult = 0; idxResult < RT_ELEMENTS(pArgs->uMaster.M2.Data.aResults); idxResult++)
3182	{
3183	uint32_t idxOther = pArgs->uMaster.M2.Data.aResults[idxResult].iSeqOther;
3184	if (idxOther & 1)
3185	{
3186	idxOther >>= 1;
3187	if (idxOther < RT_ELEMENTS(pArgs->uWorker.M2.Data.aResults))
3188	{
3189	if (pArgs->uWorker.M2.Data.aResults[idxOther].iSeqOther == pArgs->uMaster.M2.Data.aResults[idxResult].iSeqMine)
3190	{
3191	int64_t iDelta;
3192	iDelta = pArgs->uWorker.M2.Data.aResults[idxOther].uTsc
3193	- (pArgs->uMaster.M2.Data.aResults[idxResult].uTsc - iMasterTscDelta);
3194	if ( iDelta >= GIP_TSC_DELTA_INITIAL_MASTER_VALUE
3195	? iDelta < iBestDelta
3196	: iDelta > iBestDelta \|\| iBestDelta == INT64_MAX)
3197	iBestDelta = iDelta;
3198	cHits++;
3199	}
3200	}
3201	}
3202	}
3203
3204	/*
3205	* Save the results.
3206	*/
3207	if (cHits > 2)
3208	pArgs->pWorker->i64TSCDelta = iBestDelta;
3209	pArgs->uMaster.M2.cHits += cHits;
3210	}
3211
3212
3213	/**
3214	* The core function of the 2nd TSC delta measurement algorithm.
3215	*
3216	* The idea here is that we have the two CPUs execute the exact same code
3217	* collecting a largish set of TSC samples. The code has one data dependency on
3218	* the other CPU which intention it is to synchronize the execution as well as
3219	* help cross references the two sets of TSC samples (the sequence numbers).
3220	*
3221	* The @a fLag parameter is used to modify the execution a tiny bit on one or
3222	* both of the CPUs. When @a fLag differs between the CPUs, it is thought that
3223	* it will help with making the CPUs enter lock step execution occasionally.
3224	*
3225	*/
3226	static void supdrvTscDeltaMethod2CollectData(PSUPDRVTSCDELTAMETHOD2 pMyData, uint32_t volatile *piOtherSeqNo, bool fLag)
3227	{
3228	SUPDRVTSCDELTAMETHOD2ENTRY *pEntry = &pMyData->aResults[0];
3229	uint32_t cLeft = RT_ELEMENTS(pMyData->aResults);
3230
3231	ASMAtomicWriteU32(&pMyData->iCurSeqNo, 0);
3232	ASMSerializeInstruction();
3233	while (cLeft-- > 0)
3234	{
3235	uint64_t uTsc;
3236	uint32_t iSeqMine = ASMAtomicIncU32(&pMyData->iCurSeqNo);
3237	uint32_t iSeqOther = ASMAtomicReadU32(piOtherSeqNo);
3238	ASMCompilerBarrier();
3239	ASMSerializeInstruction(); /* Way better result than with ASMMemoryFenceSSE2() in this position! */
3240	uTsc = ASMReadTSC();
3241	ASMAtomicIncU32(&pMyData->iCurSeqNo);
3242	ASMCompilerBarrier();
3243	ASMSerializeInstruction();
3244	pEntry->iSeqMine = iSeqMine;
3245	pEntry->iSeqOther = iSeqOther;
3246	pEntry->uTsc = uTsc;
3247	pEntry++;
3248	ASMSerializeInstruction();
3249	if (fLag)
3250	ASMNopPause();
3251	}
3252	}
3253
3254
3255	/**
3256	* TSC delta measurement algorithm \#2 (GIP_TSC_DELTA_METHOD_2).
3257	*
3258	* See supdrvTscDeltaMethod2CollectData for algorithm details.
3259	*
3260	* @param pArgs The argument/state data.
3261	* @param pMySync My synchronization structure.
3262	* @param pOtherSync My partner's synchronization structure.
3263	* @param fIsMaster Set if master, clear if worker.
3264	* @param iTry The attempt number.
3265	*/
3266	static void supdrvTscDeltaMethod2Loop(PSUPDRVGIPTSCDELTARGS pArgs, PSUPTSCDELTASYNC2 pMySync, PSUPTSCDELTASYNC2 pOtherSync,
3267	bool fIsMaster, uint32_t iTry)
3268	{
3269	unsigned iLoop;
3270	RT_NOREF1(iTry);
3271
3272	for (iLoop = 0; iLoop < GIP_TSC_DELTA_M2_LOOPS; iLoop++)
3273	{
3274	RTCCUINTREG fEFlags;
3275	if (fIsMaster)
3276	{
3277	/*
3278	* Adjust the loop lag fudge.
3279	*/
3280	# if GIP_TSC_DELTA_M2_PRIMER_LOOPS > 0
3281	if (iLoop < GIP_TSC_DELTA_M2_PRIMER_LOOPS)
3282	{
3283	/* Lag during the priming to be nice to everyone.. */
3284	pArgs->uMaster.M2.fLag = true;
3285	pArgs->uWorker.M2.fLag = true;
3286	}
3287	else
3288	# endif
3289	if (iLoop < (GIP_TSC_DELTA_M2_LOOPS - GIP_TSC_DELTA_M2_PRIMER_LOOPS) / 4)
3290	{
3291	/* 25 % of the body without lagging. */
3292	pArgs->uMaster.M2.fLag = false;
3293	pArgs->uWorker.M2.fLag = false;
3294	}
3295	else if (iLoop < (GIP_TSC_DELTA_M2_LOOPS - GIP_TSC_DELTA_M2_PRIMER_LOOPS) / 4 * 2)
3296	{
3297	/* 25 % of the body with both lagging. */
3298	pArgs->uMaster.M2.fLag = true;
3299	pArgs->uWorker.M2.fLag = true;
3300	}
3301	else
3302	{
3303	/* 50% of the body with alternating lag. */
3304	pArgs->uMaster.M2.fLag = (iLoop & 1) == 0;
3305	pArgs->uWorker.M2.fLag= (iLoop & 1) == 1;
3306	}
3307
3308	/*
3309	* Sync up with the worker and collect data.
3310	*/
3311	TSCDELTA_MASTER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
3312	supdrvTscDeltaMethod2CollectData(&pArgs->uMaster.M2.Data, &pArgs->uWorker.M2.Data.iCurSeqNo, pArgs->uMaster.M2.fLag);
3313	TSCDELTA_MASTER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
3314
3315	/*
3316	* Process the data.
3317	*/
3318	# if GIP_TSC_DELTA_M2_PRIMER_LOOPS > 0
3319	if (iLoop >= GIP_TSC_DELTA_M2_PRIMER_LOOPS)
3320	# endif
3321	supdrvTscDeltaMethod2ProcessDataOnMaster(pArgs);
3322
3323	TSCDELTA_MASTER_KICK_OTHER_OUT_OF_AFTER(pMySync, pOtherSync);
3324	}
3325	else
3326	{
3327	/*
3328	* The worker.
3329	*/
3330	TSCDELTA_OTHER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
3331	supdrvTscDeltaMethod2CollectData(&pArgs->uWorker.M2.Data, &pArgs->uMaster.M2.Data.iCurSeqNo, pArgs->uWorker.M2.fLag);
3332	TSCDELTA_OTHER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
3333	}
3334	}
3335	}
3336
3337	#endif /* GIP_TSC_DELTA_METHOD_2 */
3338
3339
3340
3341	static int supdrvTscDeltaVerify(PSUPDRVGIPTSCDELTARGS pArgs, PSUPTSCDELTASYNC2 pMySync,
3342	PSUPTSCDELTASYNC2 pOtherSync, bool fIsMaster, int64_t iWorkerTscDelta)
3343	{
3344	/PSUPGIPCPU pGipCpuWorker = pArgs->pWorker; - unused /
3345	PSUPGIPCPU pGipCpuMaster = pArgs->pMaster;
3346	uint32_t i;
3347	TSCDELTA_DBG_VARS();
3348
3349	for (;;)
3350	{
3351	RTCCUINTREG fEFlags;
3352	AssertCompile((RT_ELEMENTS(pArgs->uMaster.Verify.auTscs) & 1) == 0);
3353	AssertCompile(RT_ELEMENTS(pArgs->uMaster.Verify.auTscs) == RT_ELEMENTS(pArgs->uWorker.Verify.auTscs));
3354
3355	if (fIsMaster)
3356	{
3357	uint64_t uTscWorker;
3358	TSCDELTA_MASTER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
3359
3360	/*
3361	* Collect TSC, master goes first.
3362	*/
3363	for (i = 0; i < RT_ELEMENTS(pArgs->uMaster.Verify.auTscs); i += 2)
3364	{
3365	/* Read, kick & wait #1. */
3366	uint64_t register uTsc = ASMReadTSC();
3367	ASMAtomicWriteU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO_GO);
3368	ASMSerializeInstruction();
3369	pArgs->uMaster.Verify.auTscs[i] = uTsc;
3370	TSCDELTA_DBG_START_LOOP();
3371	while (ASMAtomicReadU32(&pMySync->uSyncVar) == GIP_TSC_DELTA_SYNC2_GO)
3372	{
3373	TSCDELTA_DBG_CHECK_LOOP();
3374	ASMNopPause();
3375	}
3376
3377	/* Read, kick & wait #2. */
3378	uTsc = ASMReadTSC();
3379	ASMAtomicWriteU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO);
3380	ASMSerializeInstruction();
3381	pArgs->uMaster.Verify.auTscs[i + 1] = uTsc;
3382	TSCDELTA_DBG_START_LOOP();
3383	while (ASMAtomicReadU32(&pMySync->uSyncVar) == GIP_TSC_DELTA_SYNC2_GO_GO)
3384	{
3385	TSCDELTA_DBG_CHECK_LOOP();
3386	ASMNopPause();
3387	}
3388	}
3389
3390	TSCDELTA_MASTER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
3391
3392	/*
3393	* Process the data.
3394	*/
3395	#ifdef TSCDELTA_VERIFY_WITH_STATS
3396	pArgs->cMaxVerifyTscTicks = INT64_MIN;
3397	pArgs->cMinVerifyTscTicks = INT64_MAX;
3398	pArgs->iVerifyBadTscDiff = 0;
3399	#endif
3400	ASMAtomicWriteS32(&pArgs->rcVerify, VINF_SUCCESS);
3401	uTscWorker = 0;
3402	for (i = 0; i < RT_ELEMENTS(pArgs->uMaster.Verify.auTscs); i++)
3403	{
3404	/* Master vs previous worker entry. */
3405	uint64_t uTscMaster = pArgs->uMaster.Verify.auTscs[i] - pGipCpuMaster->i64TSCDelta;
3406	int64_t iDiff;
3407	if (i > 0)
3408	{
3409	iDiff = uTscMaster - uTscWorker;
3410	#ifdef TSCDELTA_VERIFY_WITH_STATS
3411	if (iDiff > pArgs->cMaxVerifyTscTicks)
3412	pArgs->cMaxVerifyTscTicks = iDiff;
3413	if (iDiff < pArgs->cMinVerifyTscTicks)
3414	pArgs->cMinVerifyTscTicks = iDiff;
3415	#endif
3416	if (iDiff < 0)
3417	{
3418	#ifdef TSCDELTA_VERIFY_WITH_STATS
3419	pArgs->iVerifyBadTscDiff = -iDiff;
3420	#endif
3421	ASMAtomicWriteS32(&pArgs->rcVerify, VERR_OUT_OF_RANGE);
3422	break;
3423	}
3424	}
3425
3426	/* Worker vs master. */
3427	uTscWorker = pArgs->uWorker.Verify.auTscs[i] - iWorkerTscDelta;
3428	iDiff = uTscWorker - uTscMaster;
3429	#ifdef TSCDELTA_VERIFY_WITH_STATS
3430	if (iDiff > pArgs->cMaxVerifyTscTicks)
3431	pArgs->cMaxVerifyTscTicks = iDiff;
3432	if (iDiff < pArgs->cMinVerifyTscTicks)
3433	pArgs->cMinVerifyTscTicks = iDiff;
3434	#endif
3435	if (iDiff < 0)
3436	{
3437	#ifdef TSCDELTA_VERIFY_WITH_STATS
3438	pArgs->iVerifyBadTscDiff = iDiff;
3439	#endif
3440	ASMAtomicWriteS32(&pArgs->rcVerify, VERR_OUT_OF_RANGE);
3441	break;
3442	}
3443	}
3444
3445	/* Done. */
3446	TSCDELTA_MASTER_KICK_OTHER_OUT_OF_AFTER(pMySync, pOtherSync);
3447	}
3448	else
3449	{
3450	/*
3451	* The worker, master leads.
3452	*/
3453	TSCDELTA_OTHER_SYNC_BEFORE(pMySync, pOtherSync, &fEFlags, pArgs);
3454
3455	for (i = 0; i < RT_ELEMENTS(pArgs->uWorker.Verify.auTscs); i += 2)
3456	{
3457	uint64_t register uTsc;
3458
3459	/* Wait, Read and Kick #1. */
3460	TSCDELTA_DBG_START_LOOP();
3461	while (ASMAtomicReadU32(&pMySync->uSyncVar) == GIP_TSC_DELTA_SYNC2_GO)
3462	{
3463	TSCDELTA_DBG_CHECK_LOOP();
3464	ASMNopPause();
3465	}
3466	uTsc = ASMReadTSC();
3467	ASMAtomicWriteU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO_GO);
3468	ASMSerializeInstruction();
3469	pArgs->uWorker.Verify.auTscs[i] = uTsc;
3470
3471	/* Wait, Read and Kick #2. */
3472	TSCDELTA_DBG_START_LOOP();
3473	while (ASMAtomicReadU32(&pMySync->uSyncVar) == GIP_TSC_DELTA_SYNC2_GO_GO)
3474	{
3475	TSCDELTA_DBG_CHECK_LOOP();
3476	ASMNopPause();
3477	}
3478	uTsc = ASMReadTSC();
3479	ASMAtomicWriteU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_GO);
3480	ASMSerializeInstruction();
3481	pArgs->uWorker.Verify.auTscs[i + 1] = uTsc;
3482	}
3483
3484	TSCDELTA_OTHER_SYNC_AFTER(pMySync, pOtherSync, fEFlags);
3485	}
3486	return pArgs->rcVerify;
3487	}
3488
3489	/*
3490	* Timed out, please retry.
3491	*/
3492	ASMAtomicWriteS32(&pArgs->rcVerify, VERR_TRY_AGAIN);
3493	return VERR_TIMEOUT;
3494	}
3495
3496
3497
3498	/**
3499	* Handles the special abort procedure during synchronization setup in
3500	* supdrvMeasureTscDeltaCallbackUnwrapped().
3501	*
3502	* @returns 0 (dummy, ignored)
3503	* @param pArgs Pointer to argument/state data.
3504	* @param pMySync Pointer to my sync structure.
3505	* @param fIsMaster Set if we're the master, clear if worker.
3506	* @param fTimeout Set if it's a timeout.
3507	*/
3508	DECL_NO_INLINE(static, int)
3509	supdrvMeasureTscDeltaCallbackAbortSyncSetup(PSUPDRVGIPTSCDELTARGS pArgs, PSUPTSCDELTASYNC2 pMySync, bool fIsMaster, bool fTimeout)
3510	{
3511	PSUPTSCDELTASYNC2 volatile *ppMySync = fIsMaster ? &pArgs->pSyncMaster : &pArgs->pSyncWorker;
3512	PSUPTSCDELTASYNC2 volatile *ppOtherSync = fIsMaster ? &pArgs->pSyncWorker : &pArgs->pSyncMaster;
3513	TSCDELTA_DBG_VARS();
3514	RT_NOREF1(pMySync);
3515
3516	/*
3517	* Clear our sync pointer and make sure the abort flag is set.
3518	*/
3519	ASMAtomicWriteNullPtr(ppMySync);
3520	ASMAtomicWriteBool(&pArgs->fAbortSetup, true);
3521	if (fTimeout)
3522	ASMAtomicWriteBool(&pArgs->fTimedOut, true);
3523
3524	/*
3525	* Make sure the other party is out of there and won't be touching our
3526	* sync state again (would cause stack corruption).
3527	*/
3528	TSCDELTA_DBG_START_LOOP();
3529	while (ASMAtomicReadPtrT(ppOtherSync, PSUPTSCDELTASYNC2) != NULL)
3530	{
3531	ASMNopPause();
3532	ASMNopPause();
3533	ASMNopPause();
3534	TSCDELTA_DBG_CHECK_LOOP();
3535	}
3536
3537	return 0;
3538	}
3539
3540
3541	/**
3542	* This is used by supdrvMeasureInitialTscDeltas() to read the TSC on two CPUs
3543	* and compute the delta between them.
3544	*
3545	* To reduce code size a good when timeout handling was added, a dummy return
3546	* value had to be added (saves 1-3 lines per timeout case), thus this
3547	* 'Unwrapped' function and the dummy 0 return value.
3548	*
3549	* @returns 0 (dummy, ignored)
3550	* @param idCpu The CPU we are current scheduled on.
3551	* @param pArgs Pointer to a parameter package.
3552	*
3553	* @remarks Measuring TSC deltas between the CPUs is tricky because we need to
3554	* read the TSC at exactly the same time on both the master and the
3555	* worker CPUs. Due to DMA, bus arbitration, cache locality,
3556	* contention, SMI, pipelining etc. there is no guaranteed way of
3557	* doing this on x86 CPUs.
3558	*/
3559	static int supdrvMeasureTscDeltaCallbackUnwrapped(RTCPUID idCpu, PSUPDRVGIPTSCDELTARGS pArgs)
3560	{
3561	PSUPDRVDEVEXT pDevExt = pArgs->pDevExt;
3562	PSUPGIPCPU pGipCpuWorker = pArgs->pWorker;
3563	PSUPGIPCPU pGipCpuMaster = pArgs->pMaster;
3564	bool const fIsMaster = idCpu == pGipCpuMaster->idCpu;
3565	uint32_t iTry;
3566	PSUPTSCDELTASYNC2 volatile *ppMySync = fIsMaster ? &pArgs->pSyncMaster : &pArgs->pSyncWorker;
3567	PSUPTSCDELTASYNC2 volatile *ppOtherSync = fIsMaster ? &pArgs->pSyncWorker : &pArgs->pSyncMaster;
3568	SUPTSCDELTASYNC2 MySync;
3569	PSUPTSCDELTASYNC2 pOtherSync;
3570	int rc;
3571	TSCDELTA_DBG_VARS();
3572
3573	/* A bit of paranoia first. */
3574	if (!pGipCpuMaster \|\| !pGipCpuWorker)
3575	return 0;
3576
3577	/*
3578	* If the CPU isn't part of the measurement, return immediately.
3579	*/
3580	if ( !fIsMaster
3581	&& idCpu != pGipCpuWorker->idCpu)
3582	return 0;
3583
3584	/*
3585	* Set up my synchronization stuff and wait for the other party to show up.
3586	*
3587	* We don't wait forever since the other party may be off fishing (offline,
3588	* spinning with ints disables, whatever), we must play nice to the rest of
3589	* the system as this context generally isn't one in which we will get
3590	* preempted and we may hold up a number of lower priority interrupts.
3591	*/
3592	ASMAtomicWriteU32(&MySync.uSyncVar, GIP_TSC_DELTA_SYNC2_PRESTART_WAIT);
3593	ASMAtomicWritePtr(ppMySync, &MySync);
3594	MySync.uTscStart = ASMReadTSC();
3595	MySync.cMaxTscTicks = pArgs->cMaxTscTicks;
3596
3597	/* Look for the partner, might not be here yet... Special abort considerations. */
3598	iTry = 0;
3599	TSCDELTA_DBG_START_LOOP();
3600	while ((pOtherSync = ASMAtomicReadPtrT(ppOtherSync, PSUPTSCDELTASYNC2)) == NULL)
3601	{
3602	ASMNopPause();
3603	if ( ASMAtomicReadBool(&pArgs->fAbortSetup)
3604	\|\| !RTMpIsCpuOnline(fIsMaster ? pGipCpuWorker->idCpu : pGipCpuMaster->idCpu) )
3605	return supdrvMeasureTscDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, false /fTimeout/);
3606	if ( (iTry++ & 0xff) == 0
3607	&& ASMReadTSC() - MySync.uTscStart > pArgs->cMaxTscTicks)
3608	return supdrvMeasureTscDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, true /fTimeout/);
3609	TSCDELTA_DBG_CHECK_LOOP();
3610	ASMNopPause();
3611	}
3612
3613	/* I found my partner, waiting to be found... Special abort considerations. */
3614	if (fIsMaster)
3615	if (!ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_PRESTART_WAIT)) /* parnaoia */
3616	return supdrvMeasureTscDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, false /fTimeout/);
3617
3618	iTry = 0;
3619	TSCDELTA_DBG_START_LOOP();
3620	while (ASMAtomicReadU32(&MySync.uSyncVar) == GIP_TSC_DELTA_SYNC2_PRESTART_WAIT)
3621	{
3622	ASMNopPause();
3623	if (ASMAtomicReadBool(&pArgs->fAbortSetup))
3624	return supdrvMeasureTscDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, false /fTimeout/);
3625	if ( (iTry++ & 0xff) == 0
3626	&& ASMReadTSC() - MySync.uTscStart > pArgs->cMaxTscTicks)
3627	{
3628	if ( fIsMaster
3629	&& !ASMAtomicCmpXchgU32(&MySync.uSyncVar, GIP_TSC_DELTA_SYNC2_PRESTART_ABORT, GIP_TSC_DELTA_SYNC2_PRESTART_WAIT))
3630	break; /* race #1: slave has moved on, handle timeout in loop instead. */
3631	return supdrvMeasureTscDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, true /fTimeout/);
3632	}
3633	TSCDELTA_DBG_CHECK_LOOP();
3634	}
3635
3636	if (!fIsMaster)
3637	if (!ASMAtomicCmpXchgU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_READY, GIP_TSC_DELTA_SYNC2_PRESTART_WAIT)) /* race #1 */
3638	return supdrvMeasureTscDeltaCallbackAbortSyncSetup(pArgs, &MySync, fIsMaster, false /fTimeout/);
3639
3640	/** @todo Add a resumable state to pArgs so we don't waste time if we time
3641	* out or something. Timeouts are legit, any of the two CPUs may get
3642	* interrupted. */
3643
3644	/*
3645	* Start by seeing if we have a zero delta between the two CPUs.
3646	* This should normally be the case.
3647	*/
3648	rc = supdrvTscDeltaVerify(pArgs, &MySync, pOtherSync, fIsMaster, GIP_TSC_DELTA_INITIAL_MASTER_VALUE);
3649	if (RT_SUCCESS(rc))
3650	{
3651	if (fIsMaster)
3652	{
3653	ASMAtomicWriteS64(&pGipCpuWorker->i64TSCDelta, GIP_TSC_DELTA_INITIAL_MASTER_VALUE);
3654	RTCpuSetDelByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet);
3655	RTCpuSetAddByIndex(&pDevExt->TscDeltaObtainedCpuSet, pGipCpuWorker->iCpuSet);
3656	}
3657	}
3658	/*
3659	* If the verification didn't time out, do regular delta measurements.
3660	* We retry this until we get a reasonable value.
3661	*/
3662	else if (rc != VERR_TIMEOUT)
3663	{
3664	Assert(pGipCpuWorker->i64TSCDelta == INT64_MAX);
3665	for (iTry = 0; iTry < 12; iTry++)
3666	{
3667	/*
3668	* Check the state before we start.
3669	*/
3670	uint32_t u32Tmp = ASMAtomicReadU32(&MySync.uSyncVar);
3671	if ( u32Tmp != GIP_TSC_DELTA_SYNC2_READY
3672	&& (fIsMaster \|\| u32Tmp != GIP_TSC_DELTA_SYNC2_STEADY) /* worker may be late prepping for the next round */ )
3673	{
3674	TSCDELTA_DBG_SYNC_MSG(("sync/loop/%s: #0 iTry=%u MyState=%#x\n", fIsMaster ? "master" : "worker", iTry, u32Tmp));
3675	break;
3676	}
3677
3678	/*
3679	* Do the measurements.
3680	*/
3681	#ifdef GIP_TSC_DELTA_METHOD_1
3682	supdrvTscDeltaMethod1Loop(pArgs, &MySync, pOtherSync, fIsMaster, iTry);
3683	#elif defined(GIP_TSC_DELTA_METHOD_2)
3684	supdrvTscDeltaMethod2Loop(pArgs, &MySync, pOtherSync, fIsMaster, iTry);
3685	#else
3686	# error "huh??"
3687	#endif
3688
3689	/*
3690	* Check the state.
3691	*/
3692	u32Tmp = ASMAtomicReadU32(&MySync.uSyncVar);
3693	if ( u32Tmp != GIP_TSC_DELTA_SYNC2_READY
3694	&& (fIsMaster \|\| u32Tmp != GIP_TSC_DELTA_SYNC2_STEADY) /* worker may be late prepping for the next round */ )
3695	{
3696	if (fIsMaster)
3697	TSCDELTA_DBG_SYNC_MSG(("sync/loop/master: #1 iTry=%u MyState=%#x\n", iTry, u32Tmp));
3698	else
3699	TSCDELTA_DBG_SYNC_MSG2(("sync/loop/worker: #1 iTry=%u MyState=%#x\n", iTry, u32Tmp));
3700	break;
3701	}
3702
3703	/*
3704	* Success? If so, stop trying. Master decides.
3705	*/
3706	if (fIsMaster)
3707	{
3708	if (pGipCpuWorker->i64TSCDelta != INT64_MAX)
3709	{
3710	RTCpuSetDelByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet);
3711	RTCpuSetAddByIndex(&pDevExt->TscDeltaObtainedCpuSet, pGipCpuWorker->iCpuSet);
3712	TSCDELTA_DBG_SYNC_MSG2(("sync/loop/master: #9 iTry=%u MyState=%#x\n", iTry, MySync.uSyncVar));
3713	break;
3714	}
3715	}
3716	}
3717	if (fIsMaster)
3718	pArgs->iTry = iTry;
3719	}
3720
3721	/*
3722	* End the synchronization dance. We tell the other that we're done,
3723	* then wait for the same kind of reply.
3724	*/
3725	ASMAtomicWriteU32(&pOtherSync->uSyncVar, GIP_TSC_DELTA_SYNC2_FINAL);
3726	ASMAtomicWriteNullPtr(ppMySync);
3727	iTry = 0;
3728	TSCDELTA_DBG_START_LOOP();
3729	while (ASMAtomicReadU32(&MySync.uSyncVar) != GIP_TSC_DELTA_SYNC2_FINAL)
3730	{
3731	iTry++;
3732	if ( iTry == 0
3733	&& !RTMpIsCpuOnline(fIsMaster ? pGipCpuWorker->idCpu : pGipCpuMaster->idCpu))
3734	break; /* this really shouldn't happen. */
3735	TSCDELTA_DBG_CHECK_LOOP();
3736	ASMNopPause();
3737	}
3738
3739	/*
3740	* Collect some runtime stats.
3741	*/
3742	if (fIsMaster)
3743	pArgs->cElapsedMasterTscTicks = ASMReadTSC() - MySync.uTscStart;
3744	else
3745	pArgs->cElapsedWorkerTscTicks = ASMReadTSC() - MySync.uTscStart;
3746	return 0;
3747	}
3748
3749	/**
3750	* Callback used by supdrvMeasureInitialTscDeltas() to read the TSC on two CPUs
3751	* and compute the delta between them.
3752	*
3753	* @param idCpu The CPU we are current scheduled on.
3754	* @param pvUser1 Pointer to a parameter package (SUPDRVGIPTSCDELTARGS).
3755	* @param pvUser2 Unused.
3756	*/
3757	static DECLCALLBACK(void) supdrvMeasureTscDeltaCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
3758	{
3759	supdrvMeasureTscDeltaCallbackUnwrapped(idCpu, (PSUPDRVGIPTSCDELTARGS)pvUser1);
3760	RT_NOREF1(pvUser2);
3761	}
3762
3763
3764	/**
3765	* Measures the TSC delta between the master GIP CPU and one specified worker
3766	* CPU.
3767	*
3768	* @returns VBox status code.
3769	* @retval VERR_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED on pure measurement
3770	* failure.
3771	* @param pDevExt Pointer to the device instance data.
3772	* @param idxWorker The index of the worker CPU from the GIP's array of
3773	* CPUs.
3774	*
3775	* @remarks This must be called with preemption enabled!
3776	*/
3777	static int supdrvMeasureTscDeltaOne(PSUPDRVDEVEXT pDevExt, uint32_t idxWorker)
3778	{
3779	int rc;
3780	int rc2;
3781	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
3782	RTCPUID idMaster = pDevExt->idGipMaster;
3783	PSUPGIPCPU pGipCpuWorker = &pGip->aCPUs[idxWorker];
3784	PSUPGIPCPU pGipCpuMaster;
3785	uint32_t iGipCpuMaster;
3786	uint32_t u32Tmp;
3787
3788	/* Validate input a bit. */
3789	AssertReturn(pGip, VERR_INVALID_PARAMETER);
3790	Assert(pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED);
3791	Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3792
3793	/*
3794	* Don't attempt measuring the delta for the GIP master.
3795	*/
3796	if (pGipCpuWorker->idCpu == idMaster)
3797	{
3798	if (pGipCpuWorker->i64TSCDelta == INT64_MAX) /* This shouldn't happen, but just in case. */
3799	ASMAtomicWriteS64(&pGipCpuWorker->i64TSCDelta, GIP_TSC_DELTA_INITIAL_MASTER_VALUE);
3800	return VINF_SUCCESS;
3801	}
3802
3803	/*
3804	* One measurement at a time, at least for now. We might be using
3805	* broadcast IPIs so, so be nice to the rest of the system.
3806	*/
3807	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
3808	rc = RTSemMutexRequest(pDevExt->mtxTscDelta, RT_INDEFINITE_WAIT);
3809	#else
3810	rc = RTSemFastMutexRequest(pDevExt->mtxTscDelta);
3811	#endif
3812	if (RT_FAILURE(rc))
3813	return rc;
3814
3815	/*
3816	* If the CPU has hyper-threading and the APIC IDs of the master and worker are adjacent,
3817	* try pick a different master. (This fudge only works with multi core systems.)
3818	* ASSUMES related threads have adjacent APIC IDs. ASSUMES two threads per core.
3819	*
3820	* We skip this on AMDs for now as their HTT is different from Intel's and
3821	* it doesn't seem to have any favorable effect on the results.
3822	*
3823	* If the master is offline, we need a new master too, so share the code.
3824	*/
3825	iGipCpuMaster = supdrvGipFindCpuIndexForCpuId(pGip, idMaster);
3826	AssertReturn(iGipCpuMaster < pGip->cCpus, VERR_INVALID_CPU_ID);
3827	pGipCpuMaster = &pGip->aCPUs[iGipCpuMaster];
3828	if ( ( (pGipCpuMaster->idApic & ~1) == (pGipCpuWorker->idApic & ~1)
3829	&& pGip->cOnlineCpus > 2
3830	&& ASMHasCpuId()
3831	&& ASMIsValidStdRange(ASMCpuId_EAX(0))
3832	&& (ASMCpuId_EDX(1) & X86_CPUID_FEATURE_EDX_HTT)
3833	&& ( !ASMIsAmdCpu()
3834	\|\| ASMGetCpuFamily(u32Tmp = ASMCpuId_EAX(1)) > 0x15
3835	\|\| ( ASMGetCpuFamily(u32Tmp) == 0x15 /* Piledriver+, not bulldozer (FX-4150 didn't like it). */
3836	&& ASMGetCpuModelAMD(u32Tmp) >= 0x02) ) )
3837	\|\| !RTMpIsCpuOnline(idMaster) )
3838	{
3839	uint32_t i;
3840	for (i = 0; i < pGip->cCpus; i++)
3841	if ( i != iGipCpuMaster
3842	&& i != idxWorker
3843	&& pGip->aCPUs[i].enmState == SUPGIPCPUSTATE_ONLINE
3844	&& pGip->aCPUs[i].i64TSCDelta != INT64_MAX
3845	&& pGip->aCPUs[i].idCpu != NIL_RTCPUID
3846	&& pGip->aCPUs[i].idCpu != idMaster /* paranoia starts here... */
3847	&& pGip->aCPUs[i].idCpu != pGipCpuWorker->idCpu
3848	&& pGip->aCPUs[i].idApic != pGipCpuWorker->idApic
3849	&& pGip->aCPUs[i].idApic != pGipCpuMaster->idApic
3850	&& RTMpIsCpuOnline(pGip->aCPUs[i].idCpu))
3851	{
3852	iGipCpuMaster = i;
3853	pGipCpuMaster = &pGip->aCPUs[i];
3854	idMaster = pGipCpuMaster->idCpu;
3855	break;
3856	}
3857	}
3858
3859	if (RTCpuSetIsMemberByIndex(&pGip->OnlineCpuSet, pGipCpuWorker->iCpuSet))
3860	{
3861	/*
3862	* Initialize data package for the RTMpOnPair callback.
3863	*/
3864	PSUPDRVGIPTSCDELTARGS pArgs = (PSUPDRVGIPTSCDELTARGS)RTMemAllocZ(sizeof(*pArgs));
3865	if (pArgs)
3866	{
3867	pArgs->pWorker = pGipCpuWorker;
3868	pArgs->pMaster = pGipCpuMaster;
3869	pArgs->pDevExt = pDevExt;
3870	pArgs->pSyncMaster = NULL;
3871	pArgs->pSyncWorker = NULL;
3872	pArgs->cMaxTscTicks = ASMAtomicReadU64(&pGip->u64CpuHz) / 512; /* 1953 us */
3873
3874	/*
3875	* Do the RTMpOnPair call. We reset i64TSCDelta first so we
3876	* and supdrvMeasureTscDeltaCallback can use it as a success check.
3877	*/
3878	/** @todo Store the i64TSCDelta result in pArgs first? Perhaps deals with
3879	* that when doing the restart loop reorg. */
3880	ASMAtomicWriteS64(&pGipCpuWorker->i64TSCDelta, INT64_MAX);
3881	rc = RTMpOnPair(pGipCpuMaster->idCpu, pGipCpuWorker->idCpu, RTMPON_F_CONCURRENT_EXEC,
3882	supdrvMeasureTscDeltaCallback, pArgs, NULL);
3883	if (RT_SUCCESS(rc))
3884	{
3885	#if 0
3886	SUPR0Printf("mponpair ticks: %9llu %9llu max: %9llu iTry: %u%s\n", pArgs->cElapsedMasterTscTicks,
3887	pArgs->cElapsedWorkerTscTicks, pArgs->cMaxTscTicks, pArgs->iTry,
3888	pArgs->fTimedOut ? " timed out" :"");
3889	#endif
3890	#if 0
3891	SUPR0Printf("rcVerify=%d iVerifyBadTscDiff=%lld cMinVerifyTscTicks=%lld cMaxVerifyTscTicks=%lld\n",
3892	pArgs->rcVerify, pArgs->iVerifyBadTscDiff, pArgs->cMinVerifyTscTicks, pArgs->cMaxVerifyTscTicks);
3893	#endif
3894	if (RT_LIKELY(pGipCpuWorker->i64TSCDelta != INT64_MAX))
3895	{
3896	/*
3897	* Work the TSC delta applicability rating. It starts
3898	* optimistic in supdrvGipInit, we downgrade it here.
3899	*/
3900	SUPGIPUSETSCDELTA enmRating;
3901	if ( pGipCpuWorker->i64TSCDelta > GIP_TSC_DELTA_THRESHOLD_ROUGHLY_ZERO
3902	\|\| pGipCpuWorker->i64TSCDelta < -GIP_TSC_DELTA_THRESHOLD_ROUGHLY_ZERO)
3903	enmRating = SUPGIPUSETSCDELTA_NOT_ZERO;
3904	else if ( pGipCpuWorker->i64TSCDelta > GIP_TSC_DELTA_THRESHOLD_PRACTICALLY_ZERO
3905	\|\| pGipCpuWorker->i64TSCDelta < -GIP_TSC_DELTA_THRESHOLD_PRACTICALLY_ZERO)
3906	enmRating = SUPGIPUSETSCDELTA_ROUGHLY_ZERO;
3907	else
3908	enmRating = SUPGIPUSETSCDELTA_PRACTICALLY_ZERO;
3909	if (pGip->enmUseTscDelta < enmRating)
3910	{
3911	AssertCompile(sizeof(pGip->enmUseTscDelta) == sizeof(uint32_t));
3912	ASMAtomicWriteU32((uint32_t volatile *)&pGip->enmUseTscDelta, enmRating);
3913	}
3914	}
3915	else
3916	rc = VERR_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED;
3917	}
3918	/** @todo return try-again if we get an offline CPU error. */
3919
3920	RTMemFree(pArgs);
3921	}
3922	else
3923	rc = VERR_NO_MEMORY;
3924	}
3925	else
3926	rc = VERR_CPU_OFFLINE;
3927
3928	/*
3929	* We're done now.
3930	*/
3931	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
3932	rc2 = RTSemMutexRelease(pDevExt->mtxTscDelta); AssertRC(rc2);
3933	#else
3934	rc2 = RTSemFastMutexRelease(pDevExt->mtxTscDelta); AssertRC(rc2);
3935	#endif
3936	return rc;
3937	}
3938
3939
3940	/**
3941	* Resets the TSC-delta related TSC samples and optionally the deltas
3942	* themselves.
3943	*
3944	* @param pDevExt Pointer to the device instance data.
3945	* @param fResetTscDeltas Whether the TSC-deltas are also to be reset.
3946	*
3947	* @remarks This might be called while holding a spinlock!
3948	*/
3949	static void supdrvTscResetSamples(PSUPDRVDEVEXT pDevExt, bool fResetTscDeltas)
3950	{
3951	unsigned iCpu;
3952	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
3953	for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
3954	{
3955	PSUPGIPCPU pGipCpu = &pGip->aCPUs[iCpu];
3956	ASMAtomicWriteU64(&pGipCpu->u64TSCSample, GIP_TSC_DELTA_RSVD);
3957	if (fResetTscDeltas)
3958	{
3959	RTCpuSetDelByIndex(&pDevExt->TscDeltaObtainedCpuSet, pGipCpu->iCpuSet);
3960	ASMAtomicWriteS64(&pGipCpu->i64TSCDelta, INT64_MAX);
3961	}
3962	}
3963	}
3964
3965
3966	/**
3967	* Picks an online CPU as the master TSC for TSC-delta computations.
3968	*
3969	* @returns VBox status code.
3970	* @param pDevExt Pointer to the device instance data.
3971	* @param pidxMaster Where to store the CPU array index of the chosen
3972	* master. Optional, can be NULL.
3973	*/
3974	static int supdrvTscPickMaster(PSUPDRVDEVEXT pDevExt, uint32_t *pidxMaster)
3975	{
3976	/*
3977	* Pick the first CPU online as the master TSC and make it the new GIP master based
3978	* on the APIC ID.
3979	*
3980	* Technically we can simply use "idGipMaster" but doing this gives us master as CPU 0
3981	* in most cases making it nicer/easier for comparisons. It is safe to update the GIP
3982	* master as this point since the sync/async timer isn't created yet.
3983	*/
3984	unsigned iCpu;
3985	uint32_t idxMaster = UINT32_MAX;
3986	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
3987	for (iCpu = 0; iCpu < RT_ELEMENTS(pGip->aiCpuFromApicId); iCpu++)
3988	{
3989	uint16_t idxCpu = pGip->aiCpuFromApicId[iCpu];
3990	if (idxCpu != UINT16_MAX)
3991	{
3992	PSUPGIPCPU pGipCpu = &pGip->aCPUs[idxCpu];
3993	if (RTCpuSetIsMemberByIndex(&pGip->OnlineCpuSet, pGipCpu->iCpuSet))
3994	{
3995	idxMaster = idxCpu;
3996	pGipCpu->i64TSCDelta = GIP_TSC_DELTA_INITIAL_MASTER_VALUE;
3997	ASMAtomicWriteSize(&pDevExt->idGipMaster, pGipCpu->idCpu);
3998	if (pidxMaster)
3999	*pidxMaster = idxMaster;
4000	return VINF_SUCCESS;
4001	}
4002	}
4003	}
4004	return VERR_CPU_OFFLINE;
4005	}
4006
4007
4008	/**
4009	* Performs the initial measurements of the TSC deltas between CPUs.
4010	*
4011	* This is called by supdrvGipCreate(), supdrvGipPowerNotificationCallback() or
4012	* triggered by it if threaded.
4013	*
4014	* @returns VBox status code.
4015	* @param pDevExt Pointer to the device instance data.
4016	*
4017	* @remarks Must be called only after supdrvGipInitOnCpu() as this function uses
4018	* idCpu, GIP's online CPU set which are populated in
4019	* supdrvGipInitOnCpu().
4020	*/
4021	static int supdrvMeasureInitialTscDeltas(PSUPDRVDEVEXT pDevExt)
4022	{
4023	PSUPGIPCPU pGipCpuMaster;
4024	unsigned iCpu;
4025	unsigned iOddEven;
4026	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
4027	uint32_t idxMaster = UINT32_MAX;
4028	uint32_t cMpOnOffEvents = ASMAtomicReadU32(&pDevExt->cMpOnOffEvents);
4029
4030	Assert(pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED);
4031	supdrvTscResetSamples(pDevExt, true /* fClearDeltas */);
4032	int rc = supdrvTscPickMaster(pDevExt, &idxMaster);
4033	if (RT_FAILURE(rc))
4034	{
4035	SUPR0Printf("Failed to pick a CPU master for TSC-delta measurements rc=%Rrc\n", rc);
4036	return rc;
4037	}
4038	AssertReturn(idxMaster < pGip->cCpus, VERR_INVALID_CPU_INDEX);
4039	pGipCpuMaster = &pGip->aCPUs[idxMaster];
4040	Assert(pDevExt->idGipMaster == pGipCpuMaster->idCpu);
4041
4042	/*
4043	* If there is only a single CPU online we have nothing to do.
4044	*/
4045	if (pGip->cOnlineCpus <= 1)
4046	{
4047	AssertReturn(pGip->cOnlineCpus > 0, VERR_INTERNAL_ERROR_5);
4048	return VINF_SUCCESS;
4049	}
4050
4051	/*
4052	* Loop thru the GIP CPU array and get deltas for each CPU (except the
4053	* master). We do the CPUs with the even numbered APIC IDs first so that
4054	* we've got alternative master CPUs to pick from on hyper-threaded systems.
4055	*/
4056	for (iOddEven = 0; iOddEven < 2; iOddEven++)
4057	{
4058	for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
4059	{
4060	PSUPGIPCPU pGipCpuWorker = &pGip->aCPUs[iCpu];
4061	if ( iCpu != idxMaster
4062	&& (iOddEven > 0 \|\| (pGipCpuWorker->idApic & 1) == 0)
4063	&& RTCpuSetIsMemberByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet))
4064	{
4065	rc = supdrvMeasureTscDeltaOne(pDevExt, iCpu);
4066	if (RT_FAILURE(rc))
4067	{
4068	SUPR0Printf("supdrvMeasureTscDeltaOne failed. rc=%d CPU[%u].idCpu=%u Master[%u].idCpu=%u\n", rc, iCpu,
4069	pGipCpuWorker->idCpu, idxMaster, pDevExt->idGipMaster, pGipCpuMaster->idCpu);
4070	break;
4071	}
4072
4073	if (ASMAtomicReadU32(&pDevExt->cMpOnOffEvents) != cMpOnOffEvents)
4074	{
4075	SUPR0Printf("One or more CPUs transitioned between online & offline states. I'm confused, retry...\n");
4076	rc = VERR_TRY_AGAIN;
4077	break;
4078	}
4079	}
4080	}
4081	}
4082
4083	return rc;
4084	}
4085
4086
4087	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
4088
4089	/**
4090	* Switches the TSC-delta measurement thread into the butchered state.
4091	*
4092	* @returns VBox status code.
4093	* @param pDevExt Pointer to the device instance data.
4094	* @param fSpinlockHeld Whether the TSC-delta spinlock is held or not.
4095	* @param pszFailed An error message to log.
4096	* @param rcFailed The error code to exit the thread with.
4097	*/
4098	static int supdrvTscDeltaThreadButchered(PSUPDRVDEVEXT pDevExt, bool fSpinlockHeld, const char *pszFailed, int rcFailed)
4099	{
4100	if (!fSpinlockHeld)
4101	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4102
4103	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Butchered;
4104	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4105	OSDBGPRINT(("supdrvTscDeltaThreadButchered: %s. rc=%Rrc\n", pszFailed, rcFailed));
4106	return rcFailed;
4107	}
4108
4109
4110	/**
4111	* The TSC-delta measurement thread.
4112	*
4113	* @returns VBox status code.
4114	* @param hThread The thread handle.
4115	* @param pvUser Opaque pointer to the device instance data.
4116	*/
4117	static DECLCALLBACK(int) supdrvTscDeltaThread(RTTHREAD hThread, void *pvUser)
4118	{
4119	PSUPDRVDEVEXT pDevExt = (PSUPDRVDEVEXT)pvUser;
4120	uint32_t cConsecutiveTimeouts = 0;
4121	int rc = VERR_INTERNAL_ERROR_2;
4122	for (;;)
4123	{
4124	/*
4125	* Switch on the current state.
4126	*/
4127	SUPDRVTSCDELTATHREADSTATE enmState;
4128	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4129	enmState = pDevExt->enmTscDeltaThreadState;
4130	switch (enmState)
4131	{
4132	case kTscDeltaThreadState_Creating:
4133	{
4134	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Listening;
4135	rc = RTSemEventSignal(pDevExt->hTscDeltaEvent);
4136	if (RT_FAILURE(rc))
4137	return supdrvTscDeltaThreadButchered(pDevExt, true /* fSpinlockHeld */, "RTSemEventSignal", rc);
4138	/* fall thru */
4139	}
4140
4141	case kTscDeltaThreadState_Listening:
4142	{
4143	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4144
4145	/*
4146	* Linux counts uninterruptible sleeps as load, hence we shall do a
4147	* regular, interruptible sleep here and ignore wake ups due to signals.
4148	* See task_contributes_to_load() in include/linux/sched.h in the Linux sources.
4149	*/
4150	rc = RTThreadUserWaitNoResume(hThread, pDevExt->cMsTscDeltaTimeout);
4151	if ( RT_FAILURE(rc)
4152	&& rc != VERR_TIMEOUT
4153	&& rc != VERR_INTERRUPTED)
4154	return supdrvTscDeltaThreadButchered(pDevExt, false /* fSpinlockHeld */, "RTThreadUserWait", rc);
4155	RTThreadUserReset(hThread);
4156	break;
4157	}
4158
4159	case kTscDeltaThreadState_WaitAndMeasure:
4160	{
4161	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Measuring;
4162	rc = RTSemEventSignal(pDevExt->hTscDeltaEvent); /* (Safe on windows as long as spinlock isn't IRQ safe.) */
4163	if (RT_FAILURE(rc))
4164	return supdrvTscDeltaThreadButchered(pDevExt, true /* fSpinlockHeld */, "RTSemEventSignal", rc);
4165	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4166	RTThreadSleep(1);
4167	/* fall thru */
4168	}
4169
4170	case kTscDeltaThreadState_Measuring:
4171	{
4172	cConsecutiveTimeouts = 0;
4173	if (pDevExt->fTscThreadRecomputeAllDeltas)
4174	{
4175	int cTries = 8;
4176	int cMsWaitPerTry = 10;
4177	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
4178	Assert(pGip);
4179	do
4180	{
4181	RTCpuSetCopy(&pDevExt->TscDeltaCpuSet, &pGip->OnlineCpuSet);
4182	rc = supdrvMeasureInitialTscDeltas(pDevExt);
4183	if ( RT_SUCCESS(rc)
4184	\|\| ( RT_FAILURE(rc)
4185	&& rc != VERR_TRY_AGAIN
4186	&& rc != VERR_CPU_OFFLINE))
4187	{
4188	break;
4189	}
4190	RTThreadSleep(cMsWaitPerTry);
4191	} while (cTries-- > 0);
4192	pDevExt->fTscThreadRecomputeAllDeltas = false;
4193	}
4194	else
4195	{
4196	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
4197	unsigned iCpu;
4198
4199	/* Measure TSC-deltas only for the CPUs that are in the set. */
4200	rc = VINF_SUCCESS;
4201	for (iCpu = 0; iCpu < pGip->cCpus; iCpu++)
4202	{
4203	PSUPGIPCPU pGipCpuWorker = &pGip->aCPUs[iCpu];
4204	if (RTCpuSetIsMemberByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet))
4205	{
4206	if (pGipCpuWorker->i64TSCDelta == INT64_MAX)
4207	{
4208	int rc2 = supdrvMeasureTscDeltaOne(pDevExt, iCpu);
4209	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
4210	rc = rc2;
4211	}
4212	else
4213	{
4214	/*
4215	* The thread/someone must've called SUPR0TscDeltaMeasureBySetIndex(),
4216	* mark the delta as fine to get the timer thread off our back.
4217	*/
4218	RTCpuSetDelByIndex(&pDevExt->TscDeltaCpuSet, pGipCpuWorker->iCpuSet);
4219	RTCpuSetAddByIndex(&pDevExt->TscDeltaObtainedCpuSet, pGipCpuWorker->iCpuSet);
4220	}
4221	}
4222	}
4223	}
4224	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4225	if (pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Measuring)
4226	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Listening;
4227	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4228	Assert(rc != VERR_NOT_AVAILABLE); /* VERR_NOT_AVAILABLE is used as init value, see supdrvTscDeltaThreadInit(). */
4229	ASMAtomicWriteS32(&pDevExt->rcTscDelta, rc);
4230	break;
4231	}
4232
4233	case kTscDeltaThreadState_Terminating:
4234	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Destroyed;
4235	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4236	return VINF_SUCCESS;
4237
4238	case kTscDeltaThreadState_Butchered:
4239	default:
4240	return supdrvTscDeltaThreadButchered(pDevExt, true /* fSpinlockHeld */, "Invalid state", VERR_INVALID_STATE);
4241	}
4242	}
4243	/* not reached */
4244	}
4245
4246
4247	/**
4248	* Waits for the TSC-delta measurement thread to respond to a state change.
4249	*
4250	* @returns VINF_SUCCESS on success, VERR_TIMEOUT if it doesn't respond in time,
4251	* other error code on internal error.
4252	*
4253	* @param pDevExt The device instance data.
4254	* @param enmCurState The current state.
4255	* @param enmNewState The new state we're waiting for it to enter.
4256	*/
4257	static int supdrvTscDeltaThreadWait(PSUPDRVDEVEXT pDevExt, SUPDRVTSCDELTATHREADSTATE enmCurState,
4258	SUPDRVTSCDELTATHREADSTATE enmNewState)
4259	{
4260	SUPDRVTSCDELTATHREADSTATE enmActualState;
4261	int rc;
4262
4263	/*
4264	* Wait a short while for the expected state transition.
4265	*/
4266	RTSemEventWait(pDevExt->hTscDeltaEvent, RT_MS_1SEC);
4267	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4268	enmActualState = pDevExt->enmTscDeltaThreadState;
4269	if (enmActualState == enmNewState)
4270	{
4271	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4272	rc = VINF_SUCCESS;
4273	}
4274	else if (enmActualState == enmCurState)
4275	{
4276	/*
4277	* Wait longer if the state has not yet transitioned to the one we want.
4278	*/
4279	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4280	rc = RTSemEventWait(pDevExt->hTscDeltaEvent, 50 * RT_MS_1SEC);
4281	if ( RT_SUCCESS(rc)
4282	\|\| rc == VERR_TIMEOUT)
4283	{
4284	/*
4285	* Check the state whether we've succeeded.
4286	*/
4287	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4288	enmActualState = pDevExt->enmTscDeltaThreadState;
4289	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4290	if (enmActualState == enmNewState)
4291	rc = VINF_SUCCESS;
4292	else if (enmActualState == enmCurState)
4293	{
4294	rc = VERR_TIMEOUT;
4295	OSDBGPRINT(("supdrvTscDeltaThreadWait: timed out state transition. enmActualState=%d enmNewState=%d\n",
4296	enmActualState, enmNewState));
4297	}
4298	else
4299	{
4300	rc = VERR_INTERNAL_ERROR;
4301	OSDBGPRINT(("supdrvTscDeltaThreadWait: invalid state transition from %d to %d, expected %d\n", enmCurState,
4302	enmActualState, enmNewState));
4303	}
4304	}
4305	else
4306	OSDBGPRINT(("supdrvTscDeltaThreadWait: RTSemEventWait failed. rc=%Rrc\n", rc));
4307	}
4308	else
4309	{
4310	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4311	OSDBGPRINT(("supdrvTscDeltaThreadWait: invalid state %d when transitioning from %d to %d\n",
4312	enmActualState, enmCurState, enmNewState));
4313	rc = VERR_INTERNAL_ERROR;
4314	}
4315
4316	return rc;
4317	}
4318
4319
4320	/**
4321	* Signals the TSC-delta thread to start measuring TSC-deltas.
4322	*
4323	* @param pDevExt Pointer to the device instance data.
4324	* @param fForceAll Force re-calculating TSC-deltas on all CPUs.
4325	*/
4326	static void supdrvTscDeltaThreadStartMeasurement(PSUPDRVDEVEXT pDevExt, bool fForceAll)
4327	{
4328	if (pDevExt->hTscDeltaThread != NIL_RTTHREAD)
4329	{
4330	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4331	if ( pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Listening
4332	\|\| pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Measuring)
4333	{
4334	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_WaitAndMeasure;
4335	if (fForceAll)
4336	pDevExt->fTscThreadRecomputeAllDeltas = true;
4337	}
4338	else if ( pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_WaitAndMeasure
4339	&& fForceAll)
4340	pDevExt->fTscThreadRecomputeAllDeltas = true;
4341	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4342	RTThreadUserSignal(pDevExt->hTscDeltaThread);
4343	}
4344	}
4345
4346
4347	/**
4348	* Terminates the actual thread running supdrvTscDeltaThread().
4349	*
4350	* This is an internal worker function for supdrvTscDeltaThreadInit() and
4351	* supdrvTscDeltaTerm().
4352	*
4353	* @param pDevExt Pointer to the device instance data.
4354	*/
4355	static void supdrvTscDeltaThreadTerminate(PSUPDRVDEVEXT pDevExt)
4356	{
4357	int rc;
4358	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4359	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Terminating;
4360	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4361	RTThreadUserSignal(pDevExt->hTscDeltaThread);
4362	rc = RTThreadWait(pDevExt->hTscDeltaThread, 50 * RT_MS_1SEC, NULL /* prc */);
4363	if (RT_FAILURE(rc))
4364	{
4365	/* Signal a few more times before giving up. */
4366	int cTriesLeft = 5;
4367	while (--cTriesLeft > 0)
4368	{
4369	RTThreadUserSignal(pDevExt->hTscDeltaThread);
4370	rc = RTThreadWait(pDevExt->hTscDeltaThread, 2 * RT_MS_1SEC, NULL /* prc */);
4371	if (rc != VERR_TIMEOUT)
4372	break;
4373	}
4374	}
4375	}
4376
4377
4378	/**
4379	* Initializes and spawns the TSC-delta measurement thread.
4380	*
4381	* A thread is required for servicing re-measurement requests from events like
4382	* CPUs coming online, suspend/resume etc. as it cannot be done synchronously
4383	* under all contexts on all OSs.
4384	*
4385	* @returns VBox status code.
4386	* @param pDevExt Pointer to the device instance data.
4387	*
4388	* @remarks Must only be called -after- initializing GIP and setting up MP
4389	* notifications!
4390	*/
4391	static int supdrvTscDeltaThreadInit(PSUPDRVDEVEXT pDevExt)
4392	{
4393	int rc;
4394	Assert(pDevExt->pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED);
4395	rc = RTSpinlockCreate(&pDevExt->hTscDeltaSpinlock, RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE, "VBoxTscSpnLck");
4396	if (RT_SUCCESS(rc))
4397	{
4398	rc = RTSemEventCreate(&pDevExt->hTscDeltaEvent);
4399	if (RT_SUCCESS(rc))
4400	{
4401	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_Creating;
4402	pDevExt->cMsTscDeltaTimeout = 60000;
4403	rc = RTThreadCreate(&pDevExt->hTscDeltaThread, supdrvTscDeltaThread, pDevExt, 0 /* cbStack */,
4404	RTTHREADTYPE_DEFAULT, RTTHREADFLAGS_WAITABLE, "VBoxTscThread");
4405	if (RT_SUCCESS(rc))
4406	{
4407	rc = supdrvTscDeltaThreadWait(pDevExt, kTscDeltaThreadState_Creating, kTscDeltaThreadState_Listening);
4408	if (RT_SUCCESS(rc))
4409	{
4410	ASMAtomicWriteS32(&pDevExt->rcTscDelta, VERR_NOT_AVAILABLE);
4411	return rc;
4412	}
4413
4414	OSDBGPRINT(("supdrvTscDeltaInit: supdrvTscDeltaThreadWait failed. rc=%Rrc\n", rc));
4415	supdrvTscDeltaThreadTerminate(pDevExt);
4416	}
4417	else
4418	OSDBGPRINT(("supdrvTscDeltaInit: RTThreadCreate failed. rc=%Rrc\n", rc));
4419	RTSemEventDestroy(pDevExt->hTscDeltaEvent);
4420	pDevExt->hTscDeltaEvent = NIL_RTSEMEVENT;
4421	}
4422	else
4423	OSDBGPRINT(("supdrvTscDeltaInit: RTSemEventCreate failed. rc=%Rrc\n", rc));
4424	RTSpinlockDestroy(pDevExt->hTscDeltaSpinlock);
4425	pDevExt->hTscDeltaSpinlock = NIL_RTSPINLOCK;
4426	}
4427	else
4428	OSDBGPRINT(("supdrvTscDeltaInit: RTSpinlockCreate failed. rc=%Rrc\n", rc));
4429
4430	return rc;
4431	}
4432
4433
4434	/**
4435	* Terminates the TSC-delta measurement thread and cleanup.
4436	*
4437	* @param pDevExt Pointer to the device instance data.
4438	*/
4439	static void supdrvTscDeltaTerm(PSUPDRVDEVEXT pDevExt)
4440	{
4441	if ( pDevExt->hTscDeltaSpinlock != NIL_RTSPINLOCK
4442	&& pDevExt->hTscDeltaEvent != NIL_RTSEMEVENT)
4443	{
4444	supdrvTscDeltaThreadTerminate(pDevExt);
4445	}
4446
4447	if (pDevExt->hTscDeltaSpinlock != NIL_RTSPINLOCK)
4448	{
4449	RTSpinlockDestroy(pDevExt->hTscDeltaSpinlock);
4450	pDevExt->hTscDeltaSpinlock = NIL_RTSPINLOCK;
4451	}
4452
4453	if (pDevExt->hTscDeltaEvent != NIL_RTSEMEVENT)
4454	{
4455	RTSemEventDestroy(pDevExt->hTscDeltaEvent);
4456	pDevExt->hTscDeltaEvent = NIL_RTSEMEVENT;
4457	}
4458
4459	ASMAtomicWriteS32(&pDevExt->rcTscDelta, VERR_NOT_AVAILABLE);
4460	}
4461
4462	#endif /* SUPDRV_USE_TSC_DELTA_THREAD */
4463
4464	/**
4465	* Measure the TSC delta for the CPU given by its CPU set index.
4466	*
4467	* @returns VBox status code.
4468	* @retval VERR_INTERRUPTED if interrupted while waiting.
4469	* @retval VERR_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED if we were unable to get a
4470	* measurement.
4471	* @retval VERR_CPU_OFFLINE if the specified CPU is offline.
4472	*
4473	* @param pSession The caller's session. GIP must've been mapped.
4474	* @param iCpuSet The CPU set index of the CPU to measure.
4475	* @param fFlags Flags, SUP_TSCDELTA_MEASURE_F_XXX.
4476	* @param cMsWaitRetry Number of milliseconds to wait between each retry.
4477	* @param cMsWaitThread Number of milliseconds to wait for the thread to get
4478	* ready.
4479	* @param cTries Number of times to try, pass 0 for the default.
4480	*/
4481	SUPR0DECL(int) SUPR0TscDeltaMeasureBySetIndex(PSUPDRVSESSION pSession, uint32_t iCpuSet, uint32_t fFlags,
4482	RTMSINTERVAL cMsWaitRetry, RTMSINTERVAL cMsWaitThread, uint32_t cTries)
4483	{
4484	PSUPDRVDEVEXT pDevExt;
4485	PSUPGLOBALINFOPAGE pGip;
4486	uint16_t iGipCpu;
4487	int rc;
4488	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
4489	uint64_t msTsStartWait;
4490	uint32_t iWaitLoop;
4491	#endif
4492
4493	/*
4494	* Validate and adjust the input.
4495	*/
4496	AssertReturn(SUP_IS_SESSION_VALID(pSession), VERR_INVALID_PARAMETER);
4497	if (!pSession->fGipReferenced)
4498	return VERR_WRONG_ORDER;
4499
4500	pDevExt = pSession->pDevExt;
4501	AssertReturn(SUP_IS_DEVEXT_VALID(pDevExt), VERR_INVALID_PARAMETER);
4502
4503	pGip = pDevExt->pGip;
4504	AssertPtrReturn(pGip, VERR_INTERNAL_ERROR_2);
4505
4506	AssertReturn(iCpuSet < RTCPUSET_MAX_CPUS, VERR_INVALID_CPU_INDEX);
4507	AssertReturn(iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx), VERR_INVALID_CPU_INDEX);
4508	iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
4509	AssertReturn(iGipCpu < pGip->cCpus, VERR_INVALID_CPU_INDEX);
4510
4511	if (fFlags & ~SUP_TSCDELTA_MEASURE_F_VALID_MASK)
4512	return VERR_INVALID_FLAGS;
4513
4514	/*
4515	* The request is a noop if the TSC delta isn't being used.
4516	*/
4517	if (pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ZERO_CLAIMED)
4518	return VINF_SUCCESS;
4519
4520	if (cTries == 0)
4521	cTries = 12;
4522	else if (cTries > 256)
4523	cTries = 256;
4524
4525	if (cMsWaitRetry == 0)
4526	cMsWaitRetry = 2;
4527	else if (cMsWaitRetry > 1000)
4528	cMsWaitRetry = 1000;
4529
4530	#ifdef SUPDRV_USE_TSC_DELTA_THREAD
4531	/*
4532	* Has the TSC already been measured and we're not forced to redo it?
4533	*/
4534	if ( pGip->aCPUs[iGipCpu].i64TSCDelta != INT64_MAX
4535	&& !(fFlags & SUP_TSCDELTA_MEASURE_F_FORCE))
4536	return VINF_SUCCESS;
4537
4538	/*
4539	* Asynchronous request? Forward it to the thread, no waiting.
4540	*/
4541	if (fFlags & SUP_TSCDELTA_MEASURE_F_ASYNC)
4542	{
4543	/** @todo Async. doesn't implement options like retries, waiting. We'll need
4544	* to pass those options to the thread somehow and implement it in the
4545	* thread. Check if anyone uses/needs fAsync before implementing this. */
4546	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4547	RTCpuSetAddByIndex(&pDevExt->TscDeltaCpuSet, iCpuSet);
4548	if ( pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Listening
4549	\|\| pDevExt->enmTscDeltaThreadState == kTscDeltaThreadState_Measuring)
4550	{
4551	pDevExt->enmTscDeltaThreadState = kTscDeltaThreadState_WaitAndMeasure;
4552	rc = VINF_SUCCESS;
4553	}
4554	else if (pDevExt->enmTscDeltaThreadState != kTscDeltaThreadState_WaitAndMeasure)
4555	rc = VERR_THREAD_IS_DEAD;
4556	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4557	RTThreadUserSignal(pDevExt->hTscDeltaThread);
4558	return VINF_SUCCESS;
4559	}
4560
4561	/*
4562	* If a TSC-delta measurement request is already being serviced by the thread,
4563	* wait 'cTries' times if a retry-timeout is provided, otherwise bail as busy.
4564	*/
4565	msTsStartWait = RTTimeSystemMilliTS();
4566	for (iWaitLoop = 0;; iWaitLoop++)
4567	{
4568	uint64_t cMsElapsed;
4569	SUPDRVTSCDELTATHREADSTATE enmState;
4570	RTSpinlockAcquire(pDevExt->hTscDeltaSpinlock);
4571	enmState = pDevExt->enmTscDeltaThreadState;
4572	RTSpinlockRelease(pDevExt->hTscDeltaSpinlock);
4573
4574	if (enmState == kTscDeltaThreadState_Measuring)
4575	{ /* Must wait, the thread is busy. */ }
4576	else if (enmState == kTscDeltaThreadState_WaitAndMeasure)
4577	{ /* Must wait, this state only says what will happen next. */ }
4578	else if (enmState == kTscDeltaThreadState_Terminating)
4579	{ /* Must wait, this state only says what should happen next. */ }
4580	else
4581	break; /* All other states, the thread is either idly listening or dead. */
4582
4583	/* Wait or fail. */
4584	if (cMsWaitThread == 0)
4585	return VERR_SUPDRV_TSC_DELTA_MEASUREMENT_BUSY;
4586	cMsElapsed = RTTimeSystemMilliTS() - msTsStartWait;
4587	if (cMsElapsed >= cMsWaitThread)
4588	return VERR_SUPDRV_TSC_DELTA_MEASUREMENT_BUSY;
4589
4590	rc = RTThreadSleep(RT_MIN((RTMSINTERVAL)(cMsWaitThread - cMsElapsed), RT_MIN(iWaitLoop + 1, 10)));
4591	if (rc == VERR_INTERRUPTED)
4592	return rc;
4593	}
4594	#endif /* SUPDRV_USE_TSC_DELTA_THREAD */
4595
4596	/*
4597	* Try measure the TSC delta the given number of times.
4598	*/
4599	for (;;)
4600	{
4601	/* Unless we're forced to measure the delta, check whether it's done already. */
4602	if ( !(fFlags & SUP_TSCDELTA_MEASURE_F_FORCE)
4603	&& pGip->aCPUs[iGipCpu].i64TSCDelta != INT64_MAX)
4604	{
4605	rc = VINF_SUCCESS;
4606	break;
4607	}
4608
4609	/* Measure it. */
4610	rc = supdrvMeasureTscDeltaOne(pDevExt, iGipCpu);
4611	if (rc != VERR_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED)
4612	{
4613	Assert(pGip->aCPUs[iGipCpu].i64TSCDelta != INT64_MAX \|\| RT_FAILURE_NP(rc));
4614	break;
4615	}
4616
4617	/* Retry? */
4618	if (cTries <= 1)
4619	break;
4620	cTries--;
4621
4622	/* Always delay between retries (be nice to the rest of the system
4623	and avoid the BSOD hounds). */
4624	rc = RTThreadSleep(cMsWaitRetry);
4625	if (rc == VERR_INTERRUPTED)
4626	break;
4627	}
4628
4629	return rc;
4630	}
4631
4632
4633	/**
4634	* Service a TSC-delta measurement request.
4635	*
4636	* @returns VBox status code.
4637	* @param pDevExt Pointer to the device instance data.
4638	* @param pSession The support driver session.
4639	* @param pReq Pointer to the TSC-delta measurement request.
4640	*/
4641	int VBOXCALL supdrvIOCtl_TscDeltaMeasure(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPTSCDELTAMEASURE pReq)
4642	{
4643	uint32_t cTries;
4644	uint32_t iCpuSet;
4645	uint32_t fFlags;
4646	RTMSINTERVAL cMsWaitRetry;
4647	RT_NOREF1(pDevExt);
4648
4649	/*
4650	* Validate and adjust/resolve the input so they can be passed onto SUPR0TscDeltaMeasureBySetIndex.
4651	*/
4652	AssertPtr(pDevExt); AssertPtr(pSession); AssertPtr(pReq); /* paranoia^2 */
4653
4654	if (pReq->u.In.idCpu == NIL_RTCPUID)
4655	return VERR_INVALID_CPU_ID;
4656	iCpuSet = RTMpCpuIdToSetIndex(pReq->u.In.idCpu);
4657	if (iCpuSet >= RTCPUSET_MAX_CPUS)
4658	return VERR_INVALID_CPU_ID;
4659
4660	cTries = pReq->u.In.cRetries == 0 ? 0 : (uint32_t)pReq->u.In.cRetries + 1;
4661
4662	cMsWaitRetry = RT_MAX(pReq->u.In.cMsWaitRetry, 5);
4663
4664	fFlags = 0;
4665	if (pReq->u.In.fAsync)
4666	fFlags \|= SUP_TSCDELTA_MEASURE_F_ASYNC;
4667	if (pReq->u.In.fForce)
4668	fFlags \|= SUP_TSCDELTA_MEASURE_F_FORCE;
4669
4670	return SUPR0TscDeltaMeasureBySetIndex(pSession, iCpuSet, fFlags, cMsWaitRetry,
4671	cTries == 0 ? 5 * RT_MS_1SEC : cMsWaitRetry * cTries /cMsWaitThread/,
4672	cTries);
4673	}
4674
4675
4676	/**
4677	* Reads TSC with delta applied.
4678	*
4679	* Will try to resolve delta value INT64_MAX before applying it. This is the
4680	* main purpose of this function, to handle the case where the delta needs to be
4681	* determined.
4682	*
4683	* @returns VBox status code.
4684	* @param pDevExt Pointer to the device instance data.
4685	* @param pSession The support driver session.
4686	* @param pReq Pointer to the TSC-read request.
4687	*/
4688	int VBOXCALL supdrvIOCtl_TscRead(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, PSUPTSCREAD pReq)
4689	{
4690	PSUPGLOBALINFOPAGE pGip;
4691	int rc;
4692
4693	/*
4694	* Validate. We require the client to have mapped GIP (no asserting on
4695	* ring-3 preconditions).
4696	*/
4697	AssertPtr(pDevExt); AssertPtr(pReq); AssertPtr(pSession); /* paranoia^2 */
4698	if (pSession->GipMapObjR3 == NIL_RTR0MEMOBJ)
4699	return VERR_WRONG_ORDER;
4700	pGip = pDevExt->pGip;
4701	AssertReturn(pGip, VERR_INTERNAL_ERROR_2);
4702
4703	/*
4704	* We're usually here because we need to apply delta, but we shouldn't be
4705	* upset if the GIP is some different mode.
4706	*/
4707	if (pGip->enmUseTscDelta > SUPGIPUSETSCDELTA_ZERO_CLAIMED)
4708	{
4709	uint32_t cTries = 0;
4710	for (;;)
4711	{
4712	/*
4713	* Start by gathering the data, using CLI for disabling preemption
4714	* while we do that.
4715	*/
4716	RTCCUINTREG fEFlags = ASMIntDisableFlags();
4717	int iCpuSet = RTMpCpuIdToSetIndex(RTMpCpuId());
4718	int iGipCpu;
4719	if (RT_LIKELY( (unsigned)iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
4720	&& (iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]) < pGip->cCpus ))
4721	{
4722	int64_t i64Delta = pGip->aCPUs[iGipCpu].i64TSCDelta;
4723	pReq->u.Out.idApic = pGip->aCPUs[iGipCpu].idApic;
4724	pReq->u.Out.u64AdjustedTsc = ASMReadTSC();
4725	ASMSetFlags(fEFlags);
4726
4727	/*
4728	* If we're lucky we've got a delta, but no predictions here
4729	* as this I/O control is normally only used when the TSC delta
4730	* is set to INT64_MAX.
4731	*/
4732	if (i64Delta != INT64_MAX)
4733	{
4734	pReq->u.Out.u64AdjustedTsc -= i64Delta;
4735	rc = VINF_SUCCESS;
4736	break;
4737	}
4738
4739	/* Give up after a few times. */
4740	if (cTries >= 4)
4741	{
4742	rc = VWRN_SUPDRV_TSC_DELTA_MEASUREMENT_FAILED;
4743	break;
4744	}
4745
4746	/* Need to measure the delta an try again. */
4747	rc = supdrvMeasureTscDeltaOne(pDevExt, iGipCpu);
4748	Assert(pGip->aCPUs[iGipCpu].i64TSCDelta != INT64_MAX \|\| RT_FAILURE_NP(rc));
4749	/** @todo should probably delay on failure... dpc watchdogs */
4750	}
4751	else
4752	{
4753	/* This really shouldn't happen. */
4754	AssertMsgFailed(("idCpu=%#x iCpuSet=%#x (%d)\n", RTMpCpuId(), iCpuSet, iCpuSet));
4755	pReq->u.Out.idApic = ASMGetApicId();
4756	pReq->u.Out.u64AdjustedTsc = ASMReadTSC();
4757	ASMSetFlags(fEFlags);
4758	rc = VERR_INTERNAL_ERROR_5; /** @todo change to warning. */
4759	break;
4760	}
4761	}
4762	}
4763	else
4764	{
4765	/*
4766	* No delta to apply. Easy. Deal with preemption the lazy way.
4767	*/
4768	RTCCUINTREG fEFlags = ASMIntDisableFlags();
4769	int iCpuSet = RTMpCpuIdToSetIndex(RTMpCpuId());
4770	int iGipCpu;
4771	if (RT_LIKELY( (unsigned)iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
4772	&& (iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet]) < pGip->cCpus ))
4773	pReq->u.Out.idApic = pGip->aCPUs[iGipCpu].idApic;
4774	else
4775	pReq->u.Out.idApic = ASMGetApicId();
4776	pReq->u.Out.u64AdjustedTsc = ASMReadTSC();
4777	ASMSetFlags(fEFlags);
4778	rc = VINF_SUCCESS;
4779	}
4780
4781	return rc;
4782	}
4783
4784
4785	/**
4786	* Worker for supdrvIOCtl_GipSetFlags.
4787	*
4788	* @returns VBox status code.
4789	* @retval VERR_WRONG_ORDER if an enable-once-per-session flag is set again for
4790	* a session.
4791	*
4792	* @param pDevExt Pointer to the device instance data.
4793	* @param pSession The support driver session.
4794	* @param fOrMask The OR mask of the GIP flags, see SUPGIP_FLAGS_XXX.
4795	* @param fAndMask The AND mask of the GIP flags, see SUPGIP_FLAGS_XXX.
4796	*
4797	* @remarks Caller must own the GIP mutex.
4798	*
4799	* @remarks This function doesn't validate any of the flags.
4800	*/
4801	static int supdrvGipSetFlags(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, uint32_t fOrMask, uint32_t fAndMask)
4802	{
4803	uint32_t cRefs;
4804	PSUPGLOBALINFOPAGE pGip = pDevExt->pGip;
4805	AssertMsg((fOrMask & fAndMask) == fOrMask, ("%#x & %#x\n", fOrMask, fAndMask)); /* ASSUMED by code below */
4806
4807	/*
4808	* Compute GIP test-mode flags.
4809	*/
4810	if (fOrMask & SUPGIP_FLAGS_TESTING_ENABLE)
4811	{
4812	if (!pSession->fGipTestMode)
4813	{
4814	Assert(pDevExt->cGipTestModeRefs < _64K);
4815	pSession->fGipTestMode = true;
4816	cRefs = ++pDevExt->cGipTestModeRefs;
4817	if (cRefs == 1)
4818	{
4819	fOrMask \|= SUPGIP_FLAGS_TESTING \| SUPGIP_FLAGS_TESTING_START;
4820	fAndMask &= ~SUPGIP_FLAGS_TESTING_STOP;
4821	}
4822	}
4823	else
4824	{
4825	LogRelMax(10, ("supdrvGipSetFlags: SUPGIP_FLAGS_TESTING_ENABLE already set for this session\n"));
4826	return VERR_WRONG_ORDER;
4827	}
4828	}
4829	else if ( !(fAndMask & SUPGIP_FLAGS_TESTING_ENABLE)
4830	&& pSession->fGipTestMode)
4831	{
4832	Assert(pDevExt->cGipTestModeRefs > 0);
4833	Assert(pDevExt->cGipTestModeRefs < _64K);
4834	pSession->fGipTestMode = false;
4835	cRefs = --pDevExt->cGipTestModeRefs;
4836	if (!cRefs)
4837	fOrMask \|= SUPGIP_FLAGS_TESTING_STOP;
4838	else
4839	fAndMask \|= SUPGIP_FLAGS_TESTING_ENABLE;
4840	}
4841
4842	/*
4843	* Commit the flags. This should be done as atomically as possible
4844	* since the flag consumers won't be holding the GIP mutex.
4845	*/
4846	ASMAtomicOrU32(&pGip->fFlags, fOrMask);
4847	ASMAtomicAndU32(&pGip->fFlags, fAndMask);
4848
4849	return VINF_SUCCESS;
4850	}
4851
4852
4853	/**
4854	* Sets GIP test mode parameters.
4855	*
4856	* @returns VBox status code.
4857	* @param pDevExt Pointer to the device instance data.
4858	* @param pSession The support driver session.
4859	* @param fOrMask The OR mask of the GIP flags, see SUPGIP_FLAGS_XXX.
4860	* @param fAndMask The AND mask of the GIP flags, see SUPGIP_FLAGS_XXX.
4861	*/
4862	int VBOXCALL supdrvIOCtl_GipSetFlags(PSUPDRVDEVEXT pDevExt, PSUPDRVSESSION pSession, uint32_t fOrMask, uint32_t fAndMask)
4863	{
4864	PSUPGLOBALINFOPAGE pGip;
4865	int rc;
4866
4867	/*
4868	* Validate. We require the client to have mapped GIP (no asserting on
4869	* ring-3 preconditions).
4870	*/
4871	AssertPtr(pDevExt); AssertPtr(pSession); /* paranoia^2 */
4872	if (pSession->GipMapObjR3 == NIL_RTR0MEMOBJ)
4873	return VERR_WRONG_ORDER;
4874	pGip = pDevExt->pGip;
4875	AssertReturn(pGip, VERR_INTERNAL_ERROR_3);
4876
4877	if (fOrMask & ~SUPGIP_FLAGS_VALID_MASK)
4878	return VERR_INVALID_PARAMETER;
4879	if ((fAndMask & ~SUPGIP_FLAGS_VALID_MASK) != ~SUPGIP_FLAGS_VALID_MASK)
4880	return VERR_INVALID_PARAMETER;
4881
4882	/*
4883	* Don't confuse supdrvGipSetFlags or anyone else by both setting
4884	* and clearing the same flags. AND takes precedence.
4885	*/
4886	fOrMask &= fAndMask;
4887
4888	/*
4889	* Take the loader lock to avoid having to think about races between two
4890	* clients changing the flags at the same time (state is not simple).
4891	*/
4892	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
4893	RTSemMutexRequest(pDevExt->mtxGip, RT_INDEFINITE_WAIT);
4894	#else
4895	RTSemFastMutexRequest(pDevExt->mtxGip);
4896	#endif
4897
4898	rc = supdrvGipSetFlags(pDevExt, pSession, fOrMask, fAndMask);
4899
4900	#ifdef SUPDRV_USE_MUTEX_FOR_GIP
4901	RTSemMutexRelease(pDevExt->mtxGip);
4902	#else
4903	RTSemFastMutexRelease(pDevExt->mtxGip);
4904	#endif
4905	return rc;
4906	}
4907

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/HostDrivers/Support/SUPDrvGip.cpp@ 62679

Download in other formats: