timesupref.h@ 93115

Last change on this file since 93115 was 93115, checked in by vboxsync, 3 years ago
scm --update-copyright-year
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 18.3 KB

Line
1	/* $Id: timesupref.h 93115 2022-01-01 11:31:46Z vboxsync $ */
2	/** @file
3	* IPRT - Time using SUPLib, the C Code Template.
4	*/
5
6	/*
7	* Copyright (C) 2006-2022 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/**
29	* The C reference implementation of the assembly routines.
30	*
31	* Calculate NanoTS using the information in the global information page (GIP)
32	* which the support library (SUPLib) exports.
33	*
34	* This function guarantees that the returned timestamp is later (in time) than
35	* any previous calls in the same thread.
36	*
37	* @remark The way the ever increasing time guarantee is currently implemented means
38	* that if you call this function at a frequency higher than 1GHz you're in for
39	* trouble. We currently assume that no idiot will do that for real life purposes.
40	*
41	* @returns Nanosecond timestamp.
42	* @param pData Pointer to the data structure.
43	* @param pExtra Where to return extra time info. Optional.
44	*/
45	RTDECL(uint64_t) rtTimeNanoTSInternalRef(PRTTIMENANOTSDATA pData, PRTITMENANOTSEXTRA pExtra)
46	{
47	#if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA && defined(IN_RING3)
48	PSUPGIPCPU pGipCpuAttemptedTscRecalibration = NULL;
49	#endif
50	AssertCompile(RT_IS_POWER_OF_TWO(RTCPUSET_MAX_CPUS));
51
52	for (;;)
53	{
54	#ifndef IN_RING3 /* This simplifies and improves everything. */
55	RTCCUINTREG const uFlags = ASMIntDisableFlags();
56	#endif
57
58	/*
59	* Check that the GIP is sane and that the premises for this worker function
60	* hasn't changed (CPU onlined with bad delta or missing features).
61	*/
62	PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage;
63	if ( RT_LIKELY(pGip)
64	&& RT_LIKELY(pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC)
65	#if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA
66	&& RT_LIKELY(pGip->enmUseTscDelta >= SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
67	#else
68	&& RT_LIKELY(pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ROUGHLY_ZERO)
69	#endif
70	#if defined(IN_RING3) && TMPL_GET_CPU_METHOD != 0
71	&& RT_LIKELY(pGip->fGetGipCpu & TMPL_GET_CPU_METHOD)
72	#endif
73	)
74	{
75	/*
76	* Resolve pGipCpu if needed. If the instruction is serializing, we
77	* read the transaction id first if possible.
78	*/
79	#if TMPL_MODE == TMPL_MODE_ASYNC \|\| TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA
80	# if defined(IN_RING0)
81	uint32_t const iCpuSet = RTMpCurSetIndex();
82	uint16_t const iGipCpu = iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
83	? pGip->aiCpuFromCpuSetIdx[iCpuSet] : UINT16_MAX;
84	# elif defined(IN_RC)
85	uint32_t const iCpuSet = VMMGetCpu(&g_VM)->iHostCpuSet;
86	uint16_t const iGipCpu = iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
87	? pGip->aiCpuFromCpuSetIdx[iCpuSet] : UINT16_MAX;
88	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID
89	# if TMPL_MODE != TMPL_MODE_ASYNC
90	uint32_t const u32TransactionId = pGip->aCPUs[0].u32TransactionId;
91	# endif
92	uint8_t const idApic = ASMGetApicId();
93	uint16_t const iGipCpu = pGip->aiCpuFromApicId[idApic];
94	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID_EXT_0B
95	# if TMPL_MODE != TMPL_MODE_ASYNC
96	uint32_t const u32TransactionId = pGip->aCPUs[0].u32TransactionId;
97	# endif
98	uint32_t const idApic = ASMGetApicIdExt0B();
99	uint16_t const iGipCpu = pGip->aiCpuFromApicId[idApic];
100	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID_EXT_8000001E
101	# if TMPL_MODE != TMPL_MODE_ASYNC
102	uint32_t const u32TransactionId = pGip->aCPUs[0].u32TransactionId;
103	# endif
104	uint32_t const idApic = ASMGetApicIdExt8000001E();
105	uint16_t const iGipCpu = pGip->aiCpuFromApicId[idApic];
106	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS \
107	\|\| TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL
108	# if TMPL_MODE != TMPL_MODE_ASYNC
109	uint32_t const u32TransactionId = pGip->aCPUs[0].u32TransactionId;
110	# endif
111	uint32_t uAux;
112	ASMReadTscWithAux(&uAux);
113	# if TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS
114	uint16_t const iCpuSet = uAux & (RTCPUSET_MAX_CPUS - 1);
115	# else
116	uint16_t iCpuSet = 0;
117	uint16_t offGipCpuGroup = pGip->aoffCpuGroup[(uAux >> 8) & UINT8_MAX];
118	if (offGipCpuGroup < pGip->cPages * PAGE_SIZE)
119	{
120	PSUPGIPCPUGROUP pGipCpuGroup = (PSUPGIPCPUGROUP)((uintptr_t)pGip + offGipCpuGroup);
121	if ( (uAux & UINT8_MAX) < pGipCpuGroup->cMaxMembers
122	&& pGipCpuGroup->aiCpuSetIdxs[uAux & UINT8_MAX] != -1)
123	iCpuSet = pGipCpuGroup->aiCpuSetIdxs[uAux & UINT8_MAX];
124	}
125	# endif
126	uint16_t const iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
127	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS
128	uint16_t const cbLim = ASMGetIdtrLimit();
129	uint16_t const iCpuSet = (cbLim - 256 * (ARCH_BITS == 64 ? 16 : 8)) & (RTCPUSET_MAX_CPUS - 1);
130	uint16_t const iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
131	# else
132	# error "What?"
133	# endif
134	if (RT_LIKELY(iGipCpu < pGip->cCpus))
135	{
136	PSUPGIPCPU pGipCpu = &pGip->aCPUs[iGipCpu];
137	#else
138	{
139	#endif
140	/*
141	* Get the transaction ID if necessary and we haven't already
142	* read it before a serializing instruction above. We can skip
143	* this for ASYNC_TSC mode in ring-0 and raw-mode context since
144	* we disable interrupts.
145	*/
146	#if TMPL_MODE == TMPL_MODE_ASYNC && defined(IN_RING3)
147	uint32_t const u32TransactionId = pGipCpu->u32TransactionId;
148	ASMCompilerBarrier();
149	TMPL_READ_FENCE();
150	#elif TMPL_MODE != TMPL_MODE_ASYNC \
151	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID \
152	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_0B \
153	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_8000001E \
154	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS \
155	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL
156	uint32_t const u32TransactionId = pGip->aCPUs[0].u32TransactionId;
157	ASMCompilerBarrier();
158	TMPL_READ_FENCE();
159	#endif
160
161	/*
162	* Gather all the data we need. The mess at the end is to make
163	* sure all loads are done before we recheck the transaction ID
164	* without triggering serializing twice.
165	*/
166	uint32_t u32NanoTSFactor0 = pGip->u32UpdateIntervalNS;
167	#if TMPL_MODE == TMPL_MODE_ASYNC
168	uint32_t u32UpdateIntervalTSC = pGipCpu->u32UpdateIntervalTSC;
169	uint64_t u64NanoTS = pGipCpu->u64NanoTS;
170	uint64_t u64TSC = pGipCpu->u64TSC;
171	#else
172	uint32_t u32UpdateIntervalTSC = pGip->aCPUs[0].u32UpdateIntervalTSC;
173	uint64_t u64NanoTS = pGip->aCPUs[0].u64NanoTS;
174	uint64_t u64TSC = pGip->aCPUs[0].u64TSC;
175	# if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA
176	int64_t i64TscDelta = pGipCpu->i64TSCDelta;
177	# endif
178	#endif
179	uint64_t u64PrevNanoTS = ASMAtomicUoReadU64(pData->pu64Prev);
180	#if TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS \
181	\|\| TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL
182	ASMCompilerBarrier();
183	uint32_t uAux2;
184	uint64_t u64Delta = ASMReadTscWithAux(&uAux2); /* serializing */
185	#else
186	uint64_t u64Delta = ASMReadTSC();
187	ASMCompilerBarrier();
188	# if TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID /* getting APIC will serialize */ \
189	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_0B \
190	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_8000001E \
191	&& (defined(IN_RING3) \|\| TMPL_MODE != TMPL_MODE_ASYNC)
192	TMPL_READ_FENCE(); /* Expensive (~30 ticks). Would like convincing argumentation that let us remove it. */
193	# endif
194	#endif
195
196	/*
197	* Check that we didn't change CPU.
198	*/
199	#if defined(IN_RING3) && ( TMPL_MODE == TMPL_MODE_ASYNC \|\| TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA )
200	# if TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID
201	if (RT_LIKELY(ASMGetApicId() == idApic))
202	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID_EXT_0B
203	if (RT_LIKELY(ASMGetApicIdExt0B() == idApic))
204	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID_EXT_8000001E
205	if (RT_LIKELY(ASMGetApicIdExt8000001E() == idApic))
206	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS \
207	\|\| TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL
208	if (RT_LIKELY(uAux2 == uAux))
209	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS
210	if (RT_LIKELY(ASMGetIdtrLimit() == cbLim))
211	# endif
212	#endif
213	{
214	/*
215	* Check the transaction ID (see above for R0/RC + ASYNC).
216	*/
217	#if defined(IN_RING3) \|\| TMPL_MODE != TMPL_MODE_ASYNC
218	# if TMPL_MODE == TMPL_MODE_ASYNC
219	if (RT_LIKELY(pGipCpu->u32TransactionId == u32TransactionId && !(u32TransactionId & 1) ))
220	# else
221	if (RT_LIKELY(pGip->aCPUs[0].u32TransactionId == u32TransactionId && !(u32TransactionId & 1) ))
222	# endif
223	#endif
224	{
225
226	/*
227	* Apply the TSC delta. If the delta is invalid and the
228	* execution allows it, try trigger delta recalibration.
229	*/
230	#if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA && defined(IN_RING3)
231	if (RT_LIKELY( i64TscDelta != INT64_MAX
232	\|\| pGipCpu == pGipCpuAttemptedTscRecalibration))
233	#endif
234	{
235	#if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA
236	# ifndef IN_RING3
237	if (RT_LIKELY(i64TscDelta != INT64_MAX))
238	# endif
239	u64Delta -= i64TscDelta;
240	#endif
241
242	/*
243	* Bingo! We've got a consistent set of data.
244	*/
245	#ifndef IN_RING3
246	ASMSetFlags(uFlags);
247	#endif
248
249	if (pExtra)
250	pExtra->uTSCValue = u64Delta;
251
252	/*
253	* Calc NanoTS delta.
254	*/
255	u64Delta -= u64TSC;
256	if (RT_LIKELY(u64Delta <= u32UpdateIntervalTSC))
257	{ /* MSVC branch hint, probably pointless. */ }
258	else
259	{
260	/*
261	* We've expired the interval, cap it. If we're here for the 2nd
262	* time without any GIP update in-between, the checks against
263	* *pu64Prev below will force 1ns stepping.
264	*/
265	ASMAtomicIncU32(&pData->cExpired);
266	u64Delta = u32UpdateIntervalTSC;
267	}
268	#if !defined(_MSC_VER) \|\| !defined(RT_ARCH_X86) /* GCC makes very pretty code from these two inline calls, while MSC cannot. */
269	u64Delta = ASMMult2xU32RetU64((uint32_t)u64Delta, u32NanoTSFactor0);
270	u64Delta = ASMDivU64ByU32RetU32(u64Delta, u32UpdateIntervalTSC);
271	#else
272	__asm
273	{
274	mov eax, dword ptr [u64Delta]
275	mul dword ptr [u32NanoTSFactor0]
276	div dword ptr [u32UpdateIntervalTSC]
277	mov dword ptr [u64Delta], eax
278	xor edx, edx
279	mov dword ptr [u64Delta + 4], edx
280	}
281	#endif
282
283	/*
284	* Calculate the time and compare it with the previously returned value.
285	*/
286	u64NanoTS += u64Delta;
287	uint64_t u64DeltaPrev = u64NanoTS - u64PrevNanoTS;
288	if (RT_LIKELY( u64DeltaPrev > 0
289	&& u64DeltaPrev < UINT64_C(86000000000000) /* 24h */))
290	{ /* Frequent - less than 24h since last call. */ }
291	else if (RT_LIKELY( (int64_t)u64DeltaPrev <= 0
292	&& (int64_t)u64DeltaPrev + u32NanoTSFactor0 * 2 >= 0))
293	{
294	/* Occasional - u64NanoTS is in the recent 'past' relative the previous call. */
295	ASMAtomicIncU32(&pData->c1nsSteps);
296	u64NanoTS = u64PrevNanoTS + 1;
297	}
298	else if (!u64PrevNanoTS)
299	/* We're resuming (see TMVirtualResume). */;
300	else
301	{
302	/* Something has gone bust, if negative offset it's real bad. */
303	ASMAtomicIncU32(&pData->cBadPrev);
304	pData->pfnBad(pData, u64NanoTS, u64DeltaPrev, u64PrevNanoTS);
305	}
306
307	/*
308	* Attempt updating the previous value, provided we're still ahead of it.
309	*
310	* There is no point in recalculating u64NanoTS because we got preempted or if
311	* we raced somebody while the GIP was updated, since these are events
312	* that might occur at any point in the return path as well.
313	*/
314	if (RT_LIKELY(ASMAtomicCmpXchgU64(pData->pu64Prev, u64NanoTS, u64PrevNanoTS)))
315	return u64NanoTS;
316
317	ASMAtomicIncU32(&pData->cUpdateRaces);
318	for (int cTries = 25; cTries > 0; cTries--)
319	{
320	u64PrevNanoTS = ASMAtomicReadU64(pData->pu64Prev);
321	if (u64PrevNanoTS >= u64NanoTS)
322	break;
323	if (ASMAtomicCmpXchgU64(pData->pu64Prev, u64NanoTS, u64PrevNanoTS))
324	break;
325	ASMNopPause();
326	}
327	return u64NanoTS;
328	}
329
330	#if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA && defined(IN_RING3)
331	/*
332	* Call into the support driver to try make it recalculate the delta. We
333	* remember which GIP CPU structure we're probably working on so we won't
334	* end up in a loop if the driver for some reason cannot get the job done.
335	*/
336	else /* else is unecessary, but helps checking the preprocessor spaghetti. */
337	{
338	pGipCpuAttemptedTscRecalibration = pGipCpu;
339	uint64_t u64TscTmp;
340	uint16_t idApicUpdate;
341	int rc = SUPR3ReadTsc(&u64TscTmp, &idApicUpdate);
342	if (RT_SUCCESS(rc) && idApicUpdate < RT_ELEMENTS(pGip->aiCpuFromApicId))
343	{
344	uint32_t iUpdateGipCpu = pGip->aiCpuFromApicId[idApicUpdate];
345	if (iUpdateGipCpu < pGip->cCpus)
346	pGipCpuAttemptedTscRecalibration = &pGip->aCPUs[iUpdateGipCpu];
347	}
348	}
349	#endif
350	}
351	}
352
353	/*
354	* No joy must try again.
355	*/
356	#ifdef _MSC_VER
357	# pragma warning(disable: 4702)
358	#endif
359	#ifndef IN_RING3
360	ASMSetFlags(uFlags);
361	#endif
362	ASMNopPause();
363	continue;
364	}
365
366	#if TMPL_MODE == TMPL_MODE_ASYNC \|\| TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA
367	/*
368	* We've got a bad CPU or APIC index of some kind.
369	*/
370	else /* else is unecessary, but helps checking the preprocessor spaghetti. */
371	{
372	# ifndef IN_RING3
373	ASMSetFlags(uFlags);
374	# endif
375	# if defined(IN_RING0) \
376	\|\| defined(IN_RC) \
377	\|\| ( TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID \
378	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_0B /?/ \
379	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_8000001E /?/)
380	return pData->pfnBadCpuIndex(pData, pExtra, UINT16_MAX-1, iCpuSet, iGipCpu);
381	# else
382	return pData->pfnBadCpuIndex(pData, pExtra, idApic, UINT16_MAX-1, iGipCpu);
383	# endif
384	}
385	#endif
386	}
387
388	/*
389	* Something changed in the GIP config or it was unmapped, figure out
390	* the right worker function to use now.
391	*/
392	#ifndef IN_RING3
393	ASMSetFlags(uFlags);
394	#endif
395	return pData->pfnRediscover(pData, pExtra);
396	}
397	}
398

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/common/time/timesupref.h@ 93115

Download in other formats: