timesupref.h@ 107192

Last change on this file since 107192 was 107192, checked in by vboxsync, 7 weeks ago
IPRT,VMM/TM: Support for GIP time on win.arm64. jiraref:VBP-1266
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 18.8 KB

Line
1	/* $Id: timesupref.h 107192 2024-11-29 14:42:15Z vboxsync $ */
2	/** @file
3	* IPRT - Time using SUPLib, the C Code Template.
4	*/
5
6	/*
7	* Copyright (C) 2006-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* The contents of this file may alternatively be used under the terms
26	* of the Common Development and Distribution License Version 1.0
27	* (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28	* in the VirtualBox distribution, in which case the provisions of the
29	* CDDL are applicable instead of those of the GPL.
30	*
31	* You may elect to license modified versions of this file under the
32	* terms and conditions of either the GPL or the CDDL or both.
33	*
34	* SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35	*/
36
37
38	/**
39	* The C reference implementation of the assembly routines.
40	*
41	* Calculate NanoTS using the information in the global information page (GIP)
42	* which the support library (SUPLib) exports.
43	*
44	* This function guarantees that the returned timestamp is later (in time) than
45	* any previous calls in the same thread.
46	*
47	* @remark The way the ever increasing time guarantee is currently implemented means
48	* that if you call this function at a frequency higher than 1GHz you're in for
49	* trouble. We currently assume that no idiot will do that for real life purposes.
50	*
51	* @returns Nanosecond timestamp.
52	* @param pData Pointer to the data structure.
53	* @param pExtra Where to return extra time info. Optional.
54	*/
55	RTDECL(uint64_t) rtTimeNanoTSInternalRef(PRTTIMENANOTSDATA pData, PRTITMENANOTSEXTRA pExtra)
56	{
57	#if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA && defined(IN_RING3)
58	PSUPGIPCPU pGipCpuAttemptedTscRecalibration = NULL;
59	#endif
60	AssertCompile(RT_IS_POWER_OF_TWO(RTCPUSET_MAX_CPUS));
61
62	for (;;)
63	{
64	#ifndef IN_RING3 /* This simplifies and improves everything. */
65	RTCCUINTREG const uFlags = ASMIntDisableFlags();
66	#endif
67
68	/*
69	* Check that the GIP is sane and that the premises for this worker function
70	* hasn't changed (CPU onlined with bad delta or missing features).
71	*/
72	PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage;
73	if ( RT_LIKELY(pGip)
74	&& RT_LIKELY(pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC)
75	#if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA
76	&& RT_LIKELY(pGip->enmUseTscDelta >= SUPGIPUSETSCDELTA_PRACTICALLY_ZERO)
77	#else
78	&& RT_LIKELY(pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ROUGHLY_ZERO)
79	#endif
80	#if defined(IN_RING3) && TMPL_GET_CPU_METHOD != 0
81	&& RT_LIKELY(pGip->fGetGipCpu & TMPL_GET_CPU_METHOD)
82	#endif
83	)
84	{
85	/*
86	* Resolve pGipCpu if needed. If the instruction is serializing, we
87	* read the transaction id first if possible.
88	*/
89	#if TMPL_MODE == TMPL_MODE_ASYNC \|\| TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA
90	# if defined(IN_RING0)
91	uint32_t const iCpuSet = RTMpCurSetIndex();
92	uint16_t const iGipCpu = iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
93	? pGip->aiCpuFromCpuSetIdx[iCpuSet] : UINT16_MAX;
94	# elif defined(IN_RC)
95	uint32_t const iCpuSet = VMMGetCpu(&g_VM)->iHostCpuSet;
96	uint16_t const iGipCpu = iCpuSet < RT_ELEMENTS(pGip->aiCpuFromCpuSetIdx)
97	? pGip->aiCpuFromCpuSetIdx[iCpuSet] : UINT16_MAX;
98	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID
99	# if TMPL_MODE != TMPL_MODE_ASYNC
100	uint32_t const u32TransactionId = pGip->aCPUs[0].u32TransactionId;
101	# endif
102	uint8_t const idApic = ASMGetApicId();
103	uint16_t const iGipCpu = pGip->aiCpuFromApicId[idApic];
104	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID_EXT_0B
105	# if TMPL_MODE != TMPL_MODE_ASYNC
106	uint32_t const u32TransactionId = pGip->aCPUs[0].u32TransactionId;
107	# endif
108	uint32_t const idApic = ASMGetApicIdExt0B();
109	uint16_t const iGipCpu = pGip->aiCpuFromApicId[idApic];
110	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID_EXT_8000001E
111	# if TMPL_MODE != TMPL_MODE_ASYNC
112	uint32_t const u32TransactionId = pGip->aCPUs[0].u32TransactionId;
113	# endif
114	uint32_t const idApic = ASMGetApicIdExt8000001E();
115	uint16_t const iGipCpu = pGip->aiCpuFromApicId[idApic];
116	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS \
117	\|\| TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL
118	# if TMPL_MODE != TMPL_MODE_ASYNC
119	uint32_t const u32TransactionId = pGip->aCPUs[0].u32TransactionId;
120	# endif
121	uint32_t uAux;
122	ASMReadTscWithAux(&uAux);
123	# if TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS
124	uint16_t const iCpuSet = uAux & (RTCPUSET_MAX_CPUS - 1);
125	# else
126	uint16_t iCpuSet = 0;
127	uint16_t offGipCpuGroup = pGip->aoffCpuGroup[(uAux >> 8) & UINT8_MAX];
128	if (offGipCpuGroup < pGip->cPages * PAGE_SIZE)
129	{
130	PSUPGIPCPUGROUP pGipCpuGroup = (PSUPGIPCPUGROUP)((uintptr_t)pGip + offGipCpuGroup);
131	if ( (uAux & UINT8_MAX) < pGipCpuGroup->cMaxMembers
132	&& pGipCpuGroup->aiCpuSetIdxs[uAux & UINT8_MAX] != -1)
133	iCpuSet = pGipCpuGroup->aiCpuSetIdxs[uAux & UINT8_MAX];
134	}
135	# endif
136	uint16_t const iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
137	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS
138	uint16_t const cbLim = ASMGetIdtrLimit();
139	uint16_t const iCpuSet = (cbLim - 256 * (ARCH_BITS == 64 ? 16 : 8)) & (RTCPUSET_MAX_CPUS - 1);
140	uint16_t const iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
141	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_TPIDRRO_EL0
142	uint32_t const iCpuSet = ASMGetThreadIdRoEL0() & (RTCPUSET_MAX_CPUS - 1);
143	uint16_t const iGipCpu = pGip->aiCpuFromCpuSetIdx[iCpuSet];
144	# else
145	# error "What?"
146	# endif
147	if (RT_LIKELY(iGipCpu < pGip->cCpus))
148	{
149	PSUPGIPCPU pGipCpu = &pGip->aCPUs[iGipCpu];
150	#else
151	{
152	#endif
153	/*
154	* Get the transaction ID if necessary and we haven't already
155	* read it before a serializing instruction above. We can skip
156	* this for ASYNC_TSC mode in ring-0 and raw-mode context since
157	* we disable interrupts.
158	*/
159	#if TMPL_MODE == TMPL_MODE_ASYNC && defined(IN_RING3)
160	uint32_t const u32TransactionId = pGipCpu->u32TransactionId;
161	ASMCompilerBarrier();
162	TMPL_READ_FENCE();
163	#elif TMPL_MODE != TMPL_MODE_ASYNC \
164	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID \
165	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_0B \
166	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_8000001E \
167	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS \
168	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL
169	uint32_t const u32TransactionId = pGip->aCPUs[0].u32TransactionId;
170	ASMCompilerBarrier();
171	TMPL_READ_FENCE();
172	#endif
173
174	/*
175	* Gather all the data we need. The mess at the end is to make
176	* sure all loads are done before we recheck the transaction ID
177	* without triggering serializing twice.
178	*/
179	uint32_t u32NanoTSFactor0 = pGip->u32UpdateIntervalNS;
180	#if TMPL_MODE == TMPL_MODE_ASYNC
181	uint32_t u32UpdateIntervalTSC = pGipCpu->u32UpdateIntervalTSC;
182	uint64_t u64NanoTS = pGipCpu->u64NanoTS;
183	uint64_t u64TSC = pGipCpu->u64TSC;
184	#else
185	uint32_t u32UpdateIntervalTSC = pGip->aCPUs[0].u32UpdateIntervalTSC;
186	uint64_t u64NanoTS = pGip->aCPUs[0].u64NanoTS;
187	uint64_t u64TSC = pGip->aCPUs[0].u64TSC;
188	# if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA
189	int64_t i64TscDelta = pGipCpu->i64TSCDelta;
190	# endif
191	#endif
192	uint64_t u64PrevNanoTS = ASMAtomicUoReadU64(pData->pu64Prev);
193	#if TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS \
194	\|\| TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL
195	ASMCompilerBarrier();
196	uint32_t uAux2;
197	uint64_t u64Delta = ASMReadTscWithAux(&uAux2); /* serializing */
198	#else
199	uint64_t u64Delta = ASMReadTSC();
200	ASMCompilerBarrier();
201	# if TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID /* getting APIC will serialize */ \
202	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_0B \
203	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_8000001E \
204	&& (defined(IN_RING3) \|\| TMPL_MODE != TMPL_MODE_ASYNC)
205	TMPL_READ_FENCE(); /* Expensive (~30 ticks). Would like convincing argumentation that let us remove it. */
206	# endif
207	#endif
208
209	/*
210	* Check that we didn't change CPU.
211	*/
212	#if defined(IN_RING3) && ( TMPL_MODE == TMPL_MODE_ASYNC \|\| TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA )
213	# if TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID
214	if (RT_LIKELY(ASMGetApicId() == idApic))
215	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID_EXT_0B
216	if (RT_LIKELY(ASMGetApicIdExt0B() == idApic))
217	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_APIC_ID_EXT_8000001E
218	if (RT_LIKELY(ASMGetApicIdExt8000001E() == idApic))
219	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS \
220	\|\| TMPL_GET_CPU_METHOD == SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL
221	if (RT_LIKELY(uAux2 == uAux))
222	# elif TMPL_GET_CPU_METHOD == SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS
223	if (RT_LIKELY(ASMGetIdtrLimit() == cbLim))
224	# endif
225	#endif
226	{
227	/*
228	* Check the transaction ID (see above for R0/RC + ASYNC).
229	*/
230	#if defined(IN_RING3) \|\| TMPL_MODE != TMPL_MODE_ASYNC
231	# if TMPL_MODE == TMPL_MODE_ASYNC
232	if (RT_LIKELY(pGipCpu->u32TransactionId == u32TransactionId && !(u32TransactionId & 1) ))
233	# else
234	if (RT_LIKELY(pGip->aCPUs[0].u32TransactionId == u32TransactionId && !(u32TransactionId & 1) ))
235	# endif
236	#endif
237	{
238
239	/*
240	* Apply the TSC delta. If the delta is invalid and the
241	* execution allows it, try trigger delta recalibration.
242	*/
243	#if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA && defined(IN_RING3)
244	if (RT_LIKELY( i64TscDelta != INT64_MAX
245	\|\| pGipCpu == pGipCpuAttemptedTscRecalibration))
246	#endif
247	{
248	#if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA
249	# ifndef IN_RING3
250	if (RT_LIKELY(i64TscDelta != INT64_MAX))
251	# endif
252	u64Delta -= i64TscDelta;
253	#endif
254
255	/*
256	* Bingo! We've got a consistent set of data.
257	*/
258	#ifndef IN_RING3
259	ASMSetFlags(uFlags);
260	#endif
261
262	if (pExtra)
263	pExtra->uTSCValue = u64Delta;
264
265	/*
266	* Calc NanoTS delta.
267	*/
268	u64Delta -= u64TSC;
269	if (RT_LIKELY(u64Delta <= u32UpdateIntervalTSC))
270	{ /* MSVC branch hint, probably pointless. */ }
271	else
272	{
273	/*
274	* We've expired the interval, cap it. If we're here for the 2nd
275	* time without any GIP update in-between, the checks against
276	* *pu64Prev below will force 1ns stepping.
277	*/
278	ASMAtomicIncU32(&pData->cExpired);
279	u64Delta = u32UpdateIntervalTSC;
280	}
281	#if !defined(_MSC_VER) \|\| !defined(RT_ARCH_X86) /* GCC makes very pretty code from these two inline calls, while MSC cannot. */
282	u64Delta = ASMMult2xU32RetU64((uint32_t)u64Delta, u32NanoTSFactor0);
283	u64Delta = ASMDivU64ByU32RetU32(u64Delta, u32UpdateIntervalTSC);
284	#else
285	__asm
286	{
287	mov eax, dword ptr [u64Delta]
288	mul dword ptr [u32NanoTSFactor0]
289	div dword ptr [u32UpdateIntervalTSC]
290	mov dword ptr [u64Delta], eax
291	xor edx, edx
292	mov dword ptr [u64Delta + 4], edx
293	}
294	#endif
295
296	/*
297	* Calculate the time and compare it with the previously returned value.
298	*/
299	u64NanoTS += u64Delta;
300	uint64_t u64DeltaPrev = u64NanoTS - u64PrevNanoTS;
301	if (RT_LIKELY( u64DeltaPrev > 0
302	&& u64DeltaPrev < UINT64_C(86000000000000) /* 24h */))
303	{ /* Frequent - less than 24h since last call. */ }
304	else if (RT_LIKELY( (int64_t)u64DeltaPrev <= 0
305	&& (int64_t)u64DeltaPrev + u32NanoTSFactor0 * 2 >= 0))
306	{
307	/* Occasional - u64NanoTS is in the recent 'past' relative the previous call. */
308	ASMAtomicIncU32(&pData->c1nsSteps);
309	u64NanoTS = u64PrevNanoTS + 1;
310	}
311	else if (!u64PrevNanoTS)
312	/* We're resuming (see TMVirtualResume). */;
313	else
314	{
315	/* Something has gone bust, if negative offset it's real bad. */
316	ASMAtomicIncU32(&pData->cBadPrev);
317	pData->pfnBad(pData, u64NanoTS, u64DeltaPrev, u64PrevNanoTS);
318	}
319
320	/*
321	* Attempt updating the previous value, provided we're still ahead of it.
322	*
323	* There is no point in recalculating u64NanoTS because we got preempted or if
324	* we raced somebody while the GIP was updated, since these are events
325	* that might occur at any point in the return path as well.
326	*/
327	if (RT_LIKELY(ASMAtomicCmpXchgU64(pData->pu64Prev, u64NanoTS, u64PrevNanoTS)))
328	return u64NanoTS;
329
330	ASMAtomicIncU32(&pData->cUpdateRaces);
331	for (int cTries = 25; cTries > 0; cTries--)
332	{
333	u64PrevNanoTS = ASMAtomicReadU64(pData->pu64Prev);
334	if (u64PrevNanoTS >= u64NanoTS)
335	break;
336	if (ASMAtomicCmpXchgU64(pData->pu64Prev, u64NanoTS, u64PrevNanoTS))
337	break;
338	ASMNopPause();
339	}
340	return u64NanoTS;
341	}
342
343	#if TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA && defined(IN_RING3)
344	/*
345	* Call into the support driver to try make it recalculate the delta. We
346	* remember which GIP CPU structure we're probably working on so we won't
347	* end up in a loop if the driver for some reason cannot get the job done.
348	*/
349	else /* else is unecessary, but helps checking the preprocessor spaghetti. */
350	{
351	pGipCpuAttemptedTscRecalibration = pGipCpu;
352	uint64_t u64TscTmp;
353	uint16_t idApicUpdate;
354	int rc = SUPR3ReadTsc(&u64TscTmp, &idApicUpdate);
355	if (RT_SUCCESS(rc) && idApicUpdate < RT_ELEMENTS(pGip->aiCpuFromApicId))
356	{
357	uint32_t iUpdateGipCpu = pGip->aiCpuFromApicId[idApicUpdate];
358	if (iUpdateGipCpu < pGip->cCpus)
359	pGipCpuAttemptedTscRecalibration = &pGip->aCPUs[iUpdateGipCpu];
360	}
361	}
362	#endif
363	}
364	}
365
366	/*
367	* No joy must try again.
368	*/
369	#ifdef _MSC_VER
370	# pragma warning(disable: 4702)
371	#endif
372	#ifndef IN_RING3
373	ASMSetFlags(uFlags);
374	#endif
375	ASMNopPause();
376	continue;
377	}
378
379	#if TMPL_MODE == TMPL_MODE_ASYNC \|\| TMPL_MODE == TMPL_MODE_SYNC_INVAR_WITH_DELTA
380	/*
381	* We've got a bad CPU or APIC index of some kind.
382	*/
383	else /* else is unecessary, but helps checking the preprocessor spaghetti. */
384	{
385	# ifndef IN_RING3
386	ASMSetFlags(uFlags);
387	# endif
388	# if defined(IN_RING0) \
389	\|\| defined(IN_RC) \
390	\|\| ( TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID \
391	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_0B /?/ \
392	&& TMPL_GET_CPU_METHOD != SUPGIPGETCPU_APIC_ID_EXT_8000001E /?/)
393	return pData->pfnBadCpuIndex(pData, pExtra, UINT16_MAX-1, iCpuSet, iGipCpu);
394	# else
395	return pData->pfnBadCpuIndex(pData, pExtra, idApic, UINT16_MAX-1, iGipCpu);
396	# endif
397	}
398	#endif
399	}
400
401	/*
402	* Something changed in the GIP config or it was unmapped, figure out
403	* the right worker function to use now.
404	*/
405	#ifndef IN_RING3
406	ASMSetFlags(uFlags);
407	#endif
408	return pData->pfnRediscover(pData, pExtra);
409	}
410	}
411

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/common/time/timesupref.h@ 107192

Download in other formats: