timesupA.mac@ 99844

Last change on this file since 99844 was 98103, checked in by vboxsync, 2 years ago
Copyright year updates by scm.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 23.6 KB

Line
1	; $Id: timesupA.mac 98103 2023-01-17 14:15:46Z vboxsync $
2	;; @file
3	; IPRT - Time using SUPLib, the Assembly Code Template.
4	;
5
6	;
7	; Copyright (C) 2006-2023 Oracle and/or its affiliates.
8	;
9	; This file is part of VirtualBox base platform packages, as
10	; available from https://www.virtualbox.org.
11	;
12	; This program is free software; you can redistribute it and/or
13	; modify it under the terms of the GNU General Public License
14	; as published by the Free Software Foundation, in version 3 of the
15	; License.
16	;
17	; This program is distributed in the hope that it will be useful, but
18	; WITHOUT ANY WARRANTY; without even the implied warranty of
19	; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	; General Public License for more details.
21	;
22	; You should have received a copy of the GNU General Public License
23	; along with this program; if not, see <https://www.gnu.org/licenses>.
24	;
25	; The contents of this file may alternatively be used under the terms
26	; of the Common Development and Distribution License Version 1.0
27	; (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28	; in the VirtualBox distribution, in which case the provisions of the
29	; CDDL are applicable instead of those of the GPL.
30	;
31	; You may elect to license modified versions of this file under the
32	; terms and conditions of either the GPL or the CDDL or both.
33	;
34	; SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35	;
36
37	%ifdef RT_ARCH_X86
38	;;
39	; The x86 assembly implementation of the assembly routines.
40	;
41	; @returns Nanosecond timestamp.
42	; @param pData Pointer to the nanosecond timestamp data.
43	;
44	BEGINPROC rtTimeNanoTSInternalAsm
45	;
46	; Variable definitions.
47	;
48	%define pData [ebp + 08h]
49	%define u64RetNanoTS_Hi [ebp - 04h]
50	%define u64RetNanoTS [ebp - 08h]
51	%define u32UpdateIntervalNS [ebp - 0ch]
52	%define u32UpdateIntervalTSC [ebp - 10h]
53	%define u64TSC_Hi [ebp - 14h]
54	%define u64TSC [ebp - 18h]
55	%define u64CurNanoTS_Hi [ebp - 1ch]
56	%define u64CurNanoTS [ebp - 20h]
57	%define u64PrevNanoTS_Hi [ebp - 24h]
58	%define u64PrevNanoTS [ebp - 28h]
59	%define u32TransactionId [ebp - 2ch]
60	%define u32ApicIdPlus [ebp - 30h]
61	%define TmpVar [ebp - 34h]
62	%define SavedEBX [ebp - 38h]
63	%define SavedEDI [ebp - 3ch]
64	%define SavedESI [ebp - 40h]
65
66	;
67	; Prolog.
68	;
69	push ebp
70	mov ebp, esp
71	sub esp, 40h
72	mov SavedEBX, ebx
73	mov SavedEDI, edi
74	mov SavedESI, esi
75
76
77	;;
78	;; Read the GIP data and the previous value.
79	;;
80	.ReadGip:
81
82
83	;
84	; Load pGip.
85	;
86	%ifdef IMPORTED_SUPLIB
87	%ifdef IN_RING0
88	mov esi, IMP(g_SUPGlobalInfoPage)
89	%else
90	mov esi, IMP(g_pSUPGlobalInfoPage)
91	mov esi, [esi]
92	%endif
93	%else
94	mov esi, [NAME(g_pSUPGlobalInfoPage)]
95	%endif
96	or esi, esi
97	jz .Rediscover
98	cmp dword [esi + SUPGLOBALINFOPAGE.u32Magic], SUPGLOBALINFOPAGE_MAGIC
99	jne .Rediscover
100
101	;
102	; Calc pGipCPU, setting u32ApicIdPlus if necessary.
103	;
104	%ifdef NEED_APIC_ID
105	; u8ApicId = ASMGetApicId();
106	mov eax, 1
107	cpuid ; expensive
108	%ifdef NEED_TRANSACTION_ID
109	mov u32ApicIdPlus, ebx
110	%endif
111	; pGipCpu/pGipCpuDelta = &pGip->aCPU[pGip->aiCpuFromApicId[u8ApicId]];
112	shr ebx, 24
113	movzx ebx, word [esi + ebx * 2 + SUPGLOBALINFOPAGE.aiCpuFromApicId]
114	mov eax, SUPGIPCPU_size
115	mul ebx
116	lea edi, [esi + eax + SUPGLOBALINFOPAGE.aCPUs] ; edi == &pGip->aCPU[u8ApicId];
117	%endif
118
119	%ifdef NEED_TRANSACTION_ID
120	;
121	; Serialized loading of u32TransactionId.
122	;
123	%ifdef ASYNC_GIP
124	mov ebx, [edi + SUPGIPCPU.u32TransactionId]
125	%else
126	mov ebx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
127	%endif
128	mov u32TransactionId, ebx
129	%ifdef USE_LFENCE
130	lfence
131	%else
132	lock xor dword TmpVar, 0
133	%endif
134	%endif
135
136	;
137	; Load the data and TSC with delta applied.
138	;
139	mov eax, [esi + SUPGLOBALINFOPAGE.u32UpdateIntervalNS]
140	mov u32UpdateIntervalNS, eax
141	%ifdef ASYNC_GIP ; esi is now free.
142	mov edx, [edi + SUPGIPCPU.u32UpdateIntervalTSC]
143	%else
144	mov edx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32UpdateIntervalTSC]
145	%endif
146	mov u32UpdateIntervalTSC, edx
147
148	rdtsc
149	%ifdef WITH_TSC_DELTA
150	cmp dword [edi + SUPGIPCPU.i64TSCDelta], 0xffffffff
151	je .TscDeltaPossiblyInvalid
152	.TscDeltaValid:
153	sub eax, dword [edi + SUPGIPCPU.i64TSCDelta]
154	sbb edx, dword [edi + SUPGIPCPU.i64TSCDelta + 4]
155	.TscDeltaNotValid: ; edi is now free.
156	%endif
157
158	%ifdef ASYNC_GIP
159	mov ecx, [edi + SUPGIPCPU.u64NanoTS]
160	mov esi, [edi + SUPGIPCPU.u64NanoTS + 4]
161	%else
162	mov ecx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64NanoTS]
163	mov ebx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64NanoTS + 4]
164	%endif
165	mov u64CurNanoTS, ecx
166	mov u64CurNanoTS_Hi, ebx
167	%ifdef ASYNC_GIP
168	mov ecx, [edi + SUPGIPCPU.u64TSC]
169	mov ebx, [edi + SUPGIPCPU.u64TSC + 4]
170	%else
171	mov ecx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64TSC]
172	mov ebx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64TSC + 4]
173	%endif
174	mov u64TSC, ecx
175	mov u64TSC_Hi, ebx
176
177	; u64PrevNanoTS = ASMAtomicReadU64(pu64Prev);
178	; This serializes load/save. And with the dependency on the
179	; RDTSC result, we try to make sure it has completed as well.
180	%ifdef ASYNC_GIP
181	mov esi, pData
182	mov esi, [esi + RTTIMENANOTSDATA.pu64Prev]
183	%else
184	mov edi, pData
185	mov edi, [esi + RTTIMENANOTSDATA.pu64Prev]
186	%endif
187	mov ebx, eax
188	mov ecx, edx
189	%ifdef ASYNC_GIP
190	lock cmpxchg8b [esi]
191	%else
192	lock cmpxchg8b [edi]
193	%endif
194	mov u64PrevNanoTS, eax
195	mov u64PrevNanoTS_Hi, edx
196
197	%undef SAVED_u64RetNanoTS
198	%ifdef NEED_TRANSACTION_ID
199	;
200	; Check that the GIP and CPU didn't change.
201	; We've already serialized all the loads and stores at this point.
202	;
203	%ifdef NEED_APIC_ID
204	mov u64RetNanoTS, ebx
205	mov u64RetNanoTS_Hi, ecx
206	%define SAVED_u64RetNanoTS
207	mov eax, 1
208	cpuid
209	cmp u32ApicIdPlus, ebx
210	jne .ReadGip
211	%endif
212	%ifdef ASYNC_GIP
213	mov esi, [edi + SUPGIPCPU.u32TransactionId]
214	%else
215	mov esi, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
216	%endif
217	cmp esi, u32TransactionId
218	jne .ReadGip
219	test esi, 1
220	jnz .ReadGip
221	%endif ; NEED_TRANSACTION_ID
222	%ifdef SAVED_u64RetNanoTS
223	mov ebx, u64RetNanoTS
224	mov ecx, u64RetNanoTS_Hi
225	%endif
226
227	;;
228	;; Calc the timestamp.
229	;;
230	; u64RetNanoTS -= u64TSC;
231	sub ebx, u64TSC
232	sbb ecx, u64TSC_Hi
233
234	; if (u64RetNanoTS > u32UpdateIntervalTSC) -> jump
235	or ecx, ecx
236	jnz .OverFlow
237	cmp ebx, u32UpdateIntervalTSC
238	ja .OverFlow
239	mov eax, ebx
240	.ContinueCalcs: ; eax <= u32UpdateIntervalTSC
241	mul dword u32UpdateIntervalNS
242	div dword u32UpdateIntervalTSC
243	xor edx, edx
244
245	; u64RetNanoTS += u64CurNanoTS;
246	add eax, u64CurNanoTS
247	adc edx, u64CurNanoTS_Hi
248
249	;;
250	;; Compare it with the previous one.
251	;;
252	; if (RT_LIKELY( u64RetNanoTS > u64PrevNanoTS
253	; && u64RetNanoTS < u64PrevNanoTS + UINT64_C(86000000000000) /* 24h */))
254	;; @todo optimize this compare (/me too tired).
255	mov ecx, u64PrevNanoTS_Hi
256	mov ebx, u64PrevNanoTS
257	cmp edx, ecx
258	ja .Compare2
259	jb .DeltaPrevTooBig
260	cmp eax, ebx
261	jbe .DeltaPrevTooBig
262
263	.Compare2:
264	add ebx, 0x6F736000
265	adc ecx, 0x00004E37
266	cmp edx, ecx
267	jb .CompareDone
268	ja .DeltaPrevTooBig
269	cmp eax, ebx
270	jae .DeltaPrevTooBig
271	.CompareDone:
272
273
274	;;
275	;; Update the previous value with the u64RetNanoTS value.
276	;;
277	.Update:
278	; if (RT_LIKELY(ASMAtomicCmpXchgU64(&pData->u64Prev, u64RetNanoTS, u64PrevNanoTS)))
279	mov ebx, eax
280	mov ecx, edx
281	mov esi, pData
282	mov esi, [esi + RTTIMENANOTSDATA.pu64Prev]
283	mov eax, u64PrevNanoTS
284	mov edx, u64PrevNanoTS_Hi
285	lock cmpxchg8b [esi]
286	jnz .UpdateFailed
287
288	.Updated:
289	mov eax, ebx
290	mov edx, ecx
291
292	.Done:
293	mov esi, SavedESI
294	mov edi, SavedEDI
295	mov ebx, SavedEBX
296	leave
297	ret
298
299
300	;;
301	;; We've expired the interval, cap it. If we're here for the 2nd
302	;; time without any GIP update in-between, the checks against
303	;; pData->u64Prev below will force 1ns stepping.
304	;;
305	.OverFlow:
306	; u64Delta = u32UpdateIntervalTSC;
307	mov esi, pData
308	inc dword [esi + RTTIMENANOTSDATA.cExpired]
309	mov eax, u32UpdateIntervalTSC
310	jmp .ContinueCalcs
311
312
313	;;
314	;; u64DeltaPrev >= 24h
315	;;
316	;; eax:edx = u64RetNanoTS (to be adjusted)
317	;;
318	.DeltaPrevTooBig:
319	; uint64_t u64DeltaPrev = u64RetNanoTS - u64PrevNanoTS;
320	mov ebx, eax
321	sub ebx, u64PrevNanoTS
322	mov ecx, edx
323	sbb ecx, u64PrevNanoTS_Hi ; ebx:ecx = u64DeltaPrev
324
325	; else if ( (int64_t)u64DeltaPrev <= 0
326	; && (int64_t)u64DeltaPrev + u32UpdateIntervalNS * 2 >= 0)
327	; {
328	; /* Occasional - u64RetNanoTS is in the recent 'past' relative the previous call. */
329	; pData->c1nsSteps++;
330	; u64RetNanoTS = u64PrevNanoTS + 1;
331	; }
332	mov esi, u32UpdateIntervalNS
333	cmp ecx, 0
334	jl .PrevNotZero2ndTest
335	jg .DeltaPrevNotInRecentPast
336	cmp ebx, 0
337	ja .DeltaPrevNotInRecentPast
338
339	.PrevNotZero2ndTest:
340	add esi, esi ; ASSUMES: u32UpdateIntervalNS * 2 <= 32-bit.
341	xor edi, edi
342	add esi, ebx
343	adc edi, ecx
344	test edi, edi
345	js .DeltaPrevNotInRecentPast
346
347	.DeltaPrevInRecentPast:
348	mov esi, pData
349	inc dword [esi + RTTIMENANOTSDATA.c1nsSteps]
350	mov eax, u64PrevNanoTS
351	mov edx, u64PrevNanoTS_Hi
352	add eax, 1
353	adc edx, 0
354	jmp .Update
355
356	.DeltaPrevNotInRecentPast:
357	; else if (!u64PrevNanoTS) /* We're resuming (see TMVirtualResume). */
358	; /* do nothing */;
359	cmp dword u64PrevNanoTS, 0
360	jne .DeltaPrevNotZero
361	cmp dword u64PrevNanoTS_Hi, 0
362	jne .DeltaPrevNotZero
363	jmp .Update
364
365	.DeltaPrevNotZero:
366	; else
367	; {
368	; /* Something has gone bust, if negative offset it's real bad. */
369	; rtTimeNanoTSInternalBitch(pVM,
370	; }
371
372	; call C function that does the bitching.
373	mov u64RetNanoTS, eax
374	mov u64RetNanoTS_Hi, edx
375
376	mov edi, u64PrevNanoTS_Hi
377	mov esi, u64PrevNanoTS
378	push edi
379	push esi ; 4 - u64PrevNanoTS
380	push ecx
381	push ebx ; 3 - u64DeltaPrev
382	push edx
383	push eax ; 2 - u64RetNanoTS
384	mov eax, pData
385	push eax ; 1 - pData
386	call dword [eax + RTTIMENANOTSDATA.pfnBad]
387	add esp, 4*7
388
389	mov eax, u64RetNanoTS
390	mov edx, u64RetNanoTS_Hi
391	jmp .Update
392
393
394	;;
395	;; Attempt updating the previous value, provided we're still ahead of it.
396	;;
397	;; There is no point in recalculating u64NanoTS because we got preempted or if
398	;; we raced somebody while the GIP was updated, since these are events
399	;; that might occur at any point in the return path as well.
400	;;
401	;; eax:edx = *pData->u64Prev
402	;; ebx:ecx = u64RetNanoTS
403	;;
404	ALIGNCODE(16)
405	.UpdateFailed:
406	mov edi, pData
407	lock inc dword [edi + RTTIMENANOTSDATA.cUpdateRaces]
408	; for (i = 0; i < 10; i++)
409	mov edi, 10
410	.UpdateLoop:
411	; if (u64PrevNanoTS >= u64NanoTS)
412	; break;
413	cmp edx, ecx
414	jg .Updated
415	jne .UpdateLoopLess
416	cmp eax, ebx
417	jae .Updated
418	.UpdateLoopLess:
419	; retry
420	lock cmpxchg8b [esi]
421	jz .Updated
422	dec edi
423	jnz .UpdateLoop
424	jmp .Updated
425
426
427	;;
428	;; The GIP is seemingly invalid, redo the discovery.
429	;;
430	.Rediscover:
431	mov eax, pData
432	push eax
433	call [eax + RTTIMENANOTSDATA.pfnRediscover]
434	add esp, 4h
435	jmp .Done
436
437
438	%ifdef WITH_TSC_DELTA
439	;;
440	;; Unlikely branch for when we think the TSC delta might be invalid.
441	;;
442	.TscDeltaPossiblyInvalid:
443	cmp dword [edi + SUPGIPCPU.i64TSCDelta + 4], 0x7fffffff
444	jne .TscDeltaValid
445	jmp .TscDeltaNotValid
446	%endif
447
448	;
449	; Cleanup variables
450	;
451	%undef pData
452	%undef u64Delta_Hi
453	%undef u64Delta
454	%undef u32UpdateIntervalNS
455	%undef u32UpdateIntervalTSC
456	%undef u64TSC_Hi
457	%undef u64TSC
458	%undef u64NanoTS_Hi
459	%undef u64NanoTS
460	%undef u64PrevNanoTS_Hi
461	%undef u64PrevNanoTS
462	%undef u32TransactionId
463	%undef u8ApicId
464
465	%else ; AMD64
466
467	;;
468	; The AMD64 assembly implementation of the assembly routines.
469	;
470	; @returns Nanosecond timestamp.
471	; @param pData gcc:rdi msc:rcx Pointer to the nanosecond timestamp data.
472	;
473	BEGINPROC rtTimeNanoTSInternalAsm
474	;
475	; Define variables and stack frame.
476	;
477	%define SavedRBX [rbp - 08h]
478	%define SavedR12 [rbp - 10h]
479	%define SavedR13 [rbp - 18h]
480	%define SavedRDI [rbp - 20h]
481	%define SavedRSI [rbp - 28h]
482	%define TmpVar [rbp - 30h]
483	%define TmpVar2 [rbp - 38h]
484	%ifdef NEED_TRANSACTION_ID
485	%ifdef NEED_APIC_ID
486	%define SavedR14 [rbp - 40h]
487	%define SavedR15 [rbp - 48h]
488	%endif
489	%endif
490
491	%define pData rdi
492
493	%ifdef ASYNC_GIP
494	%define u64TSC rsi
495	%define pGip rsi
496	%ifdef NEED_APIC_ID
497	%define pGipCPU r8
498	%endif
499	%else
500	%define u64TSC r8
501	%define pGip rsi
502	%ifdef NEED_APIC_ID
503	%define pGipCPU r8
504	%endif
505	%endif
506	%define u32TransactionId r9d
507	%define u64CurNanoTS r10
508	%define u64PrevNanoTS r11 ; not parameter register
509	%define u32UpdateIntervalTSC r12d
510	%define u32UpdateIntervalTSC_64 r12
511	%define u32UpdateIntervalNS r13d
512	%define u32UpdateIntervalNS_64 r13
513	%undef u64SavedRetNanoTS
514	%undef u32ApicIdPlus
515	%ifdef NEED_TRANSACTION_ID
516	%ifdef NEED_APIC_ID
517	%define u64SavedRetNanoTS r14
518	%define u32ApicIdPlus r15d
519	%endif
520	%endif
521
522	;
523	; The prolog.
524	;
525	push rbp
526	mov rbp, rsp
527	%ifdef ASM_CALL64_MSC
528	sub rsp, 50h+20h
529	%else
530	sub rsp, 50h
531	%endif
532	mov SavedRBX, rbx
533	mov SavedR12, r12
534	mov SavedR13, r13
535	%ifdef ASM_CALL64_MSC
536	mov SavedRDI, rdi
537	mov SavedRSI, rsi
538	mov pData, rcx
539	%else
540	;mov pData, rdi - already in rdi.
541	%endif
542	%ifdef SavedR14
543	mov SavedR14, r14
544	%endif
545	%ifdef SavedR15
546	mov SavedR15, r15
547	%endif
548
549
550	;;
551	;; Data fetch loop.
552	;; We take great pain ensuring that data consistency here.
553	;;
554	.ReadGip:
555
556	;
557	; Load pGip - finding the GIP is fun...
558	;
559	%ifdef RT_OS_WINDOWS
560	%ifdef IMPORTED_SUPLIB
561	%ifdef IN_RING0
562	mov rax, qword IMP(g_SUPGlobalInfoPage)
563	mov pGip, rax
564	%else
565	mov pGip, [IMP(g_pSUPGlobalInfoPage) wrt rip]
566	mov pGip, [pGip]
567	%endif
568	%else
569	mov pGip, [NAME(g_pSUPGlobalInfoPage) wrt rip]
570	%endif
571	%else
572	%ifdef IN_RING0
573	mov rax, qword NAME(g_SUPGlobalInfoPage)
574	mov pGip, rax
575	%else
576	mov pGip, [rel NAME(g_pSUPGlobalInfoPage) wrt ..gotpcrel]
577	mov pGip, [pGip]
578	%endif
579	%endif
580	or pGip, pGip
581	jz .Rediscover
582	cmp dword [pGip + SUPGLOBALINFOPAGE.u32Magic], SUPGLOBALINFOPAGE_MAGIC
583	jne .Rediscover
584
585	;
586	; pGipCPU, setting u32ApicIdPlus if necessary.
587	;
588	%ifdef NEED_APIC_ID
589	; u8ApicId = ASMGetApicId();
590	mov eax, 1
591	cpuid ; expensive
592	%ifdef NEED_TRANSACTION_ID
593	mov u32ApicIdPlus, ebx
594	%endif
595	; pGipCPU = &pGip->aCPU[pGip->aiCpuFromApicId[u8ApicId]];
596	shr ebx, 24
597	movzx eax, word [pGip + rbx * 2 + SUPGLOBALINFOPAGE.aiCpuFromApicId]
598	imul eax, SUPGIPCPU_size
599	lea pGipCPU, [pGip + rax + SUPGLOBALINFOPAGE.aCPUs]
600	%endif
601
602	%ifdef NEED_TRANSACTION_ID
603	;
604	; Serialized loading of u32TransactionId.
605	;
606	%ifdef ASYNC_GIP
607	mov u32TransactionId, [pGipCPU + SUPGIPCPU.u32TransactionId]
608	%else
609	mov u32TransactionId, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
610	%endif
611	%ifdef USE_LFENCE
612	lfence
613	%else
614	lock xor dword TmpVar, 0
615	%endif
616	%endif
617
618	;
619	; Load the data and TSC.
620	;
621	mov u32UpdateIntervalNS, [pGip + SUPGLOBALINFOPAGE.u32UpdateIntervalNS]
622	%ifdef ASYNC_GIP
623	mov u32UpdateIntervalTSC, [pGipCPU + SUPGIPCPU.u32UpdateIntervalTSC]
624	%else
625	mov u32UpdateIntervalTSC, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32UpdateIntervalTSC]
626	%endif
627
628	rdtsc
629	mov u64PrevNanoTS, [pData + RTTIMENANOTSDATA.pu64Prev]
630	mov u64PrevNanoTS, [u64PrevNanoTS]
631	shl rdx, 32
632	or rax, rdx ; rax is u64RetNanoTS.
633	%ifdef WITH_TSC_DELTA
634	mov rdx, [pGipCPU + SUPGIPCPU.i64TSCDelta]
635	mov u64CurNanoTS, 0x7fffffffffffffff ; INT64_MAX - temporarily borrowing u64CurNanoTS
636	cmp rdx, u64CurNanoTS
637	je .TscDeltaNotValid
638	sub rax, rdx
639	.TscDeltaNotValid:
640	%endif
641	%ifdef u64SavedRetNanoTS ; doing this here may save a tick or so?
642	mov u64SavedRetNanoTS, rax
643	%endif
644
645	%ifdef ASYNC_GIP
646	mov u64CurNanoTS, [pGipCPU + SUPGIPCPU.u64NanoTS]
647	mov u64TSC, [pGipCPU + SUPGIPCPU.u64TSC] ; transhes pGIP!
648	%else
649	mov u64CurNanoTS, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64NanoTS]
650	mov u64TSC, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64TSC] ; trashes pGipCPU!
651	%endif
652
653
654	%ifdef NEED_TRANSACTION_ID
655	;
656	; Check that the GIP and CPU didn't change.
657	;
658	; It is crucial that the rdtsc instruction has completed before
659	; we check the transaction id. The LOCK prefixed instruction with
660	; dependency on the RDTSC result should do the trick, I think.
661	; CPUID is serializing, so the async path is safe by default.
662	;
663	%ifdef NEED_APIC_ID
664	mov eax, 1
665	cpuid
666	cmp u32ApicIdPlus, ebx
667	jne .ReadGip
668	%else
669	lock xor qword TmpVar, rax
670	%endif
671	%ifdef ASYNC_GIP
672	cmp u32TransactionId, [pGipCPU + SUPGIPCPU.u32TransactionId]
673	%else
674	cmp u32TransactionId, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
675	%endif
676	jne .ReadGip
677	test u32TransactionId, 1
678	jnz .ReadGip
679	%ifdef u64SavedRetNanoTS
680	mov rax, u64SavedRetNanoTS ; rax is u64RetNanoTS.
681	%endif
682	%endif ; NEED_TRANSACTION_ID
683
684
685	;;
686	;; Calc the timestamp.
687	;;
688	; u64RetNanoTS -= u64TSC;
689	sub rax, u64TSC
690	xor edx, edx
691
692	; if (u64RetNanoTS > u32UpdateIntervalTSC) -> jump
693	cmp rax, u32UpdateIntervalTSC_64
694	ja .OverFlow
695	.ContinueCalcs: ; edx = 0; eax <= u32UpdateIntervalTSC
696	mul u32UpdateIntervalNS
697	div u32UpdateIntervalTSC
698
699	; u64RetNanoTS += u64CurNanoTS;
700	add rax, u64CurNanoTS
701
702
703	;;
704	;; Compare it with the previous one.
705	;;
706	; if (RT_LIKELY( u64RetNanoTS > u64PrevNanoTS
707	; && u64RetNanoTS < u64PrevNanoTS + UINT64_C(86000000000000) /* 24h */))
708	; /* Frequent - less than 24h since last call. */;
709	cmp rax, u64PrevNanoTS
710	jbe .DeltaPrevTooBig
711	mov ecx, 5
712	shl rcx, 44 ; close enough
713	add rcx, u64PrevNanoTS
714	cmp rax, rcx
715	jae .DeltaPrevTooBig
716
717
718	;;
719	;; Update the previous value.
720	;;
721	.Update:
722	; if (RT_LIKELY(ASMAtomicCmpXchgU64(&pData->u64Prev, u64RetNanoTS, u64PrevNanoTS)))
723	mov rbx, [pData + RTTIMENANOTSDATA.pu64Prev]
724	mov rcx, rax
725	mov rax, u64PrevNanoTS
726	lock cmpxchg [rbx], rcx
727	jnz .UpdateFailed
728
729	.Updated:
730	mov rax, rcx
731
732	.Done:
733	mov rbx, SavedRBX
734	mov r12, SavedR12
735	mov r13, SavedR13
736	%ifdef SavedR14
737	mov r14, SavedR14
738	%endif
739	%ifdef SavedR15
740	mov r15, SavedR15
741	%endif
742	%ifdef ASM_CALL64_MSC
743	mov rdi, SavedRDI
744	mov rsi, SavedRSI
745	%endif
746	leave
747	ret
748
749
750	;;
751	;; We've expired the interval, cap it. If we're here for the 2nd
752	;; time without any GIP update in-between, the checks against
753	;; pData->u64Prev below will force 1ns stepping.
754	;;
755	ALIGNCODE(16)
756	.OverFlow:
757	; u64RetNanoTS = u32UpdateIntervalTSC;
758	inc dword [pData + RTTIMENANOTSDATA.cExpired]
759	mov eax, u32UpdateIntervalTSC
760	jmp .ContinueCalcs
761
762
763	;;
764	;; u64DeltaPrev >= 24h
765	;;
766	;; rax = u64RetNanoTS (to be adjusted)
767	;;
768	ALIGNCODE(16)
769	.DeltaPrevTooBig:
770	; uint64_t u64DeltaPrev = u64RetNanoTS - u64PrevNanoTS;
771	mov rbx, rax
772	sub rbx, u64PrevNanoTS
773
774	; else if ( (int64_t)u64DeltaPrev <= 0
775	; && (int64_t)u64DeltaPrev + u32UpdateIntervalNS * 2 >= 0)
776	; {
777	; /* Occasional - u64NanoTS is in the recent 'past' relative the previous call. */
778	; pData->c1nsSteps++;
779	; u64RetNanoTS = u64PrevNanoTS + 1;
780	; }
781	test rbx, rbx
782	jg .DeltaPrevNotInRecentPast
783
784	lea rdx, [u32UpdateIntervalNS_64 + u32UpdateIntervalNS_64]
785	add rdx, rbx
786	js .DeltaPrevNotInRecentPast
787
788	; body
789	inc dword [pData + RTTIMENANOTSDATA.c1nsSteps]
790	lea rax, [u64PrevNanoTS + 1]
791	jmp .Update
792
793	; else if (!u64PrevNanoTS) /* We're resuming (see TMVirtualResume) / first call. */
794	; /* do nothing */;
795	.DeltaPrevNotInRecentPast:
796	or u64PrevNanoTS, u64PrevNanoTS
797	jz .Update
798
799	; else
800	; {
801	; /* Something has gone bust, if negative offset it's real bad. */
802	; rtTimeNanoTSInternalBitch(pVM,
803	; }
804
805	; call C function that does the bitching.
806	mov TmpVar, rax
807	mov TmpVar2, pData
808
809	%ifdef ASM_CALL64_MSC
810	mov rcx, pData ; param 1 - pData
811	mov rdx, rax ; param 2 - u64RetNanoTS
812	mov r8, rbx ; param 3 - u64DeltaPrev
813	mov r9, u64PrevNanoTS ; param 4 - u64PrevNanoTS
814	%else
815	;mov rdi, pData - already in rdi; param 1 - pData
816	mov rsi, rax ; param 2 - u64RetNanoTS
817	mov rdx, rbx ; param 3 - u64DeltaPrev
818	mov rcx, u64PrevNanoTS ; param 4 - u64PrevNanoTS
819	%endif
820	call qword [pData + RTTIMENANOTSDATA.pfnBad]
821
822	mov rax, TmpVar
823	mov pData, TmpVar2
824	jmp .Update
825
826
827	;;
828	;; Attempt updating the previous value, provided we're still ahead of it.
829	;;
830	;; There is no point in recalculating u64NanoTS because we got preempted or if
831	;; we raced somebody while the GIP was updated, since these are events
832	;; that might occur at any point in the return path as well.
833	;;
834	;; rax = *pData->u64Prev;
835	;; rcx = u64RetNanoTS
836	;;
837	ALIGNCODE(16)
838	.UpdateFailed:
839	lock inc dword [pData + RTTIMENANOTSDATA.cUpdateRaces]
840	; for (i = 0; i < 10; i++)
841	mov edx, 10
842	.UpdateLoop:
843	; if (u64PrevNanoTS >= u64RetNanoTS)
844	; break;
845	cmp rax, rcx
846	jge .Updated
847	.UpdateLoopLess:
848	; retry
849	lock cmpxchg [rbx], rcx
850	jz .Updated
851	dec edx
852	jnz .UpdateLoop
853	jmp .Updated
854
855
856	;;
857	;; The GIP is seemingly invalid, redo the discovery.
858	;;
859	.Rediscover:
860	%ifdef ASM_CALL64_MSC
861	mov rcx, pData
862	%else
863	; mov rdi, pData - already in rdi
864	%endif
865	call [pData + RTTIMENANOTSDATA.pfnRediscover]
866	jmp .Done
867
868
869	;
870	; Cleanup variables
871	;
872	%undef SavedRBX
873	%undef SavedR12
874	%undef SavedR13
875	%undef SavedR14
876	%undef SavedR15
877	%undef SavedRDI
878	%undef SavedRSI
879	%undef pData
880	%undef TmpVar
881	%undef u64TSC
882	%undef pGip
883	%undef pGipCPU
884	%undef u32TransactionId
885	%undef u64CurNanoTS
886	%undef u64PrevNanoTS
887	%undef u32UpdateIntervalTSC
888	%undef u32UpdateIntervalTSC_64
889	%undef u32UpdateIntervalNS
890	%undef u64SavedRetNanoTS
891	%undef u32ApicIdPlus
892
893	%endif ; AMD64
894	ENDPROC rtTimeNanoTSInternalAsm
895

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format