timesupA.mac@ 69474

Last change on this file since 69474 was 69219, checked in by vboxsync, 7 years ago
Runtime: scm cleanups
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 23.3 KB

Line
1	; $Id: timesupA.mac 69219 2017-10-24 15:01:30Z vboxsync $
2	;; @file
3	; IPRT - Time using SUPLib, the Assembly Code Template.
4	;
5
6	;
7	; Copyright (C) 2006-2017 Oracle Corporation
8	;
9	; This file is part of VirtualBox Open Source Edition (OSE), as
10	; available from http://www.virtualbox.org. This file is free software;
11	; you can redistribute it and/or modify it under the terms of the GNU
12	; General Public License (GPL) as published by the Free Software
13	; Foundation, in version 2 as it comes in the "COPYING" file of the
14	; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	;
17	; The contents of this file may alternatively be used under the terms
18	; of the Common Development and Distribution License Version 1.0
19	; (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	; VirtualBox OSE distribution, in which case the provisions of the
21	; CDDL are applicable instead of those of the GPL.
22	;
23	; You may elect to license modified versions of this file under the
24	; terms and conditions of either the GPL or the CDDL or both.
25	;
26
27	%ifdef RT_ARCH_X86
28	;;
29	; The x86 assembly implementation of the assembly routines.
30	;
31	; @returns Nanosecond timestamp.
32	; @param pData Pointer to the nanosecond timestamp data.
33	;
34	BEGINPROC rtTimeNanoTSInternalAsm
35	;
36	; Variable definitions.
37	;
38	%define pData [ebp + 08h]
39	%define u64RetNanoTS_Hi [ebp - 04h]
40	%define u64RetNanoTS [ebp - 08h]
41	%define u32UpdateIntervalNS [ebp - 0ch]
42	%define u32UpdateIntervalTSC [ebp - 10h]
43	%define u64TSC_Hi [ebp - 14h]
44	%define u64TSC [ebp - 18h]
45	%define u64CurNanoTS_Hi [ebp - 1ch]
46	%define u64CurNanoTS [ebp - 20h]
47	%define u64PrevNanoTS_Hi [ebp - 24h]
48	%define u64PrevNanoTS [ebp - 28h]
49	%define u32TransactionId [ebp - 2ch]
50	%define u32ApicIdPlus [ebp - 30h]
51	%define TmpVar [ebp - 34h]
52	%define SavedEBX [ebp - 38h]
53	%define SavedEDI [ebp - 3ch]
54	%define SavedESI [ebp - 40h]
55
56	;
57	; Prolog.
58	;
59	push ebp
60	mov ebp, esp
61	sub esp, 40h
62	mov SavedEBX, ebx
63	mov SavedEDI, edi
64	mov SavedESI, esi
65
66
67	;;
68	;; Read the GIP data and the previous value.
69	;;
70	.ReadGip:
71
72
73	;
74	; Load pGip.
75	;
76	%ifdef IMPORTED_SUPLIB
77	%ifdef IN_RING0
78	mov esi, IMP(g_SUPGlobalInfoPage)
79	%else
80	mov esi, IMP(g_pSUPGlobalInfoPage)
81	mov esi, [esi]
82	%endif
83	%else
84	mov esi, [NAME(g_pSUPGlobalInfoPage)]
85	%endif
86	or esi, esi
87	jz .Rediscover
88	cmp dword [esi + SUPGLOBALINFOPAGE.u32Magic], SUPGLOBALINFOPAGE_MAGIC
89	jne .Rediscover
90
91	;
92	; Calc pGipCPU, setting u32ApicIdPlus if necessary.
93	;
94	%ifdef NEED_APIC_ID
95	; u8ApicId = ASMGetApicId();
96	mov eax, 1
97	cpuid ; expensive
98	%ifdef NEED_TRANSACTION_ID
99	mov u32ApicIdPlus, ebx
100	%endif
101	; pGipCpu/pGipCpuDelta = &pGip->aCPU[pGip->aiCpuFromApicId[u8ApicId]];
102	shr ebx, 24
103	movzx ebx, word [esi + ebx * 2 + SUPGLOBALINFOPAGE.aiCpuFromApicId]
104	mov eax, SUPGIPCPU_size
105	mul ebx
106	lea edi, [esi + eax + SUPGLOBALINFOPAGE.aCPUs] ; edi == &pGip->aCPU[u8ApicId];
107	%endif
108
109	%ifdef NEED_TRANSACTION_ID
110	;
111	; Serialized loading of u32TransactionId.
112	;
113	%ifdef ASYNC_GIP
114	mov ebx, [edi + SUPGIPCPU.u32TransactionId]
115	%else
116	mov ebx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
117	%endif
118	mov u32TransactionId, ebx
119	%ifdef USE_LFENCE
120	lfence
121	%else
122	lock xor dword TmpVar, 0
123	%endif
124	%endif
125
126	;
127	; Load the data and TSC with delta applied.
128	;
129	mov eax, [esi + SUPGLOBALINFOPAGE.u32UpdateIntervalNS]
130	mov u32UpdateIntervalNS, eax
131	%ifdef ASYNC_GIP ; esi is now free.
132	mov edx, [edi + SUPGIPCPU.u32UpdateIntervalTSC]
133	%else
134	mov edx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32UpdateIntervalTSC]
135	%endif
136	mov u32UpdateIntervalTSC, edx
137
138	rdtsc
139	%ifdef WITH_TSC_DELTA
140	cmp dword [edi + SUPGIPCPU.i64TSCDelta], 0xffffffff
141	je .TscDeltaPossiblyInvalid
142	.TscDeltaValid:
143	sub eax, dword [edi + SUPGIPCPU.i64TSCDelta]
144	sbb edx, dword [edi + SUPGIPCPU.i64TSCDelta + 4]
145	.TscDeltaNotValid: ; edi is now free.
146	%endif
147
148	%ifdef ASYNC_GIP
149	mov ecx, [edi + SUPGIPCPU.u64NanoTS]
150	mov esi, [edi + SUPGIPCPU.u64NanoTS + 4]
151	%else
152	mov ecx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64NanoTS]
153	mov ebx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64NanoTS + 4]
154	%endif
155	mov u64CurNanoTS, ecx
156	mov u64CurNanoTS_Hi, ebx
157	%ifdef ASYNC_GIP
158	mov ecx, [edi + SUPGIPCPU.u64TSC]
159	mov ebx, [edi + SUPGIPCPU.u64TSC + 4]
160	%else
161	mov ecx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64TSC]
162	mov ebx, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64TSC + 4]
163	%endif
164	mov u64TSC, ecx
165	mov u64TSC_Hi, ebx
166
167	; u64PrevNanoTS = ASMAtomicReadU64(pu64Prev);
168	; This serializes load/save. And with the dependency on the
169	; RDTSC result, we try to make sure it has completed as well.
170	%ifdef ASYNC_GIP
171	mov esi, pData
172	mov esi, [esi + RTTIMENANOTSDATA.pu64Prev]
173	%else
174	mov edi, pData
175	mov edi, [esi + RTTIMENANOTSDATA.pu64Prev]
176	%endif
177	mov ebx, eax
178	mov ecx, edx
179	%ifdef ASYNC_GIP
180	lock cmpxchg8b [esi]
181	%else
182	lock cmpxchg8b [edi]
183	%endif
184	mov u64PrevNanoTS, eax
185	mov u64PrevNanoTS_Hi, edx
186
187	%undef SAVED_u64RetNanoTS
188	%ifdef NEED_TRANSACTION_ID
189	;
190	; Check that the GIP and CPU didn't change.
191	; We've already serialized all the loads and stores at this point.
192	;
193	%ifdef NEED_APIC_ID
194	mov u64RetNanoTS, ebx
195	mov u64RetNanoTS_Hi, ecx
196	%define SAVED_u64RetNanoTS
197	mov eax, 1
198	cpuid
199	cmp u32ApicIdPlus, ebx
200	jne .ReadGip
201	%endif
202	%ifdef ASYNC_GIP
203	mov esi, [edi + SUPGIPCPU.u32TransactionId]
204	%else
205	mov esi, [esi + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
206	%endif
207	cmp esi, u32TransactionId
208	jne .ReadGip
209	test esi, 1
210	jnz .ReadGip
211	%endif ; NEED_TRANSACTION_ID
212	%ifdef SAVED_u64RetNanoTS
213	mov ebx, u64RetNanoTS
214	mov ecx, u64RetNanoTS_Hi
215	%endif
216
217	;;
218	;; Calc the timestamp.
219	;;
220	; u64RetNanoTS -= u64TSC;
221	sub ebx, u64TSC
222	sbb ecx, u64TSC_Hi
223
224	; if (u64RetNanoTS > u32UpdateIntervalTSC) -> jump
225	or ecx, ecx
226	jnz .OverFlow
227	cmp ebx, u32UpdateIntervalTSC
228	ja .OverFlow
229	mov eax, ebx
230	.ContinueCalcs: ; eax <= u32UpdateIntervalTSC
231	mul dword u32UpdateIntervalNS
232	div dword u32UpdateIntervalTSC
233	xor edx, edx
234
235	; u64RetNanoTS += u64CurNanoTS;
236	add eax, u64CurNanoTS
237	adc edx, u64CurNanoTS_Hi
238
239	;;
240	;; Compare it with the previous one.
241	;;
242	; if (RT_LIKELY( u64RetNanoTS > u64PrevNanoTS
243	; && u64RetNanoTS < u64PrevNanoTS + UINT64_C(86000000000000) /* 24h */))
244	;; @todo optimize this compare (/me too tired).
245	mov ecx, u64PrevNanoTS_Hi
246	mov ebx, u64PrevNanoTS
247	cmp edx, ecx
248	ja .Compare2
249	jb .DeltaPrevTooBig
250	cmp eax, ebx
251	jbe .DeltaPrevTooBig
252
253	.Compare2:
254	add ebx, 0x6F736000
255	adc ecx, 0x00004E37
256	cmp edx, ecx
257	jb .CompareDone
258	ja .DeltaPrevTooBig
259	cmp eax, ebx
260	jae .DeltaPrevTooBig
261	.CompareDone:
262
263
264	;;
265	;; Update the previous value with the u64RetNanoTS value.
266	;;
267	.Update:
268	; if (RT_LIKELY(ASMAtomicCmpXchgU64(&pData->u64Prev, u64RetNanoTS, u64PrevNanoTS)))
269	mov ebx, eax
270	mov ecx, edx
271	mov esi, pData
272	mov esi, [esi + RTTIMENANOTSDATA.pu64Prev]
273	mov eax, u64PrevNanoTS
274	mov edx, u64PrevNanoTS_Hi
275	lock cmpxchg8b [esi]
276	jnz .UpdateFailed
277
278	.Updated:
279	mov eax, ebx
280	mov edx, ecx
281
282	.Done:
283	mov esi, SavedESI
284	mov edi, SavedEDI
285	mov ebx, SavedEBX
286	leave
287	ret
288
289
290	;;
291	;; We've expired the interval, cap it. If we're here for the 2nd
292	;; time without any GIP update in-between, the checks against
293	;; pData->u64Prev below will force 1ns stepping.
294	;;
295	.OverFlow:
296	; u64Delta = u32UpdateIntervalTSC;
297	mov esi, pData
298	inc dword [esi + RTTIMENANOTSDATA.cExpired]
299	mov eax, u32UpdateIntervalTSC
300	jmp .ContinueCalcs
301
302
303	;;
304	;; u64DeltaPrev >= 24h
305	;;
306	;; eax:edx = u64RetNanoTS (to be adjusted)
307	;;
308	.DeltaPrevTooBig:
309	; uint64_t u64DeltaPrev = u64RetNanoTS - u64PrevNanoTS;
310	mov ebx, eax
311	sub ebx, u64PrevNanoTS
312	mov ecx, edx
313	sbb ecx, u64PrevNanoTS_Hi ; ebx:ecx = u64DeltaPrev
314
315	; else if ( (int64_t)u64DeltaPrev <= 0
316	; && (int64_t)u64DeltaPrev + u32UpdateIntervalNS * 2 >= 0)
317	; {
318	; /* Occasional - u64RetNanoTS is in the recent 'past' relative the previous call. */
319	; pData->c1nsSteps++;
320	; u64RetNanoTS = u64PrevNanoTS + 1;
321	; }
322	mov esi, u32UpdateIntervalNS
323	cmp ecx, 0
324	jl .PrevNotZero2ndTest
325	jg .DeltaPrevNotInRecentPast
326	cmp ebx, 0
327	ja .DeltaPrevNotInRecentPast
328
329	.PrevNotZero2ndTest:
330	add esi, esi ; ASSUMES: u32UpdateIntervalNS * 2 <= 32-bit.
331	xor edi, edi
332	add esi, ebx
333	adc edi, ecx
334	test edi, edi
335	js .DeltaPrevNotInRecentPast
336
337	.DeltaPrevInRecentPast:
338	mov esi, pData
339	inc dword [esi + RTTIMENANOTSDATA.c1nsSteps]
340	mov eax, u64PrevNanoTS
341	mov edx, u64PrevNanoTS_Hi
342	add eax, 1
343	adc edx, 0
344	jmp .Update
345
346	.DeltaPrevNotInRecentPast:
347	; else if (!u64PrevNanoTS) /* We're resuming (see TMVirtualResume). */
348	; /* do nothing */;
349	cmp dword u64PrevNanoTS, 0
350	jne .DeltaPrevNotZero
351	cmp dword u64PrevNanoTS_Hi, 0
352	jne .DeltaPrevNotZero
353	jmp .Update
354
355	.DeltaPrevNotZero:
356	; else
357	; {
358	; /* Something has gone bust, if negative offset it's real bad. */
359	; rtTimeNanoTSInternalBitch(pVM,
360	; }
361
362	; call C function that does the bitching.
363	mov u64RetNanoTS, eax
364	mov u64RetNanoTS_Hi, edx
365
366	mov edi, u64PrevNanoTS_Hi
367	mov esi, u64PrevNanoTS
368	push edi
369	push esi ; 4 - u64PrevNanoTS
370	push ecx
371	push ebx ; 3 - u64DeltaPrev
372	push edx
373	push eax ; 2 - u64RetNanoTS
374	mov eax, pData
375	push eax ; 1 - pData
376	call dword [eax + RTTIMENANOTSDATA.pfnBad]
377	add esp, 4*7
378
379	mov eax, u64RetNanoTS
380	mov edx, u64RetNanoTS_Hi
381	jmp .Update
382
383
384	;;
385	;; Attempt updating the previous value, provided we're still ahead of it.
386	;;
387	;; There is no point in recalculating u64NanoTS because we got preempted or if
388	;; we raced somebody while the GIP was updated, since these are events
389	;; that might occur at any point in the return path as well.
390	;;
391	;; eax:edx = *pData->u64Prev
392	;; ebx:ecx = u64RetNanoTS
393	;;
394	ALIGNCODE(16)
395	.UpdateFailed:
396	mov edi, pData
397	lock inc dword [edi + RTTIMENANOTSDATA.cUpdateRaces]
398	; for (i = 0; i < 10; i++)
399	mov edi, 10
400	.UpdateLoop:
401	; if (u64PrevNanoTS >= u64NanoTS)
402	; break;
403	cmp edx, ecx
404	jg .Updated
405	jne .UpdateLoopLess
406	cmp eax, ebx
407	jae .Updated
408	.UpdateLoopLess:
409	; retry
410	lock cmpxchg8b [esi]
411	jz .Updated
412	dec edi
413	jnz .UpdateLoop
414	jmp .Updated
415
416
417	;;
418	;; The GIP is seemingly invalid, redo the discovery.
419	;;
420	.Rediscover:
421	mov eax, pData
422	push eax
423	call [eax + RTTIMENANOTSDATA.pfnRediscover]
424	add esp, 4h
425	jmp .Done
426
427
428	%ifdef WITH_TSC_DELTA
429	;;
430	;; Unlikely branch for when we think the TSC delta might be invalid.
431	;;
432	.TscDeltaPossiblyInvalid:
433	cmp dword [edi + SUPGIPCPU.i64TSCDelta + 4], 0x7fffffff
434	jne .TscDeltaValid
435	jmp .TscDeltaNotValid
436	%endif
437
438	;
439	; Cleanup variables
440	;
441	%undef pData
442	%undef u64Delta_Hi
443	%undef u64Delta
444	%undef u32UpdateIntervalNS
445	%undef u32UpdateIntervalTSC
446	%undef u64TSC_Hi
447	%undef u64TSC
448	%undef u64NanoTS_Hi
449	%undef u64NanoTS
450	%undef u64PrevNanoTS_Hi
451	%undef u64PrevNanoTS
452	%undef u32TransactionId
453	%undef u8ApicId
454
455	%else ; AMD64
456
457	;;
458	; The AMD64 assembly implementation of the assembly routines.
459	;
460	; @returns Nanosecond timestamp.
461	; @param pData gcc:rdi msc:rcx Pointer to the nanosecond timestamp data.
462	;
463	BEGINPROC rtTimeNanoTSInternalAsm
464	;
465	; Define variables and stack frame.
466	;
467	%define SavedRBX [rbp - 08h]
468	%define SavedR12 [rbp - 10h]
469	%define SavedR13 [rbp - 18h]
470	%define SavedRDI [rbp - 20h]
471	%define SavedRSI [rbp - 28h]
472	%define TmpVar [rbp - 30h]
473	%define TmpVar2 [rbp - 38h]
474	%ifdef NEED_TRANSACTION_ID
475	%ifdef NEED_APIC_ID
476	%define SavedR14 [rbp - 40h]
477	%define SavedR15 [rbp - 48h]
478	%endif
479	%endif
480
481	%define pData rdi
482
483	%ifdef ASYNC_GIP
484	%define u64TSC rsi
485	%define pGip rsi
486	%ifdef NEED_APIC_ID
487	%define pGipCPU r8
488	%endif
489	%else
490	%define u64TSC r8
491	%define pGip rsi
492	%ifdef NEED_APIC_ID
493	%define pGipCPU r8
494	%endif
495	%endif
496	%define u32TransactionId r9d
497	%define u64CurNanoTS r10
498	%define u64PrevNanoTS r11 ; not parameter register
499	%define u32UpdateIntervalTSC r12d
500	%define u32UpdateIntervalTSC_64 r12
501	%define u32UpdateIntervalNS r13d
502	%define u32UpdateIntervalNS_64 r13
503	%undef u64SavedRetNanoTS
504	%undef u32ApicIdPlus
505	%ifdef NEED_TRANSACTION_ID
506	%ifdef NEED_APIC_ID
507	%define u64SavedRetNanoTS r14
508	%define u32ApicIdPlus r15d
509	%endif
510	%endif
511
512	;
513	; The prolog.
514	;
515	push rbp
516	mov rbp, rsp
517	%ifdef ASM_CALL64_MSC
518	sub rsp, 50h+20h
519	%else
520	sub rsp, 50h
521	%endif
522	mov SavedRBX, rbx
523	mov SavedR12, r12
524	mov SavedR13, r13
525	%ifdef ASM_CALL64_MSC
526	mov SavedRDI, rdi
527	mov SavedRSI, rsi
528	mov pData, rcx
529	%else
530	;mov pData, rdi - already in rdi.
531	%endif
532	%ifdef SavedR14
533	mov SavedR14, r14
534	%endif
535	%ifdef SavedR15
536	mov SavedR15, r15
537	%endif
538
539
540	;;
541	;; Data fetch loop.
542	;; We take great pain ensuring that data consistency here.
543	;;
544	.ReadGip:
545
546	;
547	; Load pGip - finding the GIP is fun...
548	;
549	%ifdef RT_OS_WINDOWS
550	%ifdef IMPORTED_SUPLIB
551	%ifdef IN_RING0
552	mov rax, qword IMP(g_SUPGlobalInfoPage)
553	mov pGip, rax
554	%else
555	mov pGip, [IMP(g_pSUPGlobalInfoPage) wrt rip]
556	mov pGip, [pGip]
557	%endif
558	%else
559	mov pGip, [NAME(g_pSUPGlobalInfoPage) wrt rip]
560	%endif
561	%else
562	%ifdef IN_RING0
563	mov rax, qword NAME(g_SUPGlobalInfoPage)
564	mov pGip, rax
565	%else
566	mov pGip, [rel NAME(g_pSUPGlobalInfoPage) wrt ..gotpcrel]
567	mov pGip, [pGip]
568	%endif
569	%endif
570	or pGip, pGip
571	jz .Rediscover
572	cmp dword [pGip + SUPGLOBALINFOPAGE.u32Magic], SUPGLOBALINFOPAGE_MAGIC
573	jne .Rediscover
574
575	;
576	; pGipCPU, setting u32ApicIdPlus if necessary.
577	;
578	%ifdef NEED_APIC_ID
579	; u8ApicId = ASMGetApicId();
580	mov eax, 1
581	cpuid ; expensive
582	%ifdef NEED_TRANSACTION_ID
583	mov u32ApicIdPlus, ebx
584	%endif
585	; pGipCPU = &pGip->aCPU[pGip->aiCpuFromApicId[u8ApicId]];
586	shr ebx, 24
587	movzx eax, word [pGip + rbx * 2 + SUPGLOBALINFOPAGE.aiCpuFromApicId]
588	imul eax, SUPGIPCPU_size
589	lea pGipCPU, [pGip + rax + SUPGLOBALINFOPAGE.aCPUs]
590	%endif
591
592	%ifdef NEED_TRANSACTION_ID
593	;
594	; Serialized loading of u32TransactionId.
595	;
596	%ifdef ASYNC_GIP
597	mov u32TransactionId, [pGipCPU + SUPGIPCPU.u32TransactionId]
598	%else
599	mov u32TransactionId, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
600	%endif
601	%ifdef USE_LFENCE
602	lfence
603	%else
604	lock xor dword TmpVar, 0
605	%endif
606	%endif
607
608	;
609	; Load the data and TSC.
610	;
611	mov u32UpdateIntervalNS, [pGip + SUPGLOBALINFOPAGE.u32UpdateIntervalNS]
612	%ifdef ASYNC_GIP
613	mov u32UpdateIntervalTSC, [pGipCPU + SUPGIPCPU.u32UpdateIntervalTSC]
614	%else
615	mov u32UpdateIntervalTSC, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32UpdateIntervalTSC]
616	%endif
617
618	rdtsc
619	mov u64PrevNanoTS, [pData + RTTIMENANOTSDATA.pu64Prev]
620	mov u64PrevNanoTS, [u64PrevNanoTS]
621	shl rdx, 32
622	or rax, rdx ; rax is u64RetNanoTS.
623	%ifdef WITH_TSC_DELTA
624	mov rdx, [pGipCPU + SUPGIPCPU.i64TSCDelta]
625	mov u64CurNanoTS, 0x7fffffffffffffff ; INT64_MAX - temporarily borrowing u64CurNanoTS
626	cmp rdx, u64CurNanoTS
627	je .TscDeltaNotValid
628	sub rax, rdx
629	.TscDeltaNotValid:
630	%endif
631	%ifdef u64SavedRetNanoTS ; doing this here may save a tick or so?
632	mov u64SavedRetNanoTS, rax
633	%endif
634
635	%ifdef ASYNC_GIP
636	mov u64CurNanoTS, [pGipCPU + SUPGIPCPU.u64NanoTS]
637	mov u64TSC, [pGipCPU + SUPGIPCPU.u64TSC] ; transhes pGIP!
638	%else
639	mov u64CurNanoTS, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64NanoTS]
640	mov u64TSC, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u64TSC] ; trashes pGipCPU!
641	%endif
642
643
644	%ifdef NEED_TRANSACTION_ID
645	;
646	; Check that the GIP and CPU didn't change.
647	;
648	; It is crucial that the rdtsc instruction has completed before
649	; we check the transaction id. The LOCK prefixed instruction with
650	; dependency on the RDTSC result should do the trick, I think.
651	; CPUID is serializing, so the async path is safe by default.
652	;
653	%ifdef NEED_APIC_ID
654	mov eax, 1
655	cpuid
656	cmp u32ApicIdPlus, ebx
657	jne .ReadGip
658	%else
659	lock xor qword TmpVar, rax
660	%endif
661	%ifdef ASYNC_GIP
662	cmp u32TransactionId, [pGipCPU + SUPGIPCPU.u32TransactionId]
663	%else
664	cmp u32TransactionId, [pGip + SUPGLOBALINFOPAGE.aCPUs + SUPGIPCPU.u32TransactionId]
665	%endif
666	jne .ReadGip
667	test u32TransactionId, 1
668	jnz .ReadGip
669	%ifdef u64SavedRetNanoTS
670	mov rax, u64SavedRetNanoTS ; rax is u64RetNanoTS.
671	%endif
672	%endif ; NEED_TRANSACTION_ID
673
674
675	;;
676	;; Calc the timestamp.
677	;;
678	; u64RetNanoTS -= u64TSC;
679	sub rax, u64TSC
680	xor edx, edx
681
682	; if (u64RetNanoTS > u32UpdateIntervalTSC) -> jump
683	cmp rax, u32UpdateIntervalTSC_64
684	ja .OverFlow
685	.ContinueCalcs: ; edx = 0; eax <= u32UpdateIntervalTSC
686	mul u32UpdateIntervalNS
687	div u32UpdateIntervalTSC
688
689	; u64RetNanoTS += u64CurNanoTS;
690	add rax, u64CurNanoTS
691
692
693	;;
694	;; Compare it with the previous one.
695	;;
696	; if (RT_LIKELY( u64RetNanoTS > u64PrevNanoTS
697	; && u64RetNanoTS < u64PrevNanoTS + UINT64_C(86000000000000) /* 24h */))
698	; /* Frequent - less than 24h since last call. */;
699	cmp rax, u64PrevNanoTS
700	jbe .DeltaPrevTooBig
701	mov ecx, 5
702	shl rcx, 44 ; close enough
703	add rcx, u64PrevNanoTS
704	cmp rax, rcx
705	jae .DeltaPrevTooBig
706
707
708	;;
709	;; Update the previous value.
710	;;
711	.Update:
712	; if (RT_LIKELY(ASMAtomicCmpXchgU64(&pData->u64Prev, u64RetNanoTS, u64PrevNanoTS)))
713	mov rbx, [pData + RTTIMENANOTSDATA.pu64Prev]
714	mov rcx, rax
715	mov rax, u64PrevNanoTS
716	lock cmpxchg [rbx], rcx
717	jnz .UpdateFailed
718
719	.Updated:
720	mov rax, rcx
721
722	.Done:
723	mov rbx, SavedRBX
724	mov r12, SavedR12
725	mov r13, SavedR13
726	%ifdef SavedR14
727	mov r14, SavedR14
728	%endif
729	%ifdef SavedR15
730	mov r15, SavedR15
731	%endif
732	%ifdef ASM_CALL64_MSC
733	mov rdi, SavedRDI
734	mov rsi, SavedRSI
735	%endif
736	leave
737	ret
738
739
740	;;
741	;; We've expired the interval, cap it. If we're here for the 2nd
742	;; time without any GIP update in-between, the checks against
743	;; pData->u64Prev below will force 1ns stepping.
744	;;
745	ALIGNCODE(16)
746	.OverFlow:
747	; u64RetNanoTS = u32UpdateIntervalTSC;
748	inc dword [pData + RTTIMENANOTSDATA.cExpired]
749	mov eax, u32UpdateIntervalTSC
750	jmp .ContinueCalcs
751
752
753	;;
754	;; u64DeltaPrev >= 24h
755	;;
756	;; rax = u64RetNanoTS (to be adjusted)
757	;;
758	ALIGNCODE(16)
759	.DeltaPrevTooBig:
760	; uint64_t u64DeltaPrev = u64RetNanoTS - u64PrevNanoTS;
761	mov rbx, rax
762	sub rbx, u64PrevNanoTS
763
764	; else if ( (int64_t)u64DeltaPrev <= 0
765	; && (int64_t)u64DeltaPrev + u32UpdateIntervalNS * 2 >= 0)
766	; {
767	; /* Occasional - u64NanoTS is in the recent 'past' relative the previous call. */
768	; pData->c1nsSteps++;
769	; u64RetNanoTS = u64PrevNanoTS + 1;
770	; }
771	test rbx, rbx
772	jg .DeltaPrevNotInRecentPast
773
774	lea rdx, [u32UpdateIntervalNS_64 + u32UpdateIntervalNS_64]
775	add rdx, rbx
776	js .DeltaPrevNotInRecentPast
777
778	; body
779	inc dword [pData + RTTIMENANOTSDATA.c1nsSteps]
780	lea rax, [u64PrevNanoTS + 1]
781	jmp .Update
782
783	; else if (!u64PrevNanoTS) /* We're resuming (see TMVirtualResume) / first call. */
784	; /* do nothing */;
785	.DeltaPrevNotInRecentPast:
786	or u64PrevNanoTS, u64PrevNanoTS
787	jz .Update
788
789	; else
790	; {
791	; /* Something has gone bust, if negative offset it's real bad. */
792	; rtTimeNanoTSInternalBitch(pVM,
793	; }
794
795	; call C function that does the bitching.
796	mov TmpVar, rax
797	mov TmpVar2, pData
798
799	%ifdef ASM_CALL64_MSC
800	mov rcx, pData ; param 1 - pData
801	mov rdx, rax ; param 2 - u64RetNanoTS
802	mov r8, rbx ; param 3 - u64DeltaPrev
803	mov r9, u64PrevNanoTS ; param 4 - u64PrevNanoTS
804	%else
805	;mov rdi, pData - already in rdi; param 1 - pData
806	mov rsi, rax ; param 2 - u64RetNanoTS
807	mov rdx, rbx ; param 3 - u64DeltaPrev
808	mov rcx, u64PrevNanoTS ; param 4 - u64PrevNanoTS
809	%endif
810	call qword [pData + RTTIMENANOTSDATA.pfnBad]
811
812	mov rax, TmpVar
813	mov pData, TmpVar2
814	jmp .Update
815
816
817	;;
818	;; Attempt updating the previous value, provided we're still ahead of it.
819	;;
820	;; There is no point in recalculating u64NanoTS because we got preempted or if
821	;; we raced somebody while the GIP was updated, since these are events
822	;; that might occur at any point in the return path as well.
823	;;
824	;; rax = *pData->u64Prev;
825	;; rcx = u64RetNanoTS
826	;;
827	ALIGNCODE(16)
828	.UpdateFailed:
829	lock inc dword [pData + RTTIMENANOTSDATA.cUpdateRaces]
830	; for (i = 0; i < 10; i++)
831	mov edx, 10
832	.UpdateLoop:
833	; if (u64PrevNanoTS >= u64RetNanoTS)
834	; break;
835	cmp rax, rcx
836	jge .Updated
837	.UpdateLoopLess:
838	; retry
839	lock cmpxchg [rbx], rcx
840	jz .Updated
841	dec edx
842	jnz .UpdateLoop
843	jmp .Updated
844
845
846	;;
847	;; The GIP is seemingly invalid, redo the discovery.
848	;;
849	.Rediscover:
850	%ifdef ASM_CALL64_MSC
851	mov rcx, pData
852	%else
853	; mov rdi, pData - already in rdi
854	%endif
855	call [pData + RTTIMENANOTSDATA.pfnRediscover]
856	jmp .Done
857
858
859	;
860	; Cleanup variables
861	;
862	%undef SavedRBX
863	%undef SavedR12
864	%undef SavedR13
865	%undef SavedR14
866	%undef SavedR15
867	%undef SavedRDI
868	%undef SavedRSI
869	%undef pData
870	%undef TmpVar
871	%undef u64TSC
872	%undef pGip
873	%undef pGipCPU
874	%undef u32TransactionId
875	%undef u64CurNanoTS
876	%undef u64PrevNanoTS
877	%undef u32UpdateIntervalTSC
878	%undef u32UpdateIntervalTSC_64
879	%undef u32UpdateIntervalNS
880	%undef u64SavedRetNanoTS
881	%undef u32ApicIdPlus
882
883	%endif ; AMD64
884	ENDPROC rtTimeNanoTSInternalAsm
885

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format