timer-r0drv-linux.c@ 35857

Last change on this file since 35857 was 33603, checked in by vboxsync, 14 years ago
timer-r0drv-linux: typo, fixes hangs with highres timers enabled
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 53.4 KB

Line
1	/* $Id: timer-r0drv-linux.c 33603 2010-10-29 12:42:24Z vboxsync $ */
2	/** @file
3	* IPRT - Timers, Ring-0 Driver, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2010 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*******************************************************************************
29	* Header Files *
30	*******************************************************************************/
31	#include "the-linux-kernel.h"
32	#include "internal/iprt.h"
33
34	#include <iprt/timer.h>
35	#include <iprt/time.h>
36	#include <iprt/mp.h>
37	#include <iprt/cpuset.h>
38	#include <iprt/spinlock.h>
39	#include <iprt/err.h>
40	#include <iprt/asm.h>
41	#include <iprt/assert.h>
42	#include <iprt/alloc.h>
43
44	#include "internal/magics.h"
45
46	/** @def RTTIMER_LINUX_WITH_HRTIMER
47	* Whether to use high resolution timers. */
48	#if !defined(RTTIMER_LINUX_WITH_HRTIMER) \
49	&& defined(IPRT_LINUX_HAS_HRTIMER)
50	# define RTTIMER_LINUX_WITH_HRTIMER
51	#endif
52
53	#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31)
54	# define mod_timer_pinned mod_timer
55	# define HRTIMER_MODE_ABS_PINNED HRTIMER_MODE_ABS
56	#endif
57
58
59	/*******************************************************************************
60	* Structures and Typedefs *
61	*******************************************************************************/
62	/**
63	* Timer state machine.
64	*
65	* This is used to try handle the issues with MP events and
66	* timers that runs on all CPUs. It's relatively nasty :-/
67	*/
68	typedef enum RTTIMERLNXSTATE
69	{
70	/** Stopped. */
71	RTTIMERLNXSTATE_STOPPED = 0,
72	/** Transient state; next ACTIVE. */
73	RTTIMERLNXSTATE_STARTING,
74	/** Transient state; next ACTIVE. (not really necessary) */
75	RTTIMERLNXSTATE_MP_STARTING,
76	/** Active. */
77	RTTIMERLNXSTATE_ACTIVE,
78	/** Active and in callback; next ACTIVE, STOPPED or CALLBACK_DESTROYING. */
79	RTTIMERLNXSTATE_CALLBACK,
80	/** Stopped while in the callback; next STOPPED. */
81	RTTIMERLNXSTATE_CB_STOPPING,
82	/** Restarted while in the callback; next ACTIVE, STOPPED, DESTROYING. */
83	RTTIMERLNXSTATE_CB_RESTARTING,
84	/** The callback shall destroy the timer; next STOPPED. */
85	RTTIMERLNXSTATE_CB_DESTROYING,
86	/** Transient state; next STOPPED. */
87	RTTIMERLNXSTATE_STOPPING,
88	/** Transient state; next STOPPED. */
89	RTTIMERLNXSTATE_MP_STOPPING,
90	/** The usual 32-bit hack. */
91	RTTIMERLNXSTATE_32BIT_HACK = 0x7fffffff
92	} RTTIMERLNXSTATE;
93
94
95	/**
96	* A Linux sub-timer.
97	*/
98	typedef struct RTTIMERLNXSUBTIMER
99	{
100	/** Timer specific data. */
101	union
102	{
103	#if defined(RTTIMER_LINUX_WITH_HRTIMER)
104	/** High resolution timer. */
105	struct
106	{
107	/** The linux timer structure. */
108	struct hrtimer LnxTimer;
109	} Hr;
110	#endif
111	/** Standard timer. */
112	struct
113	{
114	/** The linux timer structure. */
115	struct timer_list LnxTimer;
116	/** The start of the current run (ns).
117	* This is used to calculate when the timer ought to fire the next time. */
118	uint64_t u64NextTS;
119	/** The u64NextTS in jiffies. */
120	unsigned long ulNextJiffies;
121	/** Set when starting or changing the timer so that u64StartTs
122	* and u64NextTS gets reinitialized (eliminating some jitter). */
123	bool volatile fFirstAfterChg;
124	} Std;
125	} u;
126	/** The current tick number. */
127	uint64_t iTick;
128	/** Restart the single shot timer at this specific time.
129	* Used when a single shot timer is restarted from the callback. */
130	uint64_t volatile uNsRestartAt;
131	/** Pointer to the parent timer. */
132	PRTTIMER pParent;
133	/** The current sub-timer state. */
134	RTTIMERLNXSTATE volatile enmState;
135	} RTTIMERLNXSUBTIMER;
136	/** Pointer to a linux sub-timer. */
137	typedef RTTIMERLNXSUBTIMER *PRTTIMERLNXSUBTIMER;
138
139
140	/**
141	* The internal representation of an Linux timer handle.
142	*/
143	typedef struct RTTIMER
144	{
145	/** Magic.
146	* This is RTTIMER_MAGIC, but changes to something else before the timer
147	* is destroyed to indicate clearly that thread should exit. */
148	uint32_t volatile u32Magic;
149	/** Spinlock synchronizing the fSuspended and MP event handling.
150	* This is NIL_RTSPINLOCK if cCpus == 1. */
151	RTSPINLOCK hSpinlock;
152	/** Flag indicating that the timer is suspended. */
153	bool volatile fSuspended;
154	/** Whether the timer must run on one specific CPU or not. */
155	bool fSpecificCpu;
156	#ifdef CONFIG_SMP
157	/** Whether the timer must run on all CPUs or not. */
158	bool fAllCpus;
159	#endif /* else: All -> specific on non-SMP kernels */
160	/** Whether it is a high resolution timer or a standard one. */
161	bool fHighRes;
162	/** The id of the CPU it must run on if fSpecificCpu is set. */
163	RTCPUID idCpu;
164	/** The number of CPUs this timer should run on. */
165	RTCPUID cCpus;
166	/** Callback. */
167	PFNRTTIMER pfnTimer;
168	/** User argument. */
169	void *pvUser;
170	/** The timer interval. 0 if one-shot. */
171	uint64_t volatile u64NanoInterval;
172	/** This is set to the number of jiffies between ticks if the interval is
173	* an exact number of jiffies. (Standard timers only.) */
174	unsigned long volatile cJiffies;
175	/** The change interval spinlock for standard timers only. */
176	spinlock_t ChgIntLock;
177	/** Sub-timers.
178	* Normally there is just one, but for RTTIMER_FLAGS_CPU_ALL this will contain
179	* an entry for all possible cpus. In that case the index will be the same as
180	* for the RTCpuSet. */
181	RTTIMERLNXSUBTIMER aSubTimers[1];
182	} RTTIMER;
183
184
185	/**
186	* A rtTimerLinuxStartOnCpu and rtTimerLinuxStartOnCpu argument package.
187	*/
188	typedef struct RTTIMERLINUXSTARTONCPUARGS
189	{
190	/** The current time (RTTimeSystemNanoTS). */
191	uint64_t u64Now;
192	/** When to start firing (delta). */
193	uint64_t u64First;
194	} RTTIMERLINUXSTARTONCPUARGS;
195	/** Pointer to a rtTimerLinuxStartOnCpu argument package. */
196	typedef RTTIMERLINUXSTARTONCPUARGS *PRTTIMERLINUXSTARTONCPUARGS;
197
198
199	/*******************************************************************************
200	* Internal Functions *
201	*******************************************************************************/
202	#ifdef CONFIG_SMP
203	static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser);
204	#endif
205
206	#if 0
207	#define DEBUG_HACKING
208	#include <iprt/string.h>
209	#include <iprt/asm-amd64-x86.h>
210	static void myLogBackdoorPrintf(const char *pszFormat, ...)
211	{
212	char szTmp[256];
213	va_list args;
214	size_t cb;
215
216	cb = RTStrPrintf(szTmp, sizeof(szTmp) - 10, "%d: ", RTMpCpuId());
217	va_start(args, pszFormat);
218	cb += RTStrPrintfV(&szTmp[cb], sizeof(szTmp) - cb, pszFormat, args);
219	va_end(args);
220
221	ASMOutStrU8(0x504, (uint8_t *)&szTmp[0], cb);
222	}
223	# define RTAssertMsg1Weak(pszExpr, uLine, pszFile, pszFunction) \
224	myLogBackdoorPrintf("\n!!Guest Assertion failed!!\n%s(%d) %s\n%s\n", uLine, pszFile, pszFunction, (pszExpr))
225	# define RTAssertMsg2Weak myLogBackdoorPrintf
226	# define RTTIMERLNX_LOG(a) myLogBackdoorPrintf a
227	#else
228	# define RTTIMERLNX_LOG(a) do { } while (0)
229	#endif
230
231	/**
232	* Sets the state.
233	*/
234	DECLINLINE(void) rtTimerLnxSetState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState)
235	{
236	#ifdef DEBUG_HACKING
237	RTTIMERLNX_LOG(("set %d -> %d\n", *penmState, enmNewState));
238	#endif
239	ASMAtomicWriteU32((uint32_t volatile *)penmState, enmNewState);
240	}
241
242
243	/**
244	* Sets the state if it has a certain value.
245	*
246	* @return true if xchg was done.
247	* @return false if xchg wasn't done.
248	*/
249	#ifdef DEBUG_HACKING
250	#define rtTimerLnxCmpXchgState(penmState, enmNewState, enmCurState) rtTimerLnxCmpXchgStateDebug(penmState, enmNewState, enmCurState, __LINE__)
251	static bool rtTimerLnxCmpXchgStateDebug(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState,
252	RTTIMERLNXSTATE enmCurState, uint32_t uLine)
253	{
254	RTTIMERLNXSTATE enmOldState = enmCurState;
255	bool fRc = ASMAtomicCmpXchgExU32((uint32_t volatile )penmState, enmNewState, enmCurState, (uint32_t )&enmOldState);
256	RTTIMERLNX_LOG(("cxg %d -> %d - %d at %u\n", enmOldState, enmNewState, fRc, uLine));
257	return fRc;
258	}
259	#else
260	DECLINLINE(bool) rtTimerLnxCmpXchgState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState,
261	RTTIMERLNXSTATE enmCurState)
262	{
263	return ASMAtomicCmpXchgU32((uint32_t volatile *)penmState, enmNewState, enmCurState);
264	}
265	#endif
266
267
268	/**
269	* Gets the state.
270	*/
271	DECLINLINE(RTTIMERLNXSTATE) rtTimerLnxGetState(RTTIMERLNXSTATE volatile *penmState)
272	{
273	return (RTTIMERLNXSTATE)ASMAtomicUoReadU32((uint32_t volatile *)penmState);
274	}
275
276	#ifdef RTTIMER_LINUX_WITH_HRTIMER
277
278	/**
279	* Converts a nano second time stamp to ktime_t.
280	*
281	* ASSUMES RTTimeSystemNanoTS() is implemented using ktime_get_ts().
282	*
283	* @returns ktime_t.
284	* @param cNanoSecs Nanoseconds.
285	*/
286	DECLINLINE(ktime_t) rtTimerLnxNanoToKt(uint64_t cNanoSecs)
287	{
288	/* With some luck the compiler optimizes the division out of this... (Bet it doesn't.) */
289	return ktime_set(cNanoSecs / 1000000000, cNanoSecs % 1000000000);
290	}
291
292	/**
293	* Converts ktime_t to a nano second time stamp.
294	*
295	* ASSUMES RTTimeSystemNanoTS() is implemented using ktime_get_ts().
296	*
297	* @returns nano second time stamp.
298	* @param Kt ktime_t.
299	*/
300	DECLINLINE(uint64_t) rtTimerLnxKtToNano(ktime_t Kt)
301	{
302	return ktime_to_ns(Kt);
303	}
304
305	#endif /* RTTIMER_LINUX_WITH_HRTIMER */
306
307	/**
308	* Converts a nano second interval to jiffies.
309	*
310	* @returns Jiffies.
311	* @param cNanoSecs Nanoseconds.
312	*/
313	DECLINLINE(unsigned long) rtTimerLnxNanoToJiffies(uint64_t cNanoSecs)
314	{
315	/* this can be made even better... */
316	if (cNanoSecs > (uint64_t)TICK_NSEC * MAX_JIFFY_OFFSET)
317	return MAX_JIFFY_OFFSET;
318	# if ARCH_BITS == 32
319	if (RT_LIKELY(cNanoSecs <= UINT32_MAX))
320	return ((uint32_t)cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
321	# endif
322	return (cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
323	}
324
325
326	/**
327	* Starts a sub-timer (RTTimerStart).
328	*
329	* @param pSubTimer The sub-timer to start.
330	* @param u64Now The current timestamp (RTTimeSystemNanoTS()).
331	* @param u64First The interval from u64Now to the first time the timer should fire.
332	* @param fPinned true = timer pinned to a specific CPU,
333	* false = timer can migrate between CPUs
334	* @param fHighRes Whether the user requested a high resolution timer or not.
335	* @param enmOldState The old timer state.
336	*/
337	static void rtTimerLnxStartSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, uint64_t u64Now, uint64_t u64First,
338	bool fPinned, bool fHighRes)
339	{
340	/*
341	* Calc when it should start firing.
342	*/
343	uint64_t u64NextTS = u64Now + u64First;
344	if (!fHighRes)
345	pSubTimer->u.Std.u64NextTS = u64NextTS;
346	RTTIMERLNX_LOG(("startsubtimer %p\n", pSubTimer->pParent));
347
348	pSubTimer->iTick = 0;
349
350	#ifdef RTTIMER_LINUX_WITH_HRTIMER
351	if (fHighRes)
352	hrtimer_start(&pSubTimer->u.Hr.LnxTimer, rtTimerLnxNanoToKt(u64NextTS),
353	fPinned ? HRTIMER_MODE_ABS_PINNED : HRTIMER_MODE_ABS);
354	else
355	#endif
356	{
357	unsigned long cJiffies = !u64First ? 0 : rtTimerLnxNanoToJiffies(u64First);
358	pSubTimer->u.Std.ulNextJiffies = jiffies + cJiffies;
359	pSubTimer->u.Std.fFirstAfterChg = true;
360	#ifdef CONFIG_SMP
361	if (fPinned)
362	mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
363	else
364	#endif
365	mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
366	}
367
368	/* Be a bit careful here since we could be racing the callback. */
369	if (!rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_STARTING))
370	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_MP_STARTING);
371	}
372
373
374	/**
375	* Stops a sub-timer (RTTimerStart and rtTimerLinuxMpEvent()).
376	*
377	* The caller has already changed the state, so we will not be in a callback
378	* situation wrt to the calling thread.
379	*
380	* @param pSubTimer The sub-timer.
381	* @param fHighRes Whether the user requested a high resolution timer or not.
382	*/
383	static void rtTimerLnxStopSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, bool fHighRes)
384	{
385	RTTIMERLNX_LOG(("stopsubtimer %p %d\n", pSubTimer->pParent, fHighRes));
386	#ifdef RTTIMER_LINUX_WITH_HRTIMER
387	if (fHighRes)
388	hrtimer_cancel(&pSubTimer->u.Hr.LnxTimer);
389	else
390	#endif
391	{
392	if (timer_pending(&pSubTimer->u.Std.LnxTimer))
393	del_timer_sync(&pSubTimer->u.Std.LnxTimer);
394	}
395
396	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
397	}
398
399
400	/**
401	* Used by RTTimerDestroy and rtTimerLnxCallbackDestroy to do the actual work.
402	*
403	* @param pTimer The timer in question.
404	*/
405	static void rtTimerLnxDestroyIt(PRTTIMER pTimer)
406	{
407	RTSPINLOCK hSpinlock = pTimer->hSpinlock;
408	Assert(pTimer->fSuspended);
409	RTTIMERLNX_LOG(("destroyit %p\n", pTimer));
410
411	/*
412	* Remove the MP notifications first because it'll reduce the risk of
413	* us overtaking any MP event that might theoretically be racing us here.
414	*/
415	#ifdef CONFIG_SMP
416	if ( pTimer->cCpus > 1
417	&& hSpinlock != NIL_RTSPINLOCK)
418	{
419	int rc = RTMpNotificationDeregister(rtTimerLinuxMpEvent, pTimer);
420	AssertRC(rc);
421	}
422	#endif /* CONFIG_SMP */
423
424	/*
425	* Uninitialize the structure and free the associated resources.
426	* The spinlock goes last.
427	*/
428	ASMAtomicWriteU32(&pTimer->u32Magic, ~RTTIMER_MAGIC);
429	RTMemFreeEx(pTimer, RT_OFFSETOF(RTTIMER, aSubTimers[pTimer->cCpus]));
430	if (hSpinlock != NIL_RTSPINLOCK)
431	RTSpinlockDestroy(hSpinlock);
432	}
433
434
435	/**
436	* Called when the timer was destroyed by the callback function.
437	*
438	* @param pTimer The timer.
439	* @param pSubTimer The sub-timer which we're handling, the state of this
440	* will be RTTIMERLNXSTATE_CALLBACK_DESTROYING.
441	*/
442	static void rtTimerLnxCallbackDestroy(PRTTIMER pTimer, PRTTIMERLNXSUBTIMER pSubTimer)
443	{
444	/*
445	* If it's an omni timer, the last dude does the destroying.
446	*/
447	if (pTimer->cCpus > 1)
448	{
449	uint32_t iCpu = pTimer->cCpus;
450	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
451	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
452
453	Assert(pSubTimer->enmState == RTTIMERLNXSTATE_CB_DESTROYING);
454	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
455
456	while (iCpu-- > 0)
457	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) != RTTIMERLNXSTATE_STOPPED)
458	{
459	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
460	return;
461	}
462
463	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
464	}
465
466	rtTimerLnxDestroyIt(pTimer);
467	}
468
469
470	#ifdef CONFIG_SMP
471	/**
472	* Deal with a sub-timer that has migrated.
473	*
474	* @param pTimer The timer.
475	* @param pSubTimer The sub-timer.
476	*/
477	static void rtTimerLnxCallbackHandleMigration(PRTTIMER pTimer, PRTTIMERLNXSUBTIMER pSubTimer)
478	{
479	RTTIMERLNXSTATE enmState;
480	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
481	if (pTimer->cCpus > 1)
482	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
483
484	do
485	{
486	enmState = rtTimerLnxGetState(&pSubTimer->enmState);
487	switch (enmState)
488	{
489	case RTTIMERLNXSTATE_STOPPING:
490	case RTTIMERLNXSTATE_MP_STOPPING:
491	enmState = RTTIMERLNXSTATE_STOPPED;
492	case RTTIMERLNXSTATE_STOPPED:
493	break;
494
495	default:
496	AssertMsgFailed(("%d\n", enmState));
497	case RTTIMERLNXSTATE_STARTING:
498	case RTTIMERLNXSTATE_MP_STARTING:
499	case RTTIMERLNXSTATE_ACTIVE:
500	case RTTIMERLNXSTATE_CALLBACK:
501	case RTTIMERLNXSTATE_CB_STOPPING:
502	case RTTIMERLNXSTATE_CB_RESTARTING:
503	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, enmState))
504	enmState = RTTIMERLNXSTATE_STOPPED;
505	break;
506
507	case RTTIMERLNXSTATE_CB_DESTROYING:
508	{
509	if (pTimer->cCpus > 1)
510	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
511
512	rtTimerLnxCallbackDestroy(pTimer, pSubTimer);
513	return;
514	}
515	}
516	} while (enmState != RTTIMERLNXSTATE_STOPPED);
517
518	if (pTimer->cCpus > 1)
519	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
520	}
521	#endif /* CONFIG_SMP */
522
523
524	/**
525	* The slow path of rtTimerLnxChangeToCallbackState.
526	*
527	* @returns true if changed successfully, false if not.
528	* @param pSubTimer The sub-timer.
529	*/
530	static bool rtTimerLnxChangeToCallbackStateSlow(PRTTIMERLNXSUBTIMER pSubTimer)
531	{
532	for (;;)
533	{
534	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState);
535	switch (enmState)
536	{
537	case RTTIMERLNXSTATE_ACTIVE:
538	case RTTIMERLNXSTATE_STARTING:
539	case RTTIMERLNXSTATE_MP_STARTING:
540	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CALLBACK, enmState))
541	return true;
542	break;
543
544	case RTTIMERLNXSTATE_CALLBACK:
545	case RTTIMERLNXSTATE_CB_STOPPING:
546	case RTTIMERLNXSTATE_CB_RESTARTING:
547	case RTTIMERLNXSTATE_CB_DESTROYING:
548	AssertMsgFailed(("%d\n", enmState));
549	default:
550	return false;
551	}
552	ASMNopPause();
553	}
554	}
555
556
557	/**
558	* Tries to change the sub-timer state to 'callback'.
559	*
560	* @returns true if changed successfully, false if not.
561	* @param pSubTimer The sub-timer.
562	*/
563	DECLINLINE(bool) rtTimerLnxChangeToCallbackState(PRTTIMERLNXSUBTIMER pSubTimer)
564	{
565	if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CALLBACK, RTTIMERLNXSTATE_ACTIVE)))
566	return true;
567	return rtTimerLnxChangeToCallbackStateSlow(pSubTimer);
568	}
569
570
571	#ifdef RTTIMER_LINUX_WITH_HRTIMER
572	/**
573	* Timer callback function for high resolution timers.
574	*
575	* @returns HRTIMER_NORESTART or HRTIMER_RESTART depending on whether it's a
576	* one-shot or interval timer.
577	* @param pHrTimer Pointer to the sub-timer structure.
578	*/
579	static enum hrtimer_restart rtTimerLinuxHrCallback(struct hrtimer *pHrTimer)
580	{
581	PRTTIMERLNXSUBTIMER pSubTimer = RT_FROM_MEMBER(pHrTimer, RTTIMERLNXSUBTIMER, u.Hr.LnxTimer);
582	PRTTIMER pTimer = pSubTimer->pParent;
583
584
585	RTTIMERLNX_LOG(("hrcallback %p\n", pTimer));
586	if (RT_UNLIKELY(!rtTimerLnxChangeToCallbackState(pSubTimer)))
587	return HRTIMER_NORESTART;
588
589	#ifdef CONFIG_SMP
590	/*
591	* Check for unwanted migration.
592	*/
593	if (pTimer->fAllCpus \|\| pTimer->fSpecificCpu)
594	{
595	RTCPUID idCpu = RTMpCpuId();
596	if (RT_UNLIKELY( pTimer->fAllCpus
597	? (RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != idCpu
598	: pTimer->idCpu != idCpu))
599	{
600	rtTimerLnxCallbackHandleMigration(pTimer, pSubTimer);
601	return HRTIMER_NORESTART;
602	}
603	}
604	#endif
605
606	if (pTimer->u64NanoInterval)
607	{
608	/*
609	* Periodic timer, run it and update the native timer afterwards so
610	* we can handle RTTimerStop and RTTimerChangeInterval from the
611	* callback as well as a racing control thread.
612	*/
613	pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
614	hrtimer_add_expires_ns(&pSubTimer->u.Hr.LnxTimer, ASMAtomicReadU64(&pTimer->u64NanoInterval));
615	if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CALLBACK)))
616	return HRTIMER_RESTART;
617	}
618	else
619	{
620	/*
621	* One shot timer (no omni), stop it before dispatching it.
622	* Allow RTTimerStart as well as RTTimerDestroy to be called from
623	* the callback.
624	*/
625	ASMAtomicWriteBool(&pTimer->fSuspended, true);
626	pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
627	if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CALLBACK)))
628	return HRTIMER_NORESTART;
629	}
630
631	/*
632	* Some state change occurred while we were in the callback routine.
633	*/
634	for (;;)
635	{
636	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState);
637	switch (enmState)
638	{
639	case RTTIMERLNXSTATE_CB_DESTROYING:
640	rtTimerLnxCallbackDestroy(pTimer, pSubTimer);
641	return HRTIMER_NORESTART;
642
643	case RTTIMERLNXSTATE_CB_STOPPING:
644	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CB_STOPPING))
645	return HRTIMER_NORESTART;
646	break;
647
648	case RTTIMERLNXSTATE_CB_RESTARTING:
649	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CB_RESTARTING))
650	{
651	pSubTimer->iTick = 0;
652	hrtimer_set_expires(&pSubTimer->u.Hr.LnxTimer, rtTimerLnxNanoToKt(pSubTimer->uNsRestartAt));
653	return HRTIMER_RESTART;
654	}
655	break;
656
657	default:
658	AssertMsgFailed(("%d\n", enmState));
659	return HRTIMER_NORESTART;
660	}
661	ASMNopPause();
662	}
663	}
664	#endif /* RTTIMER_LINUX_WITH_HRTIMER */
665
666
667	/**
668	* Timer callback function for standard timers.
669	*
670	* @param ulUser Address of the sub-timer structure.
671	*/
672	static void rtTimerLinuxStdCallback(unsigned long ulUser)
673	{
674	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)ulUser;
675	PRTTIMER pTimer = pSubTimer->pParent;
676
677	RTTIMERLNX_LOG(("stdcallback %p\n", pTimer));
678	if (RT_UNLIKELY(!rtTimerLnxChangeToCallbackState(pSubTimer)))
679	return;
680
681	#ifdef CONFIG_SMP
682	/*
683	* Check for unwanted migration.
684	*/
685	if (pTimer->fAllCpus \|\| pTimer->fSpecificCpu)
686	{
687	RTCPUID idCpu = RTMpCpuId();
688	if (RT_UNLIKELY( pTimer->fAllCpus
689	? (RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != idCpu
690	: pTimer->idCpu != idCpu))
691	{
692	rtTimerLnxCallbackHandleMigration(pTimer, pSubTimer);
693	return;
694	}
695	}
696	#endif
697
698	if (pTimer->u64NanoInterval)
699	{
700	/*
701	* Interval timer, calculate the next timeout.
702	*
703	* The first time around, we'll re-adjust the u.Std.u64NextTS to
704	* try prevent some jittering if we were started at a bad time.
705	*/
706	const uint64_t iTick = ++pSubTimer->iTick;
707	uint64_t u64NanoInterval;
708	unsigned long cJiffies;
709	unsigned long flFlags;
710
711	spin_lock_irqsave(&pTimer->ChgIntLock, flFlags);
712	u64NanoInterval = pTimer->u64NanoInterval;
713	cJiffies = pTimer->cJiffies;
714	if (RT_UNLIKELY(pSubTimer->u.Std.fFirstAfterChg))
715	{
716	pSubTimer->u.Std.fFirstAfterChg = false;
717	pSubTimer->u.Std.u64NextTS = RTTimeSystemNanoTS();
718	pSubTimer->u.Std.ulNextJiffies = jiffies;
719	}
720	spin_unlock_irqrestore(&pTimer->ChgIntLock, flFlags);
721
722	pSubTimer->u.Std.u64NextTS += u64NanoInterval;
723	if (cJiffies)
724	{
725	pSubTimer->u.Std.ulNextJiffies += cJiffies;
726	/* Prevent overflows when the jiffies counter wraps around.
727	* Special thanks to Ken Preslan for helping debugging! */
728	while (time_before(pSubTimer->u.Std.ulNextJiffies, jiffies))
729	{
730	pSubTimer->u.Std.ulNextJiffies += cJiffies;
731	pSubTimer->u.Std.u64NextTS += u64NanoInterval;
732	}
733	}
734	else
735	{
736	const uint64_t u64NanoTS = RTTimeSystemNanoTS();
737	while (pSubTimer->u.Std.u64NextTS < u64NanoTS)
738	pSubTimer->u.Std.u64NextTS += u64NanoInterval;
739	pSubTimer->u.Std.ulNextJiffies = jiffies + rtTimerLnxNanoToJiffies(pSubTimer->u.Std.u64NextTS - u64NanoTS);
740	}
741
742	/*
743	* Run the timer and re-arm it unless the state changed .
744	* .
745	* We must re-arm it afterwards as we're not in a position to undo this .
746	* operation if for instance someone stopped or destroyed us while we .
747	* were in the callback. (Linux takes care of any races here.)
748	*/
749	pTimer->pfnTimer(pTimer, pTimer->pvUser, iTick);
750	if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CALLBACK)))
751	{
752	#ifdef CONFIG_SMP
753	if (pTimer->fSpecificCpu \|\| pTimer->fAllCpus)
754	mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
755	else
756	#endif
757	mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
758	return;
759	}
760	}
761	else
762	{
763	/*
764	* One shot timer, stop it before dispatching it.
765	* Allow RTTimerStart as well as RTTimerDestroy to be called from
766	* the callback.
767	*/
768	ASMAtomicWriteBool(&pTimer->fSuspended, true);
769	pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
770	if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CALLBACK)))
771	return;
772	}
773
774	/*
775	* Some state change occurred while we were in the callback routine.
776	*/
777	for (;;)
778	{
779	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState);
780	switch (enmState)
781	{
782	case RTTIMERLNXSTATE_CB_DESTROYING:
783	rtTimerLnxCallbackDestroy(pTimer, pSubTimer);
784	return;
785
786	case RTTIMERLNXSTATE_CB_STOPPING:
787	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CB_STOPPING))
788	return;
789	break;
790
791	case RTTIMERLNXSTATE_CB_RESTARTING:
792	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CB_RESTARTING))
793	{
794	uint64_t u64NanoTS;
795	uint64_t u64NextTS;
796	unsigned long flFlags;
797
798	spin_lock_irqsave(&pTimer->ChgIntLock, flFlags);
799	u64NextTS = pSubTimer->uNsRestartAt;
800	u64NanoTS = RTTimeSystemNanoTS();
801	pSubTimer->iTick = 0;
802	pSubTimer->u.Std.u64NextTS = u64NextTS;
803	pSubTimer->u.Std.fFirstAfterChg = true;
804	pSubTimer->u.Std.ulNextJiffies = u64NextTS > u64NanoTS
805	? jiffies + rtTimerLnxNanoToJiffies(u64NextTS - u64NanoTS)
806	: jiffies;
807	spin_unlock_irqrestore(&pTimer->ChgIntLock, flFlags);
808
809	#ifdef CONFIG_SMP
810	if (pTimer->fSpecificCpu \|\| pTimer->fAllCpus)
811	mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
812	else
813	#endif
814	mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
815	return;
816	}
817	break;
818
819	default:
820	AssertMsgFailed(("%d\n", enmState));
821	return;
822	}
823	ASMNopPause();
824	}
825	}
826
827
828	#ifdef CONFIG_SMP
829
830	/**
831	* Per-cpu callback function (RTMpOnAll/RTMpOnSpecific).
832	*
833	* @param idCpu The current CPU.
834	* @param pvUser1 Pointer to the timer.
835	* @param pvUser2 Pointer to the argument structure.
836	*/
837	static DECLCALLBACK(void) rtTimerLnxStartAllOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
838	{
839	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
840	PRTTIMER pTimer = (PRTTIMER)pvUser1;
841	Assert(idCpu < pTimer->cCpus);
842	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[idCpu], pArgs->u64Now, pArgs->u64First, true /fPinned/, pTimer->fHighRes);
843	}
844
845
846	/**
847	* Worker for RTTimerStart() that takes care of the ugly bits.
848	*
849	* @returns RTTimerStart() return value.
850	* @param pTimer The timer.
851	* @param pArgs The argument structure.
852	*/
853	static int rtTimerLnxOmniStart(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs)
854	{
855	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
856	RTCPUID iCpu;
857	RTCPUSET OnlineSet;
858	RTCPUSET OnlineSet2;
859	int rc2;
860
861	/*
862	* Prepare all the sub-timers for the startup and then flag the timer
863	* as a whole as non-suspended, make sure we get them all before
864	* clearing fSuspended as the MP handler will be waiting on this
865	* should something happen while we're looping.
866	*/
867	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
868
869	/* Just make it a omni timer restriction that no stop/start races are allowed. */
870	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
871	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) != RTTIMERLNXSTATE_STOPPED)
872	{
873	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
874	return VERR_TIMER_BUSY;
875	}
876
877	do
878	{
879	RTMpGetOnlineSet(&OnlineSet);
880	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
881	{
882	Assert(pTimer->aSubTimers[iCpu].enmState != RTTIMERLNXSTATE_MP_STOPPING);
883	rtTimerLnxSetState(&pTimer->aSubTimers[iCpu].enmState,
884	RTCpuSetIsMember(&OnlineSet, iCpu)
885	? RTTIMERLNXSTATE_STARTING
886	: RTTIMERLNXSTATE_STOPPED);
887	}
888	} while (!RTCpuSetIsEqual(&OnlineSet, RTMpGetOnlineSet(&OnlineSet2)));
889
890	ASMAtomicWriteBool(&pTimer->fSuspended, false);
891
892	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
893
894	/*
895	* Start them (can't find any exported function that allows me to
896	* do this without the cross calls).
897	*/
898	pArgs->u64Now = RTTimeSystemNanoTS();
899	rc2 = RTMpOnAll(rtTimerLnxStartAllOnCpu, pTimer, pArgs);
900	AssertRC(rc2); /* screw this if it fails. */
901
902	/*
903	* Reset the sub-timers who didn't start up (ALL CPUs case).
904	*/
905	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
906
907	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
908	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_STARTING))
909	{
910	/** @todo very odd case for a rainy day. Cpus that temporarily went offline while
911	* we were between calls needs to nudged as the MP handler will ignore events for
912	* them because of the STARTING state. This is an extremely unlikely case - not that
913	* that means anything in my experience... ;-) */
914	RTTIMERLNX_LOG(("what!? iCpu=%u -> didn't start\n", iCpu));
915	}
916
917	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
918
919	return VINF_SUCCESS;
920	}
921
922
923	/**
924	* Worker for RTTimerStop() that takes care of the ugly SMP bits.
925	*
926	* @returns true if there was any active callbacks, false if not.
927	* @param pTimer The timer (valid).
928	* @param fForDestroy Whether this is for RTTimerDestroy or not.
929	*/
930	static bool rtTimerLnxOmniStop(PRTTIMER pTimer, bool fForDestroy)
931	{
932	bool fActiveCallbacks = false;
933	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
934	RTCPUID iCpu;
935
936
937	/*
938	* Mark the timer as suspended and flag all timers as stopping, except
939	* for those being stopped by an MP event.
940	*/
941	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
942
943	ASMAtomicWriteBool(&pTimer->fSuspended, true);
944	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
945	{
946	RTTIMERLNXSTATE enmState;
947	for (;;)
948	{
949	enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
950	if ( enmState == RTTIMERLNXSTATE_STOPPED
951	\|\| enmState == RTTIMERLNXSTATE_MP_STOPPING)
952	break;
953	if ( enmState == RTTIMERLNXSTATE_CALLBACK
954	\|\| enmState == RTTIMERLNXSTATE_CB_STOPPING
955	\|\| enmState == RTTIMERLNXSTATE_CB_RESTARTING)
956	{
957	Assert(enmState != RTTIMERLNXSTATE_CB_STOPPING \|\| fForDestroy);
958	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState,
959	!fForDestroy ? RTTIMERLNXSTATE_CB_STOPPING : RTTIMERLNXSTATE_CB_DESTROYING,
960	enmState))
961	{
962	fActiveCallbacks = true;
963	break;
964	}
965	}
966	else
967	{
968	Assert(enmState == RTTIMERLNXSTATE_ACTIVE);
969	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPING, enmState))
970	break;
971	}
972	ASMNopPause();
973	}
974	}
975
976	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
977
978	/*
979	* Do the actual stopping. Fortunately, this doesn't require any IPIs.
980	* Unfortunately it cannot be done synchronously from within the spinlock,
981	* because we might end up in an active waiting for a handler to complete.
982	*/
983	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
984	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) == RTTIMERLNXSTATE_STOPPING)
985	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[iCpu], pTimer->fHighRes);
986
987	return fActiveCallbacks;
988	}
989
990
991	/**
992	* Per-cpu callback function (RTMpOnSpecific) used by rtTimerLinuxMpEvent()
993	* to start a sub-timer on a cpu that just have come online.
994	*
995	* @param idCpu The current CPU.
996	* @param pvUser1 Pointer to the timer.
997	* @param pvUser2 Pointer to the argument structure.
998	*/
999	static DECLCALLBACK(void) rtTimerLinuxMpStartOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
1000	{
1001	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
1002	PRTTIMER pTimer = (PRTTIMER)pvUser1;
1003	RTSPINLOCK hSpinlock;
1004	Assert(idCpu < pTimer->cCpus);
1005
1006	/*
1007	* We have to be kind of careful here as we might be racing RTTimerStop
1008	* (and/or RTTimerDestroy, thus the paranoia.
1009	*/
1010	hSpinlock = pTimer->hSpinlock;
1011	if ( hSpinlock != NIL_RTSPINLOCK
1012	&& pTimer->u32Magic == RTTIMER_MAGIC)
1013	{
1014	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1015	RTSpinlockAcquire(hSpinlock, &Tmp);
1016
1017	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
1018	&& pTimer->u32Magic == RTTIMER_MAGIC)
1019	{
1020	/* We're sane and the timer is not suspended yet. */
1021	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
1022	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
1023	rtTimerLnxStartSubTimer(pSubTimer, pArgs->u64Now, pArgs->u64First, true /fPinned/, pTimer->fHighRes);
1024	}
1025
1026	RTSpinlockRelease(hSpinlock, &Tmp);
1027	}
1028	}
1029
1030
1031	/**
1032	* MP event notification callback.
1033	*
1034	* @param enmEvent The event.
1035	* @param idCpu The cpu it applies to.
1036	* @param pvUser The timer.
1037	*/
1038	static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
1039	{
1040	PRTTIMER pTimer = (PRTTIMER)pvUser;
1041	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
1042	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1043	RTSPINLOCK hSpinlock;
1044
1045	Assert(idCpu < pTimer->cCpus);
1046
1047	/*
1048	* Some initial paranoia.
1049	*/
1050	if (pTimer->u32Magic != RTTIMER_MAGIC)
1051	return;
1052	hSpinlock = pTimer->hSpinlock;
1053	if (hSpinlock == NIL_RTSPINLOCK)
1054	return;
1055
1056	RTSpinlockAcquire(hSpinlock, &Tmp);
1057
1058	/* Is it active? */
1059	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
1060	&& pTimer->u32Magic == RTTIMER_MAGIC)
1061	{
1062	switch (enmEvent)
1063	{
1064	/*
1065	* Try do it without leaving the spin lock, but if we have to, retake it
1066	* when we're on the right cpu.
1067	*/
1068	case RTMPEVENT_ONLINE:
1069	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
1070	{
1071	RTTIMERLINUXSTARTONCPUARGS Args;
1072	Args.u64Now = RTTimeSystemNanoTS();
1073	Args.u64First = 0;
1074
1075	if (RTMpCpuId() == idCpu)
1076	rtTimerLnxStartSubTimer(pSubTimer, Args.u64Now, Args.u64First, true /fPinned/, pTimer->fHighRes);
1077	else
1078	{
1079	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); /* we'll recheck it. */
1080	RTSpinlockRelease(hSpinlock, &Tmp);
1081
1082	RTMpOnSpecific(idCpu, rtTimerLinuxMpStartOnCpu, pTimer, &Args);
1083	return; /* we've left the spinlock */
1084	}
1085	}
1086	break;
1087
1088	/*
1089	* The CPU is (going) offline, make sure the sub-timer is stopped.
1090	*
1091	* Linux will migrate it to a different CPU, but we don't want this. The
1092	* timer function is checking for this.
1093	*/
1094	case RTMPEVENT_OFFLINE:
1095	{
1096	RTTIMERLNXSTATE enmState;
1097	while ( (enmState = rtTimerLnxGetState(&pSubTimer->enmState)) == RTTIMERLNXSTATE_ACTIVE
1098	\|\| enmState == RTTIMERLNXSTATE_CALLBACK
1099	\|\| enmState == RTTIMERLNXSTATE_CB_RESTARTING)
1100	{
1101	if (enmState == RTTIMERLNXSTATE_ACTIVE)
1102	{
1103	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STOPPING, RTTIMERLNXSTATE_ACTIVE))
1104	{
1105	RTSpinlockRelease(hSpinlock, &Tmp);
1106
1107	rtTimerLnxStopSubTimer(pSubTimer, pTimer->fHighRes);
1108	return; /* we've left the spinlock */
1109	}
1110	}
1111	else if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CB_STOPPING, enmState))
1112	break;
1113
1114	/* State not stable, try again. */
1115	ASMNopPause();
1116	}
1117	break;
1118	}
1119	}
1120	}
1121
1122	RTSpinlockRelease(hSpinlock, &Tmp);
1123	}
1124
1125	#endif /* CONFIG_SMP */
1126
1127
1128	/**
1129	* Callback function use by RTTimerStart via RTMpOnSpecific to start a timer
1130	* running on a specific CPU.
1131	*
1132	* @param idCpu The current CPU.
1133	* @param pvUser1 Pointer to the timer.
1134	* @param pvUser2 Pointer to the argument structure.
1135	*/
1136	static DECLCALLBACK(void) rtTimerLnxStartOnSpecificCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
1137	{
1138	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
1139	PRTTIMER pTimer = (PRTTIMER)pvUser1;
1140	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], pArgs->u64Now, pArgs->u64First, true /fPinned/, pTimer->fHighRes);
1141	}
1142
1143
1144	RTDECL(int) RTTimerStart(PRTTIMER pTimer, uint64_t u64First)
1145	{
1146	RTTIMERLINUXSTARTONCPUARGS Args;
1147	int rc2;
1148
1149	/*
1150	* Validate.
1151	*/
1152	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
1153	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
1154
1155	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
1156	return VERR_TIMER_ACTIVE;
1157	RTTIMERLNX_LOG(("start %p cCpus=%d\n", pTimer, pTimer->cCpus));
1158
1159	Args.u64First = u64First;
1160	#ifdef CONFIG_SMP
1161	/*
1162	* Omni timer?
1163	*/
1164	if (pTimer->fAllCpus)
1165	return rtTimerLnxOmniStart(pTimer, &Args);
1166	#endif
1167
1168	/*
1169	* Simple timer - Pretty straight forward if it wasn't for restarting.
1170	*/
1171	Args.u64Now = RTTimeSystemNanoTS();
1172	ASMAtomicWriteU64(&pTimer->aSubTimers[0].uNsRestartAt, Args.u64Now + u64First);
1173	for (;;)
1174	{
1175	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[0].enmState);
1176	switch (enmState)
1177	{
1178	case RTTIMERLNXSTATE_STOPPED:
1179	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STARTING, RTTIMERLNXSTATE_STOPPED))
1180	{
1181	ASMAtomicWriteBool(&pTimer->fSuspended, false);
1182	if (!pTimer->fSpecificCpu)
1183	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], Args.u64Now, Args.u64First,
1184	false /fPinned/, pTimer->fHighRes);
1185	else
1186	{
1187	rc2 = RTMpOnSpecific(pTimer->idCpu, rtTimerLnxStartOnSpecificCpu, pTimer, &Args);
1188	if (RT_FAILURE(rc2))
1189	{
1190	/* Suspend it, the cpu id is probably invalid or offline. */
1191	ASMAtomicWriteBool(&pTimer->fSuspended, true);
1192	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPED);
1193	return rc2;
1194	}
1195	}
1196	return VINF_SUCCESS;
1197	}
1198	break;
1199
1200	case RTTIMERLNXSTATE_CALLBACK:
1201	case RTTIMERLNXSTATE_CB_STOPPING:
1202	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_CB_RESTARTING, enmState))
1203	{
1204	ASMAtomicWriteBool(&pTimer->fSuspended, false);
1205	return VINF_SUCCESS;
1206	}
1207	break;
1208
1209	default:
1210	AssertMsgFailed(("%d\n", enmState));
1211	return VERR_INTERNAL_ERROR_4;
1212	}
1213	ASMNopPause();
1214	}
1215	}
1216	RT_EXPORT_SYMBOL(RTTimerStart);
1217
1218
1219	/**
1220	* Common worker for RTTimerStop and RTTimerDestroy.
1221	*
1222	* @returns true if there was any active callbacks, false if not.
1223	* @param pTimer The timer to stop.
1224	* @param fForDestroy Whether it's RTTimerDestroy calling or not.
1225	*/
1226	static bool rtTimerLnxStop(PRTTIMER pTimer, bool fForDestroy)
1227	{
1228	RTTIMERLNX_LOG(("lnxstop %p %d\n", pTimer, fForDestroy));
1229	#ifdef CONFIG_SMP
1230	/*
1231	* Omni timer?
1232	*/
1233	if (pTimer->fAllCpus)
1234	return rtTimerLnxOmniStop(pTimer, fForDestroy);
1235	#endif
1236
1237	/*
1238	* Simple timer.
1239	*/
1240	ASMAtomicWriteBool(&pTimer->fSuspended, true);
1241	for (;;)
1242	{
1243	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[0].enmState);
1244	switch (enmState)
1245	{
1246	case RTTIMERLNXSTATE_ACTIVE:
1247	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPING, RTTIMERLNXSTATE_ACTIVE))
1248	{
1249	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[0], pTimer->fHighRes);
1250	return false;
1251	}
1252	break;
1253
1254	case RTTIMERLNXSTATE_CALLBACK:
1255	case RTTIMERLNXSTATE_CB_RESTARTING:
1256	case RTTIMERLNXSTATE_CB_STOPPING:
1257	Assert(enmState != RTTIMERLNXSTATE_CB_STOPPING \|\| fForDestroy);
1258	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState,
1259	!fForDestroy ? RTTIMERLNXSTATE_CB_STOPPING : RTTIMERLNXSTATE_CB_DESTROYING,
1260	enmState))
1261	return true;
1262	break;
1263
1264	case RTTIMERLNXSTATE_STOPPED:
1265	return VINF_SUCCESS;
1266
1267	case RTTIMERLNXSTATE_CB_DESTROYING:
1268	AssertMsgFailed(("enmState=%d pTimer=%p\n", enmState, pTimer));
1269	return true;
1270
1271	default:
1272	case RTTIMERLNXSTATE_STARTING:
1273	case RTTIMERLNXSTATE_MP_STARTING:
1274	case RTTIMERLNXSTATE_STOPPING:
1275	case RTTIMERLNXSTATE_MP_STOPPING:
1276	AssertMsgFailed(("enmState=%d pTimer=%p\n", enmState, pTimer));
1277	return false;
1278	}
1279
1280	/* State not stable, try again. */
1281	ASMNopPause();
1282	}
1283	}
1284
1285
1286	RTDECL(int) RTTimerStop(PRTTIMER pTimer)
1287	{
1288	/*
1289	* Validate.
1290	*/
1291	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
1292	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
1293	RTTIMERLNX_LOG(("stop %p\n", pTimer));
1294
1295	if (ASMAtomicUoReadBool(&pTimer->fSuspended))
1296	return VERR_TIMER_SUSPENDED;
1297
1298	rtTimerLnxStop(pTimer, false /fForDestroy/);
1299	return VINF_SUCCESS;
1300	}
1301	RT_EXPORT_SYMBOL(RTTimerStop);
1302
1303
1304	RTDECL(int) RTTimerChangeInterval(PRTTIMER pTimer, uint64_t u64NanoInterval)
1305	{
1306	unsigned long cJiffies;
1307	unsigned long flFlags;
1308
1309	/*
1310	* Validate.
1311	*/
1312	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
1313	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
1314	AssertReturn(u64NanoInterval, VERR_INVALID_PARAMETER);
1315	RTTIMERLNX_LOG(("change %p %llu\n", pTimer, u64NanoInterval));
1316
1317	#ifdef RTTIMER_LINUX_WITH_HRTIMER
1318	/*
1319	* For the high resolution timers it is easy since we don't care so much
1320	* about when it is applied to the sub-timers.
1321	*/
1322	if (pTimer->fHighRes)
1323	{
1324	ASMAtomicWriteU64(&pTimer->u64NanoInterval, u64NanoInterval);
1325	return VINF_SUCCESS;
1326	}
1327	#endif
1328
1329	/*
1330	* Standard timers have a bit more complicated way of calculating
1331	* their interval and such. So, forget omni timers for now.
1332	*/
1333	if (pTimer->cCpus > 1)
1334	return VERR_NOT_SUPPORTED;
1335
1336	cJiffies = u64NanoInterval / RTTimerGetSystemGranularity();
1337	if (cJiffies * RTTimerGetSystemGranularity() != u64NanoInterval)
1338	cJiffies = 0;
1339
1340	spin_lock_irqsave(&pTimer->ChgIntLock, flFlags);
1341	pTimer->aSubTimers[0].u.Std.fFirstAfterChg = true;
1342	pTimer->cJiffies = cJiffies;
1343	ASMAtomicWriteU64(&pTimer->u64NanoInterval, u64NanoInterval);
1344	spin_unlock_irqrestore(&pTimer->ChgIntLock, flFlags);
1345	return VINF_SUCCESS;
1346	}
1347	RT_EXPORT_SYMBOL(RTTimerChangeInterval);
1348
1349
1350	RTDECL(int) RTTimerDestroy(PRTTIMER pTimer)
1351	{
1352	bool fCanDestroy;
1353
1354	/*
1355	* Validate. It's ok to pass NULL pointer.
1356	*/
1357	if (pTimer == /NIL_RTTIMER/ NULL)
1358	return VINF_SUCCESS;
1359	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
1360	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
1361	RTTIMERLNX_LOG(("destroy %p\n", pTimer));
1362
1363	/*
1364	* Stop the timer if it's still active, then destroy it if we can.
1365	*/
1366	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
1367	fCanDestroy = rtTimerLnxStop(pTimer, true /fForDestroy/);
1368	else
1369	{
1370	uint32_t iCpu = pTimer->cCpus;
1371	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1372	if (pTimer->cCpus > 1)
1373	RTSpinlockAcquireNoInts(pTimer->hSpinlock, &Tmp);
1374
1375	fCanDestroy = true;
1376	while (iCpu-- > 0)
1377	{
1378	for (;;)
1379	{
1380	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
1381	switch (enmState)
1382	{
1383	case RTTIMERLNXSTATE_CALLBACK:
1384	case RTTIMERLNXSTATE_CB_RESTARTING:
1385	case RTTIMERLNXSTATE_CB_STOPPING:
1386	if (!rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_CB_DESTROYING, enmState))
1387	continue;
1388	fCanDestroy = false;
1389	break;
1390
1391	case RTTIMERLNXSTATE_CB_DESTROYING:
1392	AssertMsgFailed(("%d\n", enmState));
1393	fCanDestroy = false;
1394	break;
1395	default:
1396	break;
1397	}
1398	break;
1399	}
1400	}
1401
1402	if (pTimer->cCpus > 1)
1403	RTSpinlockReleaseNoInts(pTimer->hSpinlock, &Tmp);
1404	}
1405
1406	if (fCanDestroy)
1407	rtTimerLnxDestroyIt(pTimer);
1408
1409	return VINF_SUCCESS;
1410	}
1411	RT_EXPORT_SYMBOL(RTTimerDestroy);
1412
1413
1414	RTDECL(int) RTTimerCreateEx(PRTTIMER ppTimer, uint64_t u64NanoInterval, uint32_t fFlags, PFNRTTIMER pfnTimer, void pvUser)
1415	{
1416	PRTTIMER pTimer;
1417	RTCPUID iCpu;
1418	unsigned cCpus;
1419	int rc;
1420
1421	*ppTimer = NULL;
1422
1423	/*
1424	* Validate flags.
1425	*/
1426	if (!RTTIMER_FLAGS_ARE_VALID(fFlags))
1427	return VERR_INVALID_PARAMETER;
1428	if ( (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC)
1429	&& (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL
1430	&& !RTMpIsCpuPossible(RTMpCpuIdFromSetIndex(fFlags & RTTIMER_FLAGS_CPU_MASK)))
1431	return VERR_CPU_NOT_FOUND;
1432
1433	/*
1434	* Allocate the timer handler.
1435	*/
1436	cCpus = 1;
1437	#ifdef CONFIG_SMP
1438	if ((fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL)
1439	{
1440	cCpus = RTMpGetMaxCpuId() + 1;
1441	Assert(cCpus <= RTCPUSET_MAX_CPUS); /* On linux we have a 1:1 relationship between cpuid and set index. */
1442	AssertReturn(u64NanoInterval, VERR_NOT_IMPLEMENTED); /* We don't implement single shot on all cpus, sorry. */
1443	}
1444	#endif
1445
1446	rc = RTMemAllocEx(RT_OFFSETOF(RTTIMER, aSubTimers[cCpus]), 0,
1447	RTMEMALLOCEX_FLAGS_ZEROED \| RTMEMALLOCEX_FLAGS_ANY_CTX_FREE, (void **)&pTimer);
1448	if (RT_FAILURE(rc))
1449	return rc;
1450
1451	/*
1452	* Initialize it.
1453	*/
1454	pTimer->u32Magic = RTTIMER_MAGIC;
1455	pTimer->hSpinlock = NIL_RTSPINLOCK;
1456	pTimer->fSuspended = true;
1457	pTimer->fHighRes = !!(fFlags & RTTIMER_FLAGS_HIGH_RES);
1458	#ifdef CONFIG_SMP
1459	pTimer->fSpecificCpu = (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC) && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL;
1460	pTimer->fAllCpus = (fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL;
1461	pTimer->idCpu = pTimer->fSpecificCpu
1462	? RTMpCpuIdFromSetIndex(fFlags & RTTIMER_FLAGS_CPU_MASK)
1463	: NIL_RTCPUID;
1464	#else
1465	pTimer->fSpecificCpu = !!(fFlags & RTTIMER_FLAGS_CPU_SPECIFIC);
1466	pTimer->idCpu = RTMpCpuId();
1467	#endif
1468	pTimer->cCpus = cCpus;
1469	pTimer->pfnTimer = pfnTimer;
1470	pTimer->pvUser = pvUser;
1471	pTimer->u64NanoInterval = u64NanoInterval;
1472	pTimer->cJiffies = u64NanoInterval / RTTimerGetSystemGranularity();
1473	if (pTimer->cJiffies * RTTimerGetSystemGranularity() != u64NanoInterval)
1474	pTimer->cJiffies = 0;
1475	spin_lock_init(&pTimer->ChgIntLock);
1476
1477	for (iCpu = 0; iCpu < cCpus; iCpu++)
1478	{
1479	#ifdef RTTIMER_LINUX_WITH_HRTIMER
1480	if (pTimer->fHighRes)
1481	{
1482	hrtimer_init(&pTimer->aSubTimers[iCpu].u.Hr.LnxTimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1483	pTimer->aSubTimers[iCpu].u.Hr.LnxTimer.function = rtTimerLinuxHrCallback;
1484	}
1485	else
1486	#endif
1487	{
1488	init_timer(&pTimer->aSubTimers[iCpu].u.Std.LnxTimer);
1489	pTimer->aSubTimers[iCpu].u.Std.LnxTimer.data = (unsigned long)&pTimer->aSubTimers[iCpu];
1490	pTimer->aSubTimers[iCpu].u.Std.LnxTimer.function = rtTimerLinuxStdCallback;
1491	pTimer->aSubTimers[iCpu].u.Std.LnxTimer.expires = jiffies;
1492	pTimer->aSubTimers[iCpu].u.Std.u64NextTS = 0;
1493	}
1494	pTimer->aSubTimers[iCpu].iTick = 0;
1495	pTimer->aSubTimers[iCpu].pParent = pTimer;
1496	pTimer->aSubTimers[iCpu].enmState = RTTIMERLNXSTATE_STOPPED;
1497	}
1498
1499	#ifdef CONFIG_SMP
1500	/*
1501	* If this is running on ALL cpus, we'll have to register a callback
1502	* for MP events (so timers can be started/stopped on cpus going
1503	* online/offline). We also create the spinlock for synchronizing
1504	* stop/start/mp-event.
1505	*/
1506	if (cCpus > 1)
1507	{
1508	int rc = RTSpinlockCreate(&pTimer->hSpinlock);
1509	if (RT_SUCCESS(rc))
1510	rc = RTMpNotificationRegister(rtTimerLinuxMpEvent, pTimer);
1511	else
1512	pTimer->hSpinlock = NIL_RTSPINLOCK;
1513	if (RT_FAILURE(rc))
1514	{
1515	RTTimerDestroy(pTimer);
1516	return rc;
1517	}
1518	}
1519	#endif /* CONFIG_SMP */
1520
1521	RTTIMERLNX_LOG(("create %p hires=%d fFlags=%#x cCpus=%u\n", pTimer, pTimer->fHighRes, fFlags, cCpus));
1522	*ppTimer = pTimer;
1523	return VINF_SUCCESS;
1524	}
1525	RT_EXPORT_SYMBOL(RTTimerCreateEx);
1526
1527
1528	RTDECL(uint32_t) RTTimerGetSystemGranularity(void)
1529	{
1530	#if 0 /** @todo Not sure if this is what we want or not... Add new API for
1531	* querying the resolution of the high res timers? */
1532	struct timespec Ts;
1533	int rc = hrtimer_get_res(CLOCK_MONOTONIC, &Ts);
1534	if (!rc)
1535	{
1536	Assert(!Ts.tv_sec);
1537	return Ts.tv_nsec;
1538	}
1539	#endif
1540	return 1000000000 / HZ; /* ns */
1541	}
1542	RT_EXPORT_SYMBOL(RTTimerGetSystemGranularity);
1543
1544
1545	RTDECL(int) RTTimerRequestSystemGranularity(uint32_t u32Request, uint32_t *pu32Granted)
1546	{
1547	return VERR_NOT_SUPPORTED;
1548	}
1549	RT_EXPORT_SYMBOL(RTTimerRequestSystemGranularity);
1550
1551
1552	RTDECL(int) RTTimerReleaseSystemGranularity(uint32_t u32Granted)
1553	{
1554	return VERR_NOT_SUPPORTED;
1555	}
1556	RT_EXPORT_SYMBOL(RTTimerReleaseSystemGranularity);
1557
1558
1559	RTDECL(bool) RTTimerCanDoHighResolution(void)
1560	{
1561	#ifdef RTTIMER_LINUX_WITH_HRTIMER
1562	return true;
1563	#else
1564	return false;
1565	#endif
1566	}
1567	RT_EXPORT_SYMBOL(RTTimerCanDoHighResolution);
1568

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/timer-r0drv-linux.c@ 35857

Download in other formats: