timer-r0drv-linux.c@ 32737

Last change on this file since 32737 was 32737, checked in by vboxsync, 14 years ago
timer-r0drv-linux.c: Fixes.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 53.1 KB

Line
1	/* $Id: timer-r0drv-linux.c 32737 2010-09-23 16:20:05Z vboxsync $ */
2	/** @file
3	* IPRT - Timers, Ring-0 Driver, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2010 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*******************************************************************************
29	* Header Files *
30	*******************************************************************************/
31	#include "the-linux-kernel.h"
32	#include "internal/iprt.h"
33
34	#include <iprt/timer.h>
35	#include <iprt/time.h>
36	#include <iprt/mp.h>
37	#include <iprt/cpuset.h>
38	#include <iprt/spinlock.h>
39	#include <iprt/err.h>
40	#include <iprt/asm.h>
41	#include <iprt/assert.h>
42	#include <iprt/alloc.h>
43
44	#include "internal/magics.h"
45
46	/** @def RTTIMER_LINUX_HAVE_HRTIMER
47	* Whether the kernel support high resolution timers (Linux kernel versions
48	* 2.6.28 and later (hrtimer_add_expires_ns()). */
49	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
50	# define RTTIMER_LINUX_HAVE_HRTIMER
51	#endif
52
53	/** @def RTTIMER_LINUX_WITH_HRTIMER
54	* Whether to use high resolution timers. */
55	#if !defined(RTTIMER_LINUX_WITH_HRTIMER) \
56	&& defined(RTTIMER_LINUX_HAVE_HRTIMER)
57	# define RTTIMER_LINUX_WITH_HRTIMER
58	#endif
59
60	#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31)
61	# define mod_timer_pinned mod_timer
62	# define HRTIMER_MODE_ABS_PINNED HRTIMER_MODE_ABS
63	#endif
64
65
66	/*******************************************************************************
67	* Structures and Typedefs *
68	*******************************************************************************/
69	/**
70	* Timer state machine.
71	*
72	* This is used to try handle the issues with MP events and
73	* timers that runs on all CPUs. It's relatively nasty :-/
74	*/
75	typedef enum RTTIMERLNXSTATE
76	{
77	/** Stopped. */
78	RTTIMERLNXSTATE_STOPPED = 0,
79	/** Transient state; next ACTIVE. */
80	RTTIMERLNXSTATE_STARTING,
81	/** Transient state; next ACTIVE. (not really necessary) */
82	RTTIMERLNXSTATE_MP_STARTING,
83	/** Active. */
84	RTTIMERLNXSTATE_ACTIVE,
85	/** Active and in callback; next ACTIVE, STOPPED or CALLBACK_DESTROYING. */
86	RTTIMERLNXSTATE_CALLBACK,
87	/** Stopped while in the callback; next STOPPED. */
88	RTTIMERLNXSTATE_CB_STOPPING,
89	/** Restarted while in the callback; next ACTIVE, STOPPED, DESTROYING. */
90	RTTIMERLNXSTATE_CB_RESTARTING,
91	/** The callback shall destroy the timer; next STOPPED. */
92	RTTIMERLNXSTATE_CB_DESTROYING,
93	/** Transient state; next STOPPED. */
94	RTTIMERLNXSTATE_STOPPING,
95	/** Transient state; next STOPPED. */
96	RTTIMERLNXSTATE_MP_STOPPING,
97	/** The usual 32-bit hack. */
98	RTTIMERLNXSTATE_32BIT_HACK = 0x7fffffff
99	} RTTIMERLNXSTATE;
100
101
102	/**
103	* A Linux sub-timer.
104	*/
105	typedef struct RTTIMERLNXSUBTIMER
106	{
107	/** Timer specific data. */
108	union
109	{
110	#if defined(RTTIMER_LINUX_WITH_HRTIMER)
111	/** High resolution timer. */
112	struct
113	{
114	/** The linux timer structure. */
115	struct hrtimer LnxTimer;
116	} Hr;
117	#endif
118	/** Standard timer. */
119	struct
120	{
121	/** The linux timer structure. */
122	struct timer_list LnxTimer;
123	/** The start of the current run (ns).
124	* This is used to calculate when the timer ought to fire the next time. */
125	uint64_t u64NextTS;
126	/** The u64NextTS in jiffies. */
127	unsigned long ulNextJiffies;
128	/** Set when starting or changing the timer so that u64StartTs
129	* and u64NextTS gets reinitialized (eliminating some jitter). */
130	bool volatile fFirstAfterChg;
131	} Std;
132	} u;
133	/** The current tick number. */
134	uint64_t iTick;
135	/** Restart the single shot timer at this specific time.
136	* Used when a single shot timer is restarted from the callback. */
137	uint64_t volatile uNsRestartAt;
138	/** Pointer to the parent timer. */
139	PRTTIMER pParent;
140	/** The current sub-timer state. */
141	RTTIMERLNXSTATE volatile enmState;
142	} RTTIMERLNXSUBTIMER;
143	/** Pointer to a linux sub-timer. */
144	typedef RTTIMERLNXSUBTIMER *PRTTIMERLNXSUBTIMER;
145
146
147	/**
148	* The internal representation of an Linux timer handle.
149	*/
150	typedef struct RTTIMER
151	{
152	/** Magic.
153	* This is RTTIMER_MAGIC, but changes to something else before the timer
154	* is destroyed to indicate clearly that thread should exit. */
155	uint32_t volatile u32Magic;
156	/** Spinlock synchronizing the fSuspended and MP event handling.
157	* This is NIL_RTSPINLOCK if cCpus == 1. */
158	RTSPINLOCK hSpinlock;
159	/** Flag indicating that the timer is suspended. */
160	bool volatile fSuspended;
161	/** Whether the timer must run on one specific CPU or not. */
162	bool fSpecificCpu;
163	#ifdef CONFIG_SMP
164	/** Whether the timer must run on all CPUs or not. */
165	bool fAllCpus;
166	#endif /* else: All -> specific on non-SMP kernels */
167	/** Whether it is a high resolution timer or a standard one. */
168	bool fHighRes;
169	/** The id of the CPU it must run on if fSpecificCpu is set. */
170	RTCPUID idCpu;
171	/** The number of CPUs this timer should run on. */
172	RTCPUID cCpus;
173	/** Callback. */
174	PFNRTTIMER pfnTimer;
175	/** User argument. */
176	void *pvUser;
177	/** The timer interval. 0 if one-shot. */
178	uint64_t volatile u64NanoInterval;
179	/** This is set to the number of jiffies between ticks if the interval is
180	* an exact number of jiffies. (Standard timers only.) */
181	unsigned long volatile cJiffies;
182	/** The change interval spinlock for standard timers only. */
183	spinlock_t ChgIntLock;
184	/** Sub-timers.
185	* Normally there is just one, but for RTTIMER_FLAGS_CPU_ALL this will contain
186	* an entry for all possible cpus. In that case the index will be the same as
187	* for the RTCpuSet. */
188	RTTIMERLNXSUBTIMER aSubTimers[1];
189	} RTTIMER;
190
191
192	/**
193	* A rtTimerLinuxStartOnCpu and rtTimerLinuxStartOnCpu argument package.
194	*/
195	typedef struct RTTIMERLINUXSTARTONCPUARGS
196	{
197	/** The current time (RTTimeSystemNanoTS). */
198	uint64_t u64Now;
199	/** When to start firing (delta). */
200	uint64_t u64First;
201	} RTTIMERLINUXSTARTONCPUARGS;
202	/** Pointer to a rtTimerLinuxStartOnCpu argument package. */
203	typedef RTTIMERLINUXSTARTONCPUARGS *PRTTIMERLINUXSTARTONCPUARGS;
204
205
206	/*******************************************************************************
207	* Internal Functions *
208	*******************************************************************************/
209	#ifdef CONFIG_SMP
210	static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser);
211	#endif
212
213	#if 0
214	#define DEBUG_HACKING
215	#include <iprt/string.h>
216	#include <iprt/asm-amd64-x86.h>
217	static void myLogBackdoorPrintf(const char *pszFormat, ...)
218	{
219	char szTmp[256];
220	va_list args;
221	size_t cb;
222
223	cb = RTStrPrintf(szTmp, sizeof(szTmp) - 10, "%d: ", RTMpCpuId());
224	va_start(args, pszFormat);
225	cb += RTStrPrintfV(&szTmp[cb], sizeof(szTmp) - cb, pszFormat, args);
226	va_end(args);
227
228	ASMOutStrU8(0x504, (uint8_t *)&szTmp[0], cb);
229	}
230	#define RTAssertMsg1Weak(pszExpr, uLine, pszFile, pszFunction) \
231	myLogBackdoorPrintf("\n!!Guest Assertion failed!!\n%s(%d) %s\n%s\n", uLine, pszFile, pszFunction, (pszExpr))
232	#define RTAssertMsg2Weak myLogBackdoorPrintf
233	#define dprintf(a) myLogBackdoorPrintf a
234	#else
235	#define dprintf(a) do { } while (0)
236	#endif
237
238	/**
239	* Sets the state.
240	*/
241	DECLINLINE(void) rtTimerLnxSetState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState)
242	{
243	#ifdef DEBUG_HACKING
244	dprintf(("set %d -> %d\n", *penmState, enmNewState));
245	#endif
246	ASMAtomicWriteU32((uint32_t volatile *)penmState, enmNewState);
247	}
248
249
250	/**
251	* Sets the state if it has a certain value.
252	*
253	* @return true if xchg was done.
254	* @return false if xchg wasn't done.
255	*/
256	#ifdef DEBUG_HACKING
257	#define rtTimerLnxCmpXchgState(penmState, enmNewState, enmCurState) rtTimerLnxCmpXchgStateDebug(penmState, enmNewState, enmCurState, __LINE__)
258	static bool rtTimerLnxCmpXchgStateDebug(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState,
259	RTTIMERLNXSTATE enmCurState, uint32_t uLine)
260	{
261	RTTIMERLNXSTATE enmOldState = enmCurState;
262	bool fRc = ASMAtomicCmpXchgExU32((uint32_t volatile )penmState, enmNewState, enmCurState, (uint32_t )&enmOldState);
263	dprintf(("cxg %d -> %d - %d at %u\n", enmOldState, enmNewState, fRc, uLine));
264	return fRc;
265	}
266	#else
267	DECLINLINE(bool) rtTimerLnxCmpXchgState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState,
268	RTTIMERLNXSTATE enmCurState)
269	{
270	return ASMAtomicCmpXchgU32((uint32_t volatile *)penmState, enmNewState, enmCurState);
271	}
272	#endif
273
274
275	/**
276	* Gets the state.
277	*/
278	DECLINLINE(RTTIMERLNXSTATE) rtTimerLnxGetState(RTTIMERLNXSTATE volatile *penmState)
279	{
280	return (RTTIMERLNXSTATE)ASMAtomicUoReadU32((uint32_t volatile *)penmState);
281	}
282
283	#ifdef RTTIMER_LINUX_WITH_HRTIMER
284
285	/**
286	* Converts a nano second time stamp to ktime_t.
287	*
288	* ASSUMES RTTimeSystemNanoTS() is implemented using ktime_get_ts().
289	*
290	* @returns ktime_t.
291	* @param cNanoSecs Nanoseconds.
292	*/
293	DECLINLINE(ktime_t) rtTimerLnxNanoToKt(uint64_t cNanoSecs)
294	{
295	/* With some luck the compiler optimizes the division out of this... (Bet it doesn't.) */
296	return ktime_set(cNanoSecs / 1000000000, cNanoSecs % 1000000000);
297	}
298
299	/**
300	* Converts ktime_t to a nano second time stamp.
301	*
302	* ASSUMES RTTimeSystemNanoTS() is implemented using ktime_get_ts().
303	*
304	* @returns nano second time stamp.
305	* @param Kt ktime_t.
306	*/
307	DECLINLINE(uint64_t) rtTimerLnxKtToNano(ktime_t Kt)
308	{
309	return ktime_to_ns(Kt);
310	}
311
312	#endif /* RTTIMER_LINUX_WITH_HRTIMER */
313
314	/**
315	* Converts a nano second interval to jiffies.
316	*
317	* @returns Jiffies.
318	* @param cNanoSecs Nanoseconds.
319	*/
320	DECLINLINE(unsigned long) rtTimerLnxNanoToJiffies(uint64_t cNanoSecs)
321	{
322	/* this can be made even better... */
323	if (cNanoSecs > (uint64_t)TICK_NSEC * MAX_JIFFY_OFFSET)
324	return MAX_JIFFY_OFFSET;
325	# if ARCH_BITS == 32
326	if (RT_LIKELY(cNanoSecs <= UINT32_MAX))
327	return ((uint32_t)cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
328	# endif
329	return (cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
330	}
331
332
333	/**
334	* Starts a sub-timer (RTTimerStart).
335	*
336	* @param pSubTimer The sub-timer to start.
337	* @param u64Now The current timestamp (RTTimeSystemNanoTS()).
338	* @param u64First The interval from u64Now to the first time the timer should fire.
339	* @param fPinned true = timer pinned to a specific CPU,
340	* false = timer can migrate between CPUs
341	* @param fHighRes Whether the user requested a high resolution timer or not.
342	* @param enmOldState The old timer state.
343	*/
344	static void rtTimerLnxStartSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, uint64_t u64Now, uint64_t u64First,
345	bool fPinned, bool fHighRes)
346	{
347	/*
348	* Calc when it should start firing.
349	*/
350	uint64_t u64NextTS = u64Now + u64First;
351	if (fHighRes)
352	pSubTimer->u.Std.u64NextTS = u64NextTS;
353	dprintf(("startsubtimer %p\n", pSubTimer->pParent));
354
355	pSubTimer->iTick = 0;
356
357	#ifdef RTTIMER_LINUX_WITH_HRTIMER
358	if (fHighRes)
359	hrtimer_start(&pSubTimer->u.Hr.LnxTimer, rtTimerLnxNanoToKt(u64NextTS),
360	fPinned ? HRTIMER_MODE_ABS_PINNED : HRTIMER_MODE_ABS);
361	else
362	#endif
363	{
364	unsigned long cJiffies = !u64First ? 0 : rtTimerLnxNanoToJiffies(u64First);
365	pSubTimer->u.Std.ulNextJiffies = jiffies + cJiffies;
366	pSubTimer->u.Std.fFirstAfterChg = true;
367	#ifdef CONFIG_SMP
368	if (fPinned)
369	mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
370	else
371	#endif
372	mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
373	}
374
375	/* Be a bit careful here since we could be racing the callback. */
376	if (!rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_STARTING))
377	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_MP_STARTING);
378	}
379
380
381	/**
382	* Stops a sub-timer (RTTimerStart and rtTimerLinuxMpEvent()).
383	*
384	* The caller has already changed the state, so we will not be in a callback
385	* situation wrt to the calling thread.
386	*
387	* @param pSubTimer The sub-timer.
388	* @param fHighRes Whether the user requested a high resolution timer or not.
389	*/
390	static void rtTimerLnxStopSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, bool fHighRes)
391	{
392	dprintf(("stopsubtimer %p %d\n", pSubTimer->pParent, fHighRes));
393	#ifdef RTTIMER_LINUX_WITH_HRTIMER
394	if (fHighRes)
395	hrtimer_cancel(&pSubTimer->u.Hr.LnxTimer);
396	else
397	#endif
398	{
399	if (timer_pending(&pSubTimer->u.Std.LnxTimer))
400	del_timer_sync(&pSubTimer->u.Std.LnxTimer);
401	}
402
403	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
404	}
405
406
407	/**
408	* Used by RTTimerDestroy and rtTimerLnxCallbackDestroy to do the actual work.
409	*
410	* @param pTimer The timer in question.
411	*/
412	static void rtTimerLnxDestroyIt(PRTTIMER pTimer)
413	{
414	RTSPINLOCK hSpinlock = pTimer->hSpinlock;
415	Assert(pTimer->fSuspended);
416	dprintf(("destroyit %p\n", pTimer));
417
418	/*
419	* Remove the MP notifications first because it'll reduce the risk of
420	* us overtaking any MP event that might theoretically be racing us here.
421	*/
422	#ifdef CONFIG_SMP
423	if ( pTimer->cCpus > 1
424	&& hSpinlock != NIL_RTSPINLOCK)
425	{
426	int rc = RTMpNotificationDeregister(rtTimerLinuxMpEvent, pTimer);
427	AssertRC(rc);
428	}
429	#endif /* CONFIG_SMP */
430
431	/*
432	* Uninitialize the structure and free the associated resources.
433	* The spinlock goes last.
434	*/
435	ASMAtomicWriteU32(&pTimer->u32Magic, ~RTTIMER_MAGIC);
436	RTMemFreeEx(pTimer, RT_OFFSETOF(RTTIMER, aSubTimers[pTimer->cCpus]));
437	if (hSpinlock != NIL_RTSPINLOCK)
438	RTSpinlockDestroy(hSpinlock);
439	}
440
441
442	/**
443	* Called when the timer was destroyed by the callback function.
444	*
445	* @param pTimer The timer.
446	* @param pSubTimer The sub-timer which we're handling, the state of this
447	* will be RTTIMERLNXSTATE_CALLBACK_DESTROYING.
448	*/
449	static void rtTimerLnxCallbackDestroy(PRTTIMER pTimer, PRTTIMERLNXSUBTIMER pSubTimer)
450	{
451	/*
452	* If it's an omni timer, the last dude does the destroying.
453	*/
454	if (pTimer->cCpus > 1)
455	{
456	bool fAllStopped = true;
457	uint32_t iCpu = pTimer->cCpus;
458	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
459	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
460
461	Assert(pSubTimer->enmState == RTTIMERLNXSTATE_CB_DESTROYING);
462	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
463
464	while (iCpu-- > 0)
465	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) != RTTIMERLNXSTATE_STOPPED)
466	{
467	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
468	return;
469	}
470
471	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
472	}
473
474	rtTimerLnxDestroyIt(pTimer);
475	}
476
477
478	#ifdef CONFIG_SMP
479	/**
480	* Deal with a sub-timer that has migrated.
481	*
482	* @param pTimer The timer.
483	* @param pSubTimer The sub-timer.
484	*/
485	static void rtTimerLnxCallbackHandleMigration(PRTTIMER pTimer, PRTTIMERLNXSUBTIMER pSubTimer)
486	{
487	RTTIMERLNXSTATE enmState;
488	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
489	if (pTimer->cCpus > 1)
490	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
491
492	do
493	{
494	enmState = rtTimerLnxGetState(&pSubTimer->enmState);
495	switch (enmState)
496	{
497	case RTTIMERLNXSTATE_STOPPING:
498	case RTTIMERLNXSTATE_MP_STOPPING:
499	enmState = RTTIMERLNXSTATE_STOPPED;
500	case RTTIMERLNXSTATE_STOPPED:
501	break;
502
503	default:
504	AssertMsgFailed(("%d\n", enmState));
505	case RTTIMERLNXSTATE_STARTING:
506	case RTTIMERLNXSTATE_MP_STARTING:
507	case RTTIMERLNXSTATE_ACTIVE:
508	case RTTIMERLNXSTATE_CALLBACK:
509	case RTTIMERLNXSTATE_CB_STOPPING:
510	case RTTIMERLNXSTATE_CB_RESTARTING:
511	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, enmState))
512	enmState = RTTIMERLNXSTATE_STOPPED;
513	break;
514
515	case RTTIMERLNXSTATE_CB_DESTROYING:
516	{
517	if (pTimer->cCpus > 1)
518	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
519
520	rtTimerLnxCallbackDestroy(pTimer, pSubTimer);
521	return;
522	}
523	}
524	} while (enmState != RTTIMERLNXSTATE_STOPPED);
525
526	if (pTimer->cCpus > 1)
527	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
528	}
529	#endif /* CONFIG_SMP */
530
531
532	/**
533	* The slow path of rtTimerLnxChangeToCallbackState.
534	*
535	* @returns true if changed successfully, false if not.
536	* @param pSubTimer The sub-timer.
537	*/
538	static bool rtTimerLnxChangeToCallbackStateSlow(PRTTIMERLNXSUBTIMER pSubTimer)
539	{
540	for (;;)
541	{
542	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState);
543	switch (enmState)
544	{
545	case RTTIMERLNXSTATE_ACTIVE:
546	case RTTIMERLNXSTATE_STARTING:
547	case RTTIMERLNXSTATE_MP_STARTING:
548	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CALLBACK, enmState))
549	return true;
550	break;
551
552	case RTTIMERLNXSTATE_CALLBACK:
553	case RTTIMERLNXSTATE_CB_STOPPING:
554	case RTTIMERLNXSTATE_CB_RESTARTING:
555	case RTTIMERLNXSTATE_CB_DESTROYING:
556	AssertMsgFailed(("%d\n", enmState));
557	default:
558	return false;
559	}
560	ASMNopPause();
561	}
562	}
563
564
565	/**
566	* Tries to change the sub-timer state to 'callback'.
567	*
568	* @returns true if changed successfully, false if not.
569	* @param pSubTimer The sub-timer.
570	*/
571	DECLINLINE(bool) rtTimerLnxChangeToCallbackState(PRTTIMERLNXSUBTIMER pSubTimer)
572	{
573	if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CALLBACK, RTTIMERLNXSTATE_ACTIVE)))
574	return true;
575	return rtTimerLnxChangeToCallbackStateSlow(pSubTimer);
576	}
577
578
579	#ifdef RTTIMER_LINUX_WITH_HRTIMER
580	/**
581	* Timer callback function for high resolution timers.
582	*
583	* @returns HRTIMER_NORESTART or HRTIMER_RESTART depending on whether it's a
584	* one-shot or interval timer.
585	* @param pHrTimer Pointer to the sub-timer structure.
586	*/
587	static enum hrtimer_restart rtTimerLinuxHrCallback(struct hrtimer *pHrTimer)
588	{
589	PRTTIMERLNXSUBTIMER pSubTimer = RT_FROM_MEMBER(pHrTimer, RTTIMERLNXSUBTIMER, u.Hr.LnxTimer);
590	PRTTIMER pTimer = pSubTimer->pParent;
591
592
593	dprintf(("hrcallback %p\n", pTimer));
594	if (RT_UNLIKELY(!rtTimerLnxChangeToCallbackState(pSubTimer)))
595	return HRTIMER_NORESTART;
596
597	#ifdef CONFIG_SMP
598	/*
599	* Check for unwanted migration.
600	*/
601	if ( (pTimer->fSpecificCpu \|\| pTimer->fAllCpus)
602	&& RT_UNLIKELY((RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != RTMpCpuId()))
603	{
604	rtTimerLnxCallbackHandleMigration(pTimer, pSubTimer);
605	return HRTIMER_NORESTART;
606	}
607	#endif
608
609	if (pTimer->u64NanoInterval)
610	{
611	/*
612	* Periodic timer, run it and update the native timer afterwards so
613	* we can handle RTTimerStop and RTTimerChangeInterval from the
614	* callback as well as a racing control thread.
615	*/
616	pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
617	hrtimer_add_expires_ns(&pSubTimer->u.Hr.LnxTimer, ASMAtomicReadU64(&pTimer->u64NanoInterval));
618	if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CALLBACK)))
619	return HRTIMER_RESTART;
620	}
621	else
622	{
623	/*
624	* One shot timer (no omni), stop it before dispatching it.
625	* Allow RTTimerStart as well as RTTimerDestroy to be called from
626	* the callback.
627	*/
628	ASMAtomicWriteBool(&pTimer->fSuspended, true);
629	pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
630	if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CALLBACK)))
631	return HRTIMER_NORESTART;
632	}
633
634	/*
635	* Some state change occured while we were in the callback routine.
636	*/
637	for (;;)
638	{
639	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState);
640	switch (enmState)
641	{
642	case RTTIMERLNXSTATE_CB_DESTROYING:
643	rtTimerLnxCallbackDestroy(pTimer, pSubTimer);
644	return HRTIMER_NORESTART;
645
646	case RTTIMERLNXSTATE_CB_STOPPING:
647	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CB_STOPPING))
648	return HRTIMER_NORESTART;
649	break;
650
651	case RTTIMERLNXSTATE_CB_RESTARTING:
652	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CB_RESTARTING))
653	{
654	pSubTimer->iTick = 0;
655	hrtimer_set_expires(&pSubTimer->u.Hr.LnxTimer, rtTimerLnxNanoToKt(pSubTimer->uNsRestartAt));
656	return HRTIMER_RESTART;
657	}
658	break;
659
660	default:
661	AssertMsgFailed(("%d\n", enmState));
662	return HRTIMER_NORESTART;
663	}
664	ASMNopPause();
665	}
666	}
667	#endif /* RTTIMER_LINUX_WITH_HRTIMER */
668
669
670	/**
671	* Timer callback function for standard timers.
672	*
673	* @param ulUser Address of the sub-timer structure.
674	*/
675	static void rtTimerLinuxStdCallback(unsigned long ulUser)
676	{
677	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)ulUser;
678	PRTTIMER pTimer = pSubTimer->pParent;
679
680	dprintf(("hrcallback %p\n", pTimer));
681	if (RT_UNLIKELY(!rtTimerLnxChangeToCallbackState(pSubTimer)))
682	return;
683
684	#ifdef CONFIG_SMP
685	/*
686	* Check for unwanted migration.
687	*/
688	if ( (pTimer->fSpecificCpu \|\| pTimer->fAllCpus)
689	&& RT_UNLIKELY((RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != RTMpCpuId()))
690	{
691	rtTimerLnxCallbackHandleMigration(pTimer, pSubTimer);
692	return;
693	}
694	#endif
695
696	if (pTimer->u64NanoInterval)
697	{
698	/*
699	* Interval timer, calculate the next timeout.
700	*
701	* The first time around, we'll re-adjust the u.Std.u64NextTS to
702	* try prevent some jittering if we were started at a bad time.
703	*/
704	const uint64_t iTick = ++pSubTimer->iTick;
705	uint64_t u64NanoInterval;
706	unsigned long cJiffies;
707	unsigned long flFlags;
708
709	spin_lock_irqsave(&pTimer->ChgIntLock, flFlags);
710	u64NanoInterval = pTimer->u64NanoInterval;
711	cJiffies = pTimer->cJiffies;
712	if (RT_UNLIKELY(pSubTimer->u.Std.fFirstAfterChg))
713	{
714	pSubTimer->u.Std.fFirstAfterChg = false;
715	pSubTimer->u.Std.u64NextTS = RTTimeSystemNanoTS();
716	pSubTimer->u.Std.ulNextJiffies = jiffies;
717	}
718	spin_unlock_irqrestore(&pTimer->ChgIntLock, flFlags);
719
720	pSubTimer->u.Std.u64NextTS += u64NanoInterval;
721	if (cJiffies)
722	{
723	pSubTimer->u.Std.ulNextJiffies += cJiffies;
724	/* Prevent overflows when the jiffies counter wraps around.
725	* Special thanks to Ken Preslan for helping debugging! */
726	while (time_before(pSubTimer->u.Std.ulNextJiffies, jiffies))
727	{
728	pSubTimer->u.Std.ulNextJiffies += cJiffies;
729	pSubTimer->u.Std.u64NextTS += u64NanoInterval;
730	}
731	}
732	else
733	{
734	const uint64_t u64NanoTS = RTTimeSystemNanoTS();
735	while (pSubTimer->u.Std.u64NextTS < u64NanoTS)
736	pSubTimer->u.Std.u64NextTS += u64NanoInterval;
737	pSubTimer->u.Std.ulNextJiffies = jiffies + rtTimerLnxNanoToJiffies(pSubTimer->u.Std.u64NextTS - u64NanoTS);
738	}
739
740	/*
741	* Run the timer and re-arm it unless the state changed .
742	* .
743	* We must re-arm it afterwards as we're not in a position to undo this .
744	* operation if for instance someone stopped or destroyed us while we .
745	* were in the callback. (Linux takes care of any races here.)
746	*/
747	pTimer->pfnTimer(pTimer, pTimer->pvUser, iTick);
748	if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CALLBACK)))
749	{
750	#ifdef CONFIG_SMP
751	if (pTimer->fSpecificCpu \|\| pTimer->fAllCpus)
752	mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
753	else
754	#endif
755	mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
756	return;
757	}
758	}
759	else
760	{
761	/*
762	* One shot timer, stop it before dispatching it.
763	* Allow RTTimerStart as well as RTTimerDestroy to be called from
764	* the callback.
765	*/
766	ASMAtomicWriteBool(&pTimer->fSuspended, true);
767	pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
768	if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CALLBACK)))
769	return;
770	}
771
772	/*
773	* Some state change occured while we were in the callback routine.
774	*/
775	for (;;)
776	{
777	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState);
778	switch (enmState)
779	{
780	case RTTIMERLNXSTATE_CB_DESTROYING:
781	rtTimerLnxCallbackDestroy(pTimer, pSubTimer);
782	return;
783
784	case RTTIMERLNXSTATE_CB_STOPPING:
785	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CB_STOPPING))
786	return;
787	break;
788
789	case RTTIMERLNXSTATE_CB_RESTARTING:
790	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CB_RESTARTING))
791	{
792	uint64_t u64NanoTS;
793	uint64_t u64NextTS;
794	unsigned long flFlags;
795
796	spin_lock_irqsave(&pTimer->ChgIntLock, flFlags);
797	u64NextTS = pSubTimer->uNsRestartAt;
798	u64NanoTS = RTTimeSystemNanoTS();
799	pSubTimer->iTick = 0;
800	pSubTimer->u.Std.u64NextTS = u64NextTS;
801	pSubTimer->u.Std.fFirstAfterChg = true;
802	pSubTimer->u.Std.ulNextJiffies = u64NextTS > u64NanoTS
803	? jiffies + rtTimerLnxNanoToJiffies(u64NextTS - u64NanoTS)
804	: jiffies;
805	spin_unlock_irqrestore(&pTimer->ChgIntLock, flFlags);
806
807	#ifdef CONFIG_SMP
808	if (pTimer->fSpecificCpu \|\| pTimer->fAllCpus)
809	mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
810	else
811	#endif
812	mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
813	return;
814	}
815	break;
816
817	default:
818	AssertMsgFailed(("%d\n", enmState));
819	return;
820	}
821	ASMNopPause();
822	}
823	}
824
825
826	#ifdef CONFIG_SMP
827
828	/**
829	* Per-cpu callback function (RTMpOnAll/RTMpOnSpecific).
830	*
831	* @param idCpu The current CPU.
832	* @param pvUser1 Pointer to the timer.
833	* @param pvUser2 Pointer to the argument structure.
834	*/
835	static DECLCALLBACK(void) rtTimerLnxStartAllOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
836	{
837	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
838	PRTTIMER pTimer = (PRTTIMER)pvUser1;
839	Assert(idCpu < pTimer->cCpus);
840	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[idCpu], pArgs->u64Now, pArgs->u64First, true /fPinned/, pTimer->fHighRes);
841	}
842
843
844	/**
845	* Worker for RTTimerStart() that takes care of the ugly bits.
846	*
847	* @returns RTTimerStart() return value.
848	* @param pTimer The timer.
849	* @param pArgs The argument structure.
850	*/
851	static int rtTimerLnxOmniStart(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs)
852	{
853	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
854	RTCPUID iCpu;
855	RTCPUSET OnlineSet;
856	RTCPUSET OnlineSet2;
857	int rc2;
858
859	/*
860	* Prepare all the sub-timers for the startup and then flag the timer
861	* as a whole as non-suspended, make sure we get them all before
862	* clearing fSuspended as the MP handler will be waiting on this
863	* should something happen while we're looping.
864	*/
865	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
866
867	/* Just make it a omni timer restriction that no stop/start races are allowed. */
868	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
869	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) != RTTIMERLNXSTATE_STOPPED)
870	{
871	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
872	return VERR_TIMER_BUSY;
873	}
874
875	do
876	{
877	RTMpGetOnlineSet(&OnlineSet);
878	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
879	{
880	Assert(pTimer->aSubTimers[iCpu].enmState != RTTIMERLNXSTATE_MP_STOPPING);
881	rtTimerLnxSetState(&pTimer->aSubTimers[iCpu].enmState,
882	RTCpuSetIsMember(&OnlineSet, iCpu)
883	? RTTIMERLNXSTATE_STARTING
884	: RTTIMERLNXSTATE_STOPPED);
885	}
886	} while (!RTCpuSetIsEqual(&OnlineSet, RTMpGetOnlineSet(&OnlineSet2)));
887
888	ASMAtomicWriteBool(&pTimer->fSuspended, false);
889
890	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
891
892	/*
893	* Start them (can't find any exported function that allows me to
894	* do this without the cross calls).
895	*/
896	pArgs->u64Now = RTTimeSystemNanoTS();
897	rc2 = RTMpOnAll(rtTimerLnxStartAllOnCpu, pTimer, pArgs);
898	AssertRC(rc2); /* screw this if it fails. */
899
900	/*
901	* Reset the sub-timers who didn't start up (ALL CPUs case).
902	*/
903	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
904
905	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
906	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_STARTING))
907	{
908	/** @todo very odd case for a rainy day. Cpus that temporarily went offline while
909	* we were between calls needs to nudged as the MP handler will ignore events for
910	* them because of the STARTING state. This is an extremely unlikely case - not that
911	* that means anything in my experience... ;-) */
912	}
913
914	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
915
916	return VINF_SUCCESS;
917	}
918
919
920	/**
921	* Worker for RTTimerStop() that takes care of the ugly SMP bits.
922	*
923	* @returns true if there was any active callbacks, false if not.
924	* @param pTimer The timer (valid).
925	* @param fForDestroy Whether this is for RTTimerDestroy or not.
926	*/
927	static bool rtTimerLnxOmniStop(PRTTIMER pTimer, bool fForDestroy)
928	{
929	bool fActiveCallbacks = false;
930	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
931	RTCPUID iCpu;
932
933
934	/*
935	* Mark the timer as suspended and flag all timers as stopping, except
936	* for those being stopped by an MP event.
937	*/
938	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
939
940	ASMAtomicWriteBool(&pTimer->fSuspended, true);
941	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
942	{
943	RTTIMERLNXSTATE enmState;
944	for (;;)
945	{
946	enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
947	if ( enmState == RTTIMERLNXSTATE_STOPPED
948	\|\| enmState == RTTIMERLNXSTATE_MP_STOPPING)
949	break;
950	if ( enmState == RTTIMERLNXSTATE_CALLBACK
951	\|\| enmState == RTTIMERLNXSTATE_CB_STOPPING
952	\|\| enmState == RTTIMERLNXSTATE_CB_RESTARTING)
953	{
954	Assert(enmState != RTTIMERLNXSTATE_CB_STOPPING \|\| fForDestroy);
955	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState,
956	!fForDestroy ? RTTIMERLNXSTATE_CB_STOPPING : RTTIMERLNXSTATE_CB_DESTROYING,
957	enmState))
958	{
959	fActiveCallbacks = true;
960	break;
961	}
962	}
963	else
964	{
965	Assert(enmState == RTTIMERLNXSTATE_ACTIVE);
966	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPING, enmState))
967	break;
968	}
969	ASMNopPause();
970	}
971	}
972
973	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
974
975	/*
976	* Do the actual stopping. Fortunately, this doesn't require any IPIs.
977	* Unfortunately it cannot be done synchronously from within the spinlock,
978	* because we might end up in an active waiting for a handler to complete.
979	*/
980	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
981	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) == RTTIMERLNXSTATE_STOPPING)
982	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[iCpu], pTimer->fHighRes);
983
984	return fActiveCallbacks;
985	}
986
987
988	/**
989	* Per-cpu callback function (RTMpOnSpecific) used by rtTimerLinuxMpEvent()
990	* to start a sub-timer on a cpu that just have come online.
991	*
992	* @param idCpu The current CPU.
993	* @param pvUser1 Pointer to the timer.
994	* @param pvUser2 Pointer to the argument structure.
995	*/
996	static DECLCALLBACK(void) rtTimerLinuxMpStartOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
997	{
998	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
999	PRTTIMER pTimer = (PRTTIMER)pvUser1;
1000	RTSPINLOCK hSpinlock;
1001	Assert(idCpu < pTimer->cCpus);
1002
1003	/*
1004	* We have to be kind of careful here as we might be racing RTTimerStop
1005	* (and/or RTTimerDestroy, thus the paranoia.
1006	*/
1007	hSpinlock = pTimer->hSpinlock;
1008	if ( hSpinlock != NIL_RTSPINLOCK
1009	&& pTimer->u32Magic == RTTIMER_MAGIC)
1010	{
1011	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1012	RTSpinlockAcquire(hSpinlock, &Tmp);
1013
1014	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
1015	&& pTimer->u32Magic == RTTIMER_MAGIC)
1016	{
1017	/* We're sane and the timer is not suspended yet. */
1018	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
1019	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
1020	rtTimerLnxStartSubTimer(pSubTimer, pArgs->u64Now, pArgs->u64First, true /fPinned/, pTimer->fHighRes);
1021	}
1022
1023	RTSpinlockRelease(hSpinlock, &Tmp);
1024	}
1025	}
1026
1027
1028	/**
1029	* MP event notification callback.
1030	*
1031	* @param enmEvent The event.
1032	* @param idCpu The cpu it applies to.
1033	* @param pvUser The timer.
1034	*/
1035	static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
1036	{
1037	PRTTIMER pTimer = (PRTTIMER)pvUser;
1038	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
1039	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1040	RTSPINLOCK hSpinlock;
1041
1042	Assert(idCpu < pTimer->cCpus);
1043
1044	/*
1045	* Some initial paranoia.
1046	*/
1047	if (pTimer->u32Magic != RTTIMER_MAGIC)
1048	return;
1049	hSpinlock = pTimer->hSpinlock;
1050	if (hSpinlock == NIL_RTSPINLOCK)
1051	return;
1052
1053	RTSpinlockAcquire(hSpinlock, &Tmp);
1054
1055	/* Is it active? */
1056	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
1057	&& pTimer->u32Magic == RTTIMER_MAGIC)
1058	{
1059	switch (enmEvent)
1060	{
1061	/*
1062	* Try do it without leaving the spin lock, but if we have to, retake it
1063	* when we're on the right cpu.
1064	*/
1065	case RTMPEVENT_ONLINE:
1066	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
1067	{
1068	RTTIMERLINUXSTARTONCPUARGS Args;
1069	Args.u64Now = RTTimeSystemNanoTS();
1070	Args.u64First = 0;
1071
1072	if (RTMpCpuId() == idCpu)
1073	rtTimerLnxStartSubTimer(pSubTimer, Args.u64Now, Args.u64First, true /fPinned/, pTimer->fHighRes);
1074	else
1075	{
1076	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); /* we'll recheck it. */
1077	RTSpinlockRelease(hSpinlock, &Tmp);
1078
1079	RTMpOnSpecific(idCpu, rtTimerLinuxMpStartOnCpu, pTimer, &Args);
1080	return; /* we've left the spinlock */
1081	}
1082	}
1083	break;
1084
1085	/*
1086	* The CPU is (going) offline, make sure the sub-timer is stopped.
1087	*
1088	* Linux will migrate it to a different CPU, but we don't want this. The
1089	* timer function is checking for this.
1090	*/
1091	case RTMPEVENT_OFFLINE:
1092	{
1093	RTTIMERLNXSTATE enmState;
1094	while ( (enmState = rtTimerLnxGetState(&pSubTimer->enmState)) == RTTIMERLNXSTATE_ACTIVE
1095	\|\| enmState == RTTIMERLNXSTATE_CALLBACK
1096	\|\| enmState == RTTIMERLNXSTATE_CB_RESTARTING)
1097	{
1098	if (enmState == RTTIMERLNXSTATE_ACTIVE)
1099	{
1100	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STOPPING, RTTIMERLNXSTATE_ACTIVE))
1101	{
1102	RTSpinlockRelease(hSpinlock, &Tmp);
1103
1104	rtTimerLnxStopSubTimer(pSubTimer, pTimer->fHighRes);
1105	return; /* we've left the spinlock */
1106	}
1107	}
1108	else if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CB_STOPPING, enmState))
1109	break;
1110
1111	/* State not stable, try again. */
1112	ASMNopPause();
1113	}
1114	break;
1115	}
1116	}
1117	}
1118
1119	RTSpinlockRelease(hSpinlock, &Tmp);
1120	}
1121
1122	#endif /* CONFIG_SMP */
1123
1124
1125	/**
1126	* Callback function use by RTTimerStart via RTMpOnSpecific to start a timer
1127	* running on a specific CPU.
1128	*
1129	* @param idCpu The current CPU.
1130	* @param pvUser1 Pointer to the timer.
1131	* @param pvUser2 Pointer to the argument structure.
1132	*/
1133	static DECLCALLBACK(void) rtTimerLnxStartOnSpecificCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
1134	{
1135	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
1136	PRTTIMER pTimer = (PRTTIMER)pvUser1;
1137	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], pArgs->u64Now, pArgs->u64First, true /fPinned/, pTimer->fHighRes);
1138	}
1139
1140
1141	RTDECL(int) RTTimerStart(PRTTIMER pTimer, uint64_t u64First)
1142	{
1143	RTTIMERLINUXSTARTONCPUARGS Args;
1144	int rc2;
1145
1146	/*
1147	* Validate.
1148	*/
1149	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
1150	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
1151
1152	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
1153	return VERR_TIMER_ACTIVE;
1154	dprintf(("start %p\n", pTimer));
1155
1156	Args.u64First = u64First;
1157	#ifdef CONFIG_SMP
1158	/*
1159	* Omni timer?
1160	*/
1161	if (pTimer->fAllCpus)
1162	return rtTimerLnxOmniStart(pTimer, &Args);
1163	#endif
1164
1165	/*
1166	* Simple timer - Pretty straight forward if it wasn't for restarting.
1167	*/
1168	Args.u64Now = RTTimeSystemNanoTS();
1169	ASMAtomicWriteU64(&pTimer->aSubTimers[0].uNsRestartAt, Args.u64Now + u64First);
1170	for (;;)
1171	{
1172	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[0].enmState);
1173	switch (enmState)
1174	{
1175	case RTTIMERLNXSTATE_STOPPED:
1176	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STARTING, RTTIMERLNXSTATE_STOPPED))
1177	{
1178	ASMAtomicWriteBool(&pTimer->fSuspended, false);
1179	if (!pTimer->fSpecificCpu)
1180	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], Args.u64Now, Args.u64First,
1181	false /fPinned/, pTimer->fHighRes);
1182	else
1183	{
1184	rc2 = RTMpOnSpecific(pTimer->idCpu, rtTimerLnxStartOnSpecificCpu, pTimer, &Args);
1185	if (RT_FAILURE(rc2))
1186	{
1187	/* Suspend it, the cpu id is probably invalid or offline. */
1188	ASMAtomicWriteBool(&pTimer->fSuspended, true);
1189	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPED);
1190	return rc2;
1191	}
1192	}
1193	return VINF_SUCCESS;
1194	}
1195	break;
1196
1197	case RTTIMERLNXSTATE_CALLBACK:
1198	case RTTIMERLNXSTATE_CB_STOPPING:
1199	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_CB_RESTARTING, enmState))
1200	{
1201	ASMAtomicWriteBool(&pTimer->fSuspended, false);
1202	return VINF_SUCCESS;
1203	}
1204	break;
1205
1206	default:
1207	AssertMsgFailed(("%d\n", enmState));
1208	return VERR_INTERNAL_ERROR_4;
1209	}
1210	ASMNopPause();
1211	}
1212	}
1213	RT_EXPORT_SYMBOL(RTTimerStart);
1214
1215
1216	/**
1217	* Common worker for RTTimerStop and RTTimerDestroy.
1218	*
1219	* @returns true if there was any active callbacks, false if not.
1220	* @param pTimer The timer to stop.
1221	* @param fForDestroy Whether it's RTTimerDestroy calling or not.
1222	*/
1223	static bool rtTimerLnxStop(PRTTIMER pTimer, bool fForDestroy)
1224	{
1225	dprintf(("lnxstop %p %d\n", pTimer, fForDestroy));
1226	#ifdef CONFIG_SMP
1227	/*
1228	* Omni timer?
1229	*/
1230	if (pTimer->fAllCpus)
1231	return rtTimerLnxOmniStop(pTimer, fForDestroy);
1232	#endif
1233
1234	/*
1235	* Simple timer.
1236	*/
1237	ASMAtomicWriteBool(&pTimer->fSuspended, true);
1238	for (;;)
1239	{
1240	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[0].enmState);
1241	switch (enmState)
1242	{
1243	case RTTIMERLNXSTATE_ACTIVE:
1244	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPING, RTTIMERLNXSTATE_ACTIVE))
1245	{
1246	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[0], pTimer->fHighRes);
1247	return false;
1248	}
1249	break;
1250
1251	case RTTIMERLNXSTATE_CALLBACK:
1252	case RTTIMERLNXSTATE_CB_RESTARTING:
1253	case RTTIMERLNXSTATE_CB_STOPPING:
1254	Assert(enmState != RTTIMERLNXSTATE_CB_STOPPING \|\| fForDestroy);
1255	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState,
1256	!fForDestroy ? RTTIMERLNXSTATE_CB_STOPPING : RTTIMERLNXSTATE_CB_DESTROYING,
1257	enmState))
1258	return true;
1259	break;
1260
1261	case RTTIMERLNXSTATE_STOPPED:
1262	return VINF_SUCCESS;
1263
1264	case RTTIMERLNXSTATE_CB_DESTROYING:
1265	AssertMsgFailed(("enmState=%d pTimer=%p\n", enmState, pTimer));
1266	return true;
1267
1268	default:
1269	case RTTIMERLNXSTATE_STARTING:
1270	case RTTIMERLNXSTATE_MP_STARTING:
1271	case RTTIMERLNXSTATE_STOPPING:
1272	case RTTIMERLNXSTATE_MP_STOPPING:
1273	AssertMsgFailed(("enmState=%d pTimer=%p\n", enmState, pTimer));
1274	return false;
1275	}
1276
1277	/* State not stable, try again. */
1278	ASMNopPause();
1279	}
1280	}
1281
1282
1283	RTDECL(int) RTTimerStop(PRTTIMER pTimer)
1284	{
1285	/*
1286	* Validate.
1287	*/
1288	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
1289	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
1290	dprintf(("stop %p\n", pTimer));
1291
1292	if (ASMAtomicUoReadBool(&pTimer->fSuspended))
1293	return VERR_TIMER_SUSPENDED;
1294
1295	rtTimerLnxStop(pTimer, false /fForDestroy/);
1296	return VINF_SUCCESS;
1297	}
1298	RT_EXPORT_SYMBOL(RTTimerStop);
1299
1300
1301	RTDECL(int) RTTimerChangeInterval(PRTTIMER pTimer, uint64_t u64NanoInterval)
1302	{
1303	unsigned long cJiffies;
1304	unsigned long flFlags;
1305
1306	/*
1307	* Validate.
1308	*/
1309	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
1310	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
1311	AssertReturn(u64NanoInterval, VERR_INVALID_PARAMETER);
1312	dprintf(("change %p %llu\n", pTimer, u64NanoInterval));
1313
1314	#ifdef RTTIMER_LINUX_WITH_HRTIMER
1315	/*
1316	* For the high resolution timers it is easy since we don't care so much
1317	* about when it is applied to the sub-timers.
1318	*/
1319	if (pTimer->fHighRes)
1320	{
1321	ASMAtomicWriteU64(&pTimer->u64NanoInterval, u64NanoInterval);
1322	return VINF_SUCCESS;
1323	}
1324	#endif
1325
1326	/*
1327	* Standard timers have a bit more complicated way of calculating
1328	* their interval and such. So, forget omni timers for now.
1329	*/
1330	if (pTimer->cCpus > 1)
1331	return VERR_NOT_SUPPORTED;
1332
1333	cJiffies = u64NanoInterval / RTTimerGetSystemGranularity();
1334	if (cJiffies * RTTimerGetSystemGranularity() != u64NanoInterval)
1335	cJiffies = 0;
1336
1337	spin_lock_irqsave(&pTimer->ChgIntLock, flFlags);
1338	pTimer->aSubTimers[0].u.Std.fFirstAfterChg = true;
1339	pTimer->cJiffies = cJiffies;
1340	ASMAtomicWriteU64(&pTimer->u64NanoInterval, u64NanoInterval);
1341	spin_unlock_irqrestore(&pTimer->ChgIntLock, flFlags);
1342	return VINF_SUCCESS;
1343	}
1344	RT_EXPORT_SYMBOL(RTTimerChangeInterval);
1345
1346
1347	RTDECL(int) RTTimerDestroy(PRTTIMER pTimer)
1348	{
1349	bool fCanDestroy;
1350
1351	/*
1352	* Validate. It's ok to pass NULL pointer.
1353	*/
1354	if (pTimer == /NIL_RTTIMER/ NULL)
1355	return VINF_SUCCESS;
1356	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
1357	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
1358	dprintf(("destroy %p\n", pTimer));
1359
1360	/*
1361	* Stop the timer if it's still active, then destroy it if we can.
1362	*/
1363	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
1364	fCanDestroy = rtTimerLnxStop(pTimer, true /fForDestroy/);
1365	else
1366	{
1367	uint32_t iCpu = pTimer->cCpus;
1368	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1369	if (pTimer->cCpus > 1)
1370	RTSpinlockAcquireNoInts(pTimer->hSpinlock, &Tmp);
1371
1372	fCanDestroy = true;
1373	while (iCpu-- > 0)
1374	{
1375	for (;;)
1376	{
1377	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
1378	switch (enmState)
1379	{
1380	case RTTIMERLNXSTATE_CALLBACK:
1381	case RTTIMERLNXSTATE_CB_RESTARTING:
1382	case RTTIMERLNXSTATE_CB_STOPPING:
1383	if (!rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_CB_DESTROYING, enmState))
1384	continue;
1385	fCanDestroy = false;
1386	break;
1387
1388	case RTTIMERLNXSTATE_CB_DESTROYING:
1389	AssertMsgFailed(("%d\n", enmState));
1390	fCanDestroy = false;
1391	break;
1392	default:
1393	break;
1394	}
1395	break;
1396	}
1397	}
1398
1399	if (pTimer->cCpus > 1)
1400	RTSpinlockReleaseNoInts(pTimer->hSpinlock, &Tmp);
1401	}
1402
1403	if (fCanDestroy)
1404	rtTimerLnxDestroyIt(pTimer);
1405
1406	return VINF_SUCCESS;
1407	}
1408	RT_EXPORT_SYMBOL(RTTimerDestroy);
1409
1410
1411	RTDECL(int) RTTimerCreateEx(PRTTIMER ppTimer, uint64_t u64NanoInterval, uint32_t fFlags, PFNRTTIMER pfnTimer, void pvUser)
1412	{
1413	PRTTIMER pTimer;
1414	RTCPUID iCpu;
1415	unsigned cCpus;
1416	int rc;
1417
1418	*ppTimer = NULL;
1419
1420	/*
1421	* Validate flags.
1422	*/
1423	if (!RTTIMER_FLAGS_ARE_VALID(fFlags))
1424	return VERR_INVALID_PARAMETER;
1425	if ( (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC)
1426	&& (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL
1427	&& !RTMpIsCpuPossible(RTMpCpuIdFromSetIndex(fFlags & RTTIMER_FLAGS_CPU_MASK)))
1428	return VERR_CPU_NOT_FOUND;
1429
1430	/*
1431	* Allocate the timer handler.
1432	*/
1433	cCpus = 1;
1434	#ifdef CONFIG_SMP
1435	if ((fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL)
1436	{
1437	cCpus = RTMpGetMaxCpuId() + 1;
1438	Assert(cCpus <= RTCPUSET_MAX_CPUS); /* On linux we have a 1:1 relationship between cpuid and set index. */
1439	AssertReturn(u64NanoInterval, VERR_NOT_IMPLEMENTED); /* We don't implement single shot on all cpus, sorry. */
1440	}
1441	#endif
1442
1443	rc = RTMemAllocEx(RT_OFFSETOF(RTTIMER, aSubTimers[cCpus]), 0,
1444	RTMEMALLOCEX_FLAGS_ZEROED \| RTMEMALLOCEX_FLAGS_ANY_CTX_FREE, (void **)&pTimer);
1445	if (RT_FAILURE(rc))
1446	return rc;
1447
1448	/*
1449	* Initialize it.
1450	*/
1451	pTimer->u32Magic = RTTIMER_MAGIC;
1452	pTimer->hSpinlock = NIL_RTSPINLOCK;
1453	pTimer->fSuspended = true;
1454	pTimer->fHighRes = !!(fFlags & RTTIMER_FLAGS_HIGH_RES);
1455	#ifdef CONFIG_SMP
1456	pTimer->fSpecificCpu = (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC) && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL;
1457	pTimer->fAllCpus = (fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL;
1458	pTimer->idCpu = pTimer->fSpecificCpu
1459	? RTMpCpuIdFromSetIndex(fFlags & RTTIMER_FLAGS_CPU_MASK)
1460	: NIL_RTCPUID;
1461	#else
1462	pTimer->fSpecificCpu = !!(fFlags & RTTIMER_FLAGS_CPU_SPECIFIC);
1463	pTimer->idCpu = RTMpCpuId();
1464	#endif
1465	pTimer->cCpus = cCpus;
1466	pTimer->pfnTimer = pfnTimer;
1467	pTimer->pvUser = pvUser;
1468	pTimer->u64NanoInterval = u64NanoInterval;
1469	pTimer->cJiffies = u64NanoInterval / RTTimerGetSystemGranularity();
1470	if (pTimer->cJiffies * RTTimerGetSystemGranularity() != u64NanoInterval)
1471	pTimer->cJiffies = 0;
1472	spin_lock_init(&pTimer->ChgIntLock);
1473
1474	for (iCpu = 0; iCpu < cCpus; iCpu++)
1475	{
1476	#ifdef RTTIMER_LINUX_WITH_HRTIMER
1477	if (pTimer->fHighRes)
1478	{
1479	hrtimer_init(&pTimer->aSubTimers[iCpu].u.Hr.LnxTimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1480	pTimer->aSubTimers[iCpu].u.Hr.LnxTimer.function = rtTimerLinuxHrCallback;
1481	}
1482	else
1483	#endif
1484	{
1485	init_timer(&pTimer->aSubTimers[iCpu].u.Std.LnxTimer);
1486	pTimer->aSubTimers[iCpu].u.Std.LnxTimer.data = (unsigned long)&pTimer->aSubTimers[iCpu];
1487	pTimer->aSubTimers[iCpu].u.Std.LnxTimer.function = rtTimerLinuxStdCallback;
1488	pTimer->aSubTimers[iCpu].u.Std.LnxTimer.expires = jiffies;
1489	pTimer->aSubTimers[iCpu].u.Std.u64NextTS = 0;
1490	}
1491	pTimer->aSubTimers[iCpu].iTick = 0;
1492	pTimer->aSubTimers[iCpu].pParent = pTimer;
1493	pTimer->aSubTimers[iCpu].enmState = RTTIMERLNXSTATE_STOPPED;
1494	}
1495
1496	#ifdef CONFIG_SMP
1497	/*
1498	* If this is running on ALL cpus, we'll have to register a callback
1499	* for MP events (so timers can be started/stopped on cpus going
1500	* online/offline). We also create the spinlock for syncrhonizing
1501	* stop/start/mp-event.
1502	*/
1503	if (cCpus > 1)
1504	{
1505	int rc = RTSpinlockCreate(&pTimer->hSpinlock);
1506	if (RT_SUCCESS(rc))
1507	rc = RTMpNotificationRegister(rtTimerLinuxMpEvent, pTimer);
1508	else
1509	pTimer->hSpinlock = NIL_RTSPINLOCK;
1510	if (RT_FAILURE(rc))
1511	{
1512	RTTimerDestroy(pTimer);
1513	return rc;
1514	}
1515	}
1516	#endif /* CONFIG_SMP */
1517
1518	dprintf(("create %p\n", pTimer));
1519	*ppTimer = pTimer;
1520	return VINF_SUCCESS;
1521	}
1522	RT_EXPORT_SYMBOL(RTTimerCreateEx);
1523
1524
1525	RTDECL(uint32_t) RTTimerGetSystemGranularity(void)
1526	{
1527	#if 0 /** @todo Not sure if this is what we want or not... Add new API for
1528	* querying the resolution of the high res timers? */
1529	struct timespec Ts;
1530	int rc = hrtimer_get_res(CLOCK_MONOTONIC, &Ts);
1531	if (!rc)
1532	{
1533	Assert(!Ts.tv_sec);
1534	return Ts.tv_nsec;
1535	}
1536	#endif
1537	return 1000000000 / HZ; /* ns */
1538	}
1539	RT_EXPORT_SYMBOL(RTTimerGetSystemGranularity);
1540
1541
1542	RTDECL(int) RTTimerRequestSystemGranularity(uint32_t u32Request, uint32_t *pu32Granted)
1543	{
1544	return VERR_NOT_SUPPORTED;
1545	}
1546	RT_EXPORT_SYMBOL(RTTimerRequestSystemGranularity);
1547
1548
1549	RTDECL(int) RTTimerReleaseSystemGranularity(uint32_t u32Granted)
1550	{
1551	return VERR_NOT_SUPPORTED;
1552	}
1553	RT_EXPORT_SYMBOL(RTTimerReleaseSystemGranularity);
1554
1555
1556	RTDECL(bool) RTTimerCanDoHighResolution(void)
1557	{
1558	#ifdef RTTIMER_LINUX_WITH_HRTIMER
1559	return true;
1560	#else
1561	return false;
1562	#endif
1563	}
1564	RT_EXPORT_SYMBOL(RTTimerCanDoHighResolution);
1565

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/timer-r0drv-linux.c@ 32737

Download in other formats: