timer-r0drv-linux.c@ 32711

Last change on this file since 32711 was 32711, checked in by vboxsync, 14 years ago
timer-r0drv-linux.c: incorrect check for cpu migration.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 51.5 KB

Line
1	/* $Id: timer-r0drv-linux.c 32711 2010-09-23 12:12:54Z vboxsync $ */
2	/** @file
3	* IPRT - Timers, Ring-0 Driver, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2010 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*******************************************************************************
29	* Header Files *
30	*******************************************************************************/
31	#include "the-linux-kernel.h"
32	#include "internal/iprt.h"
33
34	#include <iprt/timer.h>
35	#include <iprt/time.h>
36	#include <iprt/mp.h>
37	#include <iprt/cpuset.h>
38	#include <iprt/spinlock.h>
39	#include <iprt/err.h>
40	#include <iprt/asm.h>
41	#include <iprt/assert.h>
42	#include <iprt/alloc.h>
43
44	#include "internal/magics.h"
45
46	/** @def RTTIMER_LINUX_HAVE_HRTIMER
47	* Whether the kernel support high resolution timers (Linux kernel versions
48	* 2.6.28 and later (hrtimer_add_expires_ns()). */
49	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
50	# define RTTIMER_LINUX_HAVE_HRTIMER
51	#endif
52
53	/** @def RTTIMER_LINUX_WITH_HRTIMER
54	* Whether to use high resolution timers. */
55	#if !defined(RTTIMER_LINUX_WITH_HRTIMER) \
56	&& defined(RTTIMER_LINUX_HAVE_HRTIMER)
57	# define RTTIMER_LINUX_WITH_HRTIMER
58	#endif
59
60	#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31)
61	# define mod_timer_pinned mod_timer
62	# define HRTIMER_MODE_ABS_PINNED HRTIMER_MODE_ABS
63	#endif
64
65
66	/*******************************************************************************
67	* Structures and Typedefs *
68	*******************************************************************************/
69	/**
70	* Timer state machine.
71	*
72	* This is used to try handle the issues with MP events and
73	* timers that runs on all CPUs. It's relatively nasty :-/
74	*/
75	typedef enum RTTIMERLNXSTATE
76	{
77	/** Stopped. */
78	RTTIMERLNXSTATE_STOPPED = 0,
79	/** Transient state; next ACTIVE. */
80	RTTIMERLNXSTATE_STARTING,
81	/** Transient state; next ACTIVE. (not really necessary) */
82	RTTIMERLNXSTATE_MP_STARTING,
83	/** Active. */
84	RTTIMERLNXSTATE_ACTIVE,
85	/** Active and in callback; next ACTIVE, STOPPED or CALLBACK_DESTROYING. */
86	RTTIMERLNXSTATE_CALLBACK,
87	/** Stopped while in the callback; next STOPPED. */
88	RTTIMERLNXSTATE_CB_STOPPING,
89	/** Restarted while in the callback; next ACTIVE, STOPPED, DESTROYING. */
90	RTTIMERLNXSTATE_CB_RESTARTING,
91	/** The callback shall destroy the timer; next STOPPED. */
92	RTTIMERLNXSTATE_CB_DESTROYING,
93	/** Transient state; next STOPPED. */
94	RTTIMERLNXSTATE_STOPPING,
95	/** Transient state; next STOPPED. */
96	RTTIMERLNXSTATE_MP_STOPPING,
97	/** The usual 32-bit hack. */
98	RTTIMERLNXSTATE_32BIT_HACK = 0x7fffffff
99	} RTTIMERLNXSTATE;
100
101
102	/**
103	* A Linux sub-timer.
104	*/
105	typedef struct RTTIMERLNXSUBTIMER
106	{
107	/** Timer specific data. */
108	union
109	{
110	#if defined(RTTIMER_LINUX_WITH_HRTIMER)
111	/** High resolution timer. */
112	struct
113	{
114	/** The linux timer structure. */
115	struct hrtimer LnxTimer;
116	} Hr;
117	#endif
118	/** Standard timer. */
119	struct
120	{
121	/** The linux timer structure. */
122	struct timer_list LnxTimer;
123	/** The start of the current run (ns).
124	* This is used to calculate when the timer ought to fire the next time. */
125	uint64_t u64StartTS;
126	/** The start of the current run (ns).
127	* This is used to calculate when the timer ought to fire the next time. */
128	uint64_t u64NextTS;
129	/** The u64NextTS in jiffies. */
130	unsigned long ulNextJiffies;
131	} Std;
132	} u;
133	/** The current tick number (since u.Std.u64StartTS). */
134	uint64_t iTick;
135	/** Restart the single shot timer at this specific time.
136	* Used when a single shot timer is restarted from the callback. */
137	uint64_t volatile uNsRestartAt;
138	/** Pointer to the parent timer. */
139	PRTTIMER pParent;
140	/** The current sub-timer state. */
141	RTTIMERLNXSTATE volatile enmState;
142	} RTTIMERLNXSUBTIMER;
143	/** Pointer to a linux sub-timer. */
144	typedef RTTIMERLNXSUBTIMER *PRTTIMERLNXSUBTIMER;
145
146
147	/**
148	* The internal representation of an Linux timer handle.
149	*/
150	typedef struct RTTIMER
151	{
152	/** Magic.
153	* This is RTTIMER_MAGIC, but changes to something else before the timer
154	* is destroyed to indicate clearly that thread should exit. */
155	uint32_t volatile u32Magic;
156	/** Spinlock synchronizing the fSuspended and MP event handling.
157	* This is NIL_RTSPINLOCK if cCpus == 1. */
158	RTSPINLOCK hSpinlock;
159	/** Flag indicating that the timer is suspended. */
160	bool volatile fSuspended;
161	/** Whether the timer must run on one specific CPU or not. */
162	bool fSpecificCpu;
163	#ifdef CONFIG_SMP
164	/** Whether the timer must run on all CPUs or not. */
165	bool fAllCpus;
166	#endif /* else: All -> specific on non-SMP kernels */
167	/** Whether it is a high resolution timer or a standard one. */
168	bool fHighRes;
169	/** The id of the CPU it must run on if fSpecificCpu is set. */
170	RTCPUID idCpu;
171	/** The number of CPUs this timer should run on. */
172	RTCPUID cCpus;
173	/** Callback. */
174	PFNRTTIMER pfnTimer;
175	/** User argument. */
176	void *pvUser;
177	/** The timer interval. 0 if one-shot. */
178	uint64_t volatile u64NanoInterval;
179	/** This is set to the number of jiffies between ticks if the interval is
180	* an exact number of jiffies. (Standard timers only.) */
181	unsigned long cJiffies;
182	/** Sub-timers.
183	* Normally there is just one, but for RTTIMER_FLAGS_CPU_ALL this will contain
184	* an entry for all possible cpus. In that case the index will be the same as
185	* for the RTCpuSet. */
186	RTTIMERLNXSUBTIMER aSubTimers[1];
187	} RTTIMER;
188
189
190	/**
191	* A rtTimerLinuxStartOnCpu and rtTimerLinuxStartOnCpu argument package.
192	*/
193	typedef struct RTTIMERLINUXSTARTONCPUARGS
194	{
195	/** The current time (RTTimeNanoTS). */
196	uint64_t u64Now;
197	/** When to start firing (delta). */
198	uint64_t u64First;
199	} RTTIMERLINUXSTARTONCPUARGS;
200	/** Pointer to a rtTimerLinuxStartOnCpu argument package. */
201	typedef RTTIMERLINUXSTARTONCPUARGS *PRTTIMERLINUXSTARTONCPUARGS;
202
203
204	/*******************************************************************************
205	* Internal Functions *
206	*******************************************************************************/
207	#ifdef CONFIG_SMP
208	static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser);
209	#endif
210
211	#if 0
212	#define DEBUG_HACKING
213	#include <iprt/string.h>
214	#include <iprt/asm-amd64-x86.h>
215	static void myLogBackdoorPrintf(const char *pszFormat, ...)
216	{
217	char szTmp[256];
218	va_list args;
219	size_t cb;
220
221	cb = RTStrPrintf(szTmp, sizeof(szTmp) - 10, "%d: ", RTMpCpuId());
222	va_start(args, pszFormat);
223	cb += RTStrPrintfV(&szTmp[cb], sizeof(szTmp) - cb, pszFormat, args);
224	va_end(args);
225
226	ASMOutStrU8(0x504, (uint8_t *)&szTmp[0], cb);
227	}
228	#define RTAssertMsg1Weak(pszExpr, uLine, pszFile, pszFunction) \
229	myLogBackdoorPrintf("\n!!Guest Assertion failed!!\n%s(%d) %s\n%s\n", uLine, pszFile, pszFunction, (pszExpr))
230	#define RTAssertMsg2Weak myLogBackdoorPrintf
231	#define dprintf(a) myLogBackdoorPrintf a
232	#else
233	#define dprintf(a) do { } while (0)
234	#endif
235
236	/**
237	* Sets the state.
238	*/
239	DECLINLINE(void) rtTimerLnxSetState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState)
240	{
241	#ifdef DEBUG_HACKING
242	dprintf(("set %d -> %d\n", *penmState, enmNewState));
243	#endif
244	ASMAtomicWriteU32((uint32_t volatile *)penmState, enmNewState);
245	}
246
247
248	/**
249	* Sets the state if it has a certain value.
250	*
251	* @return true if xchg was done.
252	* @return false if xchg wasn't done.
253	*/
254	#ifdef DEBUG_HACKING
255	#define rtTimerLnxCmpXchgState(penmState, enmNewState, enmCurState) rtTimerLnxCmpXchgStateDebug(penmState, enmNewState, enmCurState, __LINE__)
256	static bool rtTimerLnxCmpXchgStateDebug(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState,
257	RTTIMERLNXSTATE enmCurState, uint32_t uLine)
258	{
259	RTTIMERLNXSTATE enmOldState = enmCurState;
260	bool fRc = ASMAtomicCmpXchgExU32((uint32_t volatile )penmState, enmNewState, enmCurState, (uint32_t )&enmOldState);
261	dprintf(("cxg %d -> %d - %d at %u\n", enmOldState, enmNewState, fRc, uLine));
262	return fRc;
263	}
264	#else
265	DECLINLINE(bool) rtTimerLnxCmpXchgState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState,
266	RTTIMERLNXSTATE enmCurState)
267	{
268	return ASMAtomicCmpXchgU32((uint32_t volatile *)penmState, enmNewState, enmCurState);
269	}
270	#endif
271
272
273	/**
274	* Gets the state.
275	*/
276	DECLINLINE(RTTIMERLNXSTATE) rtTimerLnxGetState(RTTIMERLNXSTATE volatile *penmState)
277	{
278	return (RTTIMERLNXSTATE)ASMAtomicUoReadU32((uint32_t volatile *)penmState);
279	}
280
281	#ifdef RTTIMER_LINUX_WITH_HRTIMER
282
283	/**
284	* Converts a nano second time stamp to ktime_t.
285	*
286	* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts().
287	*
288	* @returns ktime_t.
289	* @param cNanoSecs Nanoseconds.
290	*/
291	DECLINLINE(ktime_t) rtTimerLnxNanoToKt(uint64_t cNanoSecs)
292	{
293	/* With some luck the compiler optimizes the division out of this... (Bet it doesn't.) */
294	return ktime_set(cNanoSecs / 1000000000, cNanoSecs % 1000000000);
295	}
296
297	/**
298	* Converts ktime_t to a nano second time stamp.
299	*
300	* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts().
301	*
302	* @returns nano second time stamp.
303	* @param Kt ktime_t.
304	*/
305	DECLINLINE(uint64_t) rtTimerLnxKtToNano(ktime_t Kt)
306	{
307	return ktime_to_ns(Kt);
308	}
309
310	#endif /* RTTIMER_LINUX_WITH_HRTIMER */
311
312	/**
313	* Converts a nano second interval to jiffies.
314	*
315	* @returns Jiffies.
316	* @param cNanoSecs Nanoseconds.
317	*/
318	DECLINLINE(unsigned long) rtTimerLnxNanoToJiffies(uint64_t cNanoSecs)
319	{
320	/* this can be made even better... */
321	if (cNanoSecs > (uint64_t)TICK_NSEC * MAX_JIFFY_OFFSET)
322	return MAX_JIFFY_OFFSET;
323	# if ARCH_BITS == 32
324	if (RT_LIKELY(cNanoSecs <= UINT32_MAX))
325	return ((uint32_t)cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
326	# endif
327	return (cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
328	}
329
330
331	/**
332	* Starts a sub-timer (RTTimerStart).
333	*
334	* @param pSubTimer The sub-timer to start.
335	* @param u64Now The current timestamp (RTTimeNanoTS()).
336	* @param u64First The interval from u64Now to the first time the timer should fire.
337	* @param fPinned true = timer pinned to a specific CPU,
338	* false = timer can migrate between CPUs
339	* @param fHighRes Whether the user requested a high resolution timer or not.
340	* @param enmOldState The old timer state.
341	*/
342	static void rtTimerLnxStartSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, uint64_t u64Now, uint64_t u64First,
343	bool fPinned, bool fHighRes)
344	{
345	/*
346	* Calc when it should start firing.
347	*/
348	uint64_t u64NextTS = u64Now + u64First;
349	if (fHighRes)
350	{
351	pSubTimer->u.Std.u64StartTS = u64NextTS;
352	pSubTimer->u.Std.u64NextTS = u64NextTS;
353	}
354	dprintf(("startsubtimer %p\n", pSubTimer->pParent));
355
356	pSubTimer->iTick = 0;
357
358	#ifdef RTTIMER_LINUX_WITH_HRTIMER
359	if (fHighRes)
360	hrtimer_start(&pSubTimer->u.Hr.LnxTimer, rtTimerLnxNanoToKt(u64NextTS),
361	fPinned ? HRTIMER_MODE_ABS_PINNED : HRTIMER_MODE_ABS);
362	else
363	#endif
364	{
365	unsigned long cJiffies = !u64First ? 0 : rtTimerLnxNanoToJiffies(u64First);
366	pSubTimer->u.Std.ulNextJiffies = jiffies + cJiffies;
367	#ifdef CONFIG_SMP
368	if (fPinned)
369	mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
370	else
371	#endif
372	mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
373	}
374
375	/* Be a bit careful here since we could be racing the callback. */
376	if (!rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_STARTING))
377	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_MP_STARTING);
378	}
379
380
381	/**
382	* Stops a sub-timer (RTTimerStart and rtTimerLinuxMpEvent()).
383	*
384	* The caller has already changed the state, so we will not be in a callback
385	* situation wrt to the calling thread.
386	*
387	* @param pSubTimer The sub-timer.
388	* @param fHighRes Whether the user requested a high resolution timer or not.
389	*/
390	static void rtTimerLnxStopSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, bool fHighRes)
391	{
392	dprintf(("stopsubtimer %p %d\n", pSubTimer->pParent, fHighRes));
393	#ifdef RTTIMER_LINUX_WITH_HRTIMER
394	if (fHighRes)
395	hrtimer_cancel(&pSubTimer->u.Hr.LnxTimer);
396	else
397	#endif
398	{
399	if (timer_pending(&pSubTimer->u.Std.LnxTimer))
400	del_timer_sync(&pSubTimer->u.Std.LnxTimer);
401	}
402
403	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
404	}
405
406
407	/**
408	* Used by RTTimerDestroy and rtTimerLnxCallbackDestroy to do the actual work.
409	*
410	* @param pTimer The timer in question.
411	*/
412	static void rtTimerLnxDestroyIt(PRTTIMER pTimer)
413	{
414	RTSPINLOCK hSpinlock = pTimer->hSpinlock;
415	Assert(pTimer->fSuspended);
416	dprintf(("destroyit %p\n", pTimer));
417
418	/*
419	* Remove the MP notifications first because it'll reduce the risk of
420	* us overtaking any MP event that might theoretically be racing us here.
421	*/
422	#ifdef CONFIG_SMP
423	if ( pTimer->cCpus > 1
424	&& hSpinlock != NIL_RTSPINLOCK)
425	{
426	int rc = RTMpNotificationDeregister(rtTimerLinuxMpEvent, pTimer);
427	AssertRC(rc);
428	}
429	#endif /* CONFIG_SMP */
430
431	/*
432	* Uninitialize the structure and free the associated resources.
433	* The spinlock goes last.
434	*/
435	ASMAtomicWriteU32(&pTimer->u32Magic, ~RTTIMER_MAGIC);
436	RTMemFreeEx(pTimer, RT_OFFSETOF(RTTIMER, aSubTimers[pTimer->cCpus]));
437	if (hSpinlock != NIL_RTSPINLOCK)
438	RTSpinlockDestroy(hSpinlock);
439	}
440
441
442	/**
443	* Called when the timer was destroyed by the callback function.
444	*
445	* @param pTimer The timer.
446	* @param pSubTimer The sub-timer which we're handling, the state of this
447	* will be RTTIMERLNXSTATE_CALLBACK_DESTROYING.
448	*/
449	static void rtTimerLnxCallbackDestroy(PRTTIMER pTimer, PRTTIMERLNXSUBTIMER pSubTimer)
450	{
451	/*
452	* If it's an omni timer, the last dude does the destroying.
453	*/
454	if (pTimer->cCpus > 1)
455	{
456	bool fAllStopped = true;
457	uint32_t iCpu = pTimer->cCpus;
458	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
459	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
460
461	Assert(pSubTimer->enmState == RTTIMERLNXSTATE_CB_DESTROYING);
462	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
463
464	while (iCpu-- > 0)
465	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) != RTTIMERLNXSTATE_STOPPED)
466	{
467	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
468	return;
469	}
470
471	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
472	}
473
474	rtTimerLnxDestroyIt(pTimer);
475	}
476
477
478	#ifdef CONFIG_SMP
479	/**
480	* Deal with a sub-timer that has migrated.
481	*
482	* @param pTimer The timer.
483	* @param pSubTimer The sub-timer.
484	*/
485	static void rtTimerLnxCallbackHandleMigration(PRTTIMER pTimer, PRTTIMERLNXSUBTIMER pSubTimer)
486	{
487	RTTIMERLNXSTATE enmState;
488	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
489	if (pTimer->cCpus > 1)
490	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
491
492	do
493	{
494	enmState = rtTimerLnxGetState(&pSubTimer->enmState);
495	switch (enmState)
496	{
497	case RTTIMERLNXSTATE_STOPPING:
498	case RTTIMERLNXSTATE_MP_STOPPING:
499	enmState = RTTIMERLNXSTATE_STOPPED;
500	case RTTIMERLNXSTATE_STOPPED:
501	break;
502
503	default:
504	AssertMsgFailed(("%d\n", enmState));
505	case RTTIMERLNXSTATE_STARTING:
506	case RTTIMERLNXSTATE_MP_STARTING:
507	case RTTIMERLNXSTATE_ACTIVE:
508	case RTTIMERLNXSTATE_CALLBACK:
509	case RTTIMERLNXSTATE_CB_STOPPING:
510	case RTTIMERLNXSTATE_CB_RESTARTING:
511	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, enmState))
512	enmState = RTTIMERLNXSTATE_STOPPED;
513	break;
514
515	case RTTIMERLNXSTATE_CB_DESTROYING:
516	{
517	if (pTimer->cCpus > 1)
518	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
519
520	rtTimerLnxCallbackDestroy(pTimer, pSubTimer);
521	return;
522	}
523	}
524	} while (enmState != RTTIMERLNXSTATE_STOPPED);
525
526	if (pTimer->cCpus > 1)
527	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
528	}
529	#endif /* CONFIG_SMP */
530
531
532	/**
533	* The slow path of rtTimerLnxChangeToCallbackState.
534	*
535	* @returns true if changed successfully, false if not.
536	* @param pSubTimer The sub-timer.
537	*/
538	static bool rtTimerLnxChangeToCallbackStateSlow(PRTTIMERLNXSUBTIMER pSubTimer)
539	{
540	for (;;)
541	{
542	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState);
543	switch (enmState)
544	{
545	case RTTIMERLNXSTATE_ACTIVE:
546	case RTTIMERLNXSTATE_STARTING:
547	case RTTIMERLNXSTATE_MP_STARTING:
548	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CALLBACK, enmState))
549	return true;
550	break;
551
552	case RTTIMERLNXSTATE_CALLBACK:
553	case RTTIMERLNXSTATE_CB_STOPPING:
554	case RTTIMERLNXSTATE_CB_RESTARTING:
555	case RTTIMERLNXSTATE_CB_DESTROYING:
556	AssertMsgFailed(("%d\n", enmState));
557	default:
558	return false;
559	}
560	ASMNopPause();
561	}
562	}
563
564
565	/**
566	* Tries to change the sub-timer state to 'callback'.
567	*
568	* @returns true if changed successfully, false if not.
569	* @param pSubTimer The sub-timer.
570	*/
571	DECLINLINE(bool) rtTimerLnxChangeToCallbackState(PRTTIMERLNXSUBTIMER pSubTimer)
572	{
573	if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CALLBACK, RTTIMERLNXSTATE_ACTIVE)))
574	return true;
575	return rtTimerLnxChangeToCallbackStateSlow(pSubTimer);
576	}
577
578
579	#ifdef RTTIMER_LINUX_WITH_HRTIMER
580	/**
581	* Timer callback function for high resolution timers.
582	*
583	* @returns HRTIMER_NORESTART or HRTIMER_RESTART depending on whether it's a
584	* one-shot or interval timer.
585	* @param pHrTimer Pointer to the sub-timer structure.
586	*/
587	static enum hrtimer_restart rtTimerLinuxHrCallback(struct hrtimer *pHrTimer)
588	{
589	PRTTIMERLNXSUBTIMER pSubTimer = RT_FROM_MEMBER(pHrTimer, RTTIMERLNXSUBTIMER, u.Hr.LnxTimer);
590	PRTTIMER pTimer = pSubTimer->pParent;
591
592
593	dprintf(("hrcallback %p\n", pTimer));
594	if (RT_UNLIKELY(!rtTimerLnxChangeToCallbackState(pSubTimer)))
595	return HRTIMER_NORESTART;
596
597	#ifdef CONFIG_SMP
598	/*
599	* Check for unwanted migration.
600	*/
601	if ( (pTimer->fSpecificCpu \|\| pTimer->fAllCpus)
602	&& RT_UNLIKELY((RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != RTMpCpuId()))
603	{
604	rtTimerLnxCallbackHandleMigration(pTimer, pSubTimer);
605	return HRTIMER_NORESTART;
606	}
607	#endif
608
609	if (pTimer->u64NanoInterval)
610	{
611	/*
612	* Periodic timer, run it and update the native timer afterwards so
613	* we can handle RTTimerStop and RTTimerChangeInterval from the
614	* callback as well as a racing control thread.
615	*/
616	pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
617	hrtimer_add_expires_ns(&pSubTimer->u.Hr.LnxTimer, ASMAtomicReadU64(&pTimer->u64NanoInterval));
618	if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CALLBACK)))
619	return HRTIMER_RESTART;
620	}
621	else
622	{
623	/*
624	* One shot timer (no omni), stop it before dispatching it.
625	* Allow RTTimerStart as well as RTTimerDestroy to be called from
626	* the callback.
627	*/
628	ASMAtomicWriteBool(&pTimer->fSuspended, true);
629	pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
630	if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CALLBACK)))
631	return HRTIMER_NORESTART;
632	}
633
634	/*
635	* Some state change occured while we were in the callback routine.
636	*/
637	for (;;)
638	{
639	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState);
640	switch (enmState)
641	{
642	case RTTIMERLNXSTATE_CB_DESTROYING:
643	rtTimerLnxCallbackDestroy(pTimer, pSubTimer);
644	return HRTIMER_NORESTART;
645
646	case RTTIMERLNXSTATE_CB_STOPPING:
647	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CB_STOPPING))
648	return HRTIMER_NORESTART;
649	break;
650
651	case RTTIMERLNXSTATE_CB_RESTARTING:
652	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CB_RESTARTING))
653	{
654	pSubTimer->iTick = 0;
655	hrtimer_set_expires(&pSubTimer->u.Hr.LnxTimer, rtTimerLnxNanoToKt(pSubTimer->uNsRestartAt));
656	return HRTIMER_RESTART;
657	}
658	break;
659
660	default:
661	AssertMsgFailed(("%d\n", enmState));
662	return HRTIMER_NORESTART;
663	}
664	ASMNopPause();
665	}
666	}
667	#endif /* RTTIMER_LINUX_WITH_HRTIMER */
668
669
670	/**
671	* Timer callback function for standard timers.
672	*
673	* @param ulUser Address of the sub-timer structure.
674	*/
675	static void rtTimerLinuxStdCallback(unsigned long ulUser)
676	{
677	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)ulUser;
678	PRTTIMER pTimer = pSubTimer->pParent;
679
680	dprintf(("hrcallback %p\n", pTimer));
681	if (RT_UNLIKELY(!rtTimerLnxChangeToCallbackState(pSubTimer)))
682	return;
683
684	#ifdef CONFIG_SMP
685	/*
686	* Check for unwanted migration.
687	*/
688	if ( (pTimer->fSpecificCpu \|\| pTimer->fAllCpus)
689	&& RT_UNLIKELY((RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != RTMpCpuId()))
690	{
691	rtTimerLnxCallbackHandleMigration(pTimer, pSubTimer);
692	return;
693	}
694	#endif
695
696	if (pTimer->u64NanoInterval)
697	{
698	/*
699	* Interval timer, calculate the next timeout and re-arm it.
700	*
701	* The first time around, we'll re-adjust the u.Std.u64StartTS to
702	* try prevent some jittering if we were started at a bad time.
703	*/
704	const uint64_t u64NanoInterval = pTimer->u64NanoInterval;
705	const uint64_t iTick = ++pSubTimer->iTick;
706	const uint64_t u64NanoTS = RTTimeNanoTS();
707
708	if (RT_UNLIKELY(iTick == 1))
709	{
710	pSubTimer->u.Std.u64StartTS = u64NanoTS;
711	pSubTimer->u.Std.u64NextTS = u64NanoTS;
712	pSubTimer->u.Std.ulNextJiffies = jiffies;
713	}
714
715	pSubTimer->u.Std.u64NextTS += u64NanoInterval;
716	if (pTimer->cJiffies)
717	{
718	pSubTimer->u.Std.ulNextJiffies += pTimer->cJiffies;
719	/* Prevent overflows when the jiffies counter wraps around.
720	* Special thanks to Ken Preslan for helping debugging! */
721	while (time_before(pSubTimer->u.Std.ulNextJiffies, jiffies))
722	{
723	pSubTimer->u.Std.ulNextJiffies += pTimer->cJiffies;
724	pSubTimer->u.Std.u64NextTS += u64NanoInterval;
725	}
726	}
727	else
728	{
729	while (pSubTimer->u.Std.u64NextTS < u64NanoTS)
730	pSubTimer->u.Std.u64NextTS += u64NanoInterval;
731	pSubTimer->u.Std.ulNextJiffies = jiffies + rtTimerLnxNanoToJiffies(pSubTimer->u.Std.u64NextTS - u64NanoTS);
732	}
733
734	#ifdef CONFIG_SMP
735	if (pTimer->fSpecificCpu \|\| pTimer->fAllCpus)
736	mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
737	else
738	#endif
739	mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
740
741	/*
742	* Run the timer.
743	*/
744	pTimer->pfnTimer(pTimer, pTimer->pvUser, iTick);
745	if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CALLBACK)))
746	return;
747	}
748	else
749	{
750	/*
751	* One shot timer, stop it before dispatching it.
752	* Allow RTTimerStart as well as RTTimerDestroy to be called from
753	* the callback.
754	*/
755	ASMAtomicWriteBool(&pTimer->fSuspended, true);
756	pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
757	if (RT_LIKELY(rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CALLBACK)))
758	return;
759	}
760
761	/*
762	* Some state change occured while we were in the callback routine.
763	*/
764	if (pTimer->u64NanoInterval)
765	del_timer_sync(&pSubTimer->u.Std.LnxTimer);
766	for (;;)
767	{
768	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pSubTimer->enmState);
769	switch (enmState)
770	{
771	case RTTIMERLNXSTATE_CB_DESTROYING:
772	rtTimerLnxCallbackDestroy(pTimer, pSubTimer);
773	return;
774
775	case RTTIMERLNXSTATE_CB_STOPPING:
776	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_CB_STOPPING))
777	return;
778	break;
779
780	case RTTIMERLNXSTATE_CB_RESTARTING:
781	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE, RTTIMERLNXSTATE_CB_RESTARTING))
782	{
783	const uint64_t u64NanoTS = RTTimeNanoTS();
784	const uint64_t u64NextTS = pSubTimer->uNsRestartAt;
785	if (pTimer->fHighRes)
786	{
787	pSubTimer->u.Std.u64StartTS = u64NextTS;
788	pSubTimer->u.Std.u64NextTS = u64NextTS;
789	}
790	pSubTimer->iTick = 0;
791	pSubTimer->u.Std.ulNextJiffies = u64NextTS > u64NanoTS
792	? jiffies + rtTimerLnxNanoToJiffies(u64NextTS - u64NanoTS)
793	: jiffies;
794	#ifdef CONFIG_SMP
795	if (pTimer->fSpecificCpu \|\| pTimer->fAllCpus)
796	mod_timer_pinned(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
797	else
798	#endif
799	mod_timer(&pSubTimer->u.Std.LnxTimer, pSubTimer->u.Std.ulNextJiffies);
800	return;
801	}
802	break;
803
804	default:
805	AssertMsgFailed(("%d\n", enmState));
806	return;
807	}
808	ASMNopPause();
809	}
810	}
811
812
813	#ifdef CONFIG_SMP
814
815	/**
816	* Per-cpu callback function (RTMpOnAll/RTMpOnSpecific).
817	*
818	* @param idCpu The current CPU.
819	* @param pvUser1 Pointer to the timer.
820	* @param pvUser2 Pointer to the argument structure.
821	*/
822	static DECLCALLBACK(void) rtTimerLnxStartAllOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
823	{
824	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
825	PRTTIMER pTimer = (PRTTIMER)pvUser1;
826	Assert(idCpu < pTimer->cCpus);
827	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[idCpu], pArgs->u64Now, pArgs->u64First, true /fPinned/, pTimer->fHighRes);
828	}
829
830
831	/**
832	* Worker for RTTimerStart() that takes care of the ugly bits.
833	*
834	* @returns RTTimerStart() return value.
835	* @param pTimer The timer.
836	* @param pArgs The argument structure.
837	*/
838	static int rtTimerLnxOmniStart(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs)
839	{
840	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
841	RTCPUID iCpu;
842	RTCPUSET OnlineSet;
843	RTCPUSET OnlineSet2;
844	int rc2;
845
846	/*
847	* Prepare all the sub-timers for the startup and then flag the timer
848	* as a whole as non-suspended, make sure we get them all before
849	* clearing fSuspended as the MP handler will be waiting on this
850	* should something happen while we're looping.
851	*/
852	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
853
854	/* Just make it a omni timer restriction that no stop/start races are allowed. */
855	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
856	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) != RTTIMERLNXSTATE_STOPPED)
857	{
858	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
859	return VERR_TIMER_BUSY;
860	}
861
862	do
863	{
864	RTMpGetOnlineSet(&OnlineSet);
865	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
866	{
867	Assert(pTimer->aSubTimers[iCpu].enmState != RTTIMERLNXSTATE_MP_STOPPING);
868	rtTimerLnxSetState(&pTimer->aSubTimers[iCpu].enmState,
869	RTCpuSetIsMember(&OnlineSet, iCpu)
870	? RTTIMERLNXSTATE_STARTING
871	: RTTIMERLNXSTATE_STOPPED);
872	}
873	} while (!RTCpuSetIsEqual(&OnlineSet, RTMpGetOnlineSet(&OnlineSet2)));
874
875	ASMAtomicWriteBool(&pTimer->fSuspended, false);
876
877	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
878
879	/*
880	* Start them (can't find any exported function that allows me to
881	* do this without the cross calls).
882	*/
883	pArgs->u64Now = RTTimeNanoTS();
884	rc2 = RTMpOnAll(rtTimerLnxStartAllOnCpu, pTimer, pArgs);
885	AssertRC(rc2); /* screw this if it fails. */
886
887	/*
888	* Reset the sub-timers who didn't start up (ALL CPUs case).
889	*/
890	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
891
892	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
893	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_STARTING))
894	{
895	/** @todo very odd case for a rainy day. Cpus that temporarily went offline while
896	* we were between calls needs to nudged as the MP handler will ignore events for
897	* them because of the STARTING state. This is an extremely unlikely case - not that
898	* that means anything in my experience... ;-) */
899	}
900
901	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
902
903	return VINF_SUCCESS;
904	}
905
906
907	/**
908	* Worker for RTTimerStop() that takes care of the ugly SMP bits.
909	*
910	* @returns true if there was any active callbacks, false if not.
911	* @param pTimer The timer (valid).
912	* @param fForDestroy Whether this is for RTTimerDestroy or not.
913	*/
914	static bool rtTimerLnxOmniStop(PRTTIMER pTimer, bool fForDestroy)
915	{
916	bool fActiveCallbacks = false;
917	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
918	RTCPUID iCpu;
919
920
921	/*
922	* Mark the timer as suspended and flag all timers as stopping, except
923	* for those being stopped by an MP event.
924	*/
925	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
926
927	ASMAtomicWriteBool(&pTimer->fSuspended, true);
928	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
929	{
930	RTTIMERLNXSTATE enmState;
931	for (;;)
932	{
933	enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
934	if ( enmState == RTTIMERLNXSTATE_STOPPED
935	\|\| enmState == RTTIMERLNXSTATE_MP_STOPPING)
936	break;
937	if ( enmState == RTTIMERLNXSTATE_CALLBACK
938	\|\| enmState == RTTIMERLNXSTATE_CB_STOPPING
939	\|\| enmState == RTTIMERLNXSTATE_CB_RESTARTING)
940	{
941	Assert(enmState != RTTIMERLNXSTATE_CB_STOPPING \|\| fForDestroy);
942	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState,
943	!fForDestroy ? RTTIMERLNXSTATE_CB_STOPPING : RTTIMERLNXSTATE_CB_DESTROYING,
944	enmState))
945	{
946	fActiveCallbacks = true;
947	break;
948	}
949	}
950	else
951	{
952	Assert(enmState == RTTIMERLNXSTATE_ACTIVE);
953	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPING, enmState))
954	break;
955	}
956	ASMNopPause();
957	}
958	}
959
960	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
961
962	/*
963	* Do the actual stopping. Fortunately, this doesn't require any IPIs.
964	* Unfortunately it cannot be done synchronously from within the spinlock,
965	* because we might end up in an active waiting for a handler to complete.
966	*/
967	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
968	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) == RTTIMERLNXSTATE_STOPPING)
969	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[iCpu], pTimer->fHighRes);
970
971	return fActiveCallbacks;
972	}
973
974
975	/**
976	* Per-cpu callback function (RTMpOnSpecific) used by rtTimerLinuxMpEvent()
977	* to start a sub-timer on a cpu that just have come online.
978	*
979	* @param idCpu The current CPU.
980	* @param pvUser1 Pointer to the timer.
981	* @param pvUser2 Pointer to the argument structure.
982	*/
983	static DECLCALLBACK(void) rtTimerLinuxMpStartOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
984	{
985	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
986	PRTTIMER pTimer = (PRTTIMER)pvUser1;
987	RTSPINLOCK hSpinlock;
988	Assert(idCpu < pTimer->cCpus);
989
990	/*
991	* We have to be kind of careful here as we might be racing RTTimerStop
992	* (and/or RTTimerDestroy, thus the paranoia.
993	*/
994	hSpinlock = pTimer->hSpinlock;
995	if ( hSpinlock != NIL_RTSPINLOCK
996	&& pTimer->u32Magic == RTTIMER_MAGIC)
997	{
998	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
999	RTSpinlockAcquire(hSpinlock, &Tmp);
1000
1001	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
1002	&& pTimer->u32Magic == RTTIMER_MAGIC)
1003	{
1004	/* We're sane and the timer is not suspended yet. */
1005	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
1006	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
1007	rtTimerLnxStartSubTimer(pSubTimer, pArgs->u64Now, pArgs->u64First, true /fPinned/, pTimer->fHighRes);
1008	}
1009
1010	RTSpinlockRelease(hSpinlock, &Tmp);
1011	}
1012	}
1013
1014
1015	/**
1016	* MP event notification callback.
1017	*
1018	* @param enmEvent The event.
1019	* @param idCpu The cpu it applies to.
1020	* @param pvUser The timer.
1021	*/
1022	static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
1023	{
1024	PRTTIMER pTimer = (PRTTIMER)pvUser;
1025	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
1026	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1027	RTSPINLOCK hSpinlock;
1028
1029	Assert(idCpu < pTimer->cCpus);
1030
1031	/*
1032	* Some initial paranoia.
1033	*/
1034	if (pTimer->u32Magic != RTTIMER_MAGIC)
1035	return;
1036	hSpinlock = pTimer->hSpinlock;
1037	if (hSpinlock == NIL_RTSPINLOCK)
1038	return;
1039
1040	RTSpinlockAcquire(hSpinlock, &Tmp);
1041
1042	/* Is it active? */
1043	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
1044	&& pTimer->u32Magic == RTTIMER_MAGIC)
1045	{
1046	switch (enmEvent)
1047	{
1048	/*
1049	* Try do it without leaving the spin lock, but if we have to, retake it
1050	* when we're on the right cpu.
1051	*/
1052	case RTMPEVENT_ONLINE:
1053	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
1054	{
1055	RTTIMERLINUXSTARTONCPUARGS Args;
1056	Args.u64Now = RTTimeNanoTS();
1057	Args.u64First = 0;
1058
1059	if (RTMpCpuId() == idCpu)
1060	rtTimerLnxStartSubTimer(pSubTimer, Args.u64Now, Args.u64First, true /fPinned/, pTimer->fHighRes);
1061	else
1062	{
1063	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); /* we'll recheck it. */
1064	RTSpinlockRelease(hSpinlock, &Tmp);
1065
1066	RTMpOnSpecific(idCpu, rtTimerLinuxMpStartOnCpu, pTimer, &Args);
1067	return; /* we've left the spinlock */
1068	}
1069	}
1070	break;
1071
1072	/*
1073	* The CPU is (going) offline, make sure the sub-timer is stopped.
1074	*
1075	* Linux will migrate it to a different CPU, but we don't want this. The
1076	* timer function is checking for this.
1077	*/
1078	case RTMPEVENT_OFFLINE:
1079	{
1080	RTTIMERLNXSTATE enmState;
1081	while ( (enmState = rtTimerLnxGetState(&pSubTimer->enmState)) == RTTIMERLNXSTATE_ACTIVE
1082	\|\| enmState == RTTIMERLNXSTATE_CALLBACK
1083	\|\| enmState == RTTIMERLNXSTATE_CB_RESTARTING)
1084	{
1085	if (enmState == RTTIMERLNXSTATE_ACTIVE)
1086	{
1087	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STOPPING, RTTIMERLNXSTATE_ACTIVE))
1088	{
1089	RTSpinlockRelease(hSpinlock, &Tmp);
1090
1091	rtTimerLnxStopSubTimer(pSubTimer, pTimer->fHighRes);
1092	return; /* we've left the spinlock */
1093	}
1094	}
1095	else if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_CB_STOPPING, enmState))
1096	break;
1097
1098	/* State not stable, try again. */
1099	ASMNopPause();
1100	}
1101	break;
1102	}
1103	}
1104	}
1105
1106	RTSpinlockRelease(hSpinlock, &Tmp);
1107	}
1108
1109	#endif /* CONFIG_SMP */
1110
1111
1112	/**
1113	* Callback function use by RTTimerStart via RTMpOnSpecific to start a timer
1114	* running on a specific CPU.
1115	*
1116	* @param idCpu The current CPU.
1117	* @param pvUser1 Pointer to the timer.
1118	* @param pvUser2 Pointer to the argument structure.
1119	*/
1120	static DECLCALLBACK(void) rtTimerLnxStartOnSpecificCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
1121	{
1122	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
1123	PRTTIMER pTimer = (PRTTIMER)pvUser1;
1124	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], pArgs->u64Now, pArgs->u64First, true /fPinned/, pTimer->fHighRes);
1125	}
1126
1127
1128	RTDECL(int) RTTimerStart(PRTTIMER pTimer, uint64_t u64First)
1129	{
1130	RTTIMERLINUXSTARTONCPUARGS Args;
1131	int rc2;
1132
1133	/*
1134	* Validate.
1135	*/
1136	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
1137	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
1138
1139	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
1140	return VERR_TIMER_ACTIVE;
1141	dprintf(("start %p\n", pTimer));
1142
1143	Args.u64First = u64First;
1144	#ifdef CONFIG_SMP
1145	/*
1146	* Omni timer?
1147	*/
1148	if (pTimer->fAllCpus)
1149	return rtTimerLnxOmniStart(pTimer, &Args);
1150	#endif
1151
1152	/*
1153	* Simple timer - Pretty straight forward if it wasn't for restarting.
1154	*/
1155	Args.u64Now = RTTimeNanoTS();
1156	ASMAtomicWriteU64(&pTimer->aSubTimers[0].uNsRestartAt, Args.u64Now + u64First);
1157	for (;;)
1158	{
1159	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[0].enmState);
1160	switch (enmState)
1161	{
1162	case RTTIMERLNXSTATE_STOPPED:
1163	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STARTING, RTTIMERLNXSTATE_STOPPED))
1164	{
1165	ASMAtomicWriteBool(&pTimer->fSuspended, false);
1166	if (!pTimer->fSpecificCpu)
1167	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], Args.u64Now, Args.u64First,
1168	false /fPinned/, pTimer->fHighRes);
1169	else
1170	{
1171	rc2 = RTMpOnSpecific(pTimer->idCpu, rtTimerLnxStartOnSpecificCpu, pTimer, &Args);
1172	if (RT_FAILURE(rc2))
1173	{
1174	/* Suspend it, the cpu id is probably invalid or offline. */
1175	ASMAtomicWriteBool(&pTimer->fSuspended, true);
1176	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPED);
1177	return rc2;
1178	}
1179	}
1180	return VINF_SUCCESS;
1181	}
1182	break;
1183
1184	case RTTIMERLNXSTATE_CALLBACK:
1185	case RTTIMERLNXSTATE_CB_STOPPING:
1186	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_CB_RESTARTING, enmState))
1187	{
1188	ASMAtomicWriteBool(&pTimer->fSuspended, false);
1189	return VINF_SUCCESS;
1190	}
1191	break;
1192
1193	default:
1194	AssertMsgFailed(("%d\n", enmState));
1195	return VERR_INTERNAL_ERROR_4;
1196	}
1197	ASMNopPause();
1198	}
1199	}
1200	RT_EXPORT_SYMBOL(RTTimerStart);
1201
1202
1203	/**
1204	* Common worker for RTTimerStop and RTTimerDestroy.
1205	*
1206	* @returns true if there was any active callbacks, false if not.
1207	* @param pTimer The timer to stop.
1208	* @param fForDestroy Whether it's RTTimerDestroy calling or not.
1209	*/
1210	static bool rtTimerLnxStop(PRTTIMER pTimer, bool fForDestroy)
1211	{
1212	dprintf(("lnxstop %p %d\n", pTimer, fForDestroy));
1213	#ifdef CONFIG_SMP
1214	/*
1215	* Omni timer?
1216	*/
1217	if (pTimer->fAllCpus)
1218	return rtTimerLnxOmniStop(pTimer, fForDestroy);
1219	#endif
1220
1221	/*
1222	* Simple timer.
1223	*/
1224	ASMAtomicWriteBool(&pTimer->fSuspended, true);
1225	for (;;)
1226	{
1227	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[0].enmState);
1228	switch (enmState)
1229	{
1230	case RTTIMERLNXSTATE_ACTIVE:
1231	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_MP_STOPPING, RTTIMERLNXSTATE_ACTIVE))
1232	{
1233	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[0], pTimer->fHighRes);
1234	return false;
1235	}
1236	break;
1237
1238	case RTTIMERLNXSTATE_CALLBACK:
1239	case RTTIMERLNXSTATE_CB_RESTARTING:
1240	case RTTIMERLNXSTATE_CB_STOPPING:
1241	Assert(enmState != RTTIMERLNXSTATE_CB_STOPPING \|\| fForDestroy);
1242	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[0].enmState,
1243	!fForDestroy ? RTTIMERLNXSTATE_STOPPED : RTTIMERLNXSTATE_CB_DESTROYING,
1244	enmState))
1245	return true;
1246	break;
1247
1248	case RTTIMERLNXSTATE_STOPPED:
1249	return VINF_SUCCESS;
1250
1251	case RTTIMERLNXSTATE_CB_DESTROYING:
1252	AssertMsgFailed(("enmState=%d pTimer=%p\n", enmState, pTimer));
1253	return true;
1254
1255	default:
1256	case RTTIMERLNXSTATE_STARTING:
1257	case RTTIMERLNXSTATE_MP_STARTING:
1258	case RTTIMERLNXSTATE_STOPPING:
1259	case RTTIMERLNXSTATE_MP_STOPPING:
1260	AssertMsgFailed(("enmState=%d pTimer=%p\n", enmState, pTimer));
1261	return false;
1262	}
1263
1264	/* State not stable, try again. */
1265	ASMNopPause();
1266	}
1267	}
1268
1269
1270	RTDECL(int) RTTimerStop(PRTTIMER pTimer)
1271	{
1272	/*
1273	* Validate.
1274	*/
1275	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
1276	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
1277	dprintf(("stop %p\n", pTimer));
1278
1279	if (ASMAtomicUoReadBool(&pTimer->fSuspended))
1280	return VERR_TIMER_SUSPENDED;
1281
1282	rtTimerLnxStop(pTimer, false /fForDestroy/);
1283	return VINF_SUCCESS;
1284	}
1285	RT_EXPORT_SYMBOL(RTTimerStop);
1286
1287
1288	RTDECL(int) RTTimerChangeInterval(PRTTIMER pTimer, uint64_t u64NanoInterval)
1289	{
1290	/*
1291	* Validate.
1292	*/
1293	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
1294	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
1295	AssertReturn(u64NanoInterval, VERR_INVALID_PARAMETER);
1296	dprintf(("change %p %llu\n", pTimer, u64NanoInterval));
1297
1298	#ifdef RTTIMER_LINUX_WITH_HRTIMER
1299	/*
1300	* For the high resolution timers it is easy since we don't care so much
1301	* about when it is applied to the sub-timers.
1302	*/
1303	if (pTimer->fHighRes)
1304	{
1305	ASMAtomicWriteU64(&pTimer->u64NanoInterval, u64NanoInterval);
1306	return VINF_SUCCESS;
1307	}
1308	#endif
1309
1310	/*
1311	* Standard timers have a bit more complicated way of calculating
1312	* their interval and such. So, forget omni timers for now.
1313	*/
1314	if (pTimer->cCpus > 1)
1315	return VERR_NOT_SUPPORTED;
1316	RTTimerStop(pTimer);
1317	return RTTimerStart(pTimer, u64NanoInterval);
1318	}
1319	RT_EXPORT_SYMBOL(RTTimerChangeInterval);
1320
1321
1322	RTDECL(int) RTTimerDestroy(PRTTIMER pTimer)
1323	{
1324	bool fCanDestroy;
1325
1326	/*
1327	* Validate. It's ok to pass NULL pointer.
1328	*/
1329	if (pTimer == /NIL_RTTIMER/ NULL)
1330	return VINF_SUCCESS;
1331	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
1332	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
1333	dprintf(("destroy %p\n", pTimer));
1334
1335	/*
1336	* Stop the timer if it's still active, then destroy it if we can.
1337	*/
1338	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
1339	fCanDestroy = rtTimerLnxStop(pTimer, true /fForDestroy/);
1340	else
1341	{
1342	uint32_t iCpu = pTimer->cCpus;
1343	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
1344	if (pTimer->cCpus > 1)
1345	RTSpinlockAcquireNoInts(pTimer->hSpinlock, &Tmp);
1346
1347	fCanDestroy = true;
1348	while (iCpu-- > 0)
1349	{
1350	for (;;)
1351	{
1352	RTTIMERLNXSTATE enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
1353	switch (enmState)
1354	{
1355	case RTTIMERLNXSTATE_CALLBACK:
1356	case RTTIMERLNXSTATE_CB_RESTARTING:
1357	case RTTIMERLNXSTATE_CB_STOPPING:
1358	if (!rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_CB_DESTROYING, enmState))
1359	continue;
1360	fCanDestroy = false;
1361	break;
1362
1363	case RTTIMERLNXSTATE_CB_DESTROYING:
1364	AssertMsgFailed(("%d\n", enmState));
1365	fCanDestroy = false;
1366	break;
1367	default:
1368	break;
1369	}
1370	break;
1371	}
1372	}
1373
1374	if (pTimer->cCpus > 1)
1375	RTSpinlockReleaseNoInts(pTimer->hSpinlock, &Tmp);
1376	}
1377
1378	if (fCanDestroy)
1379	rtTimerLnxDestroyIt(pTimer);
1380
1381	return VINF_SUCCESS;
1382	}
1383	RT_EXPORT_SYMBOL(RTTimerDestroy);
1384
1385
1386	RTDECL(int) RTTimerCreateEx(PRTTIMER ppTimer, uint64_t u64NanoInterval, uint32_t fFlags, PFNRTTIMER pfnTimer, void pvUser)
1387	{
1388	PRTTIMER pTimer;
1389	RTCPUID iCpu;
1390	unsigned cCpus;
1391	int rc;
1392
1393	*ppTimer = NULL;
1394
1395	/*
1396	* Validate flags.
1397	*/
1398	if (!RTTIMER_FLAGS_ARE_VALID(fFlags))
1399	return VERR_INVALID_PARAMETER;
1400	if ( (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC)
1401	&& (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL
1402	&& !RTMpIsCpuPossible(RTMpCpuIdFromSetIndex(fFlags & RTTIMER_FLAGS_CPU_MASK)))
1403	return VERR_CPU_NOT_FOUND;
1404
1405	/*
1406	* Allocate the timer handler.
1407	*/
1408	cCpus = 1;
1409	#ifdef CONFIG_SMP
1410	if ((fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL)
1411	{
1412	cCpus = RTMpGetMaxCpuId() + 1;
1413	Assert(cCpus <= RTCPUSET_MAX_CPUS); /* On linux we have a 1:1 relationship between cpuid and set index. */
1414	AssertReturn(u64NanoInterval, VERR_NOT_IMPLEMENTED); /* We don't implement single shot on all cpus, sorry. */
1415	}
1416	#endif
1417
1418	rc = RTMemAllocEx(RT_OFFSETOF(RTTIMER, aSubTimers[cCpus]), 0,
1419	RTMEMALLOCEX_FLAGS_ZEROED \| RTMEMALLOCEX_FLAGS_ANY_CTX_FREE, (void **)&pTimer);
1420	if (RT_FAILURE(rc))
1421	return rc;
1422
1423	/*
1424	* Initialize it.
1425	*/
1426	pTimer->u32Magic = RTTIMER_MAGIC;
1427	pTimer->hSpinlock = NIL_RTSPINLOCK;
1428	pTimer->fSuspended = true;
1429	pTimer->fHighRes = !!(fFlags & RTTIMER_FLAGS_HIGH_RES);
1430	#ifdef CONFIG_SMP
1431	pTimer->fSpecificCpu = (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC) && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL;
1432	pTimer->fAllCpus = (fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL;
1433	pTimer->idCpu = pTimer->fSpecificCpu
1434	? RTMpCpuIdFromSetIndex(fFlags & RTTIMER_FLAGS_CPU_MASK)
1435	: NIL_RTCPUID;
1436	#else
1437	pTimer->fSpecificCpu = !!(fFlags & RTTIMER_FLAGS_CPU_SPECIFIC);
1438	pTimer->idCpu = RTMpCpuId();
1439	#endif
1440	pTimer->cCpus = cCpus;
1441	pTimer->pfnTimer = pfnTimer;
1442	pTimer->pvUser = pvUser;
1443	pTimer->u64NanoInterval = u64NanoInterval;
1444	pTimer->cJiffies = u64NanoInterval / RTTimerGetSystemGranularity();
1445	if (pTimer->cJiffies * RTTimerGetSystemGranularity() != u64NanoInterval)
1446	pTimer->cJiffies = 0;
1447
1448	for (iCpu = 0; iCpu < cCpus; iCpu++)
1449	{
1450	#ifdef RTTIMER_LINUX_WITH_HRTIMER
1451	if (pTimer->fHighRes)
1452	{
1453	hrtimer_init(&pTimer->aSubTimers[iCpu].u.Hr.LnxTimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1454	pTimer->aSubTimers[iCpu].u.Hr.LnxTimer.function = rtTimerLinuxHrCallback;
1455	}
1456	else
1457	#endif
1458	{
1459	init_timer(&pTimer->aSubTimers[iCpu].u.Std.LnxTimer);
1460	pTimer->aSubTimers[iCpu].u.Std.LnxTimer.data = (unsigned long)&pTimer->aSubTimers[iCpu];
1461	pTimer->aSubTimers[iCpu].u.Std.LnxTimer.function = rtTimerLinuxStdCallback;
1462	pTimer->aSubTimers[iCpu].u.Std.LnxTimer.expires = jiffies;
1463	pTimer->aSubTimers[iCpu].u.Std.u64StartTS = 0;
1464	pTimer->aSubTimers[iCpu].u.Std.u64NextTS = 0;
1465	}
1466	pTimer->aSubTimers[iCpu].iTick = 0;
1467	pTimer->aSubTimers[iCpu].pParent = pTimer;
1468	pTimer->aSubTimers[iCpu].enmState = RTTIMERLNXSTATE_STOPPED;
1469	}
1470
1471	#ifdef CONFIG_SMP
1472	/*
1473	* If this is running on ALL cpus, we'll have to register a callback
1474	* for MP events (so timers can be started/stopped on cpus going
1475	* online/offline). We also create the spinlock for syncrhonizing
1476	* stop/start/mp-event.
1477	*/
1478	if (cCpus > 1)
1479	{
1480	int rc = RTSpinlockCreate(&pTimer->hSpinlock);
1481	if (RT_SUCCESS(rc))
1482	rc = RTMpNotificationRegister(rtTimerLinuxMpEvent, pTimer);
1483	else
1484	pTimer->hSpinlock = NIL_RTSPINLOCK;
1485	if (RT_FAILURE(rc))
1486	{
1487	RTTimerDestroy(pTimer);
1488	return rc;
1489	}
1490	}
1491	#endif /* CONFIG_SMP */
1492
1493	dprintf(("create %p\n", pTimer));
1494	*ppTimer = pTimer;
1495	return VINF_SUCCESS;
1496	}
1497	RT_EXPORT_SYMBOL(RTTimerCreateEx);
1498
1499
1500	RTDECL(uint32_t) RTTimerGetSystemGranularity(void)
1501	{
1502	#if 0 /** @todo Not sure if this is what we want or not... Add new API for
1503	* querying the resolution of the high res timers? */
1504	struct timespec Ts;
1505	int rc = hrtimer_get_res(CLOCK_MONOTONIC, &Ts);
1506	if (!rc)
1507	{
1508	Assert(!Ts.tv_sec);
1509	return Ts.tv_nsec;
1510	}
1511	#endif
1512	return 1000000000 / HZ; /* ns */
1513	}
1514	RT_EXPORT_SYMBOL(RTTimerGetSystemGranularity);
1515
1516
1517	RTDECL(int) RTTimerRequestSystemGranularity(uint32_t u32Request, uint32_t *pu32Granted)
1518	{
1519	return VERR_NOT_SUPPORTED;
1520	}
1521	RT_EXPORT_SYMBOL(RTTimerRequestSystemGranularity);
1522
1523
1524	RTDECL(int) RTTimerReleaseSystemGranularity(uint32_t u32Granted)
1525	{
1526	return VERR_NOT_SUPPORTED;
1527	}
1528	RT_EXPORT_SYMBOL(RTTimerReleaseSystemGranularity);
1529
1530
1531	RTDECL(bool) RTTimerCanDoHighResolution(void)
1532	{
1533	#ifdef RTTIMER_LINUX_WITH_HRTIMER
1534	return true;
1535	#else
1536	return false;
1537	#endif
1538	}
1539	RT_EXPORT_SYMBOL(RTTimerCanDoHighResolution);
1540

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/timer-r0drv-linux.c@ 32711

Download in other formats: