timer-r0drv-linux.c@ 32043

Last change on this file since 32043 was 28800, checked in by vboxsync, 15 years ago
Automated rebranding to Oracle copyright/license strings via filemuncher
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 30.4 KB

Line
1	/* $Id: timer-r0drv-linux.c 28800 2010-04-27 08:22:32Z vboxsync $ */
2	/** @file
3	* IPRT - Timers, Ring-0 Driver, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2008 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*/
26
27
28	/*******************************************************************************
29	* Header Files *
30	*******************************************************************************/
31	#include "the-linux-kernel.h"
32	#include "internal/iprt.h"
33
34	#include <iprt/timer.h>
35	#include <iprt/time.h>
36	#include <iprt/mp.h>
37	#include <iprt/cpuset.h>
38	#include <iprt/spinlock.h>
39	#include <iprt/err.h>
40	#include <iprt/asm.h>
41	#include <iprt/assert.h>
42	#include <iprt/alloc.h>
43
44	#include "internal/magics.h"
45
46	/* We use the API of Linux 2.6.28+ (hrtimer_add_expires_ns()) */
47	#if !defined(RT_USE_LINUX_HRTIMER) \
48	&& LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28) \
49	&& 0 /* currently disabled */
50	# define RT_USE_LINUX_HRTIMER
51	#endif
52
53	/* This check must match the ktime usage in rtTimeGetSystemNanoTS() / time-r0drv-linux.c. */
54	#if defined(RT_USE_LINUX_HRTIMER) \
55	&& LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 28)
56	# error "RT_USE_LINUX_HRTIMER requires 2.6.28 or later, sorry."
57	#endif
58
59	#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31)
60	# define mod_timer_pinned mod_timer
61	# define HRTIMER_MODE_ABS_PINNED HRTIMER_MODE_ABS
62	#endif
63
64
65	/*******************************************************************************
66	* Structures and Typedefs *
67	*******************************************************************************/
68	/**
69	* Timer state machine.
70	*
71	* This is used to try handle the issues with MP events and
72	* timers that runs on all CPUs. It's relatively nasty :-/
73	*/
74	typedef enum RTTIMERLNXSTATE
75	{
76	/** Stopped. */
77	RTTIMERLNXSTATE_STOPPED = 0,
78	/** Transient state; next ACTIVE. */
79	RTTIMERLNXSTATE_STARTING,
80	/** Transient state; next ACTIVE. (not really necessary) */
81	RTTIMERLNXSTATE_MP_STARTING,
82	/** Active. */
83	RTTIMERLNXSTATE_ACTIVE,
84	/** Transient state; next STOPPED. */
85	RTTIMERLNXSTATE_STOPPING,
86	/** Transient state; next STOPPED. */
87	RTTIMERLNXSTATE_MP_STOPPING,
88	/** The usual 32-bit hack. */
89	RTTIMERLNXSTATE_32BIT_HACK = 0x7fffffff
90	} RTTIMERLNXSTATE;
91
92
93	/**
94	* A Linux sub-timer.
95	*/
96	typedef struct RTTIMERLNXSUBTIMER
97	{
98	/** The linux timer structure. */
99	#ifdef RT_USE_LINUX_HRTIMER
100	struct hrtimer LnxTimer;
101	#else
102	struct timer_list LnxTimer;
103	/** The start of the current run (ns).
104	* This is used to calculate when the timer ought to fire the next time. */
105	uint64_t u64StartTS;
106	/** The start of the current run (ns).
107	* This is used to calculate when the timer ought to fire the next time. */
108	uint64_t u64NextTS;
109	#endif
110	/** The current tick number (since u64StartTS). */
111	uint64_t iTick;
112	/** Pointer to the parent timer. */
113	PRTTIMER pParent;
114	#ifndef RT_USE_LINUX_HRTIMER
115	/** The u64NextTS in jiffies. */
116	unsigned long ulNextJiffies;
117	#endif
118	/** The current sub-timer state. */
119	RTTIMERLNXSTATE volatile enmState;
120	} RTTIMERLNXSUBTIMER;
121	/** Pointer to a linux sub-timer. */
122	typedef RTTIMERLNXSUBTIMER *PRTTIMERLNXSUBTIMER;
123	AssertCompileMemberOffset(RTTIMERLNXSUBTIMER, LnxTimer, 0);
124
125
126	/**
127	* The internal representation of an Linux timer handle.
128	*/
129	typedef struct RTTIMER
130	{
131	/** Magic.
132	* This is RTTIMER_MAGIC, but changes to something else before the timer
133	* is destroyed to indicate clearly that thread should exit. */
134	uint32_t volatile u32Magic;
135	/** Spinlock synchronizing the fSuspended and MP event handling.
136	* This is NIL_RTSPINLOCK if cCpus == 1. */
137	RTSPINLOCK hSpinlock;
138	/** Flag indicating that the timer is suspended. */
139	bool volatile fSuspended;
140	/** Whether the timer must run on one specific CPU or not. */
141	bool fSpecificCpu;
142	#ifdef CONFIG_SMP
143	/** Whether the timer must run on all CPUs or not. */
144	bool fAllCpus;
145	#endif /* else: All -> specific on non-SMP kernels */
146	/** The CPU it must run on if fSpecificCpu is set. */
147	RTCPUID idCpu;
148	/** The number of CPUs this timer should run on. */
149	RTCPUID cCpus;
150	/** Callback. */
151	PFNRTTIMER pfnTimer;
152	/** User argument. */
153	void *pvUser;
154	/** The timer interval. 0 if one-shot. */
155	uint64_t u64NanoInterval;
156	#ifndef RT_USE_LINUX_HRTIMER
157	/** This is set to the number of jiffies between ticks if the interval is
158	* an exact number of jiffies. */
159	unsigned long cJiffies;
160	#endif
161	/** Sub-timers.
162	* Normally there is just one, but for RTTIMER_FLAGS_CPU_ALL this will contain
163	* an entry for all possible cpus. In that case the index will be the same as
164	* for the RTCpuSet. */
165	RTTIMERLNXSUBTIMER aSubTimers[1];
166	} RTTIMER;
167
168
169	/**
170	* A rtTimerLinuxStartOnCpu and rtTimerLinuxStartOnCpu argument package.
171	*/
172	typedef struct RTTIMERLINUXSTARTONCPUARGS
173	{
174	/** The current time (RTTimeNanoTS). */
175	uint64_t u64Now;
176	/** When to start firing (delta). */
177	uint64_t u64First;
178	} RTTIMERLINUXSTARTONCPUARGS;
179	/** Pointer to a rtTimerLinuxStartOnCpu argument package. */
180	typedef RTTIMERLINUXSTARTONCPUARGS *PRTTIMERLINUXSTARTONCPUARGS;
181
182
183	/**
184	* Sets the state.
185	*/
186	DECLINLINE(void) rtTimerLnxSetState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState)
187	{
188	ASMAtomicWriteU32((uint32_t volatile *)penmState, enmNewState);
189	}
190
191
192	/**
193	* Sets the state if it has a certain value.
194	*
195	* @return true if xchg was done.
196	* @return false if xchg wasn't done.
197	*/
198	DECLINLINE(bool) rtTimerLnxCmpXchgState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState, RTTIMERLNXSTATE enmCurState)
199	{
200	return ASMAtomicCmpXchgU32((uint32_t volatile *)penmState, enmNewState, enmCurState);
201	}
202
203
204	/**
205	* Gets the state.
206	*/
207	DECLINLINE(RTTIMERLNXSTATE) rtTimerLnxGetState(RTTIMERLNXSTATE volatile *penmState)
208	{
209	return (RTTIMERLNXSTATE)ASMAtomicUoReadU32((uint32_t volatile *)penmState);
210	}
211
212
213	#ifdef RT_USE_LINUX_HRTIMER
214	/**
215	* Converts a nano second time stamp to ktime_t.
216	*
217	* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts().
218	*
219	* @returns ktime_t.
220	* @param cNanoSecs Nanoseconds.
221	*/
222	DECLINLINE(ktime_t) rtTimerLnxNanoToKt(uint64_t cNanoSecs)
223	{
224	/* With some luck the compiler optimizes the division out of this... (Bet it doesn't.) */
225	return ktime_set(cNanoSecs / 1000000000, cNanoSecs % 1000000000);
226	}
227
228	/**
229	* Converts ktime_t to a nano second time stamp.
230	*
231	* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts().
232	*
233	* @returns nano second time stamp.
234	* @param Kt ktime_t.
235	*/
236	DECLINLINE(uint64_t) rtTimerLnxKtToNano(ktime_t Kt)
237	{
238	return ktime_to_ns(Kt);
239	}
240
241	#else /* ! RT_USE_LINUX_HRTIMER */
242
243	/**
244	* Converts a nano second interval to jiffies.
245	*
246	* @returns Jiffies.
247	* @param cNanoSecs Nanoseconds.
248	*/
249	DECLINLINE(unsigned long) rtTimerLnxNanoToJiffies(uint64_t cNanoSecs)
250	{
251	/* this can be made even better... */
252	if (cNanoSecs > (uint64_t)TICK_NSEC * MAX_JIFFY_OFFSET)
253	return MAX_JIFFY_OFFSET;
254	# if ARCH_BITS == 32
255	if (RT_LIKELY(cNanoSecs <= UINT32_MAX))
256	return ((uint32_t)cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
257	# endif
258	return (cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
259	}
260	#endif /* ! RT_USE_LINUX_HRTIMER */
261
262
263	/**
264	* Starts a sub-timer (RTTimerStart).
265	*
266	* @param pSubTimer The sub-timer to start.
267	* @param u64Now The current timestamp (RTTimeNanoTS()).
268	* @param u64First The interval from u64Now to the first time the timer should fire.
269	* @param fPinned true = timer pinned to a specific CPU,
270	* false = timer can migrate between CPUs
271	*/
272	static void rtTimerLnxStartSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, uint64_t u64Now, uint64_t u64First, bool fPinned)
273	{
274	/*
275	* Calc when it should start firing.
276	*/
277	uint64_t u64NextTS = u64Now + u64First;
278	#ifndef RT_USE_LINUX_HRTIMER
279	pSubTimer->u64StartTS = u64NextTS;
280	pSubTimer->u64NextTS = u64NextTS;
281	#endif
282
283	pSubTimer->iTick = 0;
284
285	#ifdef RT_USE_LINUX_HRTIMER
286	hrtimer_start(&pSubTimer->LnxTimer, rtTimerLnxNanoToKt(u64NextTS),
287	fPinned ? HRTIMER_MODE_ABS_PINNED : HRTIMER_MODE_ABS);
288	#else
289	{
290	unsigned long cJiffies = !u64First ? 0 : rtTimerLnxNanoToJiffies(u64First);
291	pSubTimer->ulNextJiffies = jiffies + cJiffies;
292	# ifdef CONFIG_SMP
293	if (fPinned)
294	mod_timer_pinned(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
295	else
296	# endif
297	mod_timer(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
298	}
299	#endif
300
301	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE);
302	}
303
304
305	/**
306	* Stops a sub-timer (RTTimerStart and rtTimerLinuxMpEvent()).
307	*
308	* @param pSubTimer The sub-timer.
309	*/
310	static void rtTimerLnxStopSubTimer(PRTTIMERLNXSUBTIMER pSubTimer)
311	{
312	#ifdef RT_USE_LINUX_HRTIMER
313	hrtimer_cancel(&pSubTimer->LnxTimer);
314	#else
315	if (timer_pending(&pSubTimer->LnxTimer))
316	del_timer_sync(&pSubTimer->LnxTimer);
317	#endif
318
319	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
320	}
321
322
323	#ifdef RT_USE_LINUX_HRTIMER
324	/**
325	* Timer callback function.
326	* @returns HRTIMER_NORESTART or HRTIMER_RESTART depending on whether it's a one-shot or interval timer.
327	* @param pHrTimer Pointer to the sub-timer structure.
328	*/
329	static enum hrtimer_restart rtTimerLinuxCallback(struct hrtimer *pHrTimer)
330	#else
331	/**
332	* Timer callback function.
333	* @param ulUser Address of the sub-timer structure.
334	*/
335	static void rtTimerLinuxCallback(unsigned long ulUser)
336	#endif
337	{
338	#ifdef RT_USE_LINUX_HRTIMER
339	enum hrtimer_restart rc;
340	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)pHrTimer;
341	#else
342	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)ulUser;
343	#endif
344	PRTTIMER pTimer = pSubTimer->pParent;
345
346	/*
347	* Don't call the handler if the timer has been suspended.
348	* Also, when running on all CPUS, make sure we don't call out twice
349	* on a CPU because of timer migration.
350	*
351	* For the specific cpu case, we're just ignoring timer migration for now... (bad)
352	*/
353	if ( ASMAtomicUoReadBool(&pTimer->fSuspended)
354	#ifdef CONFIG_SMP
355	\|\| ( pTimer->fAllCpus
356	&& (RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != RTMpCpuId())
357	#endif
358	)
359	{
360	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
361	# ifdef RT_USE_LINUX_HRTIMER
362	rc = HRTIMER_NORESTART;
363	# endif
364	}
365	else if (!pTimer->u64NanoInterval)
366	{
367	/*
368	* One shot timer, stop it before dispatching it.
369	*/
370	if (pTimer->cCpus == 1)
371	ASMAtomicWriteBool(&pTimer->fSuspended, true);
372	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
373	#ifdef RT_USE_LINUX_HRTIMER
374	rc = HRTIMER_NORESTART;
375	#else
376	/* detached before we're called, nothing to do for this case. */
377	#endif
378
379	pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
380	}
381	else
382	{
383	const uint64_t iTick = ++pSubTimer->iTick;
384
385	#ifdef RT_USE_LINUX_HRTIMER
386	hrtimer_add_expires_ns(&pSubTimer->LnxTimer, pTimer->u64NanoInterval);
387	rc = HRTIMER_RESTART;
388	#else
389	const uint64_t u64NanoTS = RTTimeNanoTS();
390
391	/*
392	* Interval timer, calculate the next timeout and re-arm it.
393	*
394	* The first time around, we'll re-adjust the u64StartTS to
395	* try prevent some jittering if we were started at a bad time.
396	* This may of course backfire with highres timers...
397	*/
398	if (RT_UNLIKELY(iTick == 1))
399	{
400	pSubTimer->u64StartTS = pSubTimer->u64NextTS = u64NanoTS;
401	pSubTimer->ulNextJiffies = jiffies;
402	}
403
404	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
405	if (pTimer->cJiffies)
406	{
407	pSubTimer->ulNextJiffies += pTimer->cJiffies;
408	/* Prevent overflows when the jiffies counter wraps around.
409	* Special thanks to Ken Preslan for helping debugging! */
410	while (time_before(pSubTimer->ulNextJiffies, jiffies))
411	{
412	pSubTimer->ulNextJiffies += pTimer->cJiffies;
413	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
414	}
415	}
416	else
417	{
418	while (pSubTimer->u64NextTS < u64NanoTS)
419	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
420	pSubTimer->ulNextJiffies = jiffies + rtTimerLnxNanoToJiffies(pSubTimer->u64NextTS - u64NanoTS);
421	}
422
423	# ifdef CONFIG_SMP
424	if (pTimer->fSpecificCpu \|\| pTimer->fAllCpus)
425	mod_timer_pinned(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
426	else
427	# endif
428	mod_timer(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
429	#endif
430
431	/*
432	* Run the timer.
433	*/
434	pTimer->pfnTimer(pTimer, pTimer->pvUser, iTick);
435	}
436
437	#ifdef RT_USE_LINUX_HRTIMER
438	return rc;
439	#endif
440	}
441
442
443	#ifdef CONFIG_SMP
444
445	/**
446	* Per-cpu callback function (RTMpOnAll/RTMpOnSpecific).
447	*
448	* @param idCpu The current CPU.
449	* @param pvUser1 Pointer to the timer.
450	* @param pvUser2 Pointer to the argument structure.
451	*/
452	static DECLCALLBACK(void) rtTimerLnxStartAllOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
453	{
454	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
455	PRTTIMER pTimer = (PRTTIMER)pvUser1;
456	Assert(idCpu < pTimer->cCpus);
457	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[idCpu], pArgs->u64Now, pArgs->u64First, true /fPinned/);
458	}
459
460
461	/**
462	* Worker for RTTimerStart() that takes care of the ugly bit.s
463	*
464	* @returns RTTimerStart() return value.
465	* @param pTimer The timer.
466	* @param pArgs The argument structure.
467	*/
468	static int rtTimerLnxStartAll(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs)
469	{
470	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
471	RTCPUID iCpu;
472	RTCPUSET OnlineSet;
473	RTCPUSET OnlineSet2;
474	int rc2;
475
476	/*
477	* Prepare all the sub-timers for the startup and then flag the timer
478	* as a whole as non-suspended, make sure we get them all before
479	* clearing fSuspended as the MP handler will be waiting on this
480	* should something happen while we're looping.
481	*/
482	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
483
484	do
485	{
486	RTMpGetOnlineSet(&OnlineSet);
487	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
488	{
489	Assert(pTimer->aSubTimers[iCpu].enmState != RTTIMERLNXSTATE_MP_STOPPING);
490	rtTimerLnxSetState(&pTimer->aSubTimers[iCpu].enmState,
491	RTCpuSetIsMember(&OnlineSet, iCpu)
492	? RTTIMERLNXSTATE_STARTING
493	: RTTIMERLNXSTATE_STOPPED);
494	}
495	} while (!RTCpuSetIsEqual(&OnlineSet, RTMpGetOnlineSet(&OnlineSet2)));
496
497	ASMAtomicWriteBool(&pTimer->fSuspended, false);
498
499	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
500
501	/*
502	* Start them (can't find any exported function that allows me to
503	* do this without the cross calls).
504	*/
505	pArgs->u64Now = RTTimeNanoTS();
506	rc2 = RTMpOnAll(rtTimerLnxStartAllOnCpu, pTimer, pArgs);
507	AssertRC(rc2); /* screw this if it fails. */
508
509	/*
510	* Reset the sub-timers who didn't start up (ALL CPUs case).
511	* CPUs that comes online between the
512	*/
513	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
514
515	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
516	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_STARTING))
517	{
518	/** @todo very odd case for a rainy day. Cpus that temporarily went offline while
519	* we were between calls needs to nudged as the MP handler will ignore events for
520	* them because of the STARTING state. This is an extremely unlikely case - not that
521	* that means anything in my experience... ;-) */
522	}
523
524	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
525
526	return VINF_SUCCESS;
527	}
528
529
530	/**
531	* Worker for RTTimerStop() that takes care of the ugly SMP bits.
532	*
533	* @returns RTTimerStop() return value.
534	* @param pTimer The timer (valid).
535	*/
536	static int rtTimerLnxStopAll(PRTTIMER pTimer)
537	{
538	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
539	RTCPUID iCpu;
540
541
542	/*
543	* Mark the timer as suspended and flag all timers as stopping, except
544	* for those being stopped by an MP event.
545	*/
546	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
547
548	ASMAtomicWriteBool(&pTimer->fSuspended, true);
549	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
550	{
551	RTTIMERLNXSTATE enmState;
552	do
553	{
554	enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
555	if ( enmState == RTTIMERLNXSTATE_STOPPED
556	\|\| enmState == RTTIMERLNXSTATE_MP_STOPPING)
557	break;
558	Assert(enmState == RTTIMERLNXSTATE_ACTIVE);
559	} while (!rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPING, enmState));
560	}
561
562	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
563
564	/*
565	* Do the actual stopping. Fortunately, this doesn't require any IPIs.
566	* Unfortunately it cannot be done synchronously from within the spinlock,
567	* because we might end up in an active waiting for a handler to complete.
568	*/
569	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
570	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) == RTTIMERLNXSTATE_STOPPING)
571	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[iCpu]);
572
573	return VINF_SUCCESS;
574	}
575
576
577	/**
578	* Per-cpu callback function (RTMpOnSpecific) used by rtTimerLinuxMpEvent()
579	* to start a sub-timer on a cpu that just have come online.
580	*
581	* @param idCpu The current CPU.
582	* @param pvUser1 Pointer to the timer.
583	* @param pvUser2 Pointer to the argument structure.
584	*/
585	static DECLCALLBACK(void) rtTimerLinuxMpStartOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
586	{
587	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
588	PRTTIMER pTimer = (PRTTIMER)pvUser1;
589	RTSPINLOCK hSpinlock;
590	Assert(idCpu < pTimer->cCpus);
591
592	/*
593	* We have to be kind of careful here as we might be racing RTTimerStop
594	* (and/or RTTimerDestroy, thus the paranoia.
595	*/
596	hSpinlock = pTimer->hSpinlock;
597	if ( hSpinlock != NIL_RTSPINLOCK
598	&& pTimer->u32Magic == RTTIMER_MAGIC)
599	{
600	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
601	RTSpinlockAcquire(hSpinlock, &Tmp);
602
603	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
604	&& pTimer->u32Magic == RTTIMER_MAGIC)
605	{
606	/* We're sane and the timer is not suspended yet. */
607	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
608	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
609	rtTimerLnxStartSubTimer(pSubTimer, pArgs->u64Now, pArgs->u64First, true /fPinned/);
610	}
611
612	RTSpinlockRelease(hSpinlock, &Tmp);
613	}
614	}
615
616
617	/**
618	* MP event notification callback.
619	*
620	* @param enmEvent The event.
621	* @param idCpu The cpu it applies to.
622	* @param pvUser The timer.
623	*/
624	static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
625	{
626	PRTTIMER pTimer = (PRTTIMER)pvUser;
627	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
628	RTSPINLOCK hSpinlock;
629	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
630
631	Assert(idCpu < pTimer->cCpus);
632
633	/*
634	* Some initial paranoia.
635	*/
636	if (pTimer->u32Magic != RTTIMER_MAGIC)
637	return;
638	hSpinlock = pTimer->hSpinlock;
639	if (hSpinlock == NIL_RTSPINLOCK)
640	return;
641
642	RTSpinlockAcquire(hSpinlock, &Tmp);
643
644	/* Is it active? */
645	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
646	&& pTimer->u32Magic == RTTIMER_MAGIC)
647	{
648	switch (enmEvent)
649	{
650	/*
651	* Try do it without leaving the spin lock, but if we have to, retake it
652	* when we're on the right cpu.
653	*/
654	case RTMPEVENT_ONLINE:
655	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
656	{
657	RTTIMERLINUXSTARTONCPUARGS Args;
658	Args.u64Now = RTTimeNanoTS();
659	Args.u64First = 0;
660
661	if (RTMpCpuId() == idCpu)
662	rtTimerLnxStartSubTimer(pSubTimer, Args.u64Now, Args.u64First, true /fPinned/);
663	else
664	{
665	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); /* we'll recheck it. */
666	RTSpinlockRelease(hSpinlock, &Tmp);
667
668	RTMpOnSpecific(idCpu, rtTimerLinuxMpStartOnCpu, pTimer, &Args);
669	return; /* we've left the spinlock */
670	}
671	}
672	break;
673
674	/*
675	* The CPU is (going) offline, make sure the sub-timer is stopped.
676	*
677	* Linux will migrate it to a different CPU, but we don't want this. The
678	* timer function is checking for this.
679	*/
680	case RTMPEVENT_OFFLINE:
681	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STOPPING, RTTIMERLNXSTATE_ACTIVE))
682	{
683	RTSpinlockRelease(hSpinlock, &Tmp);
684
685	rtTimerLnxStopSubTimer(pSubTimer);
686	return; /* we've left the spinlock */
687	}
688	break;
689	}
690	}
691
692	RTSpinlockRelease(hSpinlock, &Tmp);
693	}
694
695	#endif /* CONFIG_SMP */
696
697
698	/**
699	* Callback function use by RTTimerStart via RTMpOnSpecific to start
700	* a timer running on a specific CPU.
701	*
702	* @param idCpu The current CPU.
703	* @param pvUser1 Pointer to the timer.
704	* @param pvUser2 Pointer to the argument structure.
705	*/
706	static DECLCALLBACK(void) rtTimerLnxStartOnSpecificCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
707	{
708	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
709	PRTTIMER pTimer = (PRTTIMER)pvUser1;
710	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], pArgs->u64Now, pArgs->u64First, true /fPinned/);
711	}
712
713
714	RTDECL(int) RTTimerStart(PRTTIMER pTimer, uint64_t u64First)
715	{
716	RTTIMERLINUXSTARTONCPUARGS Args;
717	int rc2;
718
719	/*
720	* Validate.
721	*/
722	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
723	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
724
725	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
726	return VERR_TIMER_ACTIVE;
727
728	Args.u64First = u64First;
729	#ifdef CONFIG_SMP
730	/*
731	* Omnit timer?
732	*/
733	if (pTimer->fAllCpus)
734	return rtTimerLnxStartAll(pTimer, &Args);
735	#endif
736
737	/*
738	* Simple timer - Pretty straight forward.
739	*/
740	Args.u64Now = RTTimeNanoTS();
741	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STARTING);
742	ASMAtomicWriteBool(&pTimer->fSuspended, false);
743	if (!pTimer->fSpecificCpu)
744	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], Args.u64Now, Args.u64First, false /fPinned/);
745	else
746	{
747	rc2 = RTMpOnSpecific(pTimer->idCpu, rtTimerLnxStartOnSpecificCpu, pTimer, &Args);
748	if (RT_FAILURE(rc2))
749	{
750	/* Suspend it, the cpu id is probably invalid or offline. */
751	ASMAtomicWriteBool(&pTimer->fSuspended, true);
752	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPED);
753	return rc2;
754	}
755	}
756
757	return VINF_SUCCESS;
758	}
759	RT_EXPORT_SYMBOL(RTTimerStart);
760
761
762	RTDECL(int) RTTimerStop(PRTTIMER pTimer)
763	{
764
765	/*
766	* Validate.
767	*/
768	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
769	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
770
771	if (ASMAtomicUoReadBool(&pTimer->fSuspended))
772	return VERR_TIMER_SUSPENDED;
773
774	#ifdef CONFIG_SMP
775	/*
776	* Omni timer?
777	*/
778	if (pTimer->fAllCpus)
779	return rtTimerLnxStopAll(pTimer);
780	#endif
781
782	/*
783	* Simple timer.
784	*/
785	ASMAtomicWriteBool(&pTimer->fSuspended, true);
786	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPING);
787	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[0]);
788
789	return VINF_SUCCESS;
790	}
791	RT_EXPORT_SYMBOL(RTTimerStop);
792
793
794	RTDECL(int) RTTimerDestroy(PRTTIMER pTimer)
795	{
796	RTSPINLOCK hSpinlock;
797
798	/* It's ok to pass NULL pointer. */
799	if (pTimer == /NIL_RTTIMER/ NULL)
800	return VINF_SUCCESS;
801	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
802	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
803
804	/*
805	* Remove the MP notifications first because it'll reduce the risk of
806	* us overtaking any MP event that might theoretically be racing us here.
807	*/
808	hSpinlock = pTimer->hSpinlock;
809	#ifdef CONFIG_SMP
810	if ( pTimer->cCpus > 1
811	&& hSpinlock != NIL_RTSPINLOCK)
812	{
813	int rc = RTMpNotificationDeregister(rtTimerLinuxMpEvent, pTimer);
814	AssertRC(rc);
815	}
816	#endif /* CONFIG_SMP */
817
818	/*
819	* Stop the timer if it's running.
820	*/
821	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
822	RTTimerStop(pTimer);
823
824	/*
825	* Uninitialize the structure and free the associated resources.
826	* The spinlock goes last.
827	*/
828	ASMAtomicWriteU32(&pTimer->u32Magic, ~RTTIMER_MAGIC);
829	RTMemFree(pTimer);
830	if (hSpinlock != NIL_RTSPINLOCK)
831	RTSpinlockDestroy(hSpinlock);
832
833	return VINF_SUCCESS;
834	}
835	RT_EXPORT_SYMBOL(RTTimerDestroy);
836
837
838	RTDECL(int) RTTimerCreateEx(PRTTIMER ppTimer, uint64_t u64NanoInterval, unsigned fFlags, PFNRTTIMER pfnTimer, void pvUser)
839	{
840	PRTTIMER pTimer;
841	RTCPUID iCpu;
842	unsigned cCpus;
843
844	*ppTimer = NULL;
845
846	/*
847	* Validate flags.
848	*/
849	if (!RTTIMER_FLAGS_ARE_VALID(fFlags))
850	return VERR_INVALID_PARAMETER;
851	if ( (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC)
852	&& (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL
853	&& !RTMpIsCpuOnline(fFlags & RTTIMER_FLAGS_CPU_MASK))
854	return (fFlags & RTTIMER_FLAGS_CPU_MASK) > RTMpGetMaxCpuId()
855	? VERR_CPU_NOT_FOUND
856	: VERR_CPU_OFFLINE;
857
858	/*
859	* Allocate the timer handler.
860	*/
861	cCpus = 1;
862	#ifdef CONFIG_SMP
863	if ((fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL)
864	{
865	cCpus = RTMpGetMaxCpuId() + 1;
866	Assert(cCpus <= RTCPUSET_MAX_CPUS); /* On linux we have a 1:1 relationship between cpuid and set index. */
867	AssertReturn(u64NanoInterval, VERR_NOT_IMPLEMENTED); /* We don't implement single shot on all cpus, sorry. */
868	}
869	#endif
870
871	pTimer = (PRTTIMER)RTMemAllocZ(RT_OFFSETOF(RTTIMER, aSubTimers[cCpus]));
872	if (!pTimer)
873	return VERR_NO_MEMORY;
874
875	/*
876	* Initialize it.
877	*/
878	pTimer->u32Magic = RTTIMER_MAGIC;
879	pTimer->hSpinlock = NIL_RTSPINLOCK;
880	pTimer->fSuspended = true;
881	#ifdef CONFIG_SMP
882	pTimer->fSpecificCpu = (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC) && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL;
883	pTimer->fAllCpus = (fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL;
884	pTimer->idCpu = fFlags & RTTIMER_FLAGS_CPU_MASK;
885	#else
886	pTimer->fSpecificCpu = !!(fFlags & RTTIMER_FLAGS_CPU_SPECIFIC);
887	pTimer->idCpu = RTMpCpuId();
888	#endif
889	pTimer->cCpus = cCpus;
890	pTimer->pfnTimer = pfnTimer;
891	pTimer->pvUser = pvUser;
892	pTimer->u64NanoInterval = u64NanoInterval;
893	#ifndef RT_USE_LINUX_HRTIMER
894	pTimer->cJiffies = u64NanoInterval / RTTimerGetSystemGranularity();
895	if (pTimer->cJiffies * RTTimerGetSystemGranularity() != u64NanoInterval)
896	pTimer->cJiffies = 0;
897	#endif
898
899	for (iCpu = 0; iCpu < cCpus; iCpu++)
900	{
901	#ifdef RT_USE_LINUX_HRTIMER
902	hrtimer_init(&pTimer->aSubTimers[iCpu].LnxTimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
903	pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
904	#else
905	init_timer(&pTimer->aSubTimers[iCpu].LnxTimer);
906	pTimer->aSubTimers[iCpu].LnxTimer.data = (unsigned long)&pTimer->aSubTimers[iCpu];
907	pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
908	pTimer->aSubTimers[iCpu].LnxTimer.expires = jiffies;
909	pTimer->aSubTimers[iCpu].u64StartTS = 0;
910	pTimer->aSubTimers[iCpu].u64NextTS = 0;
911	#endif
912	pTimer->aSubTimers[iCpu].iTick = 0;
913	pTimer->aSubTimers[iCpu].pParent = pTimer;
914	pTimer->aSubTimers[iCpu].enmState = RTTIMERLNXSTATE_STOPPED;
915	}
916
917	#ifdef CONFIG_SMP
918	/*
919	* If this is running on ALL cpus, we'll have to register a callback
920	* for MP events (so timers can be started/stopped on cpus going
921	* online/offline). We also create the spinlock for syncrhonizing
922	* stop/start/mp-event.
923	*/
924	if (cCpus > 1)
925	{
926	int rc = RTSpinlockCreate(&pTimer->hSpinlock);
927	if (RT_SUCCESS(rc))
928	rc = RTMpNotificationRegister(rtTimerLinuxMpEvent, pTimer);
929	else
930	pTimer->hSpinlock = NIL_RTSPINLOCK;
931	if (RT_FAILURE(rc))
932	{
933	RTTimerDestroy(pTimer);
934	return rc;
935	}
936	}
937	#endif /* CONFIG_SMP */
938
939	*ppTimer = pTimer;
940	return VINF_SUCCESS;
941	}
942	RT_EXPORT_SYMBOL(RTTimerCreateEx);
943
944
945	RTDECL(uint32_t) RTTimerGetSystemGranularity(void)
946	{
947	#ifdef RT_USE_LINUX_HRTIMER
948	struct timespec Ts;
949	int rc = hrtimer_get_res(CLOCK_MONOTONIC, &Ts);
950	if (!rc)
951	{
952	Assert(!Ts.tv_sec);
953	return Ts.tv_nsec;
954	}
955	#endif
956	return 1000000000 / HZ; /* ns */
957	}
958	RT_EXPORT_SYMBOL(RTTimerGetSystemGranularity);
959
960
961	RTDECL(int) RTTimerRequestSystemGranularity(uint32_t u32Request, uint32_t *pu32Granted)
962	{
963	return VERR_NOT_SUPPORTED;
964	}
965	RT_EXPORT_SYMBOL(RTTimerRequestSystemGranularity);
966
967
968	RTDECL(int) RTTimerReleaseSystemGranularity(uint32_t u32Granted)
969	{
970	return VERR_NOT_SUPPORTED;
971	}
972	RT_EXPORT_SYMBOL(RTTimerReleaseSystemGranularity);
973

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/timer-r0drv-linux.c@ 32043

Download in other formats: