timer-r0drv-linux.c@ 25645

Last change on this file since 25645 was 25427, checked in by vboxsync, 15 years ago
Runtime/r0drv/timer-linux: make the hrtimer code work, requires Linux 2.6.28+ now
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 29.8 KB

Line
1	/* $Id: timer-r0drv-linux.c 25427 2009-12-16 13:44:58Z vboxsync $ */
2	/** @file
3	* IPRT - Timers, Ring-0 Driver, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2008 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*
26	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
27	* Clara, CA 95054 USA or visit http://www.sun.com if you need
28	* additional information or have any questions.
29	*/
30
31
32	/*******************************************************************************
33	* Header Files *
34	*******************************************************************************/
35	#include "the-linux-kernel.h"
36	#include "internal/iprt.h"
37
38	#include <iprt/timer.h>
39	#include <iprt/time.h>
40	#include <iprt/mp.h>
41	#include <iprt/cpuset.h>
42	#include <iprt/spinlock.h>
43	#include <iprt/err.h>
44	#include <iprt/asm.h>
45	#include <iprt/assert.h>
46	#include <iprt/alloc.h>
47
48	#include "internal/magics.h"
49
50	/* We use the API of Linux 2.6.28+ (hrtimer_add_expires_ns()) */
51	#if !defined(RT_USE_LINUX_HRTIMER) \
52	&& LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28) \
53	&& 0 /* currently disabled */
54	# define RT_USE_LINUX_HRTIMER
55	#endif
56
57	/* This check must match the ktime usage in rtTimeGetSystemNanoTS() / time-r0drv-linux.c. */
58	#if defined(RT_USE_LINUX_HRTIMER) \
59	&& LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 28)
60	# error "RT_USE_LINUX_HRTIMER requires 2.6.28 or later, sorry."
61	#endif
62
63
64	/*******************************************************************************
65	* Structures and Typedefs *
66	*******************************************************************************/
67	/**
68	* Timer state machine.
69	*
70	* This is used to try handle the issues with MP events and
71	* timers that runs on all CPUs. It's relatively nasty :-/
72	*/
73	typedef enum RTTIMERLNXSTATE
74	{
75	/** Stopped. */
76	RTTIMERLNXSTATE_STOPPED = 0,
77	/** Transient state; next ACTIVE. */
78	RTTIMERLNXSTATE_STARTING,
79	/** Transient state; next ACTIVE. (not really necessary) */
80	RTTIMERLNXSTATE_MP_STARTING,
81	/** Active. */
82	RTTIMERLNXSTATE_ACTIVE,
83	/** Transient state; next STOPPED. */
84	RTTIMERLNXSTATE_STOPPING,
85	/** Transient state; next STOPPED. */
86	RTTIMERLNXSTATE_MP_STOPPING,
87	/** The usual 32-bit hack. */
88	RTTIMERLNXSTATE_32BIT_HACK = 0x7fffffff
89	} RTTIMERLNXSTATE;
90
91
92	/**
93	* A Linux sub-timer.
94	*/
95	typedef struct RTTIMERLNXSUBTIMER
96	{
97	/** The linux timer structure. */
98	#ifdef RT_USE_LINUX_HRTIMER
99	struct hrtimer LnxTimer;
100	#else
101	struct timer_list LnxTimer;
102	/** The start of the current run (ns).
103	* This is used to calculate when the timer ought to fire the next time. */
104	uint64_t u64StartTS;
105	/** The start of the current run (ns).
106	* This is used to calculate when the timer ought to fire the next time. */
107	uint64_t u64NextTS;
108	#endif
109	/** The current tick number (since u64StartTS). */
110	uint64_t iTick;
111	/** Pointer to the parent timer. */
112	PRTTIMER pParent;
113	#ifndef RT_USE_LINUX_HRTIMER
114	/** The u64NextTS in jiffies. */
115	unsigned long ulNextJiffies;
116	#endif
117	/** The current sub-timer state. */
118	RTTIMERLNXSTATE volatile enmState;
119	} RTTIMERLNXSUBTIMER;
120	/** Pointer to a linux sub-timer. */
121	typedef RTTIMERLNXSUBTIMER *PRTTIMERLNXSUBTIMER;
122	AssertCompileMemberOffset(RTTIMERLNXSUBTIMER, LnxTimer, 0);
123
124
125	/**
126	* The internal representation of an Linux timer handle.
127	*/
128	typedef struct RTTIMER
129	{
130	/** Magic.
131	* This is RTTIMER_MAGIC, but changes to something else before the timer
132	* is destroyed to indicate clearly that thread should exit. */
133	uint32_t volatile u32Magic;
134	/** Spinlock synchronizing the fSuspended and MP event handling.
135	* This is NIL_RTSPINLOCK if cCpus == 1. */
136	RTSPINLOCK hSpinlock;
137	/** Flag indicating that the timer is suspended. */
138	bool volatile fSuspended;
139	/** Whether the timer must run on one specific CPU or not. */
140	bool fSpecificCpu;
141	#ifdef CONFIG_SMP
142	/** Whether the timer must run on all CPUs or not. */
143	bool fAllCpus;
144	#endif /* else: All -> specific on non-SMP kernels */
145	/** The CPU it must run on if fSpecificCpu is set. */
146	RTCPUID idCpu;
147	/** The number of CPUs this timer should run on. */
148	RTCPUID cCpus;
149	/** Callback. */
150	PFNRTTIMER pfnTimer;
151	/** User argument. */
152	void *pvUser;
153	/** The timer interval. 0 if one-shot. */
154	uint64_t u64NanoInterval;
155	#ifndef RT_USE_LINUX_HRTIMER
156	/** This is set to the number of jiffies between ticks if the interval is
157	* an exact number of jiffies. */
158	unsigned long cJiffies;
159	#endif
160	/** Sub-timers.
161	* Normally there is just one, but for RTTIMER_FLAGS_CPU_ALL this will contain
162	* an entry for all possible cpus. In that case the index will be the same as
163	* for the RTCpuSet. */
164	RTTIMERLNXSUBTIMER aSubTimers[1];
165	} RTTIMER;
166
167
168	/**
169	* A rtTimerLinuxStartOnCpu and rtTimerLinuxStartOnCpu argument package.
170	*/
171	typedef struct RTTIMERLINUXSTARTONCPUARGS
172	{
173	/** The current time (RTTimeNanoTS). */
174	uint64_t u64Now;
175	/** When to start firing (delta). */
176	uint64_t u64First;
177	} RTTIMERLINUXSTARTONCPUARGS;
178	/** Pointer to a rtTimerLinuxStartOnCpu argument package. */
179	typedef RTTIMERLINUXSTARTONCPUARGS *PRTTIMERLINUXSTARTONCPUARGS;
180
181
182	/**
183	* Sets the state.
184	*/
185	DECLINLINE(void) rtTimerLnxSetState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState)
186	{
187	ASMAtomicWriteU32((uint32_t volatile *)penmState, enmNewState);
188	}
189
190
191	/**
192	* Sets the state if it has a certain value.
193	*
194	* @return true if xchg was done.
195	* @return false if xchg wasn't done.
196	*/
197	DECLINLINE(bool) rtTimerLnxCmpXchgState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState, RTTIMERLNXSTATE enmCurState)
198	{
199	return ASMAtomicCmpXchgU32((uint32_t volatile *)penmState, enmNewState, enmCurState);
200	}
201
202
203	/**
204	* Gets the state.
205	*/
206	DECLINLINE(RTTIMERLNXSTATE) rtTimerLnxGetState(RTTIMERLNXSTATE volatile *penmState)
207	{
208	return (RTTIMERLNXSTATE)ASMAtomicUoReadU32((uint32_t volatile *)penmState);
209	}
210
211
212	#ifdef RT_USE_LINUX_HRTIMER
213	/**
214	* Converts a nano second time stamp to ktime_t.
215	*
216	* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts().
217	*
218	* @returns ktime_t.
219	* @param cNanoSecs Nanoseconds.
220	*/
221	DECLINLINE(ktime_t) rtTimerLnxNanoToKt(uint64_t cNanoSecs)
222	{
223	/* With some luck the compiler optimizes the division out of this... (Bet it doesn't.) */
224	return ktime_set(cNanoSecs / 1000000000, cNanoSecs % 1000000000);
225	}
226
227	/**
228	* Converts ktime_t to a nano second time stamp.
229	*
230	* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts().
231	*
232	* @returns nano second time stamp.
233	* @param Kt ktime_t.
234	*/
235	DECLINLINE(uint64_t) rtTimerLnxKtToNano(ktime_t Kt)
236	{
237	return ktime_to_ns(Kt);
238	}
239
240	#else /* ! RT_USE_LINUX_HRTIMER */
241
242	/**
243	* Converts a nano second interval to jiffies.
244	*
245	* @returns Jiffies.
246	* @param cNanoSecs Nanoseconds.
247	*/
248	DECLINLINE(unsigned long) rtTimerLnxNanoToJiffies(uint64_t cNanoSecs)
249	{
250	/* this can be made even better... */
251	if (cNanoSecs > (uint64_t)TICK_NSEC * MAX_JIFFY_OFFSET)
252	return MAX_JIFFY_OFFSET;
253	# if ARCH_BITS == 32
254	if (RT_LIKELY(cNanoSecs <= UINT32_MAX))
255	return ((uint32_t)cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
256	# endif
257	return (cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
258	}
259	#endif /* ! RT_USE_LINUX_HRTIMER */
260
261
262	/**
263	* Starts a sub-timer (RTTimerStart).
264	*
265	* @param pSubTimer The sub-timer to start.
266	* @param u64Now The current timestamp (RTTimeNanoTS()).
267	* @param u64First The interval from u64Now to the first time the timer should fire.
268	*/
269	static void rtTimerLnxStartSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, uint64_t u64Now, uint64_t u64First)
270	{
271	/*
272	* Calc when it should start firing.
273	*/
274	uint64_t u64NextTS = u64Now + u64First;
275	#ifndef RT_USE_LINUX_HRTIMER
276	pSubTimer->u64StartTS = u64NextTS;
277	pSubTimer->u64NextTS = u64NextTS;
278	#endif
279
280	pSubTimer->iTick = 0;
281
282	#ifdef RT_USE_LINUX_HRTIMER
283	hrtimer_start(&pSubTimer->LnxTimer, rtTimerLnxNanoToKt(u64NextTS), HRTIMER_MODE_ABS);
284	#else
285	{
286	unsigned long cJiffies = !u64First ? 0 : rtTimerLnxNanoToJiffies(u64First);
287	pSubTimer->ulNextJiffies = jiffies + cJiffies;
288	mod_timer(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
289	}
290	#endif
291
292	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE);
293	}
294
295
296	/**
297	* Stops a sub-timer (RTTimerStart and rtTimerLinuxMpEvent()).
298	*
299	* @param pSubTimer The sub-timer.
300	*/
301	static void rtTimerLnxStopSubTimer(PRTTIMERLNXSUBTIMER pSubTimer)
302	{
303	#ifdef RT_USE_LINUX_HRTIMER
304	hrtimer_cancel(&pSubTimer->LnxTimer);
305	#else
306	if (timer_pending(&pSubTimer->LnxTimer))
307	del_timer_sync(&pSubTimer->LnxTimer);
308	#endif
309
310	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
311	}
312
313
314	#ifdef RT_USE_LINUX_HRTIMER
315	/**
316	* Timer callback function.
317	* @returns HRTIMER_NORESTART or HRTIMER_RESTART depending on whether it's a one-shot or interval timer.
318	* @param pHrTimer Pointer to the sub-timer structure.
319	*/
320	static enum hrtimer_restart rtTimerLinuxCallback(struct hrtimer *pHrTimer)
321	#else
322	/**
323	* Timer callback function.
324	* @param ulUser Address of the sub-timer structure.
325	*/
326	static void rtTimerLinuxCallback(unsigned long ulUser)
327	#endif
328	{
329	#ifdef RT_USE_LINUX_HRTIMER
330	enum hrtimer_restart rc;
331	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)pHrTimer;
332	#else
333	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)ulUser;
334	#endif
335	PRTTIMER pTimer = pSubTimer->pParent;
336
337	/*
338	* Don't call the handler if the timer has been suspended.
339	* Also, when running on all CPUS, make sure we don't call out twice
340	* on a CPU because of timer migration.
341	*
342	* For the specific cpu case, we're just ignoring timer migration for now... (bad)
343	*/
344	if ( ASMAtomicUoReadBool(&pTimer->fSuspended)
345	#ifdef CONFIG_SMP
346	\|\| ( pTimer->fAllCpus
347	&& (RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != RTMpCpuId())
348	#endif
349	)
350	{
351	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
352	# ifdef RT_USE_LINUX_HRTIMER
353	rc = HRTIMER_NORESTART;
354	# endif
355	}
356	else if (!pTimer->u64NanoInterval)
357	{
358	/*
359	* One shot timer, stop it before dispatching it.
360	*/
361	if (pTimer->cCpus == 1)
362	ASMAtomicWriteBool(&pTimer->fSuspended, true);
363	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
364	#ifdef RT_USE_LINUX_HRTIMER
365	rc = HRTIMER_NORESTART;
366	#else
367	/* detached before we're called, nothing to do for this case. */
368	#endif
369
370	pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
371	}
372	else
373	{
374	const uint64_t iTick = ++pSubTimer->iTick;
375
376	#ifdef RT_USE_LINUX_HRTIMER
377	hrtimer_add_expires_ns(&pSubTimer->LnxTimer, pTimer->u64NanoInterval);
378	rc = HRTIMER_RESTART;
379	#else
380	const uint64_t u64NanoTS = RTTimeNanoTS();
381
382	/*
383	* Interval timer, calculate the next timeout and re-arm it.
384	*
385	* The first time around, we'll re-adjust the u64StartTS to
386	* try prevent some jittering if we were started at a bad time.
387	* This may of course backfire with highres timers...
388	*/
389	if (RT_UNLIKELY(iTick == 1))
390	{
391	pSubTimer->u64StartTS = pSubTimer->u64NextTS = u64NanoTS;
392	pSubTimer->ulNextJiffies = jiffies;
393	}
394
395	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
396	if (pTimer->cJiffies)
397	{
398	pSubTimer->ulNextJiffies += pTimer->cJiffies;
399	/* Prevent overflows when the jiffies counter wraps around.
400	* Special thanks to Ken Preslan for helping debugging! */
401	while (time_before(pSubTimer->ulNextJiffies, jiffies))
402	{
403	pSubTimer->ulNextJiffies += pTimer->cJiffies;
404	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
405	}
406	}
407	else
408	{
409	while (pSubTimer->u64NextTS < u64NanoTS)
410	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
411	pSubTimer->ulNextJiffies = jiffies + rtTimerLnxNanoToJiffies(pSubTimer->u64NextTS - u64NanoTS);
412	}
413
414	mod_timer(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
415	#endif
416
417	/*
418	* Run the timer.
419	*/
420	pTimer->pfnTimer(pTimer, pTimer->pvUser, iTick);
421	}
422
423	#ifdef RT_USE_LINUX_HRTIMER
424	return rc;
425	#endif
426	}
427
428
429	#ifdef CONFIG_SMP
430
431	/**
432	* Per-cpu callback function (RTMpOnAll/RTMpOnSpecific).
433	*
434	* @param idCpu The current CPU.
435	* @param pvUser1 Pointer to the timer.
436	* @param pvUser2 Pointer to the argument structure.
437	*/
438	static DECLCALLBACK(void) rtTimerLnxStartAllOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
439	{
440	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
441	PRTTIMER pTimer = (PRTTIMER)pvUser1;
442	Assert(idCpu < pTimer->cCpus);
443	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[idCpu], pArgs->u64Now, pArgs->u64First);
444	}
445
446
447	/**
448	* Worker for RTTimerStart() that takes care of the ugly bit.s
449	*
450	* @returns RTTimerStart() return value.
451	* @param pTimer The timer.
452	* @param pArgs The argument structure.
453	*/
454	static int rtTimerLnxStartAll(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs)
455	{
456	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
457	RTCPUID iCpu;
458	RTCPUSET OnlineSet;
459	RTCPUSET OnlineSet2;
460	int rc2;
461
462	/*
463	* Prepare all the sub-timers for the startup and then flag the timer
464	* as a whole as non-suspended, make sure we get them all before
465	* clearing fSuspended as the MP handler will be waiting on this
466	* should something happen while we're looping.
467	*/
468	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
469
470	do
471	{
472	RTMpGetOnlineSet(&OnlineSet);
473	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
474	{
475	Assert(pTimer->aSubTimers[iCpu].enmState != RTTIMERLNXSTATE_MP_STOPPING);
476	rtTimerLnxSetState(&pTimer->aSubTimers[iCpu].enmState,
477	RTCpuSetIsMember(&OnlineSet, iCpu)
478	? RTTIMERLNXSTATE_STARTING
479	: RTTIMERLNXSTATE_STOPPED);
480	}
481	} while (!RTCpuSetIsEqual(&OnlineSet, RTMpGetOnlineSet(&OnlineSet2)));
482
483	ASMAtomicWriteBool(&pTimer->fSuspended, false);
484
485	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
486
487	/*
488	* Start them (can't find any exported function that allows me to
489	* do this without the cross calls).
490	*/
491	pArgs->u64Now = RTTimeNanoTS();
492	rc2 = RTMpOnAll(rtTimerLnxStartAllOnCpu, pTimer, pArgs);
493	AssertRC(rc2); /* screw this if it fails. */
494
495	/*
496	* Reset the sub-timers who didn't start up (ALL CPUs case).
497	* CPUs that comes online between the
498	*/
499	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
500
501	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
502	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_STARTING))
503	{
504	/** @todo very odd case for a rainy day. Cpus that temporarily went offline while
505	* we were between calls needs to nudged as the MP handler will ignore events for
506	* them because of the STARTING state. This is an extremely unlikely case - not that
507	* that means anything in my experience... ;-) */
508	}
509
510	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
511
512	return VINF_SUCCESS;
513	}
514
515
516	/**
517	* Worker for RTTimerStop() that takes care of the ugly SMP bits.
518	*
519	* @returns RTTimerStop() return value.
520	* @param pTimer The timer (valid).
521	*/
522	static int rtTimerLnxStopAll(PRTTIMER pTimer)
523	{
524	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
525	RTCPUID iCpu;
526
527
528	/*
529	* Mark the timer as suspended and flag all timers as stopping, except
530	* for those being stopped by an MP event.
531	*/
532	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
533
534	ASMAtomicWriteBool(&pTimer->fSuspended, true);
535	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
536	{
537	RTTIMERLNXSTATE enmState;
538	do
539	{
540	enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
541	if ( enmState == RTTIMERLNXSTATE_STOPPED
542	\|\| enmState == RTTIMERLNXSTATE_MP_STOPPING)
543	break;
544	Assert(enmState == RTTIMERLNXSTATE_ACTIVE);
545	} while (!rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPING, enmState));
546	}
547
548	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
549
550	/*
551	* Do the actual stopping. Fortunately, this doesn't require any IPIs.
552	* Unfortunately it cannot be done synchronously from within the spinlock,
553	* because we might end up in an active waiting for a handler to complete.
554	*/
555	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
556	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) == RTTIMERLNXSTATE_STOPPING)
557	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[iCpu]);
558
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Per-cpu callback function (RTMpOnSpecific) used by rtTimerLinuxMpEvent()
565	* to start a sub-timer on a cpu that just have come online.
566	*
567	* @param idCpu The current CPU.
568	* @param pvUser1 Pointer to the timer.
569	* @param pvUser2 Pointer to the argument structure.
570	*/
571	static DECLCALLBACK(void) rtTimerLinuxMpStartOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
572	{
573	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
574	PRTTIMER pTimer = (PRTTIMER)pvUser1;
575	RTSPINLOCK hSpinlock;
576	Assert(idCpu < pTimer->cCpus);
577
578	/*
579	* We have to be kind of careful here as we might be racing RTTimerStop
580	* (and/or RTTimerDestroy, thus the paranoia.
581	*/
582	hSpinlock = pTimer->hSpinlock;
583	if ( hSpinlock != NIL_RTSPINLOCK
584	&& pTimer->u32Magic == RTTIMER_MAGIC)
585	{
586	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
587	RTSpinlockAcquire(hSpinlock, &Tmp);
588
589	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
590	&& pTimer->u32Magic == RTTIMER_MAGIC)
591	{
592	/* We're sane and the timer is not suspended yet. */
593	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
594	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
595	rtTimerLnxStartSubTimer(pSubTimer, pArgs->u64Now, pArgs->u64First);
596	}
597
598	RTSpinlockRelease(hSpinlock, &Tmp);
599	}
600	}
601
602
603	/**
604	* MP event notification callback.
605	*
606	* @param enmEvent The event.
607	* @param idCpu The cpu it applies to.
608	* @param pvUser The timer.
609	*/
610	static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
611	{
612	PRTTIMER pTimer = (PRTTIMER)pvUser;
613	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
614	RTSPINLOCK hSpinlock;
615	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
616
617	Assert(idCpu < pTimer->cCpus);
618
619	/*
620	* Some initial paranoia.
621	*/
622	if (pTimer->u32Magic != RTTIMER_MAGIC)
623	return;
624	hSpinlock = pTimer->hSpinlock;
625	if (hSpinlock == NIL_RTSPINLOCK)
626	return;
627
628	RTSpinlockAcquire(hSpinlock, &Tmp);
629
630	/* Is it active? */
631	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
632	&& pTimer->u32Magic == RTTIMER_MAGIC)
633	{
634	switch (enmEvent)
635	{
636	/*
637	* Try do it without leaving the spin lock, but if we have to, retake it
638	* when we're on the right cpu.
639	*/
640	case RTMPEVENT_ONLINE:
641	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
642	{
643	RTTIMERLINUXSTARTONCPUARGS Args;
644	Args.u64Now = RTTimeNanoTS();
645	Args.u64First = 0;
646
647	if (RTMpCpuId() == idCpu)
648	rtTimerLnxStartSubTimer(pSubTimer, Args.u64Now, Args.u64First);
649	else
650	{
651	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); /* we'll recheck it. */
652	RTSpinlockRelease(hSpinlock, &Tmp);
653
654	RTMpOnSpecific(idCpu, rtTimerLinuxMpStartOnCpu, pTimer, &Args);
655	return; /* we've left the spinlock */
656	}
657	}
658	break;
659
660	/*
661	* The CPU is (going) offline, make sure the sub-timer is stopped.
662	*
663	* Linux will migrate it to a different CPU, but we don't want this. The
664	* timer function is checking for this.
665	*/
666	case RTMPEVENT_OFFLINE:
667	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STOPPING, RTTIMERLNXSTATE_ACTIVE))
668	{
669	RTSpinlockRelease(hSpinlock, &Tmp);
670
671	rtTimerLnxStopSubTimer(pSubTimer);
672	return; /* we've left the spinlock */
673	}
674	break;
675	}
676	}
677
678	RTSpinlockRelease(hSpinlock, &Tmp);
679	}
680
681	#endif /* CONFIG_SMP */
682
683
684	/**
685	* Callback function use by RTTimerStart via RTMpOnSpecific to start
686	* a timer running on a specific CPU.
687	*
688	* @param idCpu The current CPU.
689	* @param pvUser1 Pointer to the timer.
690	* @param pvUser2 Pointer to the argument structure.
691	*/
692	static DECLCALLBACK(void) rtTimerLnxStartOnSpecificCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
693	{
694	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
695	PRTTIMER pTimer = (PRTTIMER)pvUser1;
696	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], pArgs->u64Now, pArgs->u64First);
697	}
698
699
700	RTDECL(int) RTTimerStart(PRTTIMER pTimer, uint64_t u64First)
701	{
702	RTTIMERLINUXSTARTONCPUARGS Args;
703	int rc2;
704
705	/*
706	* Validate.
707	*/
708	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
709	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
710
711	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
712	return VERR_TIMER_ACTIVE;
713
714	Args.u64First = u64First;
715	#ifdef CONFIG_SMP
716	/*
717	* Omnit timer?
718	*/
719	if (pTimer->fAllCpus)
720	return rtTimerLnxStartAll(pTimer, &Args);
721	#endif
722
723	/*
724	* Simple timer - Pretty straight forward.
725	*/
726	Args.u64Now = RTTimeNanoTS();
727	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STARTING);
728	ASMAtomicWriteBool(&pTimer->fSuspended, false);
729	if (!pTimer->fSpecificCpu)
730	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], Args.u64Now, Args.u64First);
731	else
732	{
733	rc2 = RTMpOnSpecific(pTimer->idCpu, rtTimerLnxStartOnSpecificCpu, pTimer, &Args);
734	if (RT_FAILURE(rc2))
735	{
736	/* Suspend it, the cpu id is probably invalid or offline. */
737	ASMAtomicWriteBool(&pTimer->fSuspended, true);
738	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPED);
739	return rc2;
740	}
741	}
742
743	return VINF_SUCCESS;
744	}
745	RT_EXPORT_SYMBOL(RTTimerStart);
746
747
748	RTDECL(int) RTTimerStop(PRTTIMER pTimer)
749	{
750
751	/*
752	* Validate.
753	*/
754	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
755	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
756
757	if (ASMAtomicUoReadBool(&pTimer->fSuspended))
758	return VERR_TIMER_SUSPENDED;
759
760	#ifdef CONFIG_SMP
761	/*
762	* Omni timer?
763	*/
764	if (pTimer->fAllCpus)
765	return rtTimerLnxStopAll(pTimer);
766	#endif
767
768	/*
769	* Simple timer.
770	*/
771	ASMAtomicWriteBool(&pTimer->fSuspended, true);
772	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPING);
773	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[0]);
774
775	return VINF_SUCCESS;
776	}
777	RT_EXPORT_SYMBOL(RTTimerStop);
778
779
780	RTDECL(int) RTTimerDestroy(PRTTIMER pTimer)
781	{
782	RTSPINLOCK hSpinlock;
783
784	/* It's ok to pass NULL pointer. */
785	if (pTimer == /NIL_RTTIMER/ NULL)
786	return VINF_SUCCESS;
787	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
788	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
789
790	/*
791	* Remove the MP notifications first because it'll reduce the risk of
792	* us overtaking any MP event that might theoretically be racing us here.
793	*/
794	hSpinlock = pTimer->hSpinlock;
795	#ifdef CONFIG_SMP
796	if ( pTimer->cCpus > 1
797	&& hSpinlock != NIL_RTSPINLOCK)
798	{
799	int rc = RTMpNotificationDeregister(rtTimerLinuxMpEvent, pTimer);
800	AssertRC(rc);
801	}
802	#endif /* CONFIG_SMP */
803
804	/*
805	* Stop the timer if it's running.
806	*/
807	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
808	RTTimerStop(pTimer);
809
810	/*
811	* Uninitialize the structure and free the associated resources.
812	* The spinlock goes last.
813	*/
814	ASMAtomicWriteU32(&pTimer->u32Magic, ~RTTIMER_MAGIC);
815	RTMemFree(pTimer);
816	if (hSpinlock != NIL_RTSPINLOCK)
817	RTSpinlockDestroy(hSpinlock);
818
819	return VINF_SUCCESS;
820	}
821	RT_EXPORT_SYMBOL(RTTimerDestroy);
822
823
824	RTDECL(int) RTTimerCreateEx(PRTTIMER ppTimer, uint64_t u64NanoInterval, unsigned fFlags, PFNRTTIMER pfnTimer, void pvUser)
825	{
826	PRTTIMER pTimer;
827	RTCPUID iCpu;
828	unsigned cCpus;
829
830	*ppTimer = NULL;
831
832	/*
833	* Validate flags.
834	*/
835	if (!RTTIMER_FLAGS_ARE_VALID(fFlags))
836	return VERR_INVALID_PARAMETER;
837	if ( (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC)
838	&& (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL
839	&& !RTMpIsCpuOnline(fFlags & RTTIMER_FLAGS_CPU_MASK))
840	return (fFlags & RTTIMER_FLAGS_CPU_MASK) > RTMpGetMaxCpuId()
841	? VERR_CPU_NOT_FOUND
842	: VERR_CPU_OFFLINE;
843
844	/*
845	* Allocate the timer handler.
846	*/
847	cCpus = 1;
848	#ifdef CONFIG_SMP
849	if ((fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL)
850	{
851	cCpus = RTMpGetMaxCpuId() + 1;
852	Assert(cCpus <= RTCPUSET_MAX_CPUS); /* On linux we have a 1:1 relationship between cpuid and set index. */
853	AssertReturn(u64NanoInterval, VERR_NOT_IMPLEMENTED); /* We don't implement single shot on all cpus, sorry. */
854	}
855	#endif
856
857	pTimer = (PRTTIMER)RTMemAllocZ(RT_OFFSETOF(RTTIMER, aSubTimers[cCpus]));
858	if (!pTimer)
859	return VERR_NO_MEMORY;
860
861	/*
862	* Initialize it.
863	*/
864	pTimer->u32Magic = RTTIMER_MAGIC;
865	pTimer->hSpinlock = NIL_RTSPINLOCK;
866	pTimer->fSuspended = true;
867	#ifdef CONFIG_SMP
868	pTimer->fSpecificCpu = (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC) && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL;
869	pTimer->fAllCpus = (fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL;
870	pTimer->idCpu = fFlags & RTTIMER_FLAGS_CPU_MASK;
871	#else
872	pTimer->fSpecificCpu = !!(fFlags & RTTIMER_FLAGS_CPU_SPECIFIC);
873	pTimer->idCpu = RTMpCpuId();
874	#endif
875	pTimer->cCpus = cCpus;
876	pTimer->pfnTimer = pfnTimer;
877	pTimer->pvUser = pvUser;
878	pTimer->u64NanoInterval = u64NanoInterval;
879	#ifndef RT_USE_LINUX_HRTIMER
880	pTimer->cJiffies = u64NanoInterval / RTTimerGetSystemGranularity();
881	if (pTimer->cJiffies * RTTimerGetSystemGranularity() != u64NanoInterval)
882	pTimer->cJiffies = 0;
883	#endif
884
885	for (iCpu = 0; iCpu < cCpus; iCpu++)
886	{
887	#ifdef RT_USE_LINUX_HRTIMER
888	hrtimer_init(&pTimer->aSubTimers[iCpu].LnxTimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
889	pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
890	#else
891	init_timer(&pTimer->aSubTimers[iCpu].LnxTimer);
892	pTimer->aSubTimers[iCpu].LnxTimer.data = (unsigned long)&pTimer->aSubTimers[iCpu];
893	pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
894	pTimer->aSubTimers[iCpu].LnxTimer.expires = jiffies;
895	pTimer->aSubTimers[iCpu].u64StartTS = 0;
896	pTimer->aSubTimers[iCpu].u64NextTS = 0;
897	#endif
898	pTimer->aSubTimers[iCpu].iTick = 0;
899	pTimer->aSubTimers[iCpu].pParent = pTimer;
900	pTimer->aSubTimers[iCpu].enmState = RTTIMERLNXSTATE_STOPPED;
901	}
902
903	#ifdef CONFIG_SMP
904	/*
905	* If this is running on ALL cpus, we'll have to register a callback
906	* for MP events (so timers can be started/stopped on cpus going
907	* online/offline). We also create the spinlock for syncrhonizing
908	* stop/start/mp-event.
909	*/
910	if (cCpus > 1)
911	{
912	int rc = RTSpinlockCreate(&pTimer->hSpinlock);
913	if (RT_SUCCESS(rc))
914	rc = RTMpNotificationRegister(rtTimerLinuxMpEvent, pTimer);
915	else
916	pTimer->hSpinlock = NIL_RTSPINLOCK;
917	if (RT_FAILURE(rc))
918	{
919	RTTimerDestroy(pTimer);
920	return rc;
921	}
922	}
923	#endif /* CONFIG_SMP */
924
925	*ppTimer = pTimer;
926	return VINF_SUCCESS;
927	}
928	RT_EXPORT_SYMBOL(RTTimerCreateEx);
929
930
931	RTDECL(uint32_t) RTTimerGetSystemGranularity(void)
932	{
933	#ifdef RT_USE_LINUX_HRTIMER
934	struct timespec Ts;
935	int rc = hrtimer_get_res(CLOCK_MONOTONIC, &Ts);
936	if (!rc)
937	{
938	Assert(!Ts.tv_sec);
939	return Ts.tv_nsec;
940	}
941	#endif
942	return 1000000000 / HZ; /* ns */
943	}
944	RT_EXPORT_SYMBOL(RTTimerGetSystemGranularity);
945
946
947	RTDECL(int) RTTimerRequestSystemGranularity(uint32_t u32Request, uint32_t *pu32Granted)
948	{
949	return VERR_NOT_SUPPORTED;
950	}
951	RT_EXPORT_SYMBOL(RTTimerRequestSystemGranularity);
952
953
954	RTDECL(int) RTTimerReleaseSystemGranularity(uint32_t u32Granted)
955	{
956	return VERR_NOT_SUPPORTED;
957	}
958	RT_EXPORT_SYMBOL(RTTimerReleaseSystemGranularity);
959

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/timer-r0drv-linux.c@ 25645

Download in other formats: