timer-r0drv-linux.c@ 12591

Last change on this file since 12591 was 12591, checked in by vboxsync, 16 years ago
#2247 (public), #3193 (internal): handle jiffies wrap around
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 29.5 KB

Line
1	/* $Id: timer-r0drv-linux.c 12591 2008-09-19 08:45:10Z vboxsync $ */
2	/** @file
3	* IPRT - Timers, Ring-0 Driver, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2008 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*
26	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
27	* Clara, CA 95054 USA or visit http://www.sun.com if you need
28	* additional information or have any questions.
29	*/
30
31	/*******************************************************************************
32	* Header Files *
33	*******************************************************************************/
34	#include "the-linux-kernel.h"
35
36	#include <iprt/timer.h>
37	#include <iprt/time.h>
38	#include <iprt/mp.h>
39	#include <iprt/cpuset.h>
40	#include <iprt/spinlock.h>
41	#include <iprt/err.h>
42	#include <iprt/asm.h>
43	#include <iprt/assert.h>
44	#include <iprt/alloc.h>
45
46	#include "internal/magics.h"
47
48	#if !defined(RT_USE_LINUX_HRTIMER) \
49	&& LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23) \
50	&& 0 /* disabled because it somehow sucks. */
51	# define RT_USE_LINUX_HRTIMER
52	#endif
53
54	/* This check must match the ktime usage in rtTimeGetSystemNanoTS() / time-r0drv-linux.c. */
55	#if defined(RT_USE_LINUX_HRTIMER) \
56	&& LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 16)
57	# error "RT_USE_LINUX_HRTIMER requires 2.6.16 or later, sorry."
58	#endif
59
60
61	/*******************************************************************************
62	* Structures and Typedefs *
63	*******************************************************************************/
64	/**
65	* Timer state machine.
66	*
67	* This is used to try handle the issues with MP events and
68	* timers that runs on all CPUs. It's relatively nasty :-/
69	*/
70	typedef enum RTTIMERLNXSTATE
71	{
72	/** Stopped. */
73	RTTIMERLNXSTATE_STOPPED = 0,
74	/** Transient state; next ACTIVE. */
75	RTTIMERLNXSTATE_STARTING,
76	/** Transient state; next ACTIVE. (not really necessary) */
77	RTTIMERLNXSTATE_MP_STARTING,
78	/** Active. */
79	RTTIMERLNXSTATE_ACTIVE,
80	/** Transient state; next STOPPED. */
81	RTTIMERLNXSTATE_STOPPING,
82	/** Transient state; next STOPPED. */
83	RTTIMERLNXSTATE_MP_STOPPING,
84	/** The usual 32-bit hack. */
85	RTTIMERLNXSTATE_32BIT_HACK = 0x7fffffff
86	} RTTIMERLNXSTATE;
87
88
89	/**
90	* A Linux sub-timer.
91	*/
92	typedef struct RTTIMERLNXSUBTIMER
93	{
94	/** The linux timer structure. */
95	#ifdef RT_USE_LINUX_HRTIMER
96	struct hrtimer LnxTimer;
97	#else
98	struct timer_list LnxTimer;
99	#endif
100	/** The start of the current run (ns).
101	* This is used to calculate when the timer ought to fire the next time. */
102	uint64_t u64StartTS;
103	/** The start of the current run (ns).
104	* This is used to calculate when the timer ought to fire the next time. */
105	uint64_t u64NextTS;
106	/** The current tick number (since u64StartTS). */
107	uint64_t iTick;
108	/** Pointer to the parent timer. */
109	PRTTIMER pParent;
110	#ifndef RT_USE_LINUX_HRTIMER
111	/** The u64NextTS in jiffies. */
112	unsigned long ulNextJiffies;
113	#endif
114	/** The current sub-timer state. */
115	RTTIMERLNXSTATE volatile enmState;
116	} RTTIMERLNXSUBTIMER;
117	/** Pointer to a linux sub-timer. */
118	typedef RTTIMERLNXSUBTIMER *PRTTIMERLNXSUBTIMER;
119	AssertCompileMemberOffset(RTTIMERLNXSUBTIMER, LnxTimer, 0);
120
121
122	/**
123	* The internal representation of an Linux timer handle.
124	*/
125	typedef struct RTTIMER
126	{
127	/** Magic.
128	* This is RTTIMER_MAGIC, but changes to something else before the timer
129	* is destroyed to indicate clearly that thread should exit. */
130	uint32_t volatile u32Magic;
131	/** Spinlock synchronizing the fSuspended and MP event handling.
132	* This is NIL_RTSPINLOCK if cCpus == 1. */
133	RTSPINLOCK hSpinlock;
134	/** Flag indicating the the timer is suspended. */
135	bool volatile fSuspended;
136	/** Whether the timer must run on one specific CPU or not. */
137	bool fSpecificCpu;
138	#ifdef CONFIG_SMP
139	/** Whether the timer must run on all CPUs or not. */
140	bool fAllCpus;
141	#endif /* else: All -> specific on non-SMP kernels */
142	/** The CPU it must run on if fSpecificCpu is set. */
143	RTCPUID idCpu;
144	/** The number of CPUs this timer should run on. */
145	RTCPUID cCpus;
146	/** Callback. */
147	PFNRTTIMER pfnTimer;
148	/** User argument. */
149	void *pvUser;
150	/** The timer interval. 0 if one-shot. */
151	uint64_t u64NanoInterval;
152	#ifndef RT_USE_LINUX_HRTIMER
153	/** This is set to the number of jiffies between ticks if the interval is
154	* an exact number of jiffies. */
155	unsigned long cJiffies;
156	#endif
157	/** Sub-timers.
158	* Normally there is just one, but for RTTIMER_FLAGS_CPU_ALL this will contain
159	* an entry for all possible cpus. In that case the index will be the same as
160	* for the RTCpuSet. */
161	RTTIMERLNXSUBTIMER aSubTimers[1];
162	} RTTIMER;
163
164
165	/**
166	* A rtTimerLinuxStartOnCpu and rtTimerLinuxStartOnCpu argument package.
167	*/
168	typedef struct RTTIMERLINUXSTARTONCPUARGS
169	{
170	/** The current time (RTTimeNanoTS). */
171	uint64_t u64Now;
172	/** When to start firing (delta). */
173	uint64_t u64First;
174	} RTTIMERLINUXSTARTONCPUARGS;
175	/** Pointer to a rtTimerLinuxStartOnCpu argument package. */
176	typedef RTTIMERLINUXSTARTONCPUARGS *PRTTIMERLINUXSTARTONCPUARGS;
177
178
179	/**
180	* Sets the state.
181	*/
182	DECLINLINE(void) rtTimerLnxSetState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState)
183	{
184	ASMAtomicWriteU32((uint32_t volatile *)penmState, enmNewState);
185	}
186
187
188	/**
189	* Sets the state if it has a certain value.
190	*
191	* @return true if xchg was done.
192	* @return false if xchg wasn't done.
193	*/
194	DECLINLINE(bool) rtTimerLnxCmpXchgState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState, RTTIMERLNXSTATE enmCurState)
195	{
196	return ASMAtomicCmpXchgU32((uint32_t volatile *)penmState, enmNewState, enmCurState);
197	}
198
199
200	/**
201	* Gets the state.
202	*/
203	DECLINLINE(RTTIMERLNXSTATE) rtTimerLnxGetState(RTTIMERLNXSTATE volatile *penmState)
204	{
205	return (RTTIMERLNXSTATE)ASMAtomicUoReadU32((uint32_t volatile *)penmState);
206	}
207
208
209	#ifdef RT_USE_LINUX_HRTIMER
210	/**
211	* Converts a nano second time stamp to ktime_t.
212	*
213	* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts().
214	*
215	* @returns ktime_t.
216	* @param cNanoSecs Nanoseconds.
217	*/
218	DECLINLINE(ktime_t) rtTimerLnxNanoToKt(uint64_t cNanoSecs)
219	{
220	/* With some luck the compiler optimizes the division out of this... (Bet it doesn't.) */
221	return ktime_set(cNanoSecs / 1000000000, cNanoSecs % 1000000000);
222	}
223
224	/**
225	* Converts ktime_t to a nano second time stamp.
226	*
227	* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts().
228	*
229	* @returns nano second time stamp.
230	* @param Kt ktime_t.
231	*/
232	DECLINLINE(uint64_t) rtTimerLnxKtToNano(ktime_t Kt)
233	{
234	return ktime_to_ns(Kt);
235	}
236
237	#else /* ! RT_USE_LINUX_HRTIMER */
238
239	/**
240	* Converts a nano second interval to jiffies.
241	*
242	* @returns Jiffies.
243	* @param cNanoSecs Nanoseconds.
244	*/
245	DECLINLINE(unsigned long) rtTimerLnxNanoToJiffies(uint64_t cNanoSecs)
246	{
247	/* this can be made even better... */
248	if (cNanoSecs > (uint64_t)TICK_NSEC * MAX_JIFFY_OFFSET)
249	return MAX_JIFFY_OFFSET;
250	#if ARCH_BITS == 32
251	if (RT_LIKELY(cNanoSecs <= UINT32_MAX))
252	return ((uint32_t)cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
253	#endif
254	return (cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
255	}
256	#endif
257
258
259	/**
260	* Starts a sub-timer (RTTimerStart).
261	*
262	* @param pSubTimer The sub-timer to start.
263	* @param u64Now The current timestamp (RTTimeNanoTS()).
264	* @param u64First The interval from u64Now to the first time the timer should fire.
265	*/
266	static void rtTimerLnxStartSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, uint64_t u64Now, uint64_t u64First)
267	{
268	/*
269	* Calc when it should start firing.
270	*/
271	uint64_t u64NextTS = u64Now + u64First;
272	pSubTimer->u64StartTS = u64NextTS;
273	pSubTimer->u64NextTS = u64NextTS;
274	pSubTimer->iTick = 0;
275
276	#ifdef RT_USE_LINUX_HRTIMER
277	hrtimer_start(&pSubTimer->LnxTimer, rtTimerLnxNanoToKt(u64NextTS), HRTIMER_MODE_ABS);
278	#else
279	{
280	unsigned long cJiffies = !u64First ? 0 : rtTimerLnxNanoToJiffies(u64First);
281	pSubTimer->ulNextJiffies = jiffies + cJiffies;
282	mod_timer(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
283	}
284	#endif
285
286	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE);
287	}
288
289
290	/**
291	* Stops a sub-timer (RTTimerStart and rtTimerLinuxMpEvent()).
292	*
293	* @param pSubTimer The sub-timer.
294	*/
295	static void rtTimerLnxStopSubTimer(PRTTIMERLNXSUBTIMER pSubTimer)
296	{
297	#ifdef RT_USE_LINUX_HRTIMER
298	hrtimer_cancel(&pSubTimer->LnxTimer);
299	#else
300	if (timer_pending(&pSubTimer->LnxTimer))
301	del_timer_sync(&pSubTimer->LnxTimer);
302	#endif
303
304	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
305	}
306
307
308	#ifdef RT_USE_LINUX_HRTIMER
309	/**
310	* Timer callback function.
311	* @returns HRTIMER_NORESTART or HRTIMER_RESTART depending on whether it's a one-shot or interval timer.
312	* @param pHrTimer Pointer to the sub-timer structure.
313	*/
314	static enum hrtimer_restart rtTimerLinuxCallback(struct hrtimer *pHrTimer)
315	#else
316	/**
317	* Timer callback function.
318	* @param ulUser Address of the sub-timer structure.
319	*/
320	static void rtTimerLinuxCallback(unsigned long ulUser)
321	#endif
322	{
323	#ifdef RT_USE_LINUX_HRTIMER
324	enum hrtimer_restart rc;
325	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)pHrTimer;
326	#else
327	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)ulUser;
328	#endif
329	PRTTIMER pTimer = pSubTimer->pParent;
330
331	/*
332	* Don't call the handler if the timer has been suspended.
333	* Also, when running on all CPUS, make sure we don't call out twice
334	* on a CPU because of timer migration.
335	*
336	* For the specific cpu case, we're just ignoring timer migration for now... (bad)
337	*/
338	if ( ASMAtomicUoReadBool(&pTimer->fSuspended)
339	#ifdef CONFIG_SMP
340	\|\| ( pTimer->fAllCpus
341	&& (pSubTimer - &pTimer->aSubTimers[0]) != RTMpCpuId())
342	#endif
343	)
344	{
345	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
346	# ifdef RT_USE_LINUX_HRTIMER
347	rc = HRTIMER_NORESTART;
348	# endif
349	}
350	else if (!pTimer->u64NanoInterval)
351	{
352	/*
353	* One shot timer, stop it before dispatching it.
354	*/
355	if (pTimer->cCpus == 1)
356	ASMAtomicWriteBool(&pTimer->fSuspended, true);
357	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
358	#ifdef RT_USE_LINUX_HRTIMER
359	rc = HRTIMER_NORESTART;
360	#else
361	/* detached before we're called, nothing to do for this case. */
362	#endif
363
364	pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
365	}
366	else
367	{
368	/*
369	* Interval timer, calculate the next timeout and re-arm it.
370	*
371	* The first time around, we'll re-adjust the u64StartTS to
372	* try prevent some jittering if we were started at a bad time.
373	* This may of course backfire with highres timers...
374	*/
375	const uint64_t u64NanoTS = RTTimeNanoTS();
376	const uint64_t iTick = ++pSubTimer->iTick;
377
378	if (RT_UNLIKELY(iTick == 1))
379	{
380	#ifdef RT_USE_LINUX_HRTIMER
381	pSubTimer->u64StartTS = pSubTimer->u64NextTS = u64NanoTS;//rtTimerLnxKtToNano(pSubTimer->LnxTimer.base->softirq_time);
382	#else
383	pSubTimer->u64StartTS = pSubTimer->u64NextTS = u64NanoTS;
384	pSubTimer->ulNextJiffies = jiffies;
385	#endif
386	}
387
388	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
389
390	#ifdef RT_USE_LINUX_HRTIMER
391	while (pSubTimer->u64NextTS < u64NanoTS)
392	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
393
394	pSubTimer->LnxTimer.expires = rtTimerLnxNanoToKt(pSubTimer->u64NextTS);
395	rc = HRTIMER_RESTART;
396	#else
397	if (pTimer->cJiffies)
398	{
399	pSubTimer->ulNextJiffies += pTimer->cJiffies;
400	/* Prevent overflows when the jiffies counter wraps around.
401	* Special thanks to Ken Preslan for helping debugging! */
402	while (jiffies - pSubTimer->ulNextJiffies < 3600*HZ)
403	{
404	pSubTimer->ulNextJiffies += pTimer->cJiffies;
405	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
406	}
407	}
408	else
409	{
410	while (pSubTimer->u64NextTS < u64NanoTS)
411	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
412	pSubTimer->ulNextJiffies = jiffies + rtTimerLnxNanoToJiffies(pSubTimer->u64NextTS - u64NanoTS);
413	}
414
415	mod_timer(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
416	#endif
417
418	/*
419	* Run the timer.
420	*/
421	pTimer->pfnTimer(pTimer, pTimer->pvUser, iTick);
422	}
423
424	#ifdef RT_USE_LINUX_HRTIMER
425	return rc;
426	#endif
427	}
428
429
430	#ifdef CONFIG_SMP
431
432	/**
433	* Per-cpu callback function (RTMpOnAll/RTMpOnSpecific).
434	*
435	* @param idCpu The current CPU.
436	* @param pvUser1 Pointer to the timer.
437	* @param pvUser2 Pointer to the argument structure.
438	*/
439	static DECLCALLBACK(void) rtTimerLnxStartAllOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
440	{
441	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
442	PRTTIMER pTimer = (PRTTIMER)pvUser1;
443	Assert(idCpu < pTimer->cCpus);
444	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[idCpu], pArgs->u64Now, pArgs->u64First);
445	}
446
447
448	/**
449	* Worker for RTTimerStart() that takes care of the ugly bit.s
450	*
451	* @returns RTTimerStart() return value.
452	* @param pTimer The timer.
453	* @param pArgs The argument structure.
454	*/
455	static int rtTimerLnxStartAll(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs)
456	{
457	RTSPINLOCKTMP Tmp;
458	RTCPUID iCpu;
459	RTCPUSET OnlineSet;
460	RTCPUSET OnlineSet2;
461	int rc2;
462
463	/*
464	* Prepare all the sub-timers for the startup and then flag the timer
465	* as a whole as non-suspended, make sure we get them all before
466	* clearing fSuspended as the MP handler will be waiting on this
467	* should something happen while we're looping.
468	*/
469	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
470
471	do
472	{
473	RTMpGetOnlineSet(&OnlineSet);
474	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
475	{
476	Assert(pTimer->aSubTimers[iCpu].enmState != RTTIMERLNXSTATE_MP_STOPPING);
477	rtTimerLnxSetState(&pTimer->aSubTimers[iCpu].enmState,
478	RTCpuSetIsMember(&OnlineSet, iCpu)
479	? RTTIMERLNXSTATE_STARTING
480	: RTTIMERLNXSTATE_STOPPED);
481	}
482	} while (!RTCpuSetIsEqual(&OnlineSet, RTMpGetOnlineSet(&OnlineSet2)));
483
484	ASMAtomicWriteBool(&pTimer->fSuspended, false);
485
486	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
487
488	/*
489	* Start them (can't find any exported function that allows me to
490	* do this without the cross calls).
491	*/
492	pArgs->u64Now = RTTimeNanoTS();
493	rc2 = RTMpOnAll(rtTimerLnxStartAllOnCpu, pTimer, pArgs);
494	AssertRC(rc2); /* screw this if it fails. */
495
496	/*
497	* Reset the sub-timers who didn't start up (ALL CPUs case).
498	* CPUs that comes online between the
499	*/
500	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
501
502	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
503	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_STARTING))
504	{
505	/** @todo very odd case for a rainy day. Cpus that temporarily went offline while
506	* we were between calls needs to nudged as the MP handler will ignore events for
507	* them because of the STARTING state. This is an extremely unlikely case - not that
508	* that means anything in my experience... ;-) */
509	}
510
511	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
512
513	return VINF_SUCCESS;
514	}
515
516
517	/**
518	* Worker for RTTimerStop() that takes care of the ugly SMP bits.
519	*
520	* @returns RTTimerStop() return value.
521	* @param pTimer The timer (valid).
522	*/
523	static int rtTimerLnxStopAll(PRTTIMER pTimer)
524	{
525	RTCPUID iCpu;
526	RTSPINLOCKTMP Tmp;
527
528
529	/*
530	* Mark the timer as suspended and flag all timers as stopping, except
531	* for those being stopped by an MP event.
532	*/
533	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
534
535	ASMAtomicWriteBool(&pTimer->fSuspended, true);
536	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
537	{
538	RTTIMERLNXSTATE enmState;
539	do
540	{
541	enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
542	if ( enmState == RTTIMERLNXSTATE_STOPPED
543	\|\| enmState == RTTIMERLNXSTATE_MP_STOPPING)
544	break;
545	Assert(enmState == RTTIMERLNXSTATE_ACTIVE);
546	} while (!rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPING, enmState));
547	}
548
549	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
550
551	/*
552	* Do the actual stopping. Fortunately, this doesn't require any IPIs.
553	* Unfortunately it cannot be done synchronously from within the spinlock,
554	* because we might end up in an active waiting for a handler to complete.
555	*/
556	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
557	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) == RTTIMERLNXSTATE_STOPPING)
558	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[iCpu]);
559
560	return VINF_SUCCESS;
561	}
562
563
564	/**
565	* Per-cpu callback function (RTMpOnSpecific) used by rtTimerLinuxMpEvent()
566	* to start a sub-timer on a cpu that just have come online.
567	*
568	* @param idCpu The current CPU.
569	* @param pvUser1 Pointer to the timer.
570	* @param pvUser2 Pointer to the argument structure.
571	*/
572	static DECLCALLBACK(void) rtTimerLinuxMpStartOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
573	{
574	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
575	PRTTIMER pTimer = (PRTTIMER)pvUser1;
576	RTSPINLOCK hSpinlock;
577	Assert(idCpu < pTimer->cCpus);
578
579	/*
580	* We have to be kind of careful here as we might be racing RTTimerStop
581	* (and/or RTTimerDestroy, thus the paranoia.
582	*/
583	hSpinlock = pTimer->hSpinlock;
584	if ( hSpinlock != NIL_RTSPINLOCK
585	&& pTimer->u32Magic == RTTIMER_MAGIC)
586	{
587	RTSPINLOCKTMP Tmp;
588	RTSpinlockAcquire(hSpinlock, &Tmp);
589
590	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
591	&& pTimer->u32Magic == RTTIMER_MAGIC)
592	{
593	/* We're sane and the timer is not suspended yet. */
594	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
595	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
596	rtTimerLnxStartSubTimer(pSubTimer, pArgs->u64Now, pArgs->u64First);
597	}
598
599	RTSpinlockRelease(hSpinlock, &Tmp);
600	}
601	}
602
603
604	/**
605	* MP event notification callback.
606	*
607	* @param enmEvent The event.
608	* @param idCpu The cpu it applies to.
609	* @param pvUser The timer.
610	*/
611	static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
612	{
613	PRTTIMER pTimer = (PRTTIMER)pvUser;
614	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
615	RTSPINLOCK hSpinlock;
616	RTSPINLOCKTMP Tmp;
617
618	Assert(idCpu < pTimer->cCpus);
619
620	/*
621	* Some initial paranoia.
622	*/
623	if (pTimer->u32Magic != RTTIMER_MAGIC)
624	return;
625	hSpinlock = pTimer->hSpinlock;
626	if (hSpinlock == NIL_RTSPINLOCK)
627	return;
628
629	RTSpinlockAcquire(hSpinlock, &Tmp);
630
631	/* Is it active? */
632	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
633	&& pTimer->u32Magic == RTTIMER_MAGIC)
634	{
635	switch (enmEvent)
636	{
637	/*
638	* Try do it without leaving the spin lock, but if we have to, retake it
639	* when we're on the right cpu.
640	*/
641	case RTMPEVENT_ONLINE:
642	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
643	{
644	RTTIMERLINUXSTARTONCPUARGS Args;
645	Args.u64Now = RTTimeNanoTS();
646	Args.u64First = 0;
647
648	if (RTMpCpuId() == idCpu)
649	rtTimerLnxStartSubTimer(pSubTimer, Args.u64Now, Args.u64First);
650	else
651	{
652	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); /* we'll recheck it. */
653	RTSpinlockRelease(hSpinlock, &Tmp);
654
655	RTMpOnSpecific(idCpu, rtTimerLinuxMpStartOnCpu, pTimer, &Args);
656	return; /* we've left the spinlock */
657	}
658	}
659	break;
660
661	/*
662	* The CPU is (going) offline, make sure the sub-timer is stopped.
663	*
664	* Linux will migrate it to a different CPU, but we don't want this. The
665	* timer function is checking for this.
666	*/
667	case RTMPEVENT_OFFLINE:
668	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STOPPING, RTTIMERLNXSTATE_ACTIVE))
669	{
670	RTSpinlockRelease(hSpinlock, &Tmp);
671
672	rtTimerLnxStopSubTimer(pSubTimer);
673	return; /* we've left the spinlock */
674	}
675	break;
676	}
677	}
678
679	RTSpinlockRelease(hSpinlock, &Tmp);
680	}
681
682	#endif /* CONFIG_SMP */
683
684
685	/**
686	* Callback function use by RTTimerStart via RTMpOnSpecific to start
687	* a timer running on a specific CPU.
688	*
689	* @param idCpu The current CPU.
690	* @param pvUser1 Pointer to the timer.
691	* @param pvUser2 Pointer to the argument structure.
692	*/
693	static DECLCALLBACK(void) rtTimerLnxStartOnSpecificCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
694	{
695	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
696	PRTTIMER pTimer = (PRTTIMER)pvUser1;
697	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], pArgs->u64Now, pArgs->u64First);
698	}
699
700
701	RTDECL(int) RTTimerStart(PRTTIMER pTimer, uint64_t u64First)
702	{
703	RTTIMERLINUXSTARTONCPUARGS Args;
704	int rc2;
705
706	/*
707	* Validate.
708	*/
709	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
710	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
711
712	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
713	return VERR_TIMER_ACTIVE;
714
715	Args.u64First = u64First;
716	#ifdef CONFIG_SMP
717	/*
718	* Omnit timer?
719	*/
720	if (pTimer->fAllCpus)
721	return rtTimerLnxStartAll(pTimer, &Args);
722	#endif
723
724	/*
725	* Simple timer - Pretty straight forward.
726	*/
727	Args.u64Now = RTTimeNanoTS();
728	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STARTING);
729	ASMAtomicWriteBool(&pTimer->fSuspended, false);
730	if (!pTimer->fSpecificCpu)
731	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], Args.u64Now, Args.u64First);
732	else
733	{
734	rc2 = RTMpOnSpecific(pTimer->idCpu, rtTimerLnxStartOnSpecificCpu, pTimer, &Args);
735	if (RT_FAILURE(rc2))
736	{
737	/* Suspend it, the cpu id is probably invalid or offline. */
738	ASMAtomicWriteBool(&pTimer->fSuspended, true);
739	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPED);
740	return rc2;
741	}
742	}
743
744	return VINF_SUCCESS;
745	}
746
747
748	RTDECL(int) RTTimerStop(PRTTIMER pTimer)
749	{
750
751	/*
752	* Validate.
753	*/
754	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
755	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
756
757	if (ASMAtomicUoReadBool(&pTimer->fSuspended))
758	return VERR_TIMER_SUSPENDED;
759
760	#ifdef CONFIG_SMP
761	/*
762	* Omni timer?
763	*/
764	if (pTimer->fAllCpus)
765	return rtTimerLnxStopAll(pTimer);
766	#endif
767
768	/*
769	* Simple timer.
770	*/
771	ASMAtomicWriteBool(&pTimer->fSuspended, true);
772	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPING);
773	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[0]);
774
775	return VINF_SUCCESS;
776	}
777
778
779	RTDECL(int) RTTimerDestroy(PRTTIMER pTimer)
780	{
781	RTSPINLOCK hSpinlock;
782
783	/* It's ok to pass NULL pointer. */
784	if (pTimer == /NIL_RTTIMER/ NULL)
785	return VINF_SUCCESS;
786	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
787	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
788
789	/*
790	* Remove the MP notifications first because it'll reduce the risk of
791	* us overtaking any MP event that might theoretically be racing us here.
792	*/
793	hSpinlock = pTimer->hSpinlock;
794	#ifdef CONFIG_SMP
795	if ( pTimer->cCpus > 1
796	&& hSpinlock != NIL_RTSPINLOCK)
797	{
798	int rc = RTMpNotificationDeregister(rtTimerLinuxMpEvent, pTimer);
799	AssertRC(rc);
800	}
801	#endif /* CONFIG_SMP */
802
803	/*
804	* Stop the timer if it's running.
805	*/
806	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
807	RTTimerStop(pTimer);
808
809	/*
810	* Uninitialize the structure and free the associated resources.
811	* The spinlock goes last.
812	*/
813	ASMAtomicWriteU32(&pTimer->u32Magic, ~RTTIMER_MAGIC);
814	RTMemFree(pTimer);
815	if (hSpinlock != NIL_RTSPINLOCK)
816	RTSpinlockDestroy(hSpinlock);
817
818	return VINF_SUCCESS;
819	}
820
821
822	RTDECL(int) RTTimerCreateEx(PRTTIMER ppTimer, uint64_t u64NanoInterval, unsigned fFlags, PFNRTTIMER pfnTimer, void pvUser)
823	{
824	PRTTIMER pTimer;
825	RTCPUID iCpu;
826	unsigned cCpus;
827
828	*ppTimer = NULL;
829
830	/*
831	* Validate flags.
832	*/
833	if (!RTTIMER_FLAGS_ARE_VALID(fFlags))
834	return VERR_INVALID_PARAMETER;
835	if ( (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC)
836	&& (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL
837	&& !RTMpIsCpuOnline(fFlags & RTTIMER_FLAGS_CPU_MASK))
838	return (fFlags & RTTIMER_FLAGS_CPU_MASK) > RTMpGetMaxCpuId()
839	? VERR_CPU_NOT_FOUND
840	: VERR_CPU_OFFLINE;
841
842	/*
843	* Allocate the timer handler.
844	*/
845	cCpus = 1;
846	#ifdef CONFIG_SMP
847	if ((fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL)
848	{
849	cCpus = RTMpGetMaxCpuId() + 1;
850	Assert(cCpus <= RTCPUSET_MAX_CPUS); /* On linux we have a 1:1 relationship between cpuid and set index. */
851	AssertReturn(u64NanoInterval, VERR_NOT_IMPLEMENTED); /* We don't implement single shot on all cpus, sorry. */
852	}
853	#endif
854
855	pTimer = (PRTTIMER)RTMemAllocZ(RT_OFFSETOF(RTTIMER, aSubTimers[cCpus]));
856	if (!pTimer)
857	return VERR_NO_MEMORY;
858
859	/*
860	* Initialize it.
861	*/
862	pTimer->u32Magic = RTTIMER_MAGIC;
863	pTimer->hSpinlock = NIL_RTSPINLOCK;
864	pTimer->fSuspended = true;
865	#ifdef CONFIG_SMP
866	pTimer->fSpecificCpu = (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC) && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL;
867	pTimer->fAllCpus = (fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL;
868	pTimer->idCpu = fFlags & RTTIMER_FLAGS_CPU_MASK;
869	#else
870	pTimer->fSpecificCpu = !!(fFlags & RTTIMER_FLAGS_CPU_SPECIFIC);
871	pTimer->idCpu = RTMpCpuId();
872	#endif
873	pTimer->cCpus = cCpus;
874	pTimer->pfnTimer = pfnTimer;
875	pTimer->pvUser = pvUser;
876	pTimer->u64NanoInterval = u64NanoInterval;
877	#ifndef RT_USE_LINUX_HRTIMER
878	pTimer->cJiffies = u64NanoInterval / RTTimerGetSystemGranularity();
879	if (pTimer->cJiffies * RTTimerGetSystemGranularity() != u64NanoInterval)
880	pTimer->cJiffies = 0;
881	#endif
882
883	for (iCpu = 0; iCpu < cCpus; iCpu++)
884	{
885	#ifdef RT_USE_LINUX_HRTIMER
886	hrtimer_init(&pTimer->aSubTimers[iCpu].LnxTimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
887	pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
888	#else
889	init_timer(&pTimer->aSubTimers[iCpu].LnxTimer);
890	pTimer->aSubTimers[iCpu].LnxTimer.data = (unsigned long)&pTimer->aSubTimers[iCpu];
891	pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
892	pTimer->aSubTimers[iCpu].LnxTimer.expires = jiffies;
893	#endif
894	pTimer->aSubTimers[iCpu].u64StartTS = 0;
895	pTimer->aSubTimers[iCpu].u64NextTS = 0;
896	pTimer->aSubTimers[iCpu].iTick = 0;
897	pTimer->aSubTimers[iCpu].pParent = pTimer;
898	pTimer->aSubTimers[iCpu].enmState = RTTIMERLNXSTATE_STOPPED;
899	}
900
901	#ifdef CONFIG_SMP
902	/*
903	* If this is running on ALL cpus, we'll have to register a callback
904	* for MP events (so timers can be started/stopped on cpus going
905	* online/offline). We also create the spinlock for syncrhonizing
906	* stop/start/mp-event.
907	*/
908	if (cCpus > 1)
909	{
910	int rc = RTSpinlockCreate(&pTimer->hSpinlock);
911	if (RT_SUCCESS(rc))
912	rc = RTMpNotificationRegister(rtTimerLinuxMpEvent, pTimer);
913	else
914	pTimer->hSpinlock = NIL_RTSPINLOCK;
915	if (RT_FAILURE(rc))
916	{
917	RTTimerDestroy(pTimer);
918	return rc;
919	}
920	}
921	#endif /* CONFIG_SMP */
922
923	*ppTimer = pTimer;
924	return VINF_SUCCESS;
925	}
926
927
928	RTDECL(uint32_t) RTTimerGetSystemGranularity(void)
929	{
930	#ifdef RT_USE_LINUX_HRTIMER
931	struct timespec Ts;
932	int rc = hrtimer_get_res(CLOCK_MONOTONIC, &Ts);
933	if (!rc)
934	{
935	Assert(!Ts.tv_sec);
936	return Ts.tv_nsec;
937	}
938	#endif
939	return 1000000000 / HZ; /* ns */
940	}
941
942
943	RTDECL(int) RTTimerRequestSystemGranularity(uint32_t u32Request, uint32_t *pu32Granted)
944	{
945	return VERR_NOT_SUPPORTED;
946	}
947
948
949	RTDECL(int) RTTimerReleaseSystemGranularity(uint32_t u32Granted)
950	{
951	return VERR_NOT_SUPPORTED;
952	}
953

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/timer-r0drv-linux.c@ 12591

Download in other formats: