timer-r0drv-linux.c@ 28469

Last change on this file since 28469 was 27199, checked in by vboxsync, 15 years ago
r0drv/linux/timer: hrtimer pinning fix for Linux >= 2.6.31 (code not active)
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 30.6 KB

Line
1	/* $Id: timer-r0drv-linux.c 27199 2010-03-09 10:08:44Z vboxsync $ */
2	/** @file
3	* IPRT - Timers, Ring-0 Driver, Linux.
4	*/
5
6	/*
7	* Copyright (C) 2006-2008 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* The contents of this file may alternatively be used under the terms
18	* of the Common Development and Distribution License Version 1.0
19	* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20	* VirtualBox OSE distribution, in which case the provisions of the
21	* CDDL are applicable instead of those of the GPL.
22	*
23	* You may elect to license modified versions of this file under the
24	* terms and conditions of either the GPL or the CDDL or both.
25	*
26	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
27	* Clara, CA 95054 USA or visit http://www.sun.com if you need
28	* additional information or have any questions.
29	*/
30
31
32	/*******************************************************************************
33	* Header Files *
34	*******************************************************************************/
35	#include "the-linux-kernel.h"
36	#include "internal/iprt.h"
37
38	#include <iprt/timer.h>
39	#include <iprt/time.h>
40	#include <iprt/mp.h>
41	#include <iprt/cpuset.h>
42	#include <iprt/spinlock.h>
43	#include <iprt/err.h>
44	#include <iprt/asm.h>
45	#include <iprt/assert.h>
46	#include <iprt/alloc.h>
47
48	#include "internal/magics.h"
49
50	/* We use the API of Linux 2.6.28+ (hrtimer_add_expires_ns()) */
51	#if !defined(RT_USE_LINUX_HRTIMER) \
52	&& LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28) \
53	&& 0 /* currently disabled */
54	# define RT_USE_LINUX_HRTIMER
55	#endif
56
57	/* This check must match the ktime usage in rtTimeGetSystemNanoTS() / time-r0drv-linux.c. */
58	#if defined(RT_USE_LINUX_HRTIMER) \
59	&& LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 28)
60	# error "RT_USE_LINUX_HRTIMER requires 2.6.28 or later, sorry."
61	#endif
62
63	#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31)
64	# define mod_timer_pinned mod_timer
65	# define HRTIMER_MODE_ABS_PINNED HRTIMER_MODE_ABS
66	#endif
67
68
69	/*******************************************************************************
70	* Structures and Typedefs *
71	*******************************************************************************/
72	/**
73	* Timer state machine.
74	*
75	* This is used to try handle the issues with MP events and
76	* timers that runs on all CPUs. It's relatively nasty :-/
77	*/
78	typedef enum RTTIMERLNXSTATE
79	{
80	/** Stopped. */
81	RTTIMERLNXSTATE_STOPPED = 0,
82	/** Transient state; next ACTIVE. */
83	RTTIMERLNXSTATE_STARTING,
84	/** Transient state; next ACTIVE. (not really necessary) */
85	RTTIMERLNXSTATE_MP_STARTING,
86	/** Active. */
87	RTTIMERLNXSTATE_ACTIVE,
88	/** Transient state; next STOPPED. */
89	RTTIMERLNXSTATE_STOPPING,
90	/** Transient state; next STOPPED. */
91	RTTIMERLNXSTATE_MP_STOPPING,
92	/** The usual 32-bit hack. */
93	RTTIMERLNXSTATE_32BIT_HACK = 0x7fffffff
94	} RTTIMERLNXSTATE;
95
96
97	/**
98	* A Linux sub-timer.
99	*/
100	typedef struct RTTIMERLNXSUBTIMER
101	{
102	/** The linux timer structure. */
103	#ifdef RT_USE_LINUX_HRTIMER
104	struct hrtimer LnxTimer;
105	#else
106	struct timer_list LnxTimer;
107	/** The start of the current run (ns).
108	* This is used to calculate when the timer ought to fire the next time. */
109	uint64_t u64StartTS;
110	/** The start of the current run (ns).
111	* This is used to calculate when the timer ought to fire the next time. */
112	uint64_t u64NextTS;
113	#endif
114	/** The current tick number (since u64StartTS). */
115	uint64_t iTick;
116	/** Pointer to the parent timer. */
117	PRTTIMER pParent;
118	#ifndef RT_USE_LINUX_HRTIMER
119	/** The u64NextTS in jiffies. */
120	unsigned long ulNextJiffies;
121	#endif
122	/** The current sub-timer state. */
123	RTTIMERLNXSTATE volatile enmState;
124	} RTTIMERLNXSUBTIMER;
125	/** Pointer to a linux sub-timer. */
126	typedef RTTIMERLNXSUBTIMER *PRTTIMERLNXSUBTIMER;
127	AssertCompileMemberOffset(RTTIMERLNXSUBTIMER, LnxTimer, 0);
128
129
130	/**
131	* The internal representation of an Linux timer handle.
132	*/
133	typedef struct RTTIMER
134	{
135	/** Magic.
136	* This is RTTIMER_MAGIC, but changes to something else before the timer
137	* is destroyed to indicate clearly that thread should exit. */
138	uint32_t volatile u32Magic;
139	/** Spinlock synchronizing the fSuspended and MP event handling.
140	* This is NIL_RTSPINLOCK if cCpus == 1. */
141	RTSPINLOCK hSpinlock;
142	/** Flag indicating that the timer is suspended. */
143	bool volatile fSuspended;
144	/** Whether the timer must run on one specific CPU or not. */
145	bool fSpecificCpu;
146	#ifdef CONFIG_SMP
147	/** Whether the timer must run on all CPUs or not. */
148	bool fAllCpus;
149	#endif /* else: All -> specific on non-SMP kernels */
150	/** The CPU it must run on if fSpecificCpu is set. */
151	RTCPUID idCpu;
152	/** The number of CPUs this timer should run on. */
153	RTCPUID cCpus;
154	/** Callback. */
155	PFNRTTIMER pfnTimer;
156	/** User argument. */
157	void *pvUser;
158	/** The timer interval. 0 if one-shot. */
159	uint64_t u64NanoInterval;
160	#ifndef RT_USE_LINUX_HRTIMER
161	/** This is set to the number of jiffies between ticks if the interval is
162	* an exact number of jiffies. */
163	unsigned long cJiffies;
164	#endif
165	/** Sub-timers.
166	* Normally there is just one, but for RTTIMER_FLAGS_CPU_ALL this will contain
167	* an entry for all possible cpus. In that case the index will be the same as
168	* for the RTCpuSet. */
169	RTTIMERLNXSUBTIMER aSubTimers[1];
170	} RTTIMER;
171
172
173	/**
174	* A rtTimerLinuxStartOnCpu and rtTimerLinuxStartOnCpu argument package.
175	*/
176	typedef struct RTTIMERLINUXSTARTONCPUARGS
177	{
178	/** The current time (RTTimeNanoTS). */
179	uint64_t u64Now;
180	/** When to start firing (delta). */
181	uint64_t u64First;
182	} RTTIMERLINUXSTARTONCPUARGS;
183	/** Pointer to a rtTimerLinuxStartOnCpu argument package. */
184	typedef RTTIMERLINUXSTARTONCPUARGS *PRTTIMERLINUXSTARTONCPUARGS;
185
186
187	/**
188	* Sets the state.
189	*/
190	DECLINLINE(void) rtTimerLnxSetState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState)
191	{
192	ASMAtomicWriteU32((uint32_t volatile *)penmState, enmNewState);
193	}
194
195
196	/**
197	* Sets the state if it has a certain value.
198	*
199	* @return true if xchg was done.
200	* @return false if xchg wasn't done.
201	*/
202	DECLINLINE(bool) rtTimerLnxCmpXchgState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState, RTTIMERLNXSTATE enmCurState)
203	{
204	return ASMAtomicCmpXchgU32((uint32_t volatile *)penmState, enmNewState, enmCurState);
205	}
206
207
208	/**
209	* Gets the state.
210	*/
211	DECLINLINE(RTTIMERLNXSTATE) rtTimerLnxGetState(RTTIMERLNXSTATE volatile *penmState)
212	{
213	return (RTTIMERLNXSTATE)ASMAtomicUoReadU32((uint32_t volatile *)penmState);
214	}
215
216
217	#ifdef RT_USE_LINUX_HRTIMER
218	/**
219	* Converts a nano second time stamp to ktime_t.
220	*
221	* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts().
222	*
223	* @returns ktime_t.
224	* @param cNanoSecs Nanoseconds.
225	*/
226	DECLINLINE(ktime_t) rtTimerLnxNanoToKt(uint64_t cNanoSecs)
227	{
228	/* With some luck the compiler optimizes the division out of this... (Bet it doesn't.) */
229	return ktime_set(cNanoSecs / 1000000000, cNanoSecs % 1000000000);
230	}
231
232	/**
233	* Converts ktime_t to a nano second time stamp.
234	*
235	* ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts().
236	*
237	* @returns nano second time stamp.
238	* @param Kt ktime_t.
239	*/
240	DECLINLINE(uint64_t) rtTimerLnxKtToNano(ktime_t Kt)
241	{
242	return ktime_to_ns(Kt);
243	}
244
245	#else /* ! RT_USE_LINUX_HRTIMER */
246
247	/**
248	* Converts a nano second interval to jiffies.
249	*
250	* @returns Jiffies.
251	* @param cNanoSecs Nanoseconds.
252	*/
253	DECLINLINE(unsigned long) rtTimerLnxNanoToJiffies(uint64_t cNanoSecs)
254	{
255	/* this can be made even better... */
256	if (cNanoSecs > (uint64_t)TICK_NSEC * MAX_JIFFY_OFFSET)
257	return MAX_JIFFY_OFFSET;
258	# if ARCH_BITS == 32
259	if (RT_LIKELY(cNanoSecs <= UINT32_MAX))
260	return ((uint32_t)cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
261	# endif
262	return (cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
263	}
264	#endif /* ! RT_USE_LINUX_HRTIMER */
265
266
267	/**
268	* Starts a sub-timer (RTTimerStart).
269	*
270	* @param pSubTimer The sub-timer to start.
271	* @param u64Now The current timestamp (RTTimeNanoTS()).
272	* @param u64First The interval from u64Now to the first time the timer should fire.
273	* @param fPinned true = timer pinned to a specific CPU,
274	* false = timer can migrate between CPUs
275	*/
276	static void rtTimerLnxStartSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, uint64_t u64Now, uint64_t u64First, bool fPinned)
277	{
278	/*
279	* Calc when it should start firing.
280	*/
281	uint64_t u64NextTS = u64Now + u64First;
282	#ifndef RT_USE_LINUX_HRTIMER
283	pSubTimer->u64StartTS = u64NextTS;
284	pSubTimer->u64NextTS = u64NextTS;
285	#endif
286
287	pSubTimer->iTick = 0;
288
289	#ifdef RT_USE_LINUX_HRTIMER
290	hrtimer_start(&pSubTimer->LnxTimer, rtTimerLnxNanoToKt(u64NextTS),
291	fPinned ? HRTIMER_MODE_ABS_PINNED : HRTIMER_MODE_ABS);
292	#else
293	{
294	unsigned long cJiffies = !u64First ? 0 : rtTimerLnxNanoToJiffies(u64First);
295	pSubTimer->ulNextJiffies = jiffies + cJiffies;
296	# ifdef CONFIG_SMP
297	if (fPinned)
298	mod_timer_pinned(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
299	else
300	# endif
301	mod_timer(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
302	}
303	#endif
304
305	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE);
306	}
307
308
309	/**
310	* Stops a sub-timer (RTTimerStart and rtTimerLinuxMpEvent()).
311	*
312	* @param pSubTimer The sub-timer.
313	*/
314	static void rtTimerLnxStopSubTimer(PRTTIMERLNXSUBTIMER pSubTimer)
315	{
316	#ifdef RT_USE_LINUX_HRTIMER
317	hrtimer_cancel(&pSubTimer->LnxTimer);
318	#else
319	if (timer_pending(&pSubTimer->LnxTimer))
320	del_timer_sync(&pSubTimer->LnxTimer);
321	#endif
322
323	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
324	}
325
326
327	#ifdef RT_USE_LINUX_HRTIMER
328	/**
329	* Timer callback function.
330	* @returns HRTIMER_NORESTART or HRTIMER_RESTART depending on whether it's a one-shot or interval timer.
331	* @param pHrTimer Pointer to the sub-timer structure.
332	*/
333	static enum hrtimer_restart rtTimerLinuxCallback(struct hrtimer *pHrTimer)
334	#else
335	/**
336	* Timer callback function.
337	* @param ulUser Address of the sub-timer structure.
338	*/
339	static void rtTimerLinuxCallback(unsigned long ulUser)
340	#endif
341	{
342	#ifdef RT_USE_LINUX_HRTIMER
343	enum hrtimer_restart rc;
344	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)pHrTimer;
345	#else
346	PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)ulUser;
347	#endif
348	PRTTIMER pTimer = pSubTimer->pParent;
349
350	/*
351	* Don't call the handler if the timer has been suspended.
352	* Also, when running on all CPUS, make sure we don't call out twice
353	* on a CPU because of timer migration.
354	*
355	* For the specific cpu case, we're just ignoring timer migration for now... (bad)
356	*/
357	if ( ASMAtomicUoReadBool(&pTimer->fSuspended)
358	#ifdef CONFIG_SMP
359	\|\| ( pTimer->fAllCpus
360	&& (RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != RTMpCpuId())
361	#endif
362	)
363	{
364	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
365	# ifdef RT_USE_LINUX_HRTIMER
366	rc = HRTIMER_NORESTART;
367	# endif
368	}
369	else if (!pTimer->u64NanoInterval)
370	{
371	/*
372	* One shot timer, stop it before dispatching it.
373	*/
374	if (pTimer->cCpus == 1)
375	ASMAtomicWriteBool(&pTimer->fSuspended, true);
376	rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
377	#ifdef RT_USE_LINUX_HRTIMER
378	rc = HRTIMER_NORESTART;
379	#else
380	/* detached before we're called, nothing to do for this case. */
381	#endif
382
383	pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
384	}
385	else
386	{
387	const uint64_t iTick = ++pSubTimer->iTick;
388
389	#ifdef RT_USE_LINUX_HRTIMER
390	hrtimer_add_expires_ns(&pSubTimer->LnxTimer, pTimer->u64NanoInterval);
391	rc = HRTIMER_RESTART;
392	#else
393	const uint64_t u64NanoTS = RTTimeNanoTS();
394
395	/*
396	* Interval timer, calculate the next timeout and re-arm it.
397	*
398	* The first time around, we'll re-adjust the u64StartTS to
399	* try prevent some jittering if we were started at a bad time.
400	* This may of course backfire with highres timers...
401	*/
402	if (RT_UNLIKELY(iTick == 1))
403	{
404	pSubTimer->u64StartTS = pSubTimer->u64NextTS = u64NanoTS;
405	pSubTimer->ulNextJiffies = jiffies;
406	}
407
408	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
409	if (pTimer->cJiffies)
410	{
411	pSubTimer->ulNextJiffies += pTimer->cJiffies;
412	/* Prevent overflows when the jiffies counter wraps around.
413	* Special thanks to Ken Preslan for helping debugging! */
414	while (time_before(pSubTimer->ulNextJiffies, jiffies))
415	{
416	pSubTimer->ulNextJiffies += pTimer->cJiffies;
417	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
418	}
419	}
420	else
421	{
422	while (pSubTimer->u64NextTS < u64NanoTS)
423	pSubTimer->u64NextTS += pTimer->u64NanoInterval;
424	pSubTimer->ulNextJiffies = jiffies + rtTimerLnxNanoToJiffies(pSubTimer->u64NextTS - u64NanoTS);
425	}
426
427	# ifdef CONFIG_SMP
428	if (pTimer->fSpecificCpu \|\| pTimer->fAllCpus)
429	mod_timer_pinned(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
430	else
431	# endif
432	mod_timer(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
433	#endif
434
435	/*
436	* Run the timer.
437	*/
438	pTimer->pfnTimer(pTimer, pTimer->pvUser, iTick);
439	}
440
441	#ifdef RT_USE_LINUX_HRTIMER
442	return rc;
443	#endif
444	}
445
446
447	#ifdef CONFIG_SMP
448
449	/**
450	* Per-cpu callback function (RTMpOnAll/RTMpOnSpecific).
451	*
452	* @param idCpu The current CPU.
453	* @param pvUser1 Pointer to the timer.
454	* @param pvUser2 Pointer to the argument structure.
455	*/
456	static DECLCALLBACK(void) rtTimerLnxStartAllOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
457	{
458	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
459	PRTTIMER pTimer = (PRTTIMER)pvUser1;
460	Assert(idCpu < pTimer->cCpus);
461	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[idCpu], pArgs->u64Now, pArgs->u64First, true /fPinned/);
462	}
463
464
465	/**
466	* Worker for RTTimerStart() that takes care of the ugly bit.s
467	*
468	* @returns RTTimerStart() return value.
469	* @param pTimer The timer.
470	* @param pArgs The argument structure.
471	*/
472	static int rtTimerLnxStartAll(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs)
473	{
474	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
475	RTCPUID iCpu;
476	RTCPUSET OnlineSet;
477	RTCPUSET OnlineSet2;
478	int rc2;
479
480	/*
481	* Prepare all the sub-timers for the startup and then flag the timer
482	* as a whole as non-suspended, make sure we get them all before
483	* clearing fSuspended as the MP handler will be waiting on this
484	* should something happen while we're looping.
485	*/
486	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
487
488	do
489	{
490	RTMpGetOnlineSet(&OnlineSet);
491	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
492	{
493	Assert(pTimer->aSubTimers[iCpu].enmState != RTTIMERLNXSTATE_MP_STOPPING);
494	rtTimerLnxSetState(&pTimer->aSubTimers[iCpu].enmState,
495	RTCpuSetIsMember(&OnlineSet, iCpu)
496	? RTTIMERLNXSTATE_STARTING
497	: RTTIMERLNXSTATE_STOPPED);
498	}
499	} while (!RTCpuSetIsEqual(&OnlineSet, RTMpGetOnlineSet(&OnlineSet2)));
500
501	ASMAtomicWriteBool(&pTimer->fSuspended, false);
502
503	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
504
505	/*
506	* Start them (can't find any exported function that allows me to
507	* do this without the cross calls).
508	*/
509	pArgs->u64Now = RTTimeNanoTS();
510	rc2 = RTMpOnAll(rtTimerLnxStartAllOnCpu, pTimer, pArgs);
511	AssertRC(rc2); /* screw this if it fails. */
512
513	/*
514	* Reset the sub-timers who didn't start up (ALL CPUs case).
515	* CPUs that comes online between the
516	*/
517	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
518
519	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
520	if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_STARTING))
521	{
522	/** @todo very odd case for a rainy day. Cpus that temporarily went offline while
523	* we were between calls needs to nudged as the MP handler will ignore events for
524	* them because of the STARTING state. This is an extremely unlikely case - not that
525	* that means anything in my experience... ;-) */
526	}
527
528	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
529
530	return VINF_SUCCESS;
531	}
532
533
534	/**
535	* Worker for RTTimerStop() that takes care of the ugly SMP bits.
536	*
537	* @returns RTTimerStop() return value.
538	* @param pTimer The timer (valid).
539	*/
540	static int rtTimerLnxStopAll(PRTTIMER pTimer)
541	{
542	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
543	RTCPUID iCpu;
544
545
546	/*
547	* Mark the timer as suspended and flag all timers as stopping, except
548	* for those being stopped by an MP event.
549	*/
550	RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
551
552	ASMAtomicWriteBool(&pTimer->fSuspended, true);
553	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
554	{
555	RTTIMERLNXSTATE enmState;
556	do
557	{
558	enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
559	if ( enmState == RTTIMERLNXSTATE_STOPPED
560	\|\| enmState == RTTIMERLNXSTATE_MP_STOPPING)
561	break;
562	Assert(enmState == RTTIMERLNXSTATE_ACTIVE);
563	} while (!rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPING, enmState));
564	}
565
566	RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
567
568	/*
569	* Do the actual stopping. Fortunately, this doesn't require any IPIs.
570	* Unfortunately it cannot be done synchronously from within the spinlock,
571	* because we might end up in an active waiting for a handler to complete.
572	*/
573	for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
574	if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) == RTTIMERLNXSTATE_STOPPING)
575	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[iCpu]);
576
577	return VINF_SUCCESS;
578	}
579
580
581	/**
582	* Per-cpu callback function (RTMpOnSpecific) used by rtTimerLinuxMpEvent()
583	* to start a sub-timer on a cpu that just have come online.
584	*
585	* @param idCpu The current CPU.
586	* @param pvUser1 Pointer to the timer.
587	* @param pvUser2 Pointer to the argument structure.
588	*/
589	static DECLCALLBACK(void) rtTimerLinuxMpStartOnCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
590	{
591	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
592	PRTTIMER pTimer = (PRTTIMER)pvUser1;
593	RTSPINLOCK hSpinlock;
594	Assert(idCpu < pTimer->cCpus);
595
596	/*
597	* We have to be kind of careful here as we might be racing RTTimerStop
598	* (and/or RTTimerDestroy, thus the paranoia.
599	*/
600	hSpinlock = pTimer->hSpinlock;
601	if ( hSpinlock != NIL_RTSPINLOCK
602	&& pTimer->u32Magic == RTTIMER_MAGIC)
603	{
604	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
605	RTSpinlockAcquire(hSpinlock, &Tmp);
606
607	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
608	&& pTimer->u32Magic == RTTIMER_MAGIC)
609	{
610	/* We're sane and the timer is not suspended yet. */
611	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
612	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
613	rtTimerLnxStartSubTimer(pSubTimer, pArgs->u64Now, pArgs->u64First, true /fPinned/);
614	}
615
616	RTSpinlockRelease(hSpinlock, &Tmp);
617	}
618	}
619
620
621	/**
622	* MP event notification callback.
623	*
624	* @param enmEvent The event.
625	* @param idCpu The cpu it applies to.
626	* @param pvUser The timer.
627	*/
628	static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
629	{
630	PRTTIMER pTimer = (PRTTIMER)pvUser;
631	PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
632	RTSPINLOCK hSpinlock;
633	RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
634
635	Assert(idCpu < pTimer->cCpus);
636
637	/*
638	* Some initial paranoia.
639	*/
640	if (pTimer->u32Magic != RTTIMER_MAGIC)
641	return;
642	hSpinlock = pTimer->hSpinlock;
643	if (hSpinlock == NIL_RTSPINLOCK)
644	return;
645
646	RTSpinlockAcquire(hSpinlock, &Tmp);
647
648	/* Is it active? */
649	if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
650	&& pTimer->u32Magic == RTTIMER_MAGIC)
651	{
652	switch (enmEvent)
653	{
654	/*
655	* Try do it without leaving the spin lock, but if we have to, retake it
656	* when we're on the right cpu.
657	*/
658	case RTMPEVENT_ONLINE:
659	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
660	{
661	RTTIMERLINUXSTARTONCPUARGS Args;
662	Args.u64Now = RTTimeNanoTS();
663	Args.u64First = 0;
664
665	if (RTMpCpuId() == idCpu)
666	rtTimerLnxStartSubTimer(pSubTimer, Args.u64Now, Args.u64First, true /fPinned/);
667	else
668	{
669	rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); /* we'll recheck it. */
670	RTSpinlockRelease(hSpinlock, &Tmp);
671
672	RTMpOnSpecific(idCpu, rtTimerLinuxMpStartOnCpu, pTimer, &Args);
673	return; /* we've left the spinlock */
674	}
675	}
676	break;
677
678	/*
679	* The CPU is (going) offline, make sure the sub-timer is stopped.
680	*
681	* Linux will migrate it to a different CPU, but we don't want this. The
682	* timer function is checking for this.
683	*/
684	case RTMPEVENT_OFFLINE:
685	if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STOPPING, RTTIMERLNXSTATE_ACTIVE))
686	{
687	RTSpinlockRelease(hSpinlock, &Tmp);
688
689	rtTimerLnxStopSubTimer(pSubTimer);
690	return; /* we've left the spinlock */
691	}
692	break;
693	}
694	}
695
696	RTSpinlockRelease(hSpinlock, &Tmp);
697	}
698
699	#endif /* CONFIG_SMP */
700
701
702	/**
703	* Callback function use by RTTimerStart via RTMpOnSpecific to start
704	* a timer running on a specific CPU.
705	*
706	* @param idCpu The current CPU.
707	* @param pvUser1 Pointer to the timer.
708	* @param pvUser2 Pointer to the argument structure.
709	*/
710	static DECLCALLBACK(void) rtTimerLnxStartOnSpecificCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
711	{
712	PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
713	PRTTIMER pTimer = (PRTTIMER)pvUser1;
714	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], pArgs->u64Now, pArgs->u64First, true /fPinned/);
715	}
716
717
718	RTDECL(int) RTTimerStart(PRTTIMER pTimer, uint64_t u64First)
719	{
720	RTTIMERLINUXSTARTONCPUARGS Args;
721	int rc2;
722
723	/*
724	* Validate.
725	*/
726	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
727	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
728
729	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
730	return VERR_TIMER_ACTIVE;
731
732	Args.u64First = u64First;
733	#ifdef CONFIG_SMP
734	/*
735	* Omnit timer?
736	*/
737	if (pTimer->fAllCpus)
738	return rtTimerLnxStartAll(pTimer, &Args);
739	#endif
740
741	/*
742	* Simple timer - Pretty straight forward.
743	*/
744	Args.u64Now = RTTimeNanoTS();
745	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STARTING);
746	ASMAtomicWriteBool(&pTimer->fSuspended, false);
747	if (!pTimer->fSpecificCpu)
748	rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], Args.u64Now, Args.u64First, false /fPinned/);
749	else
750	{
751	rc2 = RTMpOnSpecific(pTimer->idCpu, rtTimerLnxStartOnSpecificCpu, pTimer, &Args);
752	if (RT_FAILURE(rc2))
753	{
754	/* Suspend it, the cpu id is probably invalid or offline. */
755	ASMAtomicWriteBool(&pTimer->fSuspended, true);
756	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPED);
757	return rc2;
758	}
759	}
760
761	return VINF_SUCCESS;
762	}
763	RT_EXPORT_SYMBOL(RTTimerStart);
764
765
766	RTDECL(int) RTTimerStop(PRTTIMER pTimer)
767	{
768
769	/*
770	* Validate.
771	*/
772	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
773	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
774
775	if (ASMAtomicUoReadBool(&pTimer->fSuspended))
776	return VERR_TIMER_SUSPENDED;
777
778	#ifdef CONFIG_SMP
779	/*
780	* Omni timer?
781	*/
782	if (pTimer->fAllCpus)
783	return rtTimerLnxStopAll(pTimer);
784	#endif
785
786	/*
787	* Simple timer.
788	*/
789	ASMAtomicWriteBool(&pTimer->fSuspended, true);
790	rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPING);
791	rtTimerLnxStopSubTimer(&pTimer->aSubTimers[0]);
792
793	return VINF_SUCCESS;
794	}
795	RT_EXPORT_SYMBOL(RTTimerStop);
796
797
798	RTDECL(int) RTTimerDestroy(PRTTIMER pTimer)
799	{
800	RTSPINLOCK hSpinlock;
801
802	/* It's ok to pass NULL pointer. */
803	if (pTimer == /NIL_RTTIMER/ NULL)
804	return VINF_SUCCESS;
805	AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
806	AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
807
808	/*
809	* Remove the MP notifications first because it'll reduce the risk of
810	* us overtaking any MP event that might theoretically be racing us here.
811	*/
812	hSpinlock = pTimer->hSpinlock;
813	#ifdef CONFIG_SMP
814	if ( pTimer->cCpus > 1
815	&& hSpinlock != NIL_RTSPINLOCK)
816	{
817	int rc = RTMpNotificationDeregister(rtTimerLinuxMpEvent, pTimer);
818	AssertRC(rc);
819	}
820	#endif /* CONFIG_SMP */
821
822	/*
823	* Stop the timer if it's running.
824	*/
825	if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
826	RTTimerStop(pTimer);
827
828	/*
829	* Uninitialize the structure and free the associated resources.
830	* The spinlock goes last.
831	*/
832	ASMAtomicWriteU32(&pTimer->u32Magic, ~RTTIMER_MAGIC);
833	RTMemFree(pTimer);
834	if (hSpinlock != NIL_RTSPINLOCK)
835	RTSpinlockDestroy(hSpinlock);
836
837	return VINF_SUCCESS;
838	}
839	RT_EXPORT_SYMBOL(RTTimerDestroy);
840
841
842	RTDECL(int) RTTimerCreateEx(PRTTIMER ppTimer, uint64_t u64NanoInterval, unsigned fFlags, PFNRTTIMER pfnTimer, void pvUser)
843	{
844	PRTTIMER pTimer;
845	RTCPUID iCpu;
846	unsigned cCpus;
847
848	*ppTimer = NULL;
849
850	/*
851	* Validate flags.
852	*/
853	if (!RTTIMER_FLAGS_ARE_VALID(fFlags))
854	return VERR_INVALID_PARAMETER;
855	if ( (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC)
856	&& (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL
857	&& !RTMpIsCpuOnline(fFlags & RTTIMER_FLAGS_CPU_MASK))
858	return (fFlags & RTTIMER_FLAGS_CPU_MASK) > RTMpGetMaxCpuId()
859	? VERR_CPU_NOT_FOUND
860	: VERR_CPU_OFFLINE;
861
862	/*
863	* Allocate the timer handler.
864	*/
865	cCpus = 1;
866	#ifdef CONFIG_SMP
867	if ((fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL)
868	{
869	cCpus = RTMpGetMaxCpuId() + 1;
870	Assert(cCpus <= RTCPUSET_MAX_CPUS); /* On linux we have a 1:1 relationship between cpuid and set index. */
871	AssertReturn(u64NanoInterval, VERR_NOT_IMPLEMENTED); /* We don't implement single shot on all cpus, sorry. */
872	}
873	#endif
874
875	pTimer = (PRTTIMER)RTMemAllocZ(RT_OFFSETOF(RTTIMER, aSubTimers[cCpus]));
876	if (!pTimer)
877	return VERR_NO_MEMORY;
878
879	/*
880	* Initialize it.
881	*/
882	pTimer->u32Magic = RTTIMER_MAGIC;
883	pTimer->hSpinlock = NIL_RTSPINLOCK;
884	pTimer->fSuspended = true;
885	#ifdef CONFIG_SMP
886	pTimer->fSpecificCpu = (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC) && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL;
887	pTimer->fAllCpus = (fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL;
888	pTimer->idCpu = fFlags & RTTIMER_FLAGS_CPU_MASK;
889	#else
890	pTimer->fSpecificCpu = !!(fFlags & RTTIMER_FLAGS_CPU_SPECIFIC);
891	pTimer->idCpu = RTMpCpuId();
892	#endif
893	pTimer->cCpus = cCpus;
894	pTimer->pfnTimer = pfnTimer;
895	pTimer->pvUser = pvUser;
896	pTimer->u64NanoInterval = u64NanoInterval;
897	#ifndef RT_USE_LINUX_HRTIMER
898	pTimer->cJiffies = u64NanoInterval / RTTimerGetSystemGranularity();
899	if (pTimer->cJiffies * RTTimerGetSystemGranularity() != u64NanoInterval)
900	pTimer->cJiffies = 0;
901	#endif
902
903	for (iCpu = 0; iCpu < cCpus; iCpu++)
904	{
905	#ifdef RT_USE_LINUX_HRTIMER
906	hrtimer_init(&pTimer->aSubTimers[iCpu].LnxTimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
907	pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
908	#else
909	init_timer(&pTimer->aSubTimers[iCpu].LnxTimer);
910	pTimer->aSubTimers[iCpu].LnxTimer.data = (unsigned long)&pTimer->aSubTimers[iCpu];
911	pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
912	pTimer->aSubTimers[iCpu].LnxTimer.expires = jiffies;
913	pTimer->aSubTimers[iCpu].u64StartTS = 0;
914	pTimer->aSubTimers[iCpu].u64NextTS = 0;
915	#endif
916	pTimer->aSubTimers[iCpu].iTick = 0;
917	pTimer->aSubTimers[iCpu].pParent = pTimer;
918	pTimer->aSubTimers[iCpu].enmState = RTTIMERLNXSTATE_STOPPED;
919	}
920
921	#ifdef CONFIG_SMP
922	/*
923	* If this is running on ALL cpus, we'll have to register a callback
924	* for MP events (so timers can be started/stopped on cpus going
925	* online/offline). We also create the spinlock for syncrhonizing
926	* stop/start/mp-event.
927	*/
928	if (cCpus > 1)
929	{
930	int rc = RTSpinlockCreate(&pTimer->hSpinlock);
931	if (RT_SUCCESS(rc))
932	rc = RTMpNotificationRegister(rtTimerLinuxMpEvent, pTimer);
933	else
934	pTimer->hSpinlock = NIL_RTSPINLOCK;
935	if (RT_FAILURE(rc))
936	{
937	RTTimerDestroy(pTimer);
938	return rc;
939	}
940	}
941	#endif /* CONFIG_SMP */
942
943	*ppTimer = pTimer;
944	return VINF_SUCCESS;
945	}
946	RT_EXPORT_SYMBOL(RTTimerCreateEx);
947
948
949	RTDECL(uint32_t) RTTimerGetSystemGranularity(void)
950	{
951	#ifdef RT_USE_LINUX_HRTIMER
952	struct timespec Ts;
953	int rc = hrtimer_get_res(CLOCK_MONOTONIC, &Ts);
954	if (!rc)
955	{
956	Assert(!Ts.tv_sec);
957	return Ts.tv_nsec;
958	}
959	#endif
960	return 1000000000 / HZ; /* ns */
961	}
962	RT_EXPORT_SYMBOL(RTTimerGetSystemGranularity);
963
964
965	RTDECL(int) RTTimerRequestSystemGranularity(uint32_t u32Request, uint32_t *pu32Granted)
966	{
967	return VERR_NOT_SUPPORTED;
968	}
969	RT_EXPORT_SYMBOL(RTTimerRequestSystemGranularity);
970
971
972	RTDECL(int) RTTimerReleaseSystemGranularity(uint32_t u32Granted)
973	{
974	return VERR_NOT_SUPPORTED;
975	}
976	RT_EXPORT_SYMBOL(RTTimerReleaseSystemGranularity);
977

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/timer-r0drv-linux.c@ 28469

Download in other formats: