VirtualBox

source: vbox/trunk/src/VBox/Runtime/r0drv/linux/mp-r0drv-linux.c@ 58591

Last change on this file since 58591 was 57358, checked in by vboxsync, 9 years ago

*: scm cleanup run.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 15.4 KB
Line 
1/* $Id: mp-r0drv-linux.c 57358 2015-08-14 15:16:38Z vboxsync $ */
2/** @file
3 * IPRT - Multiprocessor, Ring-0 Driver, Linux.
4 */
5
6/*
7 * Copyright (C) 2008-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*********************************************************************************************************************************
29* Header Files *
30*********************************************************************************************************************************/
31#include "the-linux-kernel.h"
32#include "internal/iprt.h"
33
34#include <iprt/mp.h>
35#include <iprt/cpuset.h>
36#include <iprt/err.h>
37#include <iprt/asm.h>
38#include <iprt/thread.h>
39#include "r0drv/mp-r0drv.h"
40
41
42RTDECL(RTCPUID) RTMpCpuId(void)
43{
44 return smp_processor_id();
45}
46RT_EXPORT_SYMBOL(RTMpCpuId);
47
48
49RTDECL(int) RTMpCurSetIndex(void)
50{
51 return smp_processor_id();
52}
53RT_EXPORT_SYMBOL(RTMpCurSetIndex);
54
55
56RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu)
57{
58 return *pidCpu = smp_processor_id();
59}
60RT_EXPORT_SYMBOL(RTMpCurSetIndexAndId);
61
62
63RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
64{
65 return idCpu < RTCPUSET_MAX_CPUS && idCpu < NR_CPUS ? (int)idCpu : -1;
66}
67RT_EXPORT_SYMBOL(RTMpCpuIdToSetIndex);
68
69
70RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
71{
72 return iCpu < NR_CPUS ? (RTCPUID)iCpu : NIL_RTCPUID;
73}
74RT_EXPORT_SYMBOL(RTMpCpuIdFromSetIndex);
75
76
77RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
78{
79 return NR_CPUS - 1; //???
80}
81RT_EXPORT_SYMBOL(RTMpGetMaxCpuId);
82
83
84RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
85{
86#if defined(CONFIG_SMP)
87 if (RT_UNLIKELY(idCpu >= NR_CPUS))
88 return false;
89
90# if defined(cpu_possible)
91 return cpu_possible(idCpu);
92# else /* < 2.5.29 */
93 return idCpu < (RTCPUID)smp_num_cpus;
94# endif
95#else
96 return idCpu == RTMpCpuId();
97#endif
98}
99RT_EXPORT_SYMBOL(RTMpIsCpuPossible);
100
101
102RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
103{
104 RTCPUID idCpu;
105
106 RTCpuSetEmpty(pSet);
107 idCpu = RTMpGetMaxCpuId();
108 do
109 {
110 if (RTMpIsCpuPossible(idCpu))
111 RTCpuSetAdd(pSet, idCpu);
112 } while (idCpu-- > 0);
113 return pSet;
114}
115RT_EXPORT_SYMBOL(RTMpGetSet);
116
117
118RTDECL(RTCPUID) RTMpGetCount(void)
119{
120#ifdef CONFIG_SMP
121# if defined(CONFIG_HOTPLUG_CPU) /* introduced & uses cpu_present */
122 return num_present_cpus();
123# elif defined(num_possible_cpus)
124 return num_possible_cpus();
125# elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 0)
126 return smp_num_cpus;
127# else
128 RTCPUSET Set;
129 RTMpGetSet(&Set);
130 return RTCpuSetCount(&Set);
131# endif
132#else
133 return 1;
134#endif
135}
136RT_EXPORT_SYMBOL(RTMpGetCount);
137
138
139RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
140{
141#ifdef CONFIG_SMP
142 if (RT_UNLIKELY(idCpu >= NR_CPUS))
143 return false;
144# ifdef cpu_online
145 return cpu_online(idCpu);
146# else /* 2.4: */
147 return cpu_online_map & RT_BIT_64(idCpu);
148# endif
149#else
150 return idCpu == RTMpCpuId();
151#endif
152}
153RT_EXPORT_SYMBOL(RTMpIsCpuOnline);
154
155
156RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
157{
158#ifdef CONFIG_SMP
159 RTCPUID idCpu;
160
161 RTCpuSetEmpty(pSet);
162 idCpu = RTMpGetMaxCpuId();
163 do
164 {
165 if (RTMpIsCpuOnline(idCpu))
166 RTCpuSetAdd(pSet, idCpu);
167 } while (idCpu-- > 0);
168#else
169 RTCpuSetEmpty(pSet);
170 RTCpuSetAdd(pSet, RTMpCpuId());
171#endif
172 return pSet;
173}
174RT_EXPORT_SYMBOL(RTMpGetOnlineSet);
175
176
177RTDECL(RTCPUID) RTMpGetOnlineCount(void)
178{
179#ifdef CONFIG_SMP
180# if defined(num_online_cpus)
181 return num_online_cpus();
182# else
183 RTCPUSET Set;
184 RTMpGetOnlineSet(&Set);
185 return RTCpuSetCount(&Set);
186# endif
187#else
188 return 1;
189#endif
190}
191RT_EXPORT_SYMBOL(RTMpGetOnlineCount);
192
193
194RTDECL(bool) RTMpIsCpuWorkPending(void)
195{
196 /** @todo (not used on non-Windows platforms yet). */
197 return false;
198}
199RT_EXPORT_SYMBOL(RTMpIsCpuWorkPending);
200
201
202/**
203 * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER.
204 *
205 * @param pvInfo Pointer to the RTMPARGS package.
206 */
207static void rtmpLinuxWrapper(void *pvInfo)
208{
209 PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
210 ASMAtomicIncU32(&pArgs->cHits);
211 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
212}
213
214
215/**
216 * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER, does hit
217 * increment after calling the worker.
218 *
219 * @param pvInfo Pointer to the RTMPARGS package.
220 */
221static void rtmpLinuxWrapperPostInc(void *pvInfo)
222{
223 PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
224 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
225 ASMAtomicIncU32(&pArgs->cHits);
226}
227
228
229/**
230 * Wrapper between the native linux all-cpu callbacks and PFNRTWORKER.
231 *
232 * @param pvInfo Pointer to the RTMPARGS package.
233 */
234static void rtmpLinuxAllWrapper(void *pvInfo)
235{
236 PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
237 PRTCPUSET pWorkerSet = pArgs->pWorkerSet;
238 RTCPUID idCpu = RTMpCpuId();
239 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
240
241 if (RTCpuSetIsMember(pWorkerSet, idCpu))
242 {
243 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
244 RTCpuSetDel(pWorkerSet, idCpu);
245 }
246}
247
248
249RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
250{
251 IPRT_LINUX_SAVE_EFL_AC();
252 int rc;
253 RTMPARGS Args;
254 RTCPUSET OnlineSet;
255 RTCPUID idCpu;
256 uint32_t cLoops;
257
258 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
259
260 Args.pfnWorker = pfnWorker;
261 Args.pvUser1 = pvUser1;
262 Args.pvUser2 = pvUser2;
263 Args.idCpu = NIL_RTCPUID;
264 Args.cHits = 0;
265
266 RTThreadPreemptDisable(&PreemptState);
267 RTMpGetOnlineSet(&OnlineSet);
268 Args.pWorkerSet = &OnlineSet;
269 idCpu = RTMpCpuId();
270
271 if (RTCpuSetCount(&OnlineSet) > 1)
272 {
273 /* Fire the function on all other CPUs without waiting for completion. */
274#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
275 rc = smp_call_function(rtmpLinuxAllWrapper, &Args, 0 /* wait */);
276#else
277 rc = smp_call_function(rtmpLinuxAllWrapper, &Args, 0 /* retry */, 0 /* wait */);
278#endif
279 Assert(!rc); NOREF(rc);
280 }
281
282 /* Fire the function on this CPU. */
283 Args.pfnWorker(idCpu, Args.pvUser1, Args.pvUser2);
284 RTCpuSetDel(Args.pWorkerSet, idCpu);
285
286 /* Wait for all of them finish. */
287 cLoops = 64000;
288 while (!RTCpuSetIsEmpty(Args.pWorkerSet))
289 {
290 /* Periodically check if any CPU in the wait set has gone offline, if so update the wait set. */
291 if (!cLoops--)
292 {
293 RTCPUSET OnlineSetNow;
294 RTMpGetOnlineSet(&OnlineSetNow);
295 RTCpuSetAnd(Args.pWorkerSet, &OnlineSetNow);
296
297 cLoops = 64000;
298 }
299
300 ASMNopPause();
301 }
302
303 RTThreadPreemptRestore(&PreemptState);
304 IPRT_LINUX_RESTORE_EFL_AC();
305 return VINF_SUCCESS;
306}
307RT_EXPORT_SYMBOL(RTMpOnAll);
308
309
310RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
311{
312 IPRT_LINUX_SAVE_EFL_AC();
313 int rc;
314 RTMPARGS Args;
315
316 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
317 Args.pfnWorker = pfnWorker;
318 Args.pvUser1 = pvUser1;
319 Args.pvUser2 = pvUser2;
320 Args.idCpu = NIL_RTCPUID;
321 Args.cHits = 0;
322
323 RTThreadPreemptDisable(&PreemptState);
324#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
325 rc = smp_call_function(rtmpLinuxWrapper, &Args, 1 /* wait */);
326#else /* older kernels */
327 rc = smp_call_function(rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
328#endif /* older kernels */
329 RTThreadPreemptRestore(&PreemptState);
330
331 Assert(rc == 0); NOREF(rc);
332 IPRT_LINUX_RESTORE_EFL_AC();
333 return VINF_SUCCESS;
334}
335RT_EXPORT_SYMBOL(RTMpOnOthers);
336
337
338#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27)
339/**
340 * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER
341 * employed by RTMpOnPair on older kernels that lacks smp_call_function_many.
342 *
343 * @param pvInfo Pointer to the RTMPARGS package.
344 */
345static void rtMpLinuxOnPairWrapper(void *pvInfo)
346{
347 PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
348 RTCPUID idCpu = RTMpCpuId();
349
350 if ( idCpu == pArgs->idCpu
351 || idCpu == pArgs->idCpu2)
352 {
353 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
354 ASMAtomicIncU32(&pArgs->cHits);
355 }
356}
357#endif
358
359
360RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
361{
362 IPRT_LINUX_SAVE_EFL_AC();
363 int rc;
364 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
365
366 AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
367 AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
368
369 /*
370 * Check that both CPUs are online before doing the broadcast call.
371 */
372 RTThreadPreemptDisable(&PreemptState);
373 if ( RTMpIsCpuOnline(idCpu1)
374 && RTMpIsCpuOnline(idCpu2))
375 {
376 /*
377 * Use the smp_call_function variant taking a cpu mask where available,
378 * falling back on broadcast with filter. Slight snag if one of the
379 * CPUs is the one we're running on, we must do the call and the post
380 * call wait ourselves.
381 */
382#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
383 cpumask_t DstCpuMask;
384#endif
385 RTCPUID idCpuSelf = RTMpCpuId();
386 bool const fCallSelf = idCpuSelf == idCpu1 || idCpuSelf == idCpu2;
387 RTMPARGS Args;
388 Args.pfnWorker = pfnWorker;
389 Args.pvUser1 = pvUser1;
390 Args.pvUser2 = pvUser2;
391 Args.idCpu = idCpu1;
392 Args.idCpu2 = idCpu2;
393 Args.cHits = 0;
394
395#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
396 cpumask_clear(&DstCpuMask);
397 cpumask_set_cpu(idCpu1, &DstCpuMask);
398 cpumask_set_cpu(idCpu2, &DstCpuMask);
399#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
400 cpus_clear(DstCpuMask);
401 cpu_set(idCpu1, DstCpuMask);
402 cpu_set(idCpu2, DstCpuMask);
403#endif
404
405#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
406 smp_call_function_many(&DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
407 rc = 0;
408#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
409 rc = smp_call_function_many(&DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
410#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
411 rc = smp_call_function_mask(DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
412#else /* older kernels */
413 rc = smp_call_function(rtMpLinuxOnPairWrapper, &Args, 0 /* retry */, !fCallSelf /* wait */);
414#endif /* older kernels */
415 Assert(rc == 0);
416
417 /* Call ourselves if necessary and wait for the other party to be done. */
418 if (fCallSelf)
419 {
420 uint32_t cLoops = 0;
421 rtmpLinuxWrapper(&Args);
422 while (ASMAtomicReadU32(&Args.cHits) < 2)
423 {
424 if ((cLoops & 0x1ff) == 0 && !RTMpIsCpuOnline(idCpuSelf == idCpu1 ? idCpu2 : idCpu1))
425 break;
426 cLoops++;
427 ASMNopPause();
428 }
429 }
430
431 Assert(Args.cHits <= 2);
432 if (Args.cHits == 2)
433 rc = VINF_SUCCESS;
434 else if (Args.cHits == 1)
435 rc = VERR_NOT_ALL_CPUS_SHOWED;
436 else if (Args.cHits == 0)
437 rc = VERR_CPU_OFFLINE;
438 else
439 rc = VERR_CPU_IPE_1;
440 }
441 /*
442 * A CPU must be present to be considered just offline.
443 */
444 else if ( RTMpIsCpuPresent(idCpu1)
445 && RTMpIsCpuPresent(idCpu2))
446 rc = VERR_CPU_OFFLINE;
447 else
448 rc = VERR_CPU_NOT_FOUND;
449 RTThreadPreemptRestore(&PreemptState);;
450 IPRT_LINUX_RESTORE_EFL_AC();
451 return rc;
452}
453RT_EXPORT_SYMBOL(RTMpOnPair);
454
455
456RTDECL(bool) RTMpOnPairIsConcurrentExecSupported(void)
457{
458 return true;
459}
460RT_EXPORT_SYMBOL(RTMpOnPairIsConcurrentExecSupported);
461
462
463#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)
464/**
465 * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER
466 * employed by RTMpOnSpecific on older kernels that lacks smp_call_function_single.
467 *
468 * @param pvInfo Pointer to the RTMPARGS package.
469 */
470static void rtmpOnSpecificLinuxWrapper(void *pvInfo)
471{
472 PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
473 RTCPUID idCpu = RTMpCpuId();
474
475 if (idCpu == pArgs->idCpu)
476 {
477 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
478 ASMAtomicIncU32(&pArgs->cHits);
479 }
480}
481#endif
482
483
484RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
485{
486 IPRT_LINUX_SAVE_EFL_AC();
487 int rc;
488 RTMPARGS Args;
489
490 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
491 Args.pfnWorker = pfnWorker;
492 Args.pvUser1 = pvUser1;
493 Args.pvUser2 = pvUser2;
494 Args.idCpu = idCpu;
495 Args.cHits = 0;
496
497 if (!RTMpIsCpuPossible(idCpu))
498 return VERR_CPU_NOT_FOUND;
499
500 RTThreadPreemptDisable(&PreemptState);
501 if (idCpu != RTMpCpuId())
502 {
503 if (RTMpIsCpuOnline(idCpu))
504 {
505#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
506 rc = smp_call_function_single(idCpu, rtmpLinuxWrapper, &Args, 1 /* wait */);
507#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
508 rc = smp_call_function_single(idCpu, rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
509#else /* older kernels */
510 rc = smp_call_function(rtmpOnSpecificLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
511#endif /* older kernels */
512 Assert(rc == 0);
513 rc = Args.cHits ? VINF_SUCCESS : VERR_CPU_OFFLINE;
514 }
515 else
516 rc = VERR_CPU_OFFLINE;
517 }
518 else
519 {
520 rtmpLinuxWrapper(&Args);
521 rc = VINF_SUCCESS;
522 }
523 RTThreadPreemptRestore(&PreemptState);;
524
525 NOREF(rc);
526 IPRT_LINUX_RESTORE_EFL_AC();
527 return rc;
528}
529RT_EXPORT_SYMBOL(RTMpOnSpecific);
530
531
532#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
533/**
534 * Dummy callback used by RTMpPokeCpu.
535 *
536 * @param pvInfo Ignored.
537 */
538static void rtmpLinuxPokeCpuCallback(void *pvInfo)
539{
540 NOREF(pvInfo);
541}
542#endif
543
544
545RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
546{
547#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
548 int rc;
549 IPRT_LINUX_SAVE_EFL_AC();
550
551 if (!RTMpIsCpuPossible(idCpu))
552 return VERR_CPU_NOT_FOUND;
553 if (!RTMpIsCpuOnline(idCpu))
554 return VERR_CPU_OFFLINE;
555
556# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
557 rc = smp_call_function_single(idCpu, rtmpLinuxPokeCpuCallback, NULL, 0 /* wait */);
558# elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
559 rc = smp_call_function_single(idCpu, rtmpLinuxPokeCpuCallback, NULL, 0 /* retry */, 0 /* wait */);
560# else /* older kernels */
561# error oops
562# endif /* older kernels */
563 NOREF(rc);
564 Assert(rc == 0);
565 IPRT_LINUX_RESTORE_EFL_AC();
566 return VINF_SUCCESS;
567
568#else /* older kernels */
569 /* no unicast here? */
570 return VERR_NOT_SUPPORTED;
571#endif /* older kernels */
572}
573RT_EXPORT_SYMBOL(RTMpPokeCpu);
574
575
576RTDECL(bool) RTMpOnAllIsConcurrentSafe(void)
577{
578 return true;
579}
580RT_EXPORT_SYMBOL(RTMpOnAllIsConcurrentSafe);
581
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette