blob: 9867b6bfefce7f1d20d342149984ca40e0dbd3ab [file] [log] [blame]
Jens Axboe3d442232008-06-26 11:21:34 +02001/*
2 * Generic helpers for smp ipi calls
3 *
4 * (C) Jens Axboe <jens.axboe@oracle.com> 2008
Jens Axboe3d442232008-06-26 11:21:34 +02005 */
Jens Axboe3d442232008-06-26 11:21:34 +02006#include <linux/rcupdate.h>
Linus Torvalds59190f42008-07-15 14:02:33 -07007#include <linux/rculist.h>
Ingo Molnar641cd4c2009-03-13 10:47:34 +01008#include <linux/kernel.h>
Ingo Molnar0b13fda2009-02-25 16:52:11 +01009#include <linux/module.h>
10#include <linux/percpu.h>
11#include <linux/init.h>
Jens Axboe3d442232008-06-26 11:21:34 +020012#include <linux/smp.h>
Peter Zijlstra8969a5e2009-02-25 13:59:47 +010013#include <linux/cpu.h>
Jens Axboe3d442232008-06-26 11:21:34 +020014
Peter Zijlstra8969a5e2009-02-25 13:59:47 +010015static struct {
16 struct list_head queue;
Thomas Gleixner9f5a5622009-11-17 15:40:01 +010017 raw_spinlock_t lock;
Ingo Molnar0b13fda2009-02-25 16:52:11 +010018} call_function __cacheline_aligned_in_smp =
19 {
20 .queue = LIST_HEAD_INIT(call_function.queue),
Thomas Gleixner9f5a5622009-11-17 15:40:01 +010021 .lock = __RAW_SPIN_LOCK_UNLOCKED(call_function.lock),
Ingo Molnar0b13fda2009-02-25 16:52:11 +010022 };
Jens Axboe3d442232008-06-26 11:21:34 +020023
24enum {
Peter Zijlstra6e275632009-02-25 13:59:48 +010025 CSD_FLAG_LOCK = 0x01,
Jens Axboe3d442232008-06-26 11:21:34 +020026};
27
28struct call_function_data {
Ingo Molnar0b13fda2009-02-25 16:52:11 +010029 struct call_single_data csd;
Xiao Guangrong54fdade2009-09-22 16:43:39 -070030 atomic_t refs;
Ingo Molnar0b13fda2009-02-25 16:52:11 +010031 cpumask_var_t cpumask;
Jens Axboe3d442232008-06-26 11:21:34 +020032};
33
Milton Millere03bcb62010-01-18 13:00:51 +110034static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_function_data, cfd_data);
35
Jens Axboe3d442232008-06-26 11:21:34 +020036struct call_single_queue {
Ingo Molnar0b13fda2009-02-25 16:52:11 +010037 struct list_head list;
Thomas Gleixner9f5a5622009-11-17 15:40:01 +010038 raw_spinlock_t lock;
Jens Axboe3d442232008-06-26 11:21:34 +020039};
40
Milton Millere03bcb62010-01-18 13:00:51 +110041static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_queue, call_single_queue);
Peter Zijlstra8969a5e2009-02-25 13:59:47 +010042
43static int
44hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
45{
46 long cpu = (long)hcpu;
47 struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
48
49 switch (action) {
50 case CPU_UP_PREPARE:
51 case CPU_UP_PREPARE_FROZEN:
Yinghai Lueaa95842009-06-06 14:51:36 -070052 if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
Peter Zijlstra8969a5e2009-02-25 13:59:47 +010053 cpu_to_node(cpu)))
54 return NOTIFY_BAD;
55 break;
56
Xiao Guangrong69dd6472009-08-06 15:07:29 -070057#ifdef CONFIG_HOTPLUG_CPU
Peter Zijlstra8969a5e2009-02-25 13:59:47 +010058 case CPU_UP_CANCELED:
59 case CPU_UP_CANCELED_FROZEN:
60
61 case CPU_DEAD:
62 case CPU_DEAD_FROZEN:
63 free_cpumask_var(cfd->cpumask);
64 break;
65#endif
66 };
67
68 return NOTIFY_OK;
69}
70
71static struct notifier_block __cpuinitdata hotplug_cfd_notifier = {
Ingo Molnar0b13fda2009-02-25 16:52:11 +010072 .notifier_call = hotplug_cfd,
Peter Zijlstra8969a5e2009-02-25 13:59:47 +010073};
74
Eduard - Gabriel Munteanu7babe8d2008-07-25 19:45:11 -070075static int __cpuinit init_call_single_data(void)
Jens Axboe3d442232008-06-26 11:21:34 +020076{
Peter Zijlstra8969a5e2009-02-25 13:59:47 +010077 void *cpu = (void *)(long)smp_processor_id();
Jens Axboe3d442232008-06-26 11:21:34 +020078 int i;
79
80 for_each_possible_cpu(i) {
81 struct call_single_queue *q = &per_cpu(call_single_queue, i);
82
Thomas Gleixner9f5a5622009-11-17 15:40:01 +010083 raw_spin_lock_init(&q->lock);
Jens Axboe3d442232008-06-26 11:21:34 +020084 INIT_LIST_HEAD(&q->list);
85 }
Peter Zijlstra8969a5e2009-02-25 13:59:47 +010086
87 hotplug_cfd(&hotplug_cfd_notifier, CPU_UP_PREPARE, cpu);
88 register_cpu_notifier(&hotplug_cfd_notifier);
89
Eduard - Gabriel Munteanu7babe8d2008-07-25 19:45:11 -070090 return 0;
Jens Axboe3d442232008-06-26 11:21:34 +020091}
Eduard - Gabriel Munteanu7babe8d2008-07-25 19:45:11 -070092early_initcall(init_call_single_data);
Jens Axboe3d442232008-06-26 11:21:34 +020093
Peter Zijlstra8969a5e2009-02-25 13:59:47 +010094/*
Peter Zijlstra8969a5e2009-02-25 13:59:47 +010095 * csd_lock/csd_unlock used to serialize access to per-cpu csd resources
96 *
Ingo Molnar0b13fda2009-02-25 16:52:11 +010097 * For non-synchronous ipi calls the csd can still be in use by the
98 * previous function call. For multi-cpu calls its even more interesting
99 * as we'll have to ensure no other cpu is observing our csd.
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100100 */
Peter Zijlstra6e275632009-02-25 13:59:48 +0100101static void csd_lock_wait(struct call_single_data *data)
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100102{
103 while (data->flags & CSD_FLAG_LOCK)
104 cpu_relax();
Peter Zijlstra6e275632009-02-25 13:59:48 +0100105}
106
107static void csd_lock(struct call_single_data *data)
108{
109 csd_lock_wait(data);
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100110 data->flags = CSD_FLAG_LOCK;
111
112 /*
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100113 * prevent CPU from reordering the above assignment
114 * to ->flags with any subsequent assignments to other
115 * fields of the specified call_single_data structure:
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100116 */
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100117 smp_mb();
118}
119
120static void csd_unlock(struct call_single_data *data)
121{
122 WARN_ON(!(data->flags & CSD_FLAG_LOCK));
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100123
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100124 /*
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100125 * ensure we're all done before releasing data:
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100126 */
127 smp_mb();
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100128
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100129 data->flags &= ~CSD_FLAG_LOCK;
Jens Axboe3d442232008-06-26 11:21:34 +0200130}
131
132/*
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100133 * Insert a previously allocated call_single_data element
134 * for execution on the given CPU. data must already have
135 * ->func, ->info, and ->flags set.
Jens Axboe3d442232008-06-26 11:21:34 +0200136 */
Peter Zijlstra6e275632009-02-25 13:59:48 +0100137static
138void generic_exec_single(int cpu, struct call_single_data *data, int wait)
Jens Axboe3d442232008-06-26 11:21:34 +0200139{
140 struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
Jens Axboe3d442232008-06-26 11:21:34 +0200141 unsigned long flags;
Peter Zijlstra6e275632009-02-25 13:59:48 +0100142 int ipi;
Jens Axboe3d442232008-06-26 11:21:34 +0200143
Thomas Gleixner9f5a5622009-11-17 15:40:01 +0100144 raw_spin_lock_irqsave(&dst->lock, flags);
Jens Axboe3d442232008-06-26 11:21:34 +0200145 ipi = list_empty(&dst->list);
146 list_add_tail(&data->list, &dst->list);
Thomas Gleixner9f5a5622009-11-17 15:40:01 +0100147 raw_spin_unlock_irqrestore(&dst->lock, flags);
Jens Axboe3d442232008-06-26 11:21:34 +0200148
Suresh Siddha561920a02008-10-30 18:28:41 +0100149 /*
Nick Piggin15d0d3b2009-02-25 06:22:45 +0100150 * The list addition should be visible before sending the IPI
151 * handler locks the list to pull the entry off it because of
152 * normal cache coherency rules implied by spinlocks.
153 *
154 * If IPIs can go out of order to the cache coherency protocol
155 * in an architecture, sufficient synchronisation should be added
156 * to arch code to make it appear to obey cache coherency WRT
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100157 * locking and barrier primitives. Generic code isn't really
158 * equipped to do the right thing...
Suresh Siddha561920a02008-10-30 18:28:41 +0100159 */
Jens Axboe3d442232008-06-26 11:21:34 +0200160 if (ipi)
161 arch_send_call_function_single_ipi(cpu);
162
163 if (wait)
Peter Zijlstra6e275632009-02-25 13:59:48 +0100164 csd_lock_wait(data);
Jens Axboe3d442232008-06-26 11:21:34 +0200165}
166
167/*
168 * Invoked by arch to handle an IPI for call function. Must be called with
169 * interrupts disabled.
170 */
171void generic_smp_call_function_interrupt(void)
172{
173 struct call_function_data *data;
Xiao Guangrongc0f68c22009-12-14 18:00:16 -0800174 int cpu = smp_processor_id();
Jens Axboe3d442232008-06-26 11:21:34 +0200175
176 /*
Suresh Siddha269c8612009-08-19 18:05:35 -0700177 * Shouldn't receive this interrupt on a cpu that is not yet online.
178 */
179 WARN_ON_ONCE(!cpu_online(cpu));
180
181 /*
Nick Piggin15d0d3b2009-02-25 06:22:45 +0100182 * Ensure entry is visible on call_function_queue after we have
183 * entered the IPI. See comment in smp_call_function_many.
184 * If we don't have this, then we may miss an entry on the list
185 * and never get another IPI to process it.
186 */
187 smp_mb();
188
189 /*
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100190 * It's ok to use list_for_each_rcu() here even though we may
191 * delete 'pos', since list_del_rcu() doesn't clear ->next
Jens Axboe3d442232008-06-26 11:21:34 +0200192 */
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100193 list_for_each_entry_rcu(data, &call_function.queue, csd.list) {
Jens Axboe3d442232008-06-26 11:21:34 +0200194 int refs;
195
Xiao Guangrong54fdade2009-09-22 16:43:39 -0700196 if (!cpumask_test_and_clear_cpu(cpu, data->cpumask))
Jens Axboe3d442232008-06-26 11:21:34 +0200197 continue;
198
199 data->csd.func(data->csd.info);
200
Xiao Guangrong54fdade2009-09-22 16:43:39 -0700201 refs = atomic_dec_return(&data->refs);
202 WARN_ON(refs < 0);
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100203 if (!refs) {
Thomas Gleixner9f5a5622009-11-17 15:40:01 +0100204 raw_spin_lock(&call_function.lock);
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100205 list_del_rcu(&data->csd.list);
Thomas Gleixner9f5a5622009-11-17 15:40:01 +0100206 raw_spin_unlock(&call_function.lock);
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100207 }
Jens Axboe3d442232008-06-26 11:21:34 +0200208
209 if (refs)
210 continue;
211
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100212 csd_unlock(&data->csd);
Jens Axboe3d442232008-06-26 11:21:34 +0200213 }
Jens Axboe3d442232008-06-26 11:21:34 +0200214
Jens Axboe3d442232008-06-26 11:21:34 +0200215}
216
217/*
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100218 * Invoked by arch to handle an IPI for call function single. Must be
219 * called from the arch with interrupts disabled.
Jens Axboe3d442232008-06-26 11:21:34 +0200220 */
221void generic_smp_call_function_single_interrupt(void)
222{
223 struct call_single_queue *q = &__get_cpu_var(call_single_queue);
Nick Piggin15d0d3b2009-02-25 06:22:45 +0100224 unsigned int data_flags;
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100225 LIST_HEAD(list);
Jens Axboe3d442232008-06-26 11:21:34 +0200226
Suresh Siddha269c8612009-08-19 18:05:35 -0700227 /*
228 * Shouldn't receive this interrupt on a cpu that is not yet online.
229 */
230 WARN_ON_ONCE(!cpu_online(smp_processor_id()));
231
Thomas Gleixner9f5a5622009-11-17 15:40:01 +0100232 raw_spin_lock(&q->lock);
Nick Piggin15d0d3b2009-02-25 06:22:45 +0100233 list_replace_init(&q->list, &list);
Thomas Gleixner9f5a5622009-11-17 15:40:01 +0100234 raw_spin_unlock(&q->lock);
Jens Axboe3d442232008-06-26 11:21:34 +0200235
Nick Piggin15d0d3b2009-02-25 06:22:45 +0100236 while (!list_empty(&list)) {
237 struct call_single_data *data;
Jens Axboe3d442232008-06-26 11:21:34 +0200238
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100239 data = list_entry(list.next, struct call_single_data, list);
Nick Piggin15d0d3b2009-02-25 06:22:45 +0100240 list_del(&data->list);
Jens Axboe3d442232008-06-26 11:21:34 +0200241
Jens Axboe3d442232008-06-26 11:21:34 +0200242 /*
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100243 * 'data' can be invalid after this call if flags == 0
244 * (when called through generic_exec_single()),
245 * so save them away before making the call:
Jens Axboe3d442232008-06-26 11:21:34 +0200246 */
Nick Piggin15d0d3b2009-02-25 06:22:45 +0100247 data_flags = data->flags;
248
249 data->func(data->info);
250
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100251 /*
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100252 * Unlocked CSDs are valid through generic_exec_single():
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100253 */
254 if (data_flags & CSD_FLAG_LOCK)
255 csd_unlock(data);
Jens Axboe3d442232008-06-26 11:21:34 +0200256 }
257}
258
Milton Millere03bcb62010-01-18 13:00:51 +1100259static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_data, csd_data);
Steven Rostedtd7240b92009-01-29 10:08:01 -0500260
Jens Axboe3d442232008-06-26 11:21:34 +0200261/*
262 * smp_call_function_single - Run a function on a specific CPU
263 * @func: The function to run. This must be fast and non-blocking.
264 * @info: An arbitrary pointer to pass to the function.
Jens Axboe3d442232008-06-26 11:21:34 +0200265 * @wait: If true, wait until function has completed on other CPUs.
266 *
Sheng Yang72f279b2009-10-22 19:19:34 +0800267 * Returns 0 on success, else a negative status code.
Jens Axboe3d442232008-06-26 11:21:34 +0200268 */
269int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
Jens Axboe8691e5a2008-06-06 11:18:06 +0200270 int wait)
Jens Axboe3d442232008-06-26 11:21:34 +0200271{
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100272 struct call_single_data d = {
273 .flags = 0,
274 };
Jens Axboe3d442232008-06-26 11:21:34 +0200275 unsigned long flags;
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100276 int this_cpu;
H. Peter Anvinf73be6d2008-08-25 17:07:14 -0700277 int err = 0;
Jens Axboe3d442232008-06-26 11:21:34 +0200278
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100279 /*
280 * prevent preemption and reschedule on another processor,
281 * as well as CPU removal
282 */
283 this_cpu = get_cpu();
284
Suresh Siddha269c8612009-08-19 18:05:35 -0700285 /*
286 * Can deadlock when called with interrupts disabled.
287 * We allow cpu's that are not yet online though, as no one else can
288 * send smp call function interrupt to this cpu and as such deadlocks
289 * can't happen.
290 */
291 WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
292 && !oops_in_progress);
Jens Axboe3d442232008-06-26 11:21:34 +0200293
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100294 if (cpu == this_cpu) {
Jens Axboe3d442232008-06-26 11:21:34 +0200295 local_irq_save(flags);
296 func(info);
297 local_irq_restore(flags);
H. Peter Anvinf73be6d2008-08-25 17:07:14 -0700298 } else {
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100299 if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) {
300 struct call_single_data *data = &d;
301
302 if (!wait)
303 data = &__get_cpu_var(csd_data);
304
305 csd_lock(data);
306
307 data->func = func;
308 data->info = info;
309 generic_exec_single(cpu, data, wait);
310 } else {
311 err = -ENXIO; /* CPU not online */
312 }
Jens Axboe3d442232008-06-26 11:21:34 +0200313 }
314
315 put_cpu();
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100316
H. Peter Anvinf73be6d2008-08-25 17:07:14 -0700317 return err;
Jens Axboe3d442232008-06-26 11:21:34 +0200318}
319EXPORT_SYMBOL(smp_call_function_single);
320
Rusty Russell2ea6dec2009-11-17 14:27:27 -0800321/*
322 * smp_call_function_any - Run a function on any of the given cpus
323 * @mask: The mask of cpus it can run on.
324 * @func: The function to run. This must be fast and non-blocking.
325 * @info: An arbitrary pointer to pass to the function.
326 * @wait: If true, wait until function has completed.
327 *
328 * Returns 0 on success, else a negative status code (if no cpus were online).
329 * Note that @wait will be implicitly turned on in case of allocation failures,
330 * since we fall back to on-stack allocation.
331 *
332 * Selection preference:
333 * 1) current cpu if in @mask
334 * 2) any cpu of current node if in @mask
335 * 3) any other online cpu in @mask
336 */
337int smp_call_function_any(const struct cpumask *mask,
338 void (*func)(void *info), void *info, int wait)
339{
340 unsigned int cpu;
341 const struct cpumask *nodemask;
342 int ret;
343
344 /* Try for same CPU (cheapest) */
345 cpu = get_cpu();
346 if (cpumask_test_cpu(cpu, mask))
347 goto call;
348
349 /* Try for same node. */
David Johnaf2422c2010-01-15 17:01:23 -0800350 nodemask = cpumask_of_node(cpu_to_node(cpu));
Rusty Russell2ea6dec2009-11-17 14:27:27 -0800351 for (cpu = cpumask_first_and(nodemask, mask); cpu < nr_cpu_ids;
352 cpu = cpumask_next_and(cpu, nodemask, mask)) {
353 if (cpu_online(cpu))
354 goto call;
355 }
356
357 /* Any online will do: smp_call_function_single handles nr_cpu_ids. */
358 cpu = cpumask_any_and(mask, cpu_online_mask);
359call:
360 ret = smp_call_function_single(cpu, func, info, wait);
361 put_cpu();
362 return ret;
363}
364EXPORT_SYMBOL_GPL(smp_call_function_any);
365
Jens Axboe3d442232008-06-26 11:21:34 +0200366/**
367 * __smp_call_function_single(): Run a function on another CPU
368 * @cpu: The CPU to run on.
369 * @data: Pre-allocated and setup data structure
370 *
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100371 * Like smp_call_function_single(), but allow caller to pass in a
372 * pre-allocated data structure. Useful for embedding @data inside
373 * other structures, for instance.
Jens Axboe3d442232008-06-26 11:21:34 +0200374 */
Peter Zijlstra6e275632009-02-25 13:59:48 +0100375void __smp_call_function_single(int cpu, struct call_single_data *data,
376 int wait)
Jens Axboe3d442232008-06-26 11:21:34 +0200377{
Peter Zijlstra6e275632009-02-25 13:59:48 +0100378 csd_lock(data);
Jens Axboe3d442232008-06-26 11:21:34 +0200379
Suresh Siddha269c8612009-08-19 18:05:35 -0700380 /*
381 * Can deadlock when called with interrupts disabled.
382 * We allow cpu's that are not yet online though, as no one else can
383 * send smp call function interrupt to this cpu and as such deadlocks
384 * can't happen.
385 */
386 WARN_ON_ONCE(cpu_online(smp_processor_id()) && wait && irqs_disabled()
387 && !oops_in_progress);
Peter Zijlstra6e275632009-02-25 13:59:48 +0100388
389 generic_exec_single(cpu, data, wait);
Jens Axboe3d442232008-06-26 11:21:34 +0200390}
391
392/**
Rusty Russell54b11e62008-12-30 09:05:16 +1030393 * smp_call_function_many(): Run a function on a set of other CPUs.
394 * @mask: The set of cpus to run on (only runs on online subset).
Jens Axboe3d442232008-06-26 11:21:34 +0200395 * @func: The function to run. This must be fast and non-blocking.
396 * @info: An arbitrary pointer to pass to the function.
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100397 * @wait: If true, wait (atomically) until function has completed
398 * on other CPUs.
Jens Axboe3d442232008-06-26 11:21:34 +0200399 *
Sheng Yang72f279b2009-10-22 19:19:34 +0800400 * If @wait is true, then returns once @func has returned.
Jens Axboe3d442232008-06-26 11:21:34 +0200401 *
402 * You must not call this function with disabled interrupts or from a
403 * hardware interrupt handler or from a bottom half handler. Preemption
404 * must be disabled when calling this function.
405 */
Rusty Russell54b11e62008-12-30 09:05:16 +1030406void smp_call_function_many(const struct cpumask *mask,
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100407 void (*func)(void *), void *info, bool wait)
Jens Axboe3d442232008-06-26 11:21:34 +0200408{
Rusty Russell54b11e62008-12-30 09:05:16 +1030409 struct call_function_data *data;
Jens Axboe3d442232008-06-26 11:21:34 +0200410 unsigned long flags;
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100411 int cpu, next_cpu, this_cpu = smp_processor_id();
Jens Axboe3d442232008-06-26 11:21:34 +0200412
Suresh Siddha269c8612009-08-19 18:05:35 -0700413 /*
414 * Can deadlock when called with interrupts disabled.
415 * We allow cpu's that are not yet online though, as no one else can
416 * send smp call function interrupt to this cpu and as such deadlocks
417 * can't happen.
418 */
419 WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
420 && !oops_in_progress);
Jens Axboe3d442232008-06-26 11:21:34 +0200421
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100422 /* So, what's a CPU they want? Ignoring this one. */
Rusty Russell54b11e62008-12-30 09:05:16 +1030423 cpu = cpumask_first_and(mask, cpu_online_mask);
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100424 if (cpu == this_cpu)
Rusty Russell54b11e62008-12-30 09:05:16 +1030425 cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100426
Rusty Russell54b11e62008-12-30 09:05:16 +1030427 /* No online cpus? We're done. */
428 if (cpu >= nr_cpu_ids)
429 return;
Jens Axboe3d442232008-06-26 11:21:34 +0200430
Rusty Russell54b11e62008-12-30 09:05:16 +1030431 /* Do we have another CPU which isn't us? */
432 next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100433 if (next_cpu == this_cpu)
Rusty Russell54b11e62008-12-30 09:05:16 +1030434 next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask);
435
436 /* Fastpath: do that cpu by itself. */
437 if (next_cpu >= nr_cpu_ids) {
438 smp_call_function_single(cpu, func, info, wait);
439 return;
Jens Axboe3d442232008-06-26 11:21:34 +0200440 }
441
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100442 data = &__get_cpu_var(cfd_data);
443 csd_lock(&data->csd);
Jens Axboe3d442232008-06-26 11:21:34 +0200444
Jens Axboe3d442232008-06-26 11:21:34 +0200445 data->csd.func = func;
446 data->csd.info = info;
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100447 cpumask_and(data->cpumask, mask, cpu_online_mask);
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100448 cpumask_clear_cpu(this_cpu, data->cpumask);
Xiao Guangrong54fdade2009-09-22 16:43:39 -0700449 atomic_set(&data->refs, cpumask_weight(data->cpumask));
Jens Axboe3d442232008-06-26 11:21:34 +0200450
Thomas Gleixner9f5a5622009-11-17 15:40:01 +0100451 raw_spin_lock_irqsave(&call_function.lock, flags);
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100452 /*
453 * Place entry at the _HEAD_ of the list, so that any cpu still
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100454 * observing the entry in generic_smp_call_function_interrupt()
455 * will not miss any other list entries:
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100456 */
457 list_add_rcu(&data->csd.list, &call_function.queue);
Thomas Gleixner9f5a5622009-11-17 15:40:01 +0100458 raw_spin_unlock_irqrestore(&call_function.lock, flags);
Jens Axboe3d442232008-06-26 11:21:34 +0200459
Suresh Siddha561920a02008-10-30 18:28:41 +0100460 /*
461 * Make the list addition visible before sending the ipi.
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100462 * (IPIs must obey or appear to obey normal Linux cache
463 * coherency rules -- see comment in generic_exec_single).
Suresh Siddha561920a02008-10-30 18:28:41 +0100464 */
465 smp_mb();
466
Jens Axboe3d442232008-06-26 11:21:34 +0200467 /* Send a message to all CPUs in the map */
Peter Zijlstra8969a5e2009-02-25 13:59:47 +0100468 arch_send_call_function_ipi_mask(data->cpumask);
Jens Axboe3d442232008-06-26 11:21:34 +0200469
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100470 /* Optionally wait for the CPUs to complete */
Rusty Russell54b11e62008-12-30 09:05:16 +1030471 if (wait)
Peter Zijlstra6e275632009-02-25 13:59:48 +0100472 csd_lock_wait(&data->csd);
Jens Axboe3d442232008-06-26 11:21:34 +0200473}
Rusty Russell54b11e62008-12-30 09:05:16 +1030474EXPORT_SYMBOL(smp_call_function_many);
Jens Axboe3d442232008-06-26 11:21:34 +0200475
476/**
477 * smp_call_function(): Run a function on all other CPUs.
478 * @func: The function to run. This must be fast and non-blocking.
479 * @info: An arbitrary pointer to pass to the function.
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100480 * @wait: If true, wait (atomically) until function has completed
481 * on other CPUs.
Jens Axboe3d442232008-06-26 11:21:34 +0200482 *
Rusty Russell54b11e62008-12-30 09:05:16 +1030483 * Returns 0.
Jens Axboe3d442232008-06-26 11:21:34 +0200484 *
485 * If @wait is true, then returns once @func has returned; otherwise
Sheng Yang72f279b2009-10-22 19:19:34 +0800486 * it returns just before the target cpu calls @func.
Jens Axboe3d442232008-06-26 11:21:34 +0200487 *
488 * You must not call this function with disabled interrupts or from a
489 * hardware interrupt handler or from a bottom half handler.
490 */
Jens Axboe8691e5a2008-06-06 11:18:06 +0200491int smp_call_function(void (*func)(void *), void *info, int wait)
Jens Axboe3d442232008-06-26 11:21:34 +0200492{
Jens Axboe3d442232008-06-26 11:21:34 +0200493 preempt_disable();
Rusty Russell54b11e62008-12-30 09:05:16 +1030494 smp_call_function_many(cpu_online_mask, func, info, wait);
Jens Axboe3d442232008-06-26 11:21:34 +0200495 preempt_enable();
Ingo Molnar0b13fda2009-02-25 16:52:11 +0100496
Rusty Russell54b11e62008-12-30 09:05:16 +1030497 return 0;
Jens Axboe3d442232008-06-26 11:21:34 +0200498}
499EXPORT_SYMBOL(smp_call_function);
500
501void ipi_call_lock(void)
502{
Thomas Gleixner9f5a5622009-11-17 15:40:01 +0100503 raw_spin_lock(&call_function.lock);
Jens Axboe3d442232008-06-26 11:21:34 +0200504}
505
506void ipi_call_unlock(void)
507{
Thomas Gleixner9f5a5622009-11-17 15:40:01 +0100508 raw_spin_unlock(&call_function.lock);
Jens Axboe3d442232008-06-26 11:21:34 +0200509}
510
511void ipi_call_lock_irq(void)
512{
Thomas Gleixner9f5a5622009-11-17 15:40:01 +0100513 raw_spin_lock_irq(&call_function.lock);
Jens Axboe3d442232008-06-26 11:21:34 +0200514}
515
516void ipi_call_unlock_irq(void)
517{
Thomas Gleixner9f5a5622009-11-17 15:40:01 +0100518 raw_spin_unlock_irq(&call_function.lock);
Jens Axboe3d442232008-06-26 11:21:34 +0200519}