blob: ba7989ffaeeed50245562c429af4871abaca3f4c [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * SMP support for ppc.
3 *
4 * Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great
5 * deal of code from the sparc and intel versions.
6 *
7 * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
8 *
9 * PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and
10 * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version
15 * 2 of the License, or (at your option) any later version.
16 */
17
18#undef DEBUG
19
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/kernel.h>
21#include <linux/module.h>
22#include <linux/sched.h>
23#include <linux/smp.h>
24#include <linux/interrupt.h>
25#include <linux/delay.h>
26#include <linux/init.h>
27#include <linux/spinlock.h>
28#include <linux/cache.h>
29#include <linux/err.h>
30#include <linux/sysdev.h>
31#include <linux/cpu.h>
32#include <linux/notifier.h>
Anton Blanchard4b703a22005-12-13 06:56:47 +110033#include <linux/topology.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070034
35#include <asm/ptrace.h>
36#include <asm/atomic.h>
37#include <asm/irq.h>
38#include <asm/page.h>
39#include <asm/pgtable.h>
40#include <asm/prom.h>
41#include <asm/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070042#include <asm/time.h>
43#include <asm/machdep.h>
44#include <asm/cputable.h>
45#include <asm/system.h>
Stephen Rothwellbbeb3f42005-09-27 13:51:59 +100046#include <asm/mpic.h>
Benjamin Herrenschmidta7f290d2005-11-11 21:15:21 +110047#include <asm/vdso_datapage.h>
Paul Mackerras5ad57072005-11-05 10:33:55 +110048#ifdef CONFIG_PPC64
49#include <asm/paca.h>
50#endif
51
Linus Torvalds1da177e2005-04-16 15:20:36 -070052#ifdef DEBUG
Michael Ellermanf9e4ec52005-11-15 15:16:38 +110053#include <asm/udbg.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070054#define DBG(fmt...) udbg_printf(fmt)
55#else
56#define DBG(fmt...)
57#endif
58
Michael Ellermanf9e4ec52005-11-15 15:16:38 +110059int smp_hw_index[NR_CPUS];
60struct thread_info *secondary_ti;
61
Linus Torvalds1da177e2005-04-16 15:20:36 -070062cpumask_t cpu_possible_map = CPU_MASK_NONE;
63cpumask_t cpu_online_map = CPU_MASK_NONE;
Mike Travisd5a74302007-10-16 01:24:05 -070064DEFINE_PER_CPU(cpumask_t, cpu_sibling_map) = CPU_MASK_NONE;
Linus Torvalds1da177e2005-04-16 15:20:36 -070065
66EXPORT_SYMBOL(cpu_online_map);
67EXPORT_SYMBOL(cpu_possible_map);
Mike Travisd5a74302007-10-16 01:24:05 -070068EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
Linus Torvalds1da177e2005-04-16 15:20:36 -070069
Paul Mackerras5ad57072005-11-05 10:33:55 +110070/* SMP operations for this machine */
Linus Torvalds1da177e2005-04-16 15:20:36 -070071struct smp_ops_t *smp_ops;
72
73static volatile unsigned int cpu_callin_map[NR_CPUS];
74
Linus Torvalds1da177e2005-04-16 15:20:36 -070075void smp_call_function_interrupt(void);
76
77int smt_enabled_at_boot = 1;
78
Olof Johanssone057d982007-12-28 15:11:09 +110079static int ipi_fail_ok;
80
Michael Ellermancc532912005-12-04 18:39:43 +110081static void (*crash_ipi_function_ptr)(struct pt_regs *) = NULL;
82
Paul Mackerras5ad57072005-11-05 10:33:55 +110083#ifdef CONFIG_PPC64
Linus Torvalds1da177e2005-04-16 15:20:36 -070084void __devinit smp_generic_kick_cpu(int nr)
85{
86 BUG_ON(nr < 0 || nr >= NR_CPUS);
87
88 /*
89 * The processor is currently spinning, waiting for the
90 * cpu_start field to become non-zero After we set cpu_start,
91 * the processor will continue on to secondary_start
92 */
93 paca[nr].cpu_start = 1;
Anton Blanchard0d8d4d42005-05-01 08:58:47 -070094 smp_mb();
Linus Torvalds1da177e2005-04-16 15:20:36 -070095}
Paul Mackerras5ad57072005-11-05 10:33:55 +110096#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070097
David Howells7d12e782006-10-05 14:55:46 +010098void smp_message_recv(int msg)
Linus Torvalds1da177e2005-04-16 15:20:36 -070099{
100 switch(msg) {
101 case PPC_MSG_CALL_FUNCTION:
102 smp_call_function_interrupt();
103 break;
Paul Mackerras5ad57072005-11-05 10:33:55 +1100104 case PPC_MSG_RESCHEDULE:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105 /* XXX Do we have to do this? */
106 set_need_resched();
107 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108 case PPC_MSG_DEBUGGER_BREAK:
Michael Ellermancc532912005-12-04 18:39:43 +1100109 if (crash_ipi_function_ptr) {
David Howells7d12e782006-10-05 14:55:46 +0100110 crash_ipi_function_ptr(get_irq_regs());
Michael Ellermancc532912005-12-04 18:39:43 +1100111 break;
112 }
113#ifdef CONFIG_DEBUGGER
David Howells7d12e782006-10-05 14:55:46 +0100114 debugger_ipi(get_irq_regs());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115 break;
Michael Ellermancc532912005-12-04 18:39:43 +1100116#endif /* CONFIG_DEBUGGER */
117 /* FALLTHROUGH */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118 default:
119 printk("SMP %d: smp_message_recv(): unknown msg %d\n",
120 smp_processor_id(), msg);
121 break;
122 }
123}
124
125void smp_send_reschedule(int cpu)
126{
Benjamin Herrenschmidt8cffc6a2006-07-04 14:09:36 +1000127 if (likely(smp_ops))
128 smp_ops->message_pass(cpu, PPC_MSG_RESCHEDULE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129}
130
131#ifdef CONFIG_DEBUGGER
132void smp_send_debugger_break(int cpu)
133{
Benjamin Herrenschmidt8cffc6a2006-07-04 14:09:36 +1000134 if (likely(smp_ops))
135 smp_ops->message_pass(cpu, PPC_MSG_DEBUGGER_BREAK);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136}
137#endif
138
Michael Ellermancc532912005-12-04 18:39:43 +1100139#ifdef CONFIG_KEXEC
140void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *))
141{
142 crash_ipi_function_ptr = crash_ipi_callback;
Benjamin Herrenschmidt8cffc6a2006-07-04 14:09:36 +1000143 if (crash_ipi_callback && smp_ops) {
Michael Ellermancc532912005-12-04 18:39:43 +1100144 mb();
145 smp_ops->message_pass(MSG_ALL_BUT_SELF, PPC_MSG_DEBUGGER_BREAK);
146 }
147}
148#endif
149
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150static void stop_this_cpu(void *dummy)
151{
152 local_irq_disable();
153 while (1)
154 ;
155}
156
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157/*
158 * Structure and data for smp_call_function(). This is designed to minimise
159 * static memory requirements. It also looks cleaner.
160 * Stolen from the i386 version.
161 */
162static __cacheline_aligned_in_smp DEFINE_SPINLOCK(call_lock);
163
164static struct call_data_struct {
165 void (*func) (void *info);
166 void *info;
167 atomic_t started;
168 atomic_t finished;
169 int wait;
170} *call_data;
171
Paul Mackerras5ad57072005-11-05 10:33:55 +1100172/* delay of at least 8 seconds */
173#define SMP_CALL_TIMEOUT 8
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174
175/*
will schmidt44755d12007-05-03 03:12:34 +1000176 * These functions send a 'generic call function' IPI to other online
177 * CPUS in the system.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178 *
will schmidt44755d12007-05-03 03:12:34 +1000179 * [SUMMARY] Run a function on other CPUs.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180 * <func> The function to run. This must be fast and non-blocking.
181 * <info> An arbitrary pointer to pass to the function.
182 * <nonatomic> currently unused.
183 * <wait> If true, wait (atomically) until function has completed on other CPUs.
184 * [RETURNS] 0 on success, else a negative status code. Does not return until
185 * remote CPUs are nearly ready to execute <<func>> or are or have executed.
Olof Johanssonb616de52007-12-28 15:08:36 +1100186 * <map> is a cpu map of the cpus to send IPI to.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187 *
188 * You must not call this function with disabled interrupts or from a
189 * hardware interrupt handler or from a bottom half handler.
190 */
Olof Johanssonb616de52007-12-28 15:08:36 +1100191static int __smp_call_function_map(void (*func) (void *info), void *info,
192 int nonatomic, int wait, cpumask_t map)
will schmidt44755d12007-05-03 03:12:34 +1000193{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194 struct call_data_struct data;
will schmidt44755d12007-05-03 03:12:34 +1000195 int ret = -1, num_cpus;
196 int cpu;
Paul Mackerras5ad57072005-11-05 10:33:55 +1100197 u64 timeout;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198
Benjamin Herrenschmidt8cffc6a2006-07-04 14:09:36 +1000199 if (unlikely(smp_ops == NULL))
will schmidt44755d12007-05-03 03:12:34 +1000200 return ret;
Benjamin Herrenschmidt8cffc6a2006-07-04 14:09:36 +1000201
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 data.func = func;
203 data.info = info;
204 atomic_set(&data.started, 0);
205 data.wait = wait;
206 if (wait)
207 atomic_set(&data.finished, 0);
208
Hugh Dickinsd3fdaed2007-05-19 02:47:01 +1000209 /* remove 'self' from the map */
210 if (cpu_isset(smp_processor_id(), map))
211 cpu_clear(smp_processor_id(), map);
212
213 /* sanity check the map, remove any non-online processors. */
214 cpus_and(map, map, cpu_online_map);
Kevin Corry17aa3a82007-08-01 06:19:46 +1000215
216 num_cpus = cpus_weight(map);
217 if (!num_cpus)
Hugh Dickinsd3fdaed2007-05-19 02:47:01 +1000218 goto done;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219
220 call_data = &data;
Anton Blanchard0d8d4d42005-05-01 08:58:47 -0700221 smp_wmb();
will schmidt44755d12007-05-03 03:12:34 +1000222 /* Send a message to all CPUs in the map */
223 for_each_cpu_mask(cpu, map)
224 smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNCTION);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225
Paul Mackerras5ad57072005-11-05 10:33:55 +1100226 timeout = get_tb() + (u64) SMP_CALL_TIMEOUT * tb_ticks_per_sec;
227
will schmidt44755d12007-05-03 03:12:34 +1000228 /* Wait for indication that they have received the message */
229 while (atomic_read(&data.started) != num_cpus) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230 HMT_low();
Paul Mackerras5ad57072005-11-05 10:33:55 +1100231 if (get_tb() >= timeout) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232 printk("smp_call_function on cpu %d: other cpus not "
will schmidt44755d12007-05-03 03:12:34 +1000233 "responding (%d)\n", smp_processor_id(),
234 atomic_read(&data.started));
Olof Johanssone057d982007-12-28 15:11:09 +1100235 if (!ipi_fail_ok)
236 debugger(NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237 goto out;
238 }
239 }
240
will schmidt44755d12007-05-03 03:12:34 +1000241 /* optionally wait for the CPUs to complete */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242 if (wait) {
will schmidt44755d12007-05-03 03:12:34 +1000243 while (atomic_read(&data.finished) != num_cpus) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244 HMT_low();
Paul Mackerras5ad57072005-11-05 10:33:55 +1100245 if (get_tb() >= timeout) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246 printk("smp_call_function on cpu %d: other "
will schmidt44755d12007-05-03 03:12:34 +1000247 "cpus not finishing (%d/%d)\n",
248 smp_processor_id(),
249 atomic_read(&data.finished),
250 atomic_read(&data.started));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 debugger(NULL);
252 goto out;
253 }
254 }
255 }
256
Hugh Dickinsd3fdaed2007-05-19 02:47:01 +1000257 done:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258 ret = 0;
259
Paul Mackerras5ad57072005-11-05 10:33:55 +1100260 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261 call_data = NULL;
262 HMT_medium();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263 return ret;
264}
265
Satyam Sharma8fd76752007-09-18 09:43:40 +1000266static int __smp_call_function(void (*func)(void *info), void *info,
267 int nonatomic, int wait)
268{
Olof Johanssone057d982007-12-28 15:11:09 +1100269 int ret;
270 spin_lock(&call_lock);
271 ret =__smp_call_function_map(func, info, nonatomic, wait,
Olof Johanssonb616de52007-12-28 15:08:36 +1100272 cpu_online_map);
Olof Johanssone057d982007-12-28 15:11:09 +1100273 spin_unlock(&call_lock);
274 return ret;
Satyam Sharma8fd76752007-09-18 09:43:40 +1000275}
276
will schmidt44755d12007-05-03 03:12:34 +1000277int smp_call_function(void (*func) (void *info), void *info, int nonatomic,
278 int wait)
279{
Satyam Sharma8fd76752007-09-18 09:43:40 +1000280 /* Can deadlock when called with interrupts disabled */
281 WARN_ON(irqs_disabled());
282
283 return __smp_call_function(func, info, nonatomic, wait);
will schmidt44755d12007-05-03 03:12:34 +1000284}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285EXPORT_SYMBOL(smp_call_function);
286
Olof Johanssonb616de52007-12-28 15:08:36 +1100287int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
288 int nonatomic, int wait)
will schmidt44755d12007-05-03 03:12:34 +1000289{
Hugh Dickinsd3fdaed2007-05-19 02:47:01 +1000290 cpumask_t map = CPU_MASK_NONE;
Avi Kivityadff0932007-07-20 01:33:48 +1000291 int ret = 0;
will schmidt44755d12007-05-03 03:12:34 +1000292
Satyam Sharma8fd76752007-09-18 09:43:40 +1000293 /* Can deadlock when called with interrupts disabled */
294 WARN_ON(irqs_disabled());
295
will schmidt44755d12007-05-03 03:12:34 +1000296 if (!cpu_online(cpu))
297 return -EINVAL;
298
will schmidt44755d12007-05-03 03:12:34 +1000299 cpu_set(cpu, map);
Olof Johanssone057d982007-12-28 15:11:09 +1100300 if (cpu != get_cpu()) {
301 spin_lock(&call_lock);
Olof Johanssonb616de52007-12-28 15:08:36 +1100302 ret = __smp_call_function_map(func, info, nonatomic, wait, map);
Olof Johanssone057d982007-12-28 15:11:09 +1100303 spin_unlock(&call_lock);
304 } else {
Avi Kivityadff0932007-07-20 01:33:48 +1000305 local_irq_disable();
306 func(info);
307 local_irq_enable();
308 }
Hugh Dickinsd3fdaed2007-05-19 02:47:01 +1000309 put_cpu();
310 return ret;
will schmidt44755d12007-05-03 03:12:34 +1000311}
312EXPORT_SYMBOL(smp_call_function_single);
313
Satyam Sharma8fd76752007-09-18 09:43:40 +1000314void smp_send_stop(void)
315{
Olof Johanssone057d982007-12-28 15:11:09 +1100316 int nolock;
317
318 /* It's OK to fail sending the IPI, since the alternative is to
319 * be stuck forever waiting on the other CPU to take the interrupt.
320 *
321 * It's better to at least continue and go through reboot, since this
322 * function is usually called at panic or reboot time in the first
323 * place.
324 */
325 ipi_fail_ok = 1;
326
327 /* Don't deadlock in case we got called through panic */
328 nolock = !spin_trylock(&call_lock);
329 __smp_call_function_map(stop_this_cpu, NULL, 1, 0, cpu_online_map);
330 if (!nolock)
331 spin_unlock(&call_lock);
Satyam Sharma8fd76752007-09-18 09:43:40 +1000332}
333
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334void smp_call_function_interrupt(void)
335{
336 void (*func) (void *info);
337 void *info;
338 int wait;
339
340 /* call_data will be NULL if the sender timed out while
341 * waiting on us to receive the call.
342 */
343 if (!call_data)
344 return;
345
346 func = call_data->func;
347 info = call_data->info;
348 wait = call_data->wait;
349
350 if (!wait)
351 smp_mb__before_atomic_inc();
352
353 /*
354 * Notify initiating CPU that I've grabbed the data and am
355 * about to execute the function
356 */
357 atomic_inc(&call_data->started);
358 /*
359 * At this point the info structure may be out of scope unless wait==1
360 */
361 (*func)(info);
362 if (wait) {
363 smp_mb__before_atomic_inc();
364 atomic_inc(&call_data->finished);
365 }
366}
367
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368struct thread_info *current_set[NR_CPUS];
369
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370static void __devinit smp_store_cpu_info(int id)
371{
372 per_cpu(pvr, id) = mfspr(SPRN_PVR);
373}
374
375static void __init smp_create_idle(unsigned int cpu)
376{
377 struct task_struct *p;
378
379 /* create a process for the processor */
380 p = fork_idle(cpu);
381 if (IS_ERR(p))
382 panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p));
Paul Mackerras5ad57072005-11-05 10:33:55 +1100383#ifdef CONFIG_PPC64
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384 paca[cpu].__current = p;
Paul Mackerras3b575062008-05-02 14:29:12 +1000385 paca[cpu].kstack = (unsigned long) task_thread_info(p)
386 + THREAD_SIZE - STACK_FRAME_OVERHEAD;
Paul Mackerras5ad57072005-11-05 10:33:55 +1100387#endif
Al Virob5e2fc12006-01-12 01:06:01 -0800388 current_set[cpu] = task_thread_info(p);
389 task_thread_info(p)->cpu = cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390}
391
392void __init smp_prepare_cpus(unsigned int max_cpus)
393{
394 unsigned int cpu;
395
396 DBG("smp_prepare_cpus\n");
397
398 /*
399 * setup_cpu may need to be called on the boot cpu. We havent
400 * spun any cpus up but lets be paranoid.
401 */
402 BUG_ON(boot_cpuid != smp_processor_id());
403
404 /* Fixup boot cpu */
405 smp_store_cpu_info(boot_cpuid);
406 cpu_callin_map[boot_cpuid] = 1;
407
Benjamin Herrenschmidt8cffc6a2006-07-04 14:09:36 +1000408 if (smp_ops)
409 max_cpus = smp_ops->probe();
410 else
411 max_cpus = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700412
413 smp_space_timers(max_cpus);
414
KAMEZAWA Hiroyuki0e551952006-03-28 14:50:51 -0800415 for_each_possible_cpu(cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 if (cpu != boot_cpuid)
417 smp_create_idle(cpu);
418}
419
420void __devinit smp_prepare_boot_cpu(void)
421{
422 BUG_ON(smp_processor_id() != boot_cpuid);
423
424 cpu_set(boot_cpuid, cpu_online_map);
Paul Mackerras5ad57072005-11-05 10:33:55 +1100425#ifdef CONFIG_PPC64
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 paca[boot_cpuid].__current = current;
Paul Mackerras5ad57072005-11-05 10:33:55 +1100427#endif
Al Virob5e2fc12006-01-12 01:06:01 -0800428 current_set[boot_cpuid] = task_thread_info(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700429}
430
431#ifdef CONFIG_HOTPLUG_CPU
432/* State of each CPU during hotplug phases */
433DEFINE_PER_CPU(int, cpu_state) = { 0 };
434
435int generic_cpu_disable(void)
436{
437 unsigned int cpu = smp_processor_id();
438
439 if (cpu == boot_cpuid)
440 return -EBUSY;
441
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442 cpu_clear(cpu, cpu_online_map);
Paul Mackerras799d6042005-11-10 13:37:51 +1100443#ifdef CONFIG_PPC64
Benjamin Herrenschmidta7f290d2005-11-11 21:15:21 +1100444 vdso_data->processorCount--;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445 fixup_irqs(cpu_online_map);
Paul Mackerras094fe2e2005-11-10 14:26:12 +1100446#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 return 0;
448}
449
450int generic_cpu_enable(unsigned int cpu)
451{
452 /* Do the normal bootup if we haven't
453 * already bootstrapped. */
454 if (system_state != SYSTEM_RUNNING)
455 return -ENOSYS;
456
457 /* get the target out of it's holding state */
458 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
Anton Blanchard0d8d4d42005-05-01 08:58:47 -0700459 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460
461 while (!cpu_online(cpu))
462 cpu_relax();
463
Paul Mackerras094fe2e2005-11-10 14:26:12 +1100464#ifdef CONFIG_PPC64
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465 fixup_irqs(cpu_online_map);
466 /* counter the irq disable in fixup_irqs */
467 local_irq_enable();
Paul Mackerras094fe2e2005-11-10 14:26:12 +1100468#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 return 0;
470}
471
472void generic_cpu_die(unsigned int cpu)
473{
474 int i;
475
476 for (i = 0; i < 100; i++) {
Anton Blanchard0d8d4d42005-05-01 08:58:47 -0700477 smp_rmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478 if (per_cpu(cpu_state, cpu) == CPU_DEAD)
479 return;
480 msleep(100);
481 }
482 printk(KERN_ERR "CPU%d didn't die...\n", cpu);
483}
484
485void generic_mach_cpu_die(void)
486{
487 unsigned int cpu;
488
489 local_irq_disable();
490 cpu = smp_processor_id();
491 printk(KERN_DEBUG "CPU%d offline\n", cpu);
492 __get_cpu_var(cpu_state) = CPU_DEAD;
Anton Blanchard0d8d4d42005-05-01 08:58:47 -0700493 smp_wmb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494 while (__get_cpu_var(cpu_state) != CPU_UP_PREPARE)
495 cpu_relax();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496 cpu_set(cpu, cpu_online_map);
497 local_irq_enable();
498}
499#endif
500
501static int __devinit cpu_enable(unsigned int cpu)
502{
Benjamin Herrenschmidt8cffc6a2006-07-04 14:09:36 +1000503 if (smp_ops && smp_ops->cpu_enable)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 return smp_ops->cpu_enable(cpu);
505
506 return -ENOSYS;
507}
508
Gautham R Shenoyb282b6f2007-01-10 23:15:34 -0800509int __cpuinit __cpu_up(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510{
511 int c;
512
Paul Mackerras5ad57072005-11-05 10:33:55 +1100513 secondary_ti = current_set[cpu];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 if (!cpu_enable(cpu))
515 return 0;
516
Benjamin Herrenschmidt8cffc6a2006-07-04 14:09:36 +1000517 if (smp_ops == NULL ||
518 (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519 return -EINVAL;
520
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521 /* Make sure callin-map entry is 0 (can be leftover a CPU
522 * hotplug
523 */
524 cpu_callin_map[cpu] = 0;
525
526 /* The information for processor bringup must
527 * be written out to main store before we release
528 * the processor.
529 */
Anton Blanchard0d8d4d42005-05-01 08:58:47 -0700530 smp_mb();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531
532 /* wake up cpus */
533 DBG("smp: kicking cpu %d\n", cpu);
534 smp_ops->kick_cpu(cpu);
535
536 /*
537 * wait to see if the cpu made a callin (is actually up).
538 * use this value that I found through experimentation.
539 * -- Cort
540 */
541 if (system_state < SYSTEM_RUNNING)
Jon Loeligeree0339f2006-06-17 17:52:44 -0500542 for (c = 50000; c && !cpu_callin_map[cpu]; c--)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 udelay(100);
544#ifdef CONFIG_HOTPLUG_CPU
545 else
546 /*
547 * CPUs can take much longer to come up in the
548 * hotplug case. Wait five seconds.
549 */
550 for (c = 25; c && !cpu_callin_map[cpu]; c--) {
551 msleep(200);
552 }
553#endif
554
555 if (!cpu_callin_map[cpu]) {
556 printk("Processor %u is stuck.\n", cpu);
557 return -ENOENT;
558 }
559
560 printk("Processor %u found.\n", cpu);
561
562 if (smp_ops->give_timebase)
563 smp_ops->give_timebase();
564
565 /* Wait until cpu puts itself in the online map */
566 while (!cpu_online(cpu))
567 cpu_relax();
568
569 return 0;
570}
571
572
573/* Activate a secondary processor. */
574int __devinit start_secondary(void *unused)
575{
576 unsigned int cpu = smp_processor_id();
577
578 atomic_inc(&init_mm.mm_count);
579 current->active_mm = &init_mm;
580
581 smp_store_cpu_info(cpu);
Paul Mackerras5ad57072005-11-05 10:33:55 +1100582 set_dec(tb_ticks_per_jiffy);
Andrew Mortone4d76e12005-11-09 15:45:30 -0800583 preempt_disable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584 cpu_callin_map[cpu] = 1;
585
586 smp_ops->setup_cpu(cpu);
587 if (smp_ops->take_timebase)
588 smp_ops->take_timebase();
589
Nathan Lynch7d4d6152006-02-06 22:44:23 -0600590 if (system_state > SYSTEM_BOOTING)
Paul Mackerrasc6622f62006-02-24 10:06:59 +1100591 snapshot_timebase();
Nathan Lynch7d4d6152006-02-06 22:44:23 -0600592
Tony Breedsd831d0b2007-09-21 13:26:03 +1000593 secondary_cpu_time_init();
594
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595 spin_lock(&call_lock);
596 cpu_set(cpu, cpu_online_map);
597 spin_unlock(&call_lock);
598
599 local_irq_enable();
600
601 cpu_idle();
602 return 0;
603}
604
605int setup_profiling_timer(unsigned int multiplier)
606{
607 return 0;
608}
609
610void __init smp_cpus_done(unsigned int max_cpus)
611{
612 cpumask_t old_mask;
613
614 /* We want the setup_cpu() here to be called from CPU 0, but our
615 * init thread may have been "borrowed" by another CPU in the meantime
616 * se we pin us down to CPU 0 for a short while
617 */
618 old_mask = current->cpus_allowed;
619 set_cpus_allowed(current, cpumask_of_cpu(boot_cpuid));
620
Benjamin Herrenschmidt8cffc6a2006-07-04 14:09:36 +1000621 if (smp_ops)
622 smp_ops->setup_cpu(boot_cpuid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623
624 set_cpus_allowed(current, old_mask);
Anton Blanchard4b703a22005-12-13 06:56:47 +1100625
Paul Mackerrasc6622f62006-02-24 10:06:59 +1100626 snapshot_timebases();
627
Anton Blanchard4b703a22005-12-13 06:56:47 +1100628 dump_numa_cpu_topology();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629}
630
631#ifdef CONFIG_HOTPLUG_CPU
632int __cpu_disable(void)
633{
634 if (smp_ops->cpu_disable)
635 return smp_ops->cpu_disable();
636
637 return -ENOSYS;
638}
639
640void __cpu_die(unsigned int cpu)
641{
642 if (smp_ops->cpu_die)
643 smp_ops->cpu_die(cpu);
644}
645#endif