blob: f6e726f184916029e2d1cfdbcd4acb2b26f14e69 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* CPU control.
2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
3 *
4 * This code is licenced under the GPL.
5 */
6#include <linux/proc_fs.h>
7#include <linux/smp.h>
8#include <linux/init.h>
9#include <linux/notifier.h>
10#include <linux/sched.h>
11#include <linux/unistd.h>
12#include <linux/cpu.h>
13#include <linux/module.h>
14#include <linux/kthread.h>
15#include <linux/stop_machine.h>
Ingo Molnar81615b62006-06-26 00:24:32 -070016#include <linux/mutex.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090017#include <linux/gfp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018
Rusty Russell98a79d62008-12-13 21:19:41 +103019#ifdef CONFIG_SMP
Rusty Russellb3199c02008-12-30 09:05:14 +103020/* Serializes the updates to cpu_online_mask, cpu_present_mask */
Linus Torvaldsaa953872006-07-23 12:12:16 -070021static DEFINE_MUTEX(cpu_add_remove_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
Lai Jiangshan79a6cde2010-05-26 14:43:36 -070023/*
24 * The following two API's must be used when attempting
25 * to serialize the updates to cpu_online_mask, cpu_present_mask.
26 */
27void cpu_maps_update_begin(void)
28{
29 mutex_lock(&cpu_add_remove_lock);
30}
31
32void cpu_maps_update_done(void)
33{
34 mutex_unlock(&cpu_add_remove_lock);
35}
36
Daniel J Blueman5c113fb2010-06-01 12:15:11 +010037static RAW_NOTIFIER_HEAD(cpu_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -070038
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -070039/* If set, cpu_up and cpu_down will return -EBUSY and do nothing.
40 * Should always be manipulated under cpu_add_remove_lock
41 */
42static int cpu_hotplug_disabled;
43
Lai Jiangshan79a6cde2010-05-26 14:43:36 -070044#ifdef CONFIG_HOTPLUG_CPU
45
Gautham R Shenoyd2219382008-01-25 21:08:01 +010046static struct {
47 struct task_struct *active_writer;
48 struct mutex lock; /* Synchronizes accesses to refcount, */
49 /*
50 * Also blocks the new readers during
51 * an ongoing cpu hotplug operation.
52 */
53 int refcount;
Linus Torvalds31950eb2009-06-22 21:18:12 -070054} cpu_hotplug = {
55 .active_writer = NULL,
56 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
57 .refcount = 0,
58};
Gautham R Shenoyd2219382008-01-25 21:08:01 +010059
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +010060void get_online_cpus(void)
Ashok Raja9d9baa2005-11-28 13:43:46 -080061{
Gautham R Shenoyd2219382008-01-25 21:08:01 +010062 might_sleep();
63 if (cpu_hotplug.active_writer == current)
Linus Torvaldsaa953872006-07-23 12:12:16 -070064 return;
Gautham R Shenoyd2219382008-01-25 21:08:01 +010065 mutex_lock(&cpu_hotplug.lock);
66 cpu_hotplug.refcount++;
67 mutex_unlock(&cpu_hotplug.lock);
68
Ashok Raja9d9baa2005-11-28 13:43:46 -080069}
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +010070EXPORT_SYMBOL_GPL(get_online_cpus);
Ashok Raj90d45d12005-11-08 21:34:24 -080071
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +010072void put_online_cpus(void)
Ashok Raja9d9baa2005-11-28 13:43:46 -080073{
Gautham R Shenoyd2219382008-01-25 21:08:01 +010074 if (cpu_hotplug.active_writer == current)
Linus Torvaldsaa953872006-07-23 12:12:16 -070075 return;
Gautham R Shenoyd2219382008-01-25 21:08:01 +010076 mutex_lock(&cpu_hotplug.lock);
Oleg Nesterovd2ba7e22008-04-29 01:00:29 -070077 if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer))
78 wake_up_process(cpu_hotplug.active_writer);
Gautham R Shenoyd2219382008-01-25 21:08:01 +010079 mutex_unlock(&cpu_hotplug.lock);
80
Ashok Raja9d9baa2005-11-28 13:43:46 -080081}
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +010082EXPORT_SYMBOL_GPL(put_online_cpus);
Ashok Raja9d9baa2005-11-28 13:43:46 -080083
Gautham R Shenoyd2219382008-01-25 21:08:01 +010084/*
85 * This ensures that the hotplug operation can begin only when the
86 * refcount goes to zero.
87 *
88 * Note that during a cpu-hotplug operation, the new readers, if any,
89 * will be blocked by the cpu_hotplug.lock
90 *
Oleg Nesterovd2ba7e22008-04-29 01:00:29 -070091 * Since cpu_hotplug_begin() is always called after invoking
92 * cpu_maps_update_begin(), we can be sure that only one writer is active.
Gautham R Shenoyd2219382008-01-25 21:08:01 +010093 *
94 * Note that theoretically, there is a possibility of a livelock:
95 * - Refcount goes to zero, last reader wakes up the sleeping
96 * writer.
97 * - Last reader unlocks the cpu_hotplug.lock.
98 * - A new reader arrives at this moment, bumps up the refcount.
99 * - The writer acquires the cpu_hotplug.lock finds the refcount
100 * non zero and goes to sleep again.
101 *
102 * However, this is very difficult to achieve in practice since
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100103 * get_online_cpus() not an api which is called all that often.
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100104 *
105 */
106static void cpu_hotplug_begin(void)
107{
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100108 cpu_hotplug.active_writer = current;
Oleg Nesterovd2ba7e22008-04-29 01:00:29 -0700109
110 for (;;) {
111 mutex_lock(&cpu_hotplug.lock);
112 if (likely(!cpu_hotplug.refcount))
113 break;
114 __set_current_state(TASK_UNINTERRUPTIBLE);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100115 mutex_unlock(&cpu_hotplug.lock);
116 schedule();
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100117 }
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100118}
119
120static void cpu_hotplug_done(void)
121{
122 cpu_hotplug.active_writer = NULL;
123 mutex_unlock(&cpu_hotplug.lock);
124}
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700125
126#else /* #if CONFIG_HOTPLUG_CPU */
127static void cpu_hotplug_begin(void) {}
128static void cpu_hotplug_done(void) {}
129#endif /* #esle #if CONFIG_HOTPLUG_CPU */
130
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131/* Need to know about CPUs going up/down? */
Sam Ravnborgf7b16c12008-04-29 00:58:51 -0700132int __ref register_cpu_notifier(struct notifier_block *nb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133{
Neil Brownbd5349c2006-10-17 00:10:35 -0700134 int ret;
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100135 cpu_maps_update_begin();
Neil Brownbd5349c2006-10-17 00:10:35 -0700136 ret = raw_notifier_chain_register(&cpu_chain, nb);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100137 cpu_maps_update_done();
Neil Brownbd5349c2006-10-17 00:10:35 -0700138 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139}
Chandra Seetharaman65edc682006-06-27 02:54:08 -0700140
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700141static int __cpu_notify(unsigned long val, void *v, int nr_to_call,
142 int *nr_calls)
143{
Akinobu Mitae6bde732010-05-26 14:43:29 -0700144 int ret;
145
146 ret = __raw_notifier_call_chain(&cpu_chain, val, v, nr_to_call,
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700147 nr_calls);
Akinobu Mitae6bde732010-05-26 14:43:29 -0700148
149 return notifier_to_errno(ret);
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700150}
151
152static int cpu_notify(unsigned long val, void *v)
153{
154 return __cpu_notify(val, v, -1, NULL);
155}
156
Linus Torvalds00b9b0a2010-05-27 10:32:08 -0700157#ifdef CONFIG_HOTPLUG_CPU
158
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700159static void cpu_notify_nofail(unsigned long val, void *v)
160{
Linus Torvalds00b9b0a2010-05-27 10:32:08 -0700161 BUG_ON(cpu_notify(val, v));
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700162}
163
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164EXPORT_SYMBOL(register_cpu_notifier);
165
Sam Ravnborg96471552008-04-29 00:58:48 -0700166void __ref unregister_cpu_notifier(struct notifier_block *nb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167{
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100168 cpu_maps_update_begin();
Neil Brownbd5349c2006-10-17 00:10:35 -0700169 raw_notifier_chain_unregister(&cpu_chain, nb);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100170 cpu_maps_update_done();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171}
172EXPORT_SYMBOL(unregister_cpu_notifier);
173
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174static inline void check_for_tasks(int cpu)
175{
176 struct task_struct *p;
177
178 write_lock_irq(&tasklist_lock);
179 for_each_process(p) {
Peter Zijlstra11854242010-01-21 16:34:27 +0100180 if (task_cpu(p) == cpu && p->state == TASK_RUNNING &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 (!cputime_eq(p->utime, cputime_zero) ||
182 !cputime_eq(p->stime, cputime_zero)))
Frans Pop9d3cfc42010-01-25 14:56:34 +0100183 printk(KERN_WARNING "Task %s (pid = %d) is on cpu %d "
184 "(state = %ld, flags = %x)\n",
185 p->comm, task_pid_nr(p), cpu,
186 p->state, p->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187 }
188 write_unlock_irq(&tasklist_lock);
189}
190
Avi Kivitydb912f92007-05-24 12:23:10 +0300191struct take_cpu_down_param {
Oleg Nesterov6a1bdc12010-03-15 10:10:23 +0100192 struct task_struct *caller;
Avi Kivitydb912f92007-05-24 12:23:10 +0300193 unsigned long mod;
194 void *hcpu;
195};
196
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197/* Take this CPU down. */
Sam Ravnborg514a20a2008-04-29 00:58:50 -0700198static int __ref take_cpu_down(void *_param)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199{
Avi Kivitydb912f92007-05-24 12:23:10 +0300200 struct take_cpu_down_param *param = _param;
Oleg Nesterov6a1bdc12010-03-15 10:10:23 +0100201 unsigned int cpu = (unsigned long)param->hcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 int err;
203
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204 /* Ensure this CPU doesn't handle any more interrupts. */
205 err = __cpu_disable();
206 if (err < 0)
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700207 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700209 cpu_notify(CPU_DYING | param->mod, param->hcpu);
Manfred Spraul3ba35572008-08-31 19:58:49 +0200210
Oleg Nesterov6a1bdc12010-03-15 10:10:23 +0100211 if (task_cpu(param->caller) == cpu)
212 move_task_off_dead_cpu(cpu, param->caller);
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700213 /* Force idle task to run as soon as we yield: it should
214 immediately notice cpu is offline and die quickly. */
215 sched_idle_next();
216 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217}
218
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700219/* Requires cpu_add_remove_lock to be held */
Sam Ravnborg514a20a2008-04-29 00:58:50 -0700220static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221{
Heiko Carstense7407dc2007-05-09 02:34:04 -0700222 int err, nr_calls = 0;
Heiko Carstense7407dc2007-05-09 02:34:04 -0700223 void *hcpu = (void *)(long)cpu;
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700224 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
Avi Kivitydb912f92007-05-24 12:23:10 +0300225 struct take_cpu_down_param tcd_param = {
Oleg Nesterov6a1bdc12010-03-15 10:10:23 +0100226 .caller = current,
Avi Kivitydb912f92007-05-24 12:23:10 +0300227 .mod = mod,
228 .hcpu = hcpu,
229 };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700231 if (num_online_cpus() == 1)
232 return -EBUSY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700234 if (!cpu_online(cpu))
235 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100237 cpu_hotplug_begin();
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700238 err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls);
Akinobu Mitae6bde732010-05-26 14:43:29 -0700239 if (err) {
Akinobu Mitaa0d8cdb2007-10-18 03:05:12 -0700240 nr_calls--;
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700241 __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242 printk("%s: attempt to take down CPU %u failed\n",
Harvey Harrisonaf1f16d2008-04-30 00:55:08 -0700243 __func__, cpu);
Gautham R Shenoybaaca492007-05-09 02:34:03 -0700244 goto out_release;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 }
246
Rusty Russelle0b582e2009-01-01 10:12:28 +1030247 err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu));
Rusty Russell04321582008-07-28 12:16:29 -0500248 if (err) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249 /* CPU didn't die: tell everyone. Can't complain. */
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700250 cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251
Oleg Nesterov6a1bdc12010-03-15 10:10:23 +0100252 goto out_release;
Satoru Takeuchi8fa1d7d2006-10-28 10:38:57 -0700253 }
Rusty Russell04321582008-07-28 12:16:29 -0500254 BUG_ON(cpu_online(cpu));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255
256 /* Wait for it to sleep (leaving idle task). */
257 while (!idle_cpu(cpu))
258 yield();
259
260 /* This actually kills the CPU. */
261 __cpu_die(cpu);
262
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263 /* CPU is completely dead: tell everyone. Too late to complain. */
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700264 cpu_notify_nofail(CPU_DEAD | mod, hcpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265
266 check_for_tasks(cpu);
267
Gautham R Shenoybaaca492007-05-09 02:34:03 -0700268out_release:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100269 cpu_hotplug_done();
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700270 if (!err)
271 cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700272 return err;
273}
274
Sam Ravnborg514a20a2008-04-29 00:58:50 -0700275int __ref cpu_down(unsigned int cpu)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700276{
Heiko Carstens9ea09af2008-12-22 12:36:30 +0100277 int err;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700278
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100279 cpu_maps_update_begin();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700280
Max Krasnyanskye761b772008-07-15 04:43:49 -0700281 if (cpu_hotplug_disabled) {
282 err = -EBUSY;
283 goto out;
284 }
285
Max Krasnyanskye761b772008-07-15 04:43:49 -0700286 err = _cpu_down(cpu, 0);
287
Max Krasnyanskye761b772008-07-15 04:43:49 -0700288out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100289 cpu_maps_update_done();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290 return err;
291}
Zhang Ruib62b8ef2008-04-29 02:35:56 -0400292EXPORT_SYMBOL(cpu_down);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293#endif /*CONFIG_HOTPLUG_CPU*/
294
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700295/* Requires cpu_add_remove_lock to be held */
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700296static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297{
Gautham R Shenoybaaca492007-05-09 02:34:03 -0700298 int ret, nr_calls = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299 void *hcpu = (void *)(long)cpu;
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700300 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700302 if (cpu_online(cpu) || !cpu_present(cpu))
303 return -EINVAL;
Ashok Raj90d45d12005-11-08 21:34:24 -0800304
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100305 cpu_hotplug_begin();
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700306 ret = __cpu_notify(CPU_UP_PREPARE | mod, hcpu, -1, &nr_calls);
Akinobu Mitae6bde732010-05-26 14:43:29 -0700307 if (ret) {
Akinobu Mitaa0d8cdb2007-10-18 03:05:12 -0700308 nr_calls--;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309 printk("%s: attempt to bring up CPU %u failed\n",
Harvey Harrisonaf1f16d2008-04-30 00:55:08 -0700310 __func__, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 goto out_notify;
312 }
313
314 /* Arch-specific enabling code. */
315 ret = __cpu_up(cpu);
316 if (ret != 0)
317 goto out_notify;
Eric Sesterhenn6978c702006-03-24 18:45:21 +0100318 BUG_ON(!cpu_online(cpu));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319
320 /* Now call notifier in preparation. */
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700321 cpu_notify(CPU_ONLINE | mod, hcpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322
323out_notify:
324 if (ret != 0)
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700325 __cpu_notify(CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100326 cpu_hotplug_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700327
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328 return ret;
329}
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700330
Gautham R Shenoyb282b6f2007-01-10 23:15:34 -0800331int __cpuinit cpu_up(unsigned int cpu)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700332{
333 int err = 0;
minskey guocf234222010-05-24 14:32:41 -0700334
335#ifdef CONFIG_MEMORY_HOTPLUG
336 int nid;
337 pg_data_t *pgdat;
338#endif
339
Rusty Russelle0b582e2009-01-01 10:12:28 +1030340 if (!cpu_possible(cpu)) {
KAMEZAWA Hiroyuki73e753a2007-10-18 23:40:47 -0700341 printk(KERN_ERR "can't online cpu %d because it is not "
342 "configured as may-hotadd at boot time\n", cpu);
Chen Gong87d5e022010-03-05 13:42:38 -0800343#if defined(CONFIG_IA64)
KAMEZAWA Hiroyuki73e753a2007-10-18 23:40:47 -0700344 printk(KERN_ERR "please check additional_cpus= boot "
345 "parameter\n");
346#endif
347 return -EINVAL;
348 }
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700349
minskey guocf234222010-05-24 14:32:41 -0700350#ifdef CONFIG_MEMORY_HOTPLUG
351 nid = cpu_to_node(cpu);
352 if (!node_online(nid)) {
353 err = mem_online_node(nid);
354 if (err)
355 return err;
356 }
357
358 pgdat = NODE_DATA(nid);
359 if (!pgdat) {
360 printk(KERN_ERR
361 "Can't online cpu %d due to NULL pgdat\n", cpu);
362 return -ENOMEM;
363 }
364
Haicheng Li4eaf3f62010-05-24 14:32:52 -0700365 if (pgdat->node_zonelists->_zonerefs->zone == NULL) {
366 mutex_lock(&zonelists_mutex);
Haicheng Li1f522502010-05-24 14:32:51 -0700367 build_all_zonelists(NULL);
Haicheng Li4eaf3f62010-05-24 14:32:52 -0700368 mutex_unlock(&zonelists_mutex);
369 }
minskey guocf234222010-05-24 14:32:41 -0700370#endif
371
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100372 cpu_maps_update_begin();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700373
Max Krasnyanskye761b772008-07-15 04:43:49 -0700374 if (cpu_hotplug_disabled) {
375 err = -EBUSY;
376 goto out;
377 }
378
379 err = _cpu_up(cpu, 0);
380
Max Krasnyanskye761b772008-07-15 04:43:49 -0700381out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100382 cpu_maps_update_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700383 return err;
384}
385
Rafael J. Wysockif3de4be2007-08-30 23:56:29 -0700386#ifdef CONFIG_PM_SLEEP_SMP
Rusty Russelle0b582e2009-01-01 10:12:28 +1030387static cpumask_var_t frozen_cpus;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700388
389int disable_nonboot_cpus(void)
390{
Rafael J. Wysockie9a5f422010-05-27 22:16:22 +0200391 int cpu, first_cpu, error = 0;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700392
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100393 cpu_maps_update_begin();
Rusty Russelle0b582e2009-01-01 10:12:28 +1030394 first_cpu = cpumask_first(cpu_online_mask);
Xiaotian Feng9ee349a2009-12-16 18:04:32 +0100395 /*
396 * We take down all of the non-boot CPUs in one shot to avoid races
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700397 * with the userspace trying to use the CPU hotplug at the same time
398 */
Rusty Russelle0b582e2009-01-01 10:12:28 +1030399 cpumask_clear(frozen_cpus);
Peter Zijlstra6ad4c182009-11-25 13:31:39 +0100400
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700401 printk("Disabling non-boot CPUs ...\n");
402 for_each_online_cpu(cpu) {
403 if (cpu == first_cpu)
404 continue;
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700405 error = _cpu_down(cpu, 1);
Mike Travisfeae3202009-11-17 18:22:13 -0600406 if (!error)
Rusty Russelle0b582e2009-01-01 10:12:28 +1030407 cpumask_set_cpu(cpu, frozen_cpus);
Mike Travisfeae3202009-11-17 18:22:13 -0600408 else {
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700409 printk(KERN_ERR "Error taking CPU%d down: %d\n",
410 cpu, error);
411 break;
412 }
413 }
Joseph Cihula86886e52009-06-30 19:31:07 -0700414
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700415 if (!error) {
416 BUG_ON(num_online_cpus() > 1);
417 /* Make sure the CPUs won't be enabled by someone else */
418 cpu_hotplug_disabled = 1;
419 } else {
Ingo Molnare1d9fd22006-12-23 16:55:29 +0100420 printk(KERN_ERR "Non-boot CPUs are not disabled\n");
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700421 }
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100422 cpu_maps_update_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700423 return error;
424}
425
Suresh Siddhad0af9ee2009-08-19 18:05:36 -0700426void __weak arch_enable_nonboot_cpus_begin(void)
427{
428}
429
430void __weak arch_enable_nonboot_cpus_end(void)
431{
432}
433
Sam Ravnborgfa7303e2008-02-08 04:21:55 -0800434void __ref enable_nonboot_cpus(void)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700435{
436 int cpu, error;
437
438 /* Allow everyone to use the CPU hotplug again */
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100439 cpu_maps_update_begin();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700440 cpu_hotplug_disabled = 0;
Rusty Russelle0b582e2009-01-01 10:12:28 +1030441 if (cpumask_empty(frozen_cpus))
Rafael J. Wysocki1d64b9c2007-04-01 23:49:49 -0700442 goto out;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700443
444 printk("Enabling non-boot CPUs ...\n");
Suresh Siddhad0af9ee2009-08-19 18:05:36 -0700445
446 arch_enable_nonboot_cpus_begin();
447
Rusty Russelle0b582e2009-01-01 10:12:28 +1030448 for_each_cpu(cpu, frozen_cpus) {
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700449 error = _cpu_up(cpu, 1);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700450 if (!error) {
451 printk("CPU%d is up\n", cpu);
452 continue;
453 }
Rafael J. Wysocki1d64b9c2007-04-01 23:49:49 -0700454 printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700455 }
Suresh Siddhad0af9ee2009-08-19 18:05:36 -0700456
457 arch_enable_nonboot_cpus_end();
458
Rusty Russelle0b582e2009-01-01 10:12:28 +1030459 cpumask_clear(frozen_cpus);
Rafael J. Wysocki1d64b9c2007-04-01 23:49:49 -0700460out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100461 cpu_maps_update_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700462}
Rusty Russelle0b582e2009-01-01 10:12:28 +1030463
464static int alloc_frozen_cpus(void)
465{
466 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
467 return -ENOMEM;
468 return 0;
469}
470core_initcall(alloc_frozen_cpus);
Rafael J. Wysockif3de4be2007-08-30 23:56:29 -0700471#endif /* CONFIG_PM_SLEEP_SMP */
Max Krasnyansky68f4f1e2008-05-29 11:17:02 -0700472
Manfred Spraule545a612008-09-07 16:57:22 +0200473/**
474 * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers
475 * @cpu: cpu that just started
476 *
477 * This function calls the cpu_chain notifiers with CPU_STARTING.
478 * It must be called by the arch code on the new cpu, before the new cpu
479 * enables interrupts and before the "boot" cpu returns from __cpu_up().
480 */
Al Viro84196412008-11-22 17:36:44 +0000481void __cpuinit notify_cpu_starting(unsigned int cpu)
Manfred Spraule545a612008-09-07 16:57:22 +0200482{
483 unsigned long val = CPU_STARTING;
484
485#ifdef CONFIG_PM_SLEEP_SMP
Rusty Russelle0b582e2009-01-01 10:12:28 +1030486 if (frozen_cpus != NULL && cpumask_test_cpu(cpu, frozen_cpus))
Manfred Spraule545a612008-09-07 16:57:22 +0200487 val = CPU_STARTING_FROZEN;
488#endif /* CONFIG_PM_SLEEP_SMP */
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700489 cpu_notify(val, (void *)(long)cpu);
Manfred Spraule545a612008-09-07 16:57:22 +0200490}
491
Max Krasnyansky68f4f1e2008-05-29 11:17:02 -0700492#endif /* CONFIG_SMP */
Mike Travisb8d317d2008-07-24 18:21:29 -0700493
Linus Torvaldse56b3bc2008-07-28 11:32:33 -0700494/*
495 * cpu_bit_bitmap[] is a special, "compressed" data structure that
496 * represents all NR_CPUS bits binary values of 1<<nr.
497 *
Rusty Russelle0b582e2009-01-01 10:12:28 +1030498 * It is used by cpumask_of() to get a constant address to a CPU
Linus Torvaldse56b3bc2008-07-28 11:32:33 -0700499 * mask value that has a single bit set only.
500 */
Mike Travisb8d317d2008-07-24 18:21:29 -0700501
Linus Torvaldse56b3bc2008-07-28 11:32:33 -0700502/* cpu_bit_bitmap[0] is empty - so we can back into it */
503#define MASK_DECLARE_1(x) [x+1][0] = 1UL << (x)
504#define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
505#define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
506#define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
Mike Travisb8d317d2008-07-24 18:21:29 -0700507
Linus Torvaldse56b3bc2008-07-28 11:32:33 -0700508const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
Mike Travisb8d317d2008-07-24 18:21:29 -0700509
Linus Torvaldse56b3bc2008-07-28 11:32:33 -0700510 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
511 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
512#if BITS_PER_LONG > 32
513 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
514 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
Mike Travisb8d317d2008-07-24 18:21:29 -0700515#endif
516};
Linus Torvaldse56b3bc2008-07-28 11:32:33 -0700517EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
Rusty Russell2d3854a2008-11-05 13:39:10 +1100518
519const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
520EXPORT_SYMBOL(cpu_all_bits);
Rusty Russellb3199c02008-12-30 09:05:14 +1030521
522#ifdef CONFIG_INIT_ALL_POSSIBLE
523static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly
524 = CPU_BITS_ALL;
525#else
526static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly;
527#endif
528const struct cpumask *const cpu_possible_mask = to_cpumask(cpu_possible_bits);
529EXPORT_SYMBOL(cpu_possible_mask);
530
531static DECLARE_BITMAP(cpu_online_bits, CONFIG_NR_CPUS) __read_mostly;
532const struct cpumask *const cpu_online_mask = to_cpumask(cpu_online_bits);
533EXPORT_SYMBOL(cpu_online_mask);
534
535static DECLARE_BITMAP(cpu_present_bits, CONFIG_NR_CPUS) __read_mostly;
536const struct cpumask *const cpu_present_mask = to_cpumask(cpu_present_bits);
537EXPORT_SYMBOL(cpu_present_mask);
538
539static DECLARE_BITMAP(cpu_active_bits, CONFIG_NR_CPUS) __read_mostly;
540const struct cpumask *const cpu_active_mask = to_cpumask(cpu_active_bits);
541EXPORT_SYMBOL(cpu_active_mask);
Rusty Russell3fa41522008-12-30 09:05:16 +1030542
543void set_cpu_possible(unsigned int cpu, bool possible)
544{
545 if (possible)
546 cpumask_set_cpu(cpu, to_cpumask(cpu_possible_bits));
547 else
548 cpumask_clear_cpu(cpu, to_cpumask(cpu_possible_bits));
549}
550
551void set_cpu_present(unsigned int cpu, bool present)
552{
553 if (present)
554 cpumask_set_cpu(cpu, to_cpumask(cpu_present_bits));
555 else
556 cpumask_clear_cpu(cpu, to_cpumask(cpu_present_bits));
557}
558
559void set_cpu_online(unsigned int cpu, bool online)
560{
561 if (online)
562 cpumask_set_cpu(cpu, to_cpumask(cpu_online_bits));
563 else
564 cpumask_clear_cpu(cpu, to_cpumask(cpu_online_bits));
565}
566
567void set_cpu_active(unsigned int cpu, bool active)
568{
569 if (active)
570 cpumask_set_cpu(cpu, to_cpumask(cpu_active_bits));
571 else
572 cpumask_clear_cpu(cpu, to_cpumask(cpu_active_bits));
573}
574
575void init_cpu_present(const struct cpumask *src)
576{
577 cpumask_copy(to_cpumask(cpu_present_bits), src);
578}
579
580void init_cpu_possible(const struct cpumask *src)
581{
582 cpumask_copy(to_cpumask(cpu_possible_bits), src);
583}
584
585void init_cpu_online(const struct cpumask *src)
586{
587 cpumask_copy(to_cpumask(cpu_online_bits), src);
588}