Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 1 | /* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation. |
Rusty Russell | e5582ca | 2006-09-29 02:01:35 -0700 | [diff] [blame] | 2 | * GPL v2 and any later version. |
| 3 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4 | #include <linux/cpu.h> |
| 5 | #include <linux/err.h> |
Prarit Bhargava | ee527cd | 2007-05-08 00:25:08 -0700 | [diff] [blame] | 6 | #include <linux/kthread.h> |
| 7 | #include <linux/module.h> |
| 8 | #include <linux/sched.h> |
| 9 | #include <linux/stop_machine.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10 | #include <linux/syscalls.h> |
Benjamin Herrenschmidt | a12bb44 | 2007-05-10 22:22:47 -0700 | [diff] [blame] | 11 | #include <linux/interrupt.h> |
| 12 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 13 | #include <asm/atomic.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 14 | #include <asm/uaccess.h> |
| 15 | |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 16 | /* This controls the threads on each CPU. */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | enum stopmachine_state { |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 18 | /* Dummy starting state for thread. */ |
| 19 | STOPMACHINE_NONE, |
| 20 | /* Awaiting everyone to be scheduled. */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | STOPMACHINE_PREPARE, |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 22 | /* Disable interrupts. */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 23 | STOPMACHINE_DISABLE_IRQ, |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 24 | /* Run the function */ |
Jason Baron | 5c2aed6 | 2008-02-28 11:33:03 -0500 | [diff] [blame] | 25 | STOPMACHINE_RUN, |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 26 | /* Exit */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 27 | STOPMACHINE_EXIT, |
| 28 | }; |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 29 | static enum stopmachine_state state; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 30 | |
Jason Baron | 5c2aed6 | 2008-02-28 11:33:03 -0500 | [diff] [blame] | 31 | struct stop_machine_data { |
| 32 | int (*fn)(void *); |
| 33 | void *data; |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 34 | int fnret; |
| 35 | }; |
Jason Baron | 5c2aed6 | 2008-02-28 11:33:03 -0500 | [diff] [blame] | 36 | |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 37 | /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */ |
| 38 | static unsigned int num_threads; |
| 39 | static atomic_t thread_ack; |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 40 | static DEFINE_MUTEX(lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 41 | |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 42 | static struct workqueue_struct *stop_machine_wq; |
| 43 | static struct stop_machine_data active, idle; |
| 44 | static const cpumask_t *active_cpus; |
| 45 | static void *stop_machine_work; |
| 46 | |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 47 | static void set_state(enum stopmachine_state newstate) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 48 | { |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 49 | /* Reset ack counter. */ |
| 50 | atomic_set(&thread_ack, num_threads); |
| 51 | smp_wmb(); |
| 52 | state = newstate; |
| 53 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 54 | |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 55 | /* Last one to ack a state moves to the next state. */ |
| 56 | static void ack_state(void) |
| 57 | { |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 58 | if (atomic_dec_and_test(&thread_ack)) |
| 59 | set_state(state + 1); |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 60 | } |
Andrew Morton | d8cb7c1 | 2006-07-03 17:32:22 -0700 | [diff] [blame] | 61 | |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 62 | /* This is the actual function which stops the CPU. It runs |
| 63 | * in the context of a dedicated stopmachine workqueue. */ |
| 64 | static void stop_cpu(struct work_struct *unused) |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 65 | { |
| 66 | enum stopmachine_state curstate = STOPMACHINE_NONE; |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 67 | struct stop_machine_data *smdata = &idle; |
| 68 | int cpu = smp_processor_id(); |
Heiko Carstens | 8163bca | 2008-10-22 10:00:26 -0500 | [diff] [blame] | 69 | int err; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 70 | |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 71 | if (!active_cpus) { |
| 72 | if (cpu == first_cpu(cpu_online_map)) |
| 73 | smdata = &active; |
| 74 | } else { |
| 75 | if (cpu_isset(cpu, *active_cpus)) |
| 76 | smdata = &active; |
| 77 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 78 | /* Simple state machine */ |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 79 | do { |
| 80 | /* Chill out and ensure we re-read stopmachine_state. */ |
Christian Borntraeger | 3401a61e | 2008-05-08 15:20:38 +0200 | [diff] [blame] | 81 | cpu_relax(); |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 82 | if (state != curstate) { |
| 83 | curstate = state; |
| 84 | switch (curstate) { |
| 85 | case STOPMACHINE_DISABLE_IRQ: |
| 86 | local_irq_disable(); |
| 87 | hard_irq_disable(); |
| 88 | break; |
| 89 | case STOPMACHINE_RUN: |
Heiko Carstens | 8163bca | 2008-10-22 10:00:26 -0500 | [diff] [blame] | 90 | /* On multiple CPUs only a single error code |
| 91 | * is needed to tell that something failed. */ |
| 92 | err = smdata->fn(smdata->data); |
| 93 | if (err) |
| 94 | smdata->fnret = err; |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 95 | break; |
| 96 | default: |
| 97 | break; |
| 98 | } |
| 99 | ack_state(); |
| 100 | } |
| 101 | } while (curstate != STOPMACHINE_EXIT); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 102 | |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 103 | local_irq_enable(); |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 104 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 105 | |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 106 | /* Callback for CPUs which aren't supposed to do anything. */ |
| 107 | static int chill(void *unused) |
| 108 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 109 | return 0; |
| 110 | } |
| 111 | |
Rusty Russell | eeec4fa | 2008-07-28 12:16:30 -0500 | [diff] [blame] | 112 | int __stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 113 | { |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 114 | struct work_struct *sm_work; |
Rusty Russell | e14c8bf | 2008-11-17 08:22:18 +1030 | [diff] [blame] | 115 | int i, ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 116 | |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 117 | /* Set up initial state. */ |
| 118 | mutex_lock(&lock); |
| 119 | num_threads = num_online_cpus(); |
| 120 | active_cpus = cpus; |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 121 | active.fn = fn; |
| 122 | active.data = data; |
| 123 | active.fnret = 0; |
| 124 | idle.fn = chill; |
| 125 | idle.data = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 126 | |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 127 | set_state(STOPMACHINE_PREPARE); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 128 | |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 129 | /* Schedule the stop_cpu work on all cpus: hold this CPU so one |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 130 | * doesn't hit this CPU until we're ready. */ |
Rusty Russell | eeec4fa | 2008-07-28 12:16:30 -0500 | [diff] [blame] | 131 | get_cpu(); |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 132 | for_each_online_cpu(i) { |
| 133 | sm_work = percpu_ptr(stop_machine_work, i); |
| 134 | INIT_WORK(sm_work, stop_cpu); |
| 135 | queue_work_on(i, stop_machine_wq, sm_work); |
| 136 | } |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 137 | /* This will release the thread on our CPU. */ |
| 138 | put_cpu(); |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 139 | flush_workqueue(stop_machine_wq); |
Rusty Russell | e14c8bf | 2008-11-17 08:22:18 +1030 | [diff] [blame] | 140 | ret = active.fnret; |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 141 | mutex_unlock(&lock); |
Rusty Russell | e14c8bf | 2008-11-17 08:22:18 +1030 | [diff] [blame] | 142 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 143 | } |
| 144 | |
Rusty Russell | eeec4fa | 2008-07-28 12:16:30 -0500 | [diff] [blame] | 145 | int stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 146 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 147 | int ret; |
| 148 | |
| 149 | /* No CPUs can come up or down during this. */ |
Gautham R Shenoy | 86ef5c9 | 2008-01-25 21:08:02 +0100 | [diff] [blame] | 150 | get_online_cpus(); |
Rusty Russell | eeec4fa | 2008-07-28 12:16:30 -0500 | [diff] [blame] | 151 | ret = __stop_machine(fn, data, cpus); |
Gautham R Shenoy | 86ef5c9 | 2008-01-25 21:08:02 +0100 | [diff] [blame] | 152 | put_online_cpus(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 153 | |
| 154 | return ret; |
| 155 | } |
Rusty Russell | eeec4fa | 2008-07-28 12:16:30 -0500 | [diff] [blame] | 156 | EXPORT_SYMBOL_GPL(stop_machine); |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 157 | |
| 158 | static int __init stop_machine_init(void) |
| 159 | { |
| 160 | stop_machine_wq = create_rt_workqueue("kstop"); |
| 161 | stop_machine_work = alloc_percpu(struct work_struct); |
| 162 | return 0; |
| 163 | } |
Linus Torvalds | 4403b40 | 2008-10-25 19:53:38 -0700 | [diff] [blame] | 164 | core_initcall(stop_machine_init); |