Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * linux/arch/m32r/kernel/smp.c |
| 3 | * |
| 4 | * M32R SMP support routines. |
| 5 | * |
| 6 | * Copyright (c) 2001, 2002 Hitoshi Yamamoto |
| 7 | * |
| 8 | * Taken from i386 version. |
| 9 | * (c) 1995 Alan Cox, Building #3 <alan@redhat.com> |
| 10 | * (c) 1998-99, 2000 Ingo Molnar <mingo@redhat.com> |
| 11 | * |
| 12 | * This code is released under the GNU General Public License version 2 or |
| 13 | * later. |
| 14 | */ |
| 15 | |
| 16 | #undef DEBUG_SMP |
| 17 | |
| 18 | #include <linux/irq.h> |
| 19 | #include <linux/interrupt.h> |
Alexey Dobriyan | d43c36d | 2009-10-07 17:09:06 +0400 | [diff] [blame] | 20 | #include <linux/sched.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | #include <linux/spinlock.h> |
| 22 | #include <linux/mm.h> |
| 23 | #include <linux/smp.h> |
| 24 | #include <linux/profile.h> |
| 25 | #include <linux/cpu.h> |
| 26 | |
| 27 | #include <asm/cacheflush.h> |
| 28 | #include <asm/pgalloc.h> |
| 29 | #include <asm/atomic.h> |
| 30 | #include <asm/io.h> |
| 31 | #include <asm/mmu_context.h> |
| 32 | #include <asm/m32r.h> |
| 33 | |
| 34 | /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ |
| 35 | /* Data structures and variables */ |
| 36 | /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ |
| 37 | |
| 38 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 39 | * For flush_cache_all() |
| 40 | */ |
| 41 | static DEFINE_SPINLOCK(flushcache_lock); |
| 42 | static volatile unsigned long flushcache_cpumask = 0; |
| 43 | |
| 44 | /* |
| 45 | * For flush_tlb_others() |
| 46 | */ |
| 47 | static volatile cpumask_t flush_cpumask; |
| 48 | static struct mm_struct *flush_mm; |
| 49 | static struct vm_area_struct *flush_vma; |
| 50 | static volatile unsigned long flush_va; |
| 51 | static DEFINE_SPINLOCK(tlbstate_lock); |
| 52 | #define FLUSH_ALL 0xffffffff |
| 53 | |
| 54 | DECLARE_PER_CPU(int, prof_multiplier); |
| 55 | DECLARE_PER_CPU(int, prof_old_multiplier); |
| 56 | DECLARE_PER_CPU(int, prof_counter); |
| 57 | |
| 58 | extern spinlock_t ipi_lock[]; |
| 59 | |
| 60 | /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ |
| 61 | /* Function Prototypes */ |
| 62 | /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ |
| 63 | |
| 64 | void smp_send_reschedule(int); |
| 65 | void smp_reschedule_interrupt(void); |
| 66 | |
| 67 | void smp_flush_cache_all(void); |
| 68 | void smp_flush_cache_all_interrupt(void); |
| 69 | |
| 70 | void smp_flush_tlb_all(void); |
| 71 | static void flush_tlb_all_ipi(void *); |
| 72 | |
| 73 | void smp_flush_tlb_mm(struct mm_struct *); |
| 74 | void smp_flush_tlb_range(struct vm_area_struct *, unsigned long, \ |
| 75 | unsigned long); |
| 76 | void smp_flush_tlb_page(struct vm_area_struct *, unsigned long); |
| 77 | static void flush_tlb_others(cpumask_t, struct mm_struct *, |
| 78 | struct vm_area_struct *, unsigned long); |
| 79 | void smp_invalidate_interrupt(void); |
| 80 | |
| 81 | void smp_send_stop(void); |
| 82 | static void stop_this_cpu(void *); |
| 83 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 84 | void smp_send_timer(void); |
| 85 | void smp_ipi_timer_interrupt(struct pt_regs *); |
Al Viro | 9c8e7f5 | 2006-10-07 16:29:18 +0100 | [diff] [blame] | 86 | void smp_local_timer_interrupt(void); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 87 | |
Adrian Bunk | 81e4807 | 2008-09-24 15:01:47 +0900 | [diff] [blame] | 88 | static void send_IPI_allbutself(int, int); |
Rusty Russell | c2a3a48 | 2009-09-24 09:34:43 -0600 | [diff] [blame] | 89 | static void send_IPI_mask(const struct cpumask *, int, int); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 90 | unsigned long send_IPI_mask_phys(cpumask_t, int, int); |
| 91 | |
| 92 | /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ |
| 93 | /* Rescheduling request Routines */ |
| 94 | /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ |
| 95 | |
| 96 | /*==========================================================================* |
| 97 | * Name: smp_send_reschedule |
| 98 | * |
| 99 | * Description: This routine requests other CPU to execute rescheduling. |
| 100 | * 1.Send 'RESCHEDULE_IPI' to other CPU. |
| 101 | * Request other CPU to execute 'smp_reschedule_interrupt()'. |
| 102 | * |
| 103 | * Born on Date: 2002.02.05 |
| 104 | * |
| 105 | * Arguments: cpu_id - Target CPU ID |
| 106 | * |
| 107 | * Returns: void (cannot fail) |
| 108 | * |
| 109 | * Modification log: |
| 110 | * Date Who Description |
| 111 | * ---------- --- -------------------------------------------------------- |
| 112 | * |
| 113 | *==========================================================================*/ |
| 114 | void smp_send_reschedule(int cpu_id) |
| 115 | { |
| 116 | WARN_ON(cpu_is_offline(cpu_id)); |
Rusty Russell | c2a3a48 | 2009-09-24 09:34:43 -0600 | [diff] [blame] | 117 | send_IPI_mask(cpumask_of(cpu_id), RESCHEDULE_IPI, 1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 118 | } |
| 119 | |
| 120 | /*==========================================================================* |
| 121 | * Name: smp_reschedule_interrupt |
| 122 | * |
| 123 | * Description: This routine executes on CPU which received |
| 124 | * 'RESCHEDULE_IPI'. |
| 125 | * Rescheduling is processed at the exit of interrupt |
| 126 | * operation. |
| 127 | * |
| 128 | * Born on Date: 2002.02.05 |
| 129 | * |
| 130 | * Arguments: NONE |
| 131 | * |
| 132 | * Returns: void (cannot fail) |
| 133 | * |
| 134 | * Modification log: |
| 135 | * Date Who Description |
| 136 | * ---------- --- -------------------------------------------------------- |
| 137 | * |
| 138 | *==========================================================================*/ |
| 139 | void smp_reschedule_interrupt(void) |
| 140 | { |
| 141 | /* nothing to do */ |
| 142 | } |
| 143 | |
| 144 | /*==========================================================================* |
| 145 | * Name: smp_flush_cache_all |
| 146 | * |
| 147 | * Description: This routine sends a 'INVALIDATE_CACHE_IPI' to all other |
| 148 | * CPUs in the system. |
| 149 | * |
| 150 | * Born on Date: 2003-05-28 |
| 151 | * |
| 152 | * Arguments: NONE |
| 153 | * |
| 154 | * Returns: void (cannot fail) |
| 155 | * |
| 156 | * Modification log: |
| 157 | * Date Who Description |
| 158 | * ---------- --- -------------------------------------------------------- |
| 159 | * |
| 160 | *==========================================================================*/ |
| 161 | void smp_flush_cache_all(void) |
| 162 | { |
| 163 | cpumask_t cpumask; |
| 164 | unsigned long *mask; |
| 165 | |
| 166 | preempt_disable(); |
| 167 | cpumask = cpu_online_map; |
| 168 | cpu_clear(smp_processor_id(), cpumask); |
| 169 | spin_lock(&flushcache_lock); |
| 170 | mask=cpus_addr(cpumask); |
| 171 | atomic_set_mask(*mask, (atomic_t *)&flushcache_cpumask); |
Rusty Russell | c2a3a48 | 2009-09-24 09:34:43 -0600 | [diff] [blame] | 172 | send_IPI_mask(&cpumask, INVALIDATE_CACHE_IPI, 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 173 | _flush_cache_copyback_all(); |
| 174 | while (flushcache_cpumask) |
| 175 | mb(); |
| 176 | spin_unlock(&flushcache_lock); |
| 177 | preempt_enable(); |
| 178 | } |
| 179 | |
| 180 | void smp_flush_cache_all_interrupt(void) |
| 181 | { |
| 182 | _flush_cache_copyback_all(); |
| 183 | clear_bit(smp_processor_id(), &flushcache_cpumask); |
| 184 | } |
| 185 | |
| 186 | /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ |
Simon Arlott | 5aa8b6c | 2007-10-20 01:14:39 +0200 | [diff] [blame] | 187 | /* TLB flush request Routines */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 188 | /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ |
| 189 | |
| 190 | /*==========================================================================* |
| 191 | * Name: smp_flush_tlb_all |
| 192 | * |
| 193 | * Description: This routine flushes all processes TLBs. |
| 194 | * 1.Request other CPU to execute 'flush_tlb_all_ipi()'. |
| 195 | * 2.Execute 'do_flush_tlb_all_local()'. |
| 196 | * |
| 197 | * Born on Date: 2002.02.05 |
| 198 | * |
| 199 | * Arguments: NONE |
| 200 | * |
| 201 | * Returns: void (cannot fail) |
| 202 | * |
| 203 | * Modification log: |
| 204 | * Date Who Description |
| 205 | * ---------- --- -------------------------------------------------------- |
| 206 | * |
| 207 | *==========================================================================*/ |
| 208 | void smp_flush_tlb_all(void) |
| 209 | { |
| 210 | unsigned long flags; |
| 211 | |
| 212 | preempt_disable(); |
| 213 | local_irq_save(flags); |
| 214 | __flush_tlb_all(); |
| 215 | local_irq_restore(flags); |
Jens Axboe | 8691e5a | 2008-06-06 11:18:06 +0200 | [diff] [blame] | 216 | smp_call_function(flush_tlb_all_ipi, NULL, 1); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 217 | preempt_enable(); |
| 218 | } |
| 219 | |
| 220 | /*==========================================================================* |
| 221 | * Name: flush_tlb_all_ipi |
| 222 | * |
| 223 | * Description: This routine flushes all local TLBs. |
| 224 | * 1.Execute 'do_flush_tlb_all_local()'. |
| 225 | * |
| 226 | * Born on Date: 2002.02.05 |
| 227 | * |
| 228 | * Arguments: *info - not used |
| 229 | * |
| 230 | * Returns: void (cannot fail) |
| 231 | * |
| 232 | * Modification log: |
| 233 | * Date Who Description |
| 234 | * ---------- --- -------------------------------------------------------- |
| 235 | * |
| 236 | *==========================================================================*/ |
| 237 | static void flush_tlb_all_ipi(void *info) |
| 238 | { |
| 239 | __flush_tlb_all(); |
| 240 | } |
| 241 | |
| 242 | /*==========================================================================* |
| 243 | * Name: smp_flush_tlb_mm |
| 244 | * |
| 245 | * Description: This routine flushes the specified mm context TLB's. |
| 246 | * |
| 247 | * Born on Date: 2002.02.05 |
| 248 | * |
| 249 | * Arguments: *mm - a pointer to the mm struct for flush TLB |
| 250 | * |
| 251 | * Returns: void (cannot fail) |
| 252 | * |
| 253 | * Modification log: |
| 254 | * Date Who Description |
| 255 | * ---------- --- -------------------------------------------------------- |
| 256 | * |
| 257 | *==========================================================================*/ |
| 258 | void smp_flush_tlb_mm(struct mm_struct *mm) |
| 259 | { |
Hirokazu Takata | a90933f | 2005-10-14 15:59:07 -0700 | [diff] [blame] | 260 | int cpu_id; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 261 | cpumask_t cpu_mask; |
Hirokazu Takata | a90933f | 2005-10-14 15:59:07 -0700 | [diff] [blame] | 262 | unsigned long *mmc; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 263 | unsigned long flags; |
| 264 | |
| 265 | preempt_disable(); |
Hirokazu Takata | a90933f | 2005-10-14 15:59:07 -0700 | [diff] [blame] | 266 | cpu_id = smp_processor_id(); |
| 267 | mmc = &mm->context[cpu_id]; |
Rusty Russell | 49b9205 | 2009-09-24 09:34:49 -0600 | [diff] [blame] | 268 | cpu_mask = *mm_cpumask(mm); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 269 | cpu_clear(cpu_id, cpu_mask); |
| 270 | |
| 271 | if (*mmc != NO_CONTEXT) { |
| 272 | local_irq_save(flags); |
| 273 | *mmc = NO_CONTEXT; |
| 274 | if (mm == current->mm) |
| 275 | activate_context(mm); |
| 276 | else |
Rusty Russell | 49b9205 | 2009-09-24 09:34:49 -0600 | [diff] [blame] | 277 | cpumask_clear_cpu(cpu_id, mm_cpumask(mm)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 278 | local_irq_restore(flags); |
| 279 | } |
| 280 | if (!cpus_empty(cpu_mask)) |
| 281 | flush_tlb_others(cpu_mask, mm, NULL, FLUSH_ALL); |
| 282 | |
| 283 | preempt_enable(); |
| 284 | } |
| 285 | |
| 286 | /*==========================================================================* |
| 287 | * Name: smp_flush_tlb_range |
| 288 | * |
| 289 | * Description: This routine flushes a range of pages. |
| 290 | * |
| 291 | * Born on Date: 2002.02.05 |
| 292 | * |
| 293 | * Arguments: *mm - a pointer to the mm struct for flush TLB |
| 294 | * start - not used |
| 295 | * end - not used |
| 296 | * |
| 297 | * Returns: void (cannot fail) |
| 298 | * |
| 299 | * Modification log: |
| 300 | * Date Who Description |
| 301 | * ---------- --- -------------------------------------------------------- |
| 302 | * |
| 303 | *==========================================================================*/ |
| 304 | void smp_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, |
| 305 | unsigned long end) |
| 306 | { |
| 307 | smp_flush_tlb_mm(vma->vm_mm); |
| 308 | } |
| 309 | |
| 310 | /*==========================================================================* |
| 311 | * Name: smp_flush_tlb_page |
| 312 | * |
| 313 | * Description: This routine flushes one page. |
| 314 | * |
| 315 | * Born on Date: 2002.02.05 |
| 316 | * |
| 317 | * Arguments: *vma - a pointer to the vma struct include va |
| 318 | * va - virtual address for flush TLB |
| 319 | * |
| 320 | * Returns: void (cannot fail) |
| 321 | * |
| 322 | * Modification log: |
| 323 | * Date Who Description |
| 324 | * ---------- --- -------------------------------------------------------- |
| 325 | * |
| 326 | *==========================================================================*/ |
| 327 | void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long va) |
| 328 | { |
| 329 | struct mm_struct *mm = vma->vm_mm; |
Hirokazu Takata | a90933f | 2005-10-14 15:59:07 -0700 | [diff] [blame] | 330 | int cpu_id; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 331 | cpumask_t cpu_mask; |
Hirokazu Takata | a90933f | 2005-10-14 15:59:07 -0700 | [diff] [blame] | 332 | unsigned long *mmc; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 333 | unsigned long flags; |
| 334 | |
| 335 | preempt_disable(); |
Hirokazu Takata | a90933f | 2005-10-14 15:59:07 -0700 | [diff] [blame] | 336 | cpu_id = smp_processor_id(); |
| 337 | mmc = &mm->context[cpu_id]; |
Rusty Russell | 49b9205 | 2009-09-24 09:34:49 -0600 | [diff] [blame] | 338 | cpu_mask = *mm_cpumask(mm); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 339 | cpu_clear(cpu_id, cpu_mask); |
| 340 | |
| 341 | #ifdef DEBUG_SMP |
| 342 | if (!mm) |
| 343 | BUG(); |
| 344 | #endif |
| 345 | |
| 346 | if (*mmc != NO_CONTEXT) { |
| 347 | local_irq_save(flags); |
| 348 | va &= PAGE_MASK; |
| 349 | va |= (*mmc & MMU_CONTEXT_ASID_MASK); |
| 350 | __flush_tlb_page(va); |
| 351 | local_irq_restore(flags); |
| 352 | } |
| 353 | if (!cpus_empty(cpu_mask)) |
| 354 | flush_tlb_others(cpu_mask, mm, vma, va); |
| 355 | |
| 356 | preempt_enable(); |
| 357 | } |
| 358 | |
| 359 | /*==========================================================================* |
| 360 | * Name: flush_tlb_others |
| 361 | * |
| 362 | * Description: This routine requests other CPU to execute flush TLB. |
Simon Arlott | 5aa8b6c | 2007-10-20 01:14:39 +0200 | [diff] [blame] | 363 | * 1.Setup parameters. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 364 | * 2.Send 'INVALIDATE_TLB_IPI' to other CPU. |
| 365 | * Request other CPU to execute 'smp_invalidate_interrupt()'. |
| 366 | * 3.Wait for other CPUs operation finished. |
| 367 | * |
| 368 | * Born on Date: 2002.02.05 |
| 369 | * |
| 370 | * Arguments: cpumask - bitmap of target CPUs |
| 371 | * *mm - a pointer to the mm struct for flush TLB |
| 372 | * *vma - a pointer to the vma struct include va |
| 373 | * va - virtual address for flush TLB |
| 374 | * |
| 375 | * Returns: void (cannot fail) |
| 376 | * |
| 377 | * Modification log: |
| 378 | * Date Who Description |
| 379 | * ---------- --- -------------------------------------------------------- |
| 380 | * |
| 381 | *==========================================================================*/ |
| 382 | static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, |
| 383 | struct vm_area_struct *vma, unsigned long va) |
| 384 | { |
| 385 | unsigned long *mask; |
| 386 | #ifdef DEBUG_SMP |
| 387 | unsigned long flags; |
| 388 | __save_flags(flags); |
| 389 | if (!(flags & 0x0040)) /* Interrupt Disable NONONO */ |
| 390 | BUG(); |
| 391 | #endif /* DEBUG_SMP */ |
| 392 | |
| 393 | /* |
| 394 | * A couple of (to be removed) sanity checks: |
| 395 | * |
| 396 | * - we do not send IPIs to not-yet booted CPUs. |
| 397 | * - current CPU must not be in mask |
| 398 | * - mask must exist :) |
| 399 | */ |
| 400 | BUG_ON(cpus_empty(cpumask)); |
| 401 | |
| 402 | BUG_ON(cpu_isset(smp_processor_id(), cpumask)); |
| 403 | BUG_ON(!mm); |
| 404 | |
| 405 | /* If a CPU which we ran on has gone down, OK. */ |
| 406 | cpus_and(cpumask, cpumask, cpu_online_map); |
| 407 | if (cpus_empty(cpumask)) |
| 408 | return; |
| 409 | |
| 410 | /* |
| 411 | * i'm not happy about this global shared spinlock in the |
| 412 | * MM hot path, but we'll see how contended it is. |
| 413 | * Temporarily this turns IRQs off, so that lockups are |
| 414 | * detected by the NMI watchdog. |
| 415 | */ |
| 416 | spin_lock(&tlbstate_lock); |
| 417 | |
| 418 | flush_mm = mm; |
| 419 | flush_vma = vma; |
| 420 | flush_va = va; |
| 421 | mask=cpus_addr(cpumask); |
| 422 | atomic_set_mask(*mask, (atomic_t *)&flush_cpumask); |
| 423 | |
| 424 | /* |
| 425 | * We have to send the IPI only to |
| 426 | * CPUs affected. |
| 427 | */ |
Rusty Russell | c2a3a48 | 2009-09-24 09:34:43 -0600 | [diff] [blame] | 428 | send_IPI_mask(&cpumask, INVALIDATE_TLB_IPI, 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 429 | |
| 430 | while (!cpus_empty(flush_cpumask)) { |
| 431 | /* nothing. lockup detection does not belong here */ |
| 432 | mb(); |
| 433 | } |
| 434 | |
| 435 | flush_mm = NULL; |
| 436 | flush_vma = NULL; |
| 437 | flush_va = 0; |
| 438 | spin_unlock(&tlbstate_lock); |
| 439 | } |
| 440 | |
| 441 | /*==========================================================================* |
| 442 | * Name: smp_invalidate_interrupt |
| 443 | * |
| 444 | * Description: This routine executes on CPU which received |
| 445 | * 'INVALIDATE_TLB_IPI'. |
| 446 | * 1.Flush local TLB. |
| 447 | * 2.Report flush TLB process was finished. |
| 448 | * |
| 449 | * Born on Date: 2002.02.05 |
| 450 | * |
| 451 | * Arguments: NONE |
| 452 | * |
| 453 | * Returns: void (cannot fail) |
| 454 | * |
| 455 | * Modification log: |
| 456 | * Date Who Description |
| 457 | * ---------- --- -------------------------------------------------------- |
| 458 | * |
| 459 | *==========================================================================*/ |
| 460 | void smp_invalidate_interrupt(void) |
| 461 | { |
| 462 | int cpu_id = smp_processor_id(); |
| 463 | unsigned long *mmc = &flush_mm->context[cpu_id]; |
| 464 | |
| 465 | if (!cpu_isset(cpu_id, flush_cpumask)) |
| 466 | return; |
| 467 | |
| 468 | if (flush_va == FLUSH_ALL) { |
| 469 | *mmc = NO_CONTEXT; |
| 470 | if (flush_mm == current->active_mm) |
| 471 | activate_context(flush_mm); |
| 472 | else |
Rusty Russell | 49b9205 | 2009-09-24 09:34:49 -0600 | [diff] [blame] | 473 | cpumask_clear_cpu(cpu_id, mm_cpumask(flush_mm)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 474 | } else { |
| 475 | unsigned long va = flush_va; |
| 476 | |
| 477 | if (*mmc != NO_CONTEXT) { |
| 478 | va &= PAGE_MASK; |
| 479 | va |= (*mmc & MMU_CONTEXT_ASID_MASK); |
| 480 | __flush_tlb_page(va); |
| 481 | } |
| 482 | } |
| 483 | cpu_clear(cpu_id, flush_cpumask); |
| 484 | } |
| 485 | |
| 486 | /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ |
Simon Arlott | 5aa8b6c | 2007-10-20 01:14:39 +0200 | [diff] [blame] | 487 | /* Stop CPU request Routines */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 488 | /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ |
| 489 | |
| 490 | /*==========================================================================* |
| 491 | * Name: smp_send_stop |
| 492 | * |
| 493 | * Description: This routine requests stop all CPUs. |
| 494 | * 1.Request other CPU to execute 'stop_this_cpu()'. |
| 495 | * |
| 496 | * Born on Date: 2002.02.05 |
| 497 | * |
| 498 | * Arguments: NONE |
| 499 | * |
| 500 | * Returns: void (cannot fail) |
| 501 | * |
| 502 | * Modification log: |
| 503 | * Date Who Description |
| 504 | * ---------- --- -------------------------------------------------------- |
| 505 | * |
| 506 | *==========================================================================*/ |
| 507 | void smp_send_stop(void) |
| 508 | { |
Jens Axboe | 8691e5a | 2008-06-06 11:18:06 +0200 | [diff] [blame] | 509 | smp_call_function(stop_this_cpu, NULL, 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 510 | } |
| 511 | |
| 512 | /*==========================================================================* |
| 513 | * Name: stop_this_cpu |
| 514 | * |
| 515 | * Description: This routine halt CPU. |
| 516 | * |
| 517 | * Born on Date: 2002.02.05 |
| 518 | * |
| 519 | * Arguments: NONE |
| 520 | * |
| 521 | * Returns: void (cannot fail) |
| 522 | * |
| 523 | * Modification log: |
| 524 | * Date Who Description |
| 525 | * ---------- --- -------------------------------------------------------- |
| 526 | * |
| 527 | *==========================================================================*/ |
| 528 | static void stop_this_cpu(void *dummy) |
| 529 | { |
| 530 | int cpu_id = smp_processor_id(); |
| 531 | |
| 532 | /* |
| 533 | * Remove this CPU: |
| 534 | */ |
| 535 | cpu_clear(cpu_id, cpu_online_map); |
| 536 | |
| 537 | /* |
| 538 | * PSW IE = 1; |
| 539 | * IMASK = 0; |
| 540 | * goto SLEEP |
| 541 | */ |
| 542 | local_irq_disable(); |
| 543 | outl(0, M32R_ICU_IMASK_PORTL); |
| 544 | inl(M32R_ICU_IMASK_PORTL); /* dummy read */ |
| 545 | local_irq_enable(); |
| 546 | |
| 547 | for ( ; ; ); |
| 548 | } |
| 549 | |
Rusty Russell | c2a3a48 | 2009-09-24 09:34:43 -0600 | [diff] [blame] | 550 | void arch_send_call_function_ipi_mask(const struct cpumask *mask) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 551 | { |
Jens Axboe | 7b7426c | 2008-06-10 20:49:30 +0200 | [diff] [blame] | 552 | send_IPI_mask(mask, CALL_FUNCTION_IPI, 0); |
| 553 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 554 | |
Jens Axboe | 7b7426c | 2008-06-10 20:49:30 +0200 | [diff] [blame] | 555 | void arch_send_call_function_single_ipi(int cpu) |
| 556 | { |
Rusty Russell | c2a3a48 | 2009-09-24 09:34:43 -0600 | [diff] [blame] | 557 | send_IPI_mask(cpumask_of(cpu), CALL_FUNC_SINGLE_IPI, 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 558 | } |
| 559 | |
| 560 | /*==========================================================================* |
| 561 | * Name: smp_call_function_interrupt |
| 562 | * |
| 563 | * Description: This routine executes on CPU which received |
| 564 | * 'CALL_FUNCTION_IPI'. |
| 565 | * |
| 566 | * Born on Date: 2002.02.05 |
| 567 | * |
| 568 | * Arguments: NONE |
| 569 | * |
| 570 | * Returns: void (cannot fail) |
| 571 | * |
| 572 | * Modification log: |
| 573 | * Date Who Description |
| 574 | * ---------- --- -------------------------------------------------------- |
| 575 | * |
| 576 | *==========================================================================*/ |
| 577 | void smp_call_function_interrupt(void) |
| 578 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 579 | irq_enter(); |
Jens Axboe | 7b7426c | 2008-06-10 20:49:30 +0200 | [diff] [blame] | 580 | generic_smp_call_function_interrupt(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 581 | irq_exit(); |
Jens Axboe | 7b7426c | 2008-06-10 20:49:30 +0200 | [diff] [blame] | 582 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 583 | |
Jens Axboe | 7b7426c | 2008-06-10 20:49:30 +0200 | [diff] [blame] | 584 | void smp_call_function_single_interrupt(void) |
| 585 | { |
| 586 | irq_enter(); |
| 587 | generic_smp_call_function_single_interrupt(); |
| 588 | irq_exit(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 589 | } |
| 590 | |
| 591 | /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ |
Simon Arlott | 5aa8b6c | 2007-10-20 01:14:39 +0200 | [diff] [blame] | 592 | /* Timer Routines */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 593 | /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ |
| 594 | |
| 595 | /*==========================================================================* |
| 596 | * Name: smp_send_timer |
| 597 | * |
| 598 | * Description: This routine sends a 'LOCAL_TIMER_IPI' to all other CPUs |
| 599 | * in the system. |
| 600 | * |
| 601 | * Born on Date: 2002.02.05 |
| 602 | * |
| 603 | * Arguments: NONE |
| 604 | * |
| 605 | * Returns: void (cannot fail) |
| 606 | * |
| 607 | * Modification log: |
| 608 | * Date Who Description |
| 609 | * ---------- --- -------------------------------------------------------- |
| 610 | * |
| 611 | *==========================================================================*/ |
| 612 | void smp_send_timer(void) |
| 613 | { |
| 614 | send_IPI_allbutself(LOCAL_TIMER_IPI, 1); |
| 615 | } |
| 616 | |
| 617 | /*==========================================================================* |
| 618 | * Name: smp_send_timer |
| 619 | * |
| 620 | * Description: This routine executes on CPU which received |
| 621 | * 'LOCAL_TIMER_IPI'. |
| 622 | * |
| 623 | * Born on Date: 2002.02.05 |
| 624 | * |
| 625 | * Arguments: *regs - a pointer to the saved regster info |
| 626 | * |
| 627 | * Returns: void (cannot fail) |
| 628 | * |
| 629 | * Modification log: |
| 630 | * Date Who Description |
| 631 | * ---------- --- -------------------------------------------------------- |
| 632 | * |
| 633 | *==========================================================================*/ |
| 634 | void smp_ipi_timer_interrupt(struct pt_regs *regs) |
| 635 | { |
Al Viro | 9c8e7f5 | 2006-10-07 16:29:18 +0100 | [diff] [blame] | 636 | struct pt_regs *old_regs; |
| 637 | old_regs = set_irq_regs(regs); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 638 | irq_enter(); |
Al Viro | 9c8e7f5 | 2006-10-07 16:29:18 +0100 | [diff] [blame] | 639 | smp_local_timer_interrupt(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 640 | irq_exit(); |
Al Viro | 9c8e7f5 | 2006-10-07 16:29:18 +0100 | [diff] [blame] | 641 | set_irq_regs(old_regs); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 642 | } |
| 643 | |
| 644 | /*==========================================================================* |
| 645 | * Name: smp_local_timer_interrupt |
| 646 | * |
| 647 | * Description: Local timer interrupt handler. It does both profiling and |
| 648 | * process statistics/rescheduling. |
| 649 | * We do profiling in every local tick, statistics/rescheduling |
| 650 | * happen only every 'profiling multiplier' ticks. The default |
| 651 | * multiplier is 1 and it can be changed by writing the new |
| 652 | * multiplier value into /proc/profile. |
| 653 | * |
| 654 | * Born on Date: 2002.02.05 |
| 655 | * |
| 656 | * Arguments: *regs - a pointer to the saved regster info |
| 657 | * |
| 658 | * Returns: void (cannot fail) |
| 659 | * |
| 660 | * Original: arch/i386/kernel/apic.c |
| 661 | * |
| 662 | * Modification log: |
| 663 | * Date Who Description |
| 664 | * ---------- --- -------------------------------------------------------- |
| 665 | * 2003-06-24 hy use per_cpu structure. |
| 666 | *==========================================================================*/ |
Al Viro | 9c8e7f5 | 2006-10-07 16:29:18 +0100 | [diff] [blame] | 667 | void smp_local_timer_interrupt(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 668 | { |
Al Viro | 9c8e7f5 | 2006-10-07 16:29:18 +0100 | [diff] [blame] | 669 | int user = user_mode(get_irq_regs()); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 670 | int cpu_id = smp_processor_id(); |
| 671 | |
| 672 | /* |
| 673 | * The profiling function is SMP safe. (nothing can mess |
| 674 | * around with "current", and the profiling counters are |
| 675 | * updated with atomic operations). This is especially |
| 676 | * useful with a profiling multiplier != 1 |
| 677 | */ |
| 678 | |
Al Viro | 9c8e7f5 | 2006-10-07 16:29:18 +0100 | [diff] [blame] | 679 | profile_tick(CPU_PROFILING); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 680 | |
| 681 | if (--per_cpu(prof_counter, cpu_id) <= 0) { |
| 682 | /* |
| 683 | * The multiplier may have changed since the last time we got |
| 684 | * to this point as a result of the user writing to |
| 685 | * /proc/profile. In this case we need to adjust the APIC |
| 686 | * timer accordingly. |
| 687 | * |
| 688 | * Interrupts are already masked off at this point. |
| 689 | */ |
| 690 | per_cpu(prof_counter, cpu_id) |
| 691 | = per_cpu(prof_multiplier, cpu_id); |
| 692 | if (per_cpu(prof_counter, cpu_id) |
| 693 | != per_cpu(prof_old_multiplier, cpu_id)) |
| 694 | { |
| 695 | per_cpu(prof_old_multiplier, cpu_id) |
| 696 | = per_cpu(prof_counter, cpu_id); |
| 697 | } |
| 698 | |
| 699 | update_process_times(user); |
| 700 | } |
| 701 | } |
| 702 | |
| 703 | /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ |
Simon Arlott | 5aa8b6c | 2007-10-20 01:14:39 +0200 | [diff] [blame] | 704 | /* Send IPI Routines */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 705 | /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ |
| 706 | |
| 707 | /*==========================================================================* |
| 708 | * Name: send_IPI_allbutself |
| 709 | * |
| 710 | * Description: This routine sends a IPI to all other CPUs in the system. |
| 711 | * |
| 712 | * Born on Date: 2002.02.05 |
| 713 | * |
| 714 | * Arguments: ipi_num - Number of IPI |
| 715 | * try - 0 : Send IPI certainly. |
Simon Arlott | 5aa8b6c | 2007-10-20 01:14:39 +0200 | [diff] [blame] | 716 | * !0 : The following IPI is not sent when Target CPU |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 717 | * has not received the before IPI. |
| 718 | * |
| 719 | * Returns: void (cannot fail) |
| 720 | * |
| 721 | * Modification log: |
| 722 | * Date Who Description |
| 723 | * ---------- --- -------------------------------------------------------- |
| 724 | * |
| 725 | *==========================================================================*/ |
Adrian Bunk | 81e4807 | 2008-09-24 15:01:47 +0900 | [diff] [blame] | 726 | static void send_IPI_allbutself(int ipi_num, int try) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 727 | { |
| 728 | cpumask_t cpumask; |
| 729 | |
| 730 | cpumask = cpu_online_map; |
| 731 | cpu_clear(smp_processor_id(), cpumask); |
| 732 | |
Rusty Russell | c2a3a48 | 2009-09-24 09:34:43 -0600 | [diff] [blame] | 733 | send_IPI_mask(&cpumask, ipi_num, try); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 734 | } |
| 735 | |
| 736 | /*==========================================================================* |
| 737 | * Name: send_IPI_mask |
| 738 | * |
| 739 | * Description: This routine sends a IPI to CPUs in the system. |
| 740 | * |
| 741 | * Born on Date: 2002.02.05 |
| 742 | * |
| 743 | * Arguments: cpu_mask - Bitmap of target CPUs logical ID |
| 744 | * ipi_num - Number of IPI |
| 745 | * try - 0 : Send IPI certainly. |
Simon Arlott | 5aa8b6c | 2007-10-20 01:14:39 +0200 | [diff] [blame] | 746 | * !0 : The following IPI is not sent when Target CPU |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 747 | * has not received the before IPI. |
| 748 | * |
| 749 | * Returns: void (cannot fail) |
| 750 | * |
| 751 | * Modification log: |
| 752 | * Date Who Description |
| 753 | * ---------- --- -------------------------------------------------------- |
| 754 | * |
| 755 | *==========================================================================*/ |
Rusty Russell | c2a3a48 | 2009-09-24 09:34:43 -0600 | [diff] [blame] | 756 | static void send_IPI_mask(const struct cpumask *cpumask, int ipi_num, int try) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 757 | { |
| 758 | cpumask_t physid_mask, tmp; |
| 759 | int cpu_id, phys_id; |
| 760 | int num_cpus = num_online_cpus(); |
| 761 | |
| 762 | if (num_cpus <= 1) /* NO MP */ |
| 763 | return; |
| 764 | |
Rusty Russell | c2a3a48 | 2009-09-24 09:34:43 -0600 | [diff] [blame] | 765 | cpumask_and(&tmp, cpumask, cpu_online_mask); |
| 766 | BUG_ON(!cpumask_equal(cpumask, &tmp)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 767 | |
| 768 | physid_mask = CPU_MASK_NONE; |
Rusty Russell | c2a3a48 | 2009-09-24 09:34:43 -0600 | [diff] [blame] | 769 | for_each_cpu(cpu_id, cpumask) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 770 | if ((phys_id = cpu_to_physid(cpu_id)) != -1) |
| 771 | cpu_set(phys_id, physid_mask); |
| 772 | } |
| 773 | |
| 774 | send_IPI_mask_phys(physid_mask, ipi_num, try); |
| 775 | } |
| 776 | |
| 777 | /*==========================================================================* |
| 778 | * Name: send_IPI_mask_phys |
| 779 | * |
| 780 | * Description: This routine sends a IPI to other CPUs in the system. |
| 781 | * |
| 782 | * Born on Date: 2002.02.05 |
| 783 | * |
| 784 | * Arguments: cpu_mask - Bitmap of target CPUs physical ID |
| 785 | * ipi_num - Number of IPI |
| 786 | * try - 0 : Send IPI certainly. |
Simon Arlott | 5aa8b6c | 2007-10-20 01:14:39 +0200 | [diff] [blame] | 787 | * !0 : The following IPI is not sent when Target CPU |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 788 | * has not received the before IPI. |
| 789 | * |
| 790 | * Returns: IPICRi regster value. |
| 791 | * |
| 792 | * Modification log: |
| 793 | * Date Who Description |
| 794 | * ---------- --- -------------------------------------------------------- |
| 795 | * |
| 796 | *==========================================================================*/ |
| 797 | unsigned long send_IPI_mask_phys(cpumask_t physid_mask, int ipi_num, |
| 798 | int try) |
| 799 | { |
| 800 | spinlock_t *ipilock; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 801 | volatile unsigned long *ipicr_addr; |
| 802 | unsigned long ipicr_val; |
| 803 | unsigned long my_physid_mask; |
| 804 | unsigned long mask = cpus_addr(physid_mask)[0]; |
| 805 | |
| 806 | |
| 807 | if (mask & ~physids_coerce(phys_cpu_present_map)) |
| 808 | BUG(); |
Roel Kluin | 45cdd47 | 2009-11-01 15:33:06 +0100 | [diff] [blame] | 809 | if (ipi_num >= NR_IPIS || ipi_num < 0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 810 | BUG(); |
| 811 | |
| 812 | mask <<= IPI_SHIFT; |
| 813 | ipilock = &ipi_lock[ipi_num]; |
| 814 | ipicr_addr = (volatile unsigned long *)(M32R_ICU_IPICR_ADDR |
| 815 | + (ipi_num << 2)); |
| 816 | my_physid_mask = ~(1 << smp_processor_id()); |
| 817 | |
| 818 | /* |
| 819 | * lock ipi_lock[i] |
| 820 | * check IPICRi == 0 |
| 821 | * write IPICRi (send IPIi) |
| 822 | * unlock ipi_lock[i] |
| 823 | */ |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 824 | spin_lock(ipilock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 825 | __asm__ __volatile__ ( |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 826 | ";; CHECK IPICRi == 0 \n\t" |
| 827 | ".fillinsn \n" |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 828 | "1: \n\t" |
| 829 | "ld %0, @%1 \n\t" |
| 830 | "and %0, %4 \n\t" |
| 831 | "beqz %0, 2f \n\t" |
| 832 | "bnez %3, 3f \n\t" |
| 833 | "bra 1b \n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 834 | ";; WRITE IPICRi (send IPIi) \n\t" |
| 835 | ".fillinsn \n" |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 836 | "2: \n\t" |
| 837 | "st %2, @%1 \n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 838 | ".fillinsn \n" |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 839 | "3: \n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 840 | : "=&r"(ipicr_val) |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 841 | : "r"(ipicr_addr), "r"(mask), "r"(try), "r"(my_physid_mask) |
| 842 | : "memory" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 843 | ); |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 844 | spin_unlock(ipilock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 845 | |
| 846 | return ipicr_val; |
| 847 | } |