blob: 3b09634a51535768cb3f99781dfa58c6a16b6ba4 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 * Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar
3 *
4 * This file contains the lowest level x86-specific interrupt
5 * entry, irq-stacks and irq statistics code. All the remaining
6 * irq logic is done by the generic kernel/irq/ code and
7 * by the x86-specific irq controller code. (e.g. i8259.c and
8 * io_apic.c.)
9 */
10
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/module.h>
12#include <linux/seq_file.h>
13#include <linux/interrupt.h>
14#include <linux/kernel_stat.h>
Zwane Mwaikambof3705132005-06-25 14:54:50 -070015#include <linux/notifier.h>
16#include <linux/cpu.h>
17#include <linux/delay.h>
Jaswinder Singh Rajput72ade5f2009-01-04 16:32:36 +053018#include <linux/uaccess.h>
Lai Jiangshan42f8fae2009-02-17 11:46:42 +080019#include <linux/percpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020
Thomas Gleixnere05d7232007-02-16 01:27:58 -080021#include <asm/apic.h>
Thomas Gleixnere05d7232007-02-16 01:27:58 -080022
Fenghua Yuf34e3b62007-07-19 01:48:13 -070023DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
Linus Torvalds1da177e2005-04-16 15:20:36 -070024EXPORT_PER_CPU_SYMBOL(irq_stat);
25
Jeremy Fitzhardinge7c3576d2007-05-02 19:27:16 +020026DEFINE_PER_CPU(struct pt_regs *, irq_regs);
27EXPORT_PER_CPU_SYMBOL(irq_regs);
28
Thomas Gleixnerde9b10a2008-05-05 15:58:15 +020029#ifdef CONFIG_DEBUG_STACKOVERFLOW
30/* Debugging check for stack overflow: is there less than 1KB free? */
31static int check_stack_overflow(void)
32{
33 long sp;
34
35 __asm__ __volatile__("andl %%esp,%0" :
36 "=r" (sp) : "0" (THREAD_SIZE - 1));
37
38 return sp < (sizeof(struct thread_info) + STACK_WARN);
39}
40
41static void print_stack_overflow(void)
42{
43 printk(KERN_WARNING "low stack detected by irq handler\n");
44 dump_stack();
45}
46
47#else
48static inline int check_stack_overflow(void) { return 0; }
49static inline void print_stack_overflow(void) { }
50#endif
51
Linus Torvalds1da177e2005-04-16 15:20:36 -070052#ifdef CONFIG_4KSTACKS
53/*
54 * per-CPU IRQ handling contexts (thread information and stack)
55 */
56union irq_ctx {
57 struct thread_info tinfo;
58 u32 stack[THREAD_SIZE/sizeof(u32)];
Lai Jiangshan42f8fae2009-02-17 11:46:42 +080059} __attribute__((aligned(PAGE_SIZE)));
Linus Torvalds1da177e2005-04-16 15:20:36 -070060
Lai Jiangshan42f8fae2009-02-17 11:46:42 +080061static DEFINE_PER_CPU(union irq_ctx *, hardirq_ctx);
62static DEFINE_PER_CPU(union irq_ctx *, softirq_ctx);
Linus Torvalds1da177e2005-04-16 15:20:36 -070063
Lai Jiangshan42f8fae2009-02-17 11:46:42 +080064static DEFINE_PER_CPU_PAGE_ALIGNED(union irq_ctx, hardirq_stack);
65static DEFINE_PER_CPU_PAGE_ALIGNED(union irq_ctx, softirq_stack);
Thomas Gleixner403d8ef2008-05-05 18:13:50 +020066
67static void call_on_stack(void *func, void *stack)
68{
69 asm volatile("xchgl %%ebx,%%esp \n"
70 "call *%%edi \n"
71 "movl %%ebx,%%esp \n"
72 : "=b" (stack)
73 : "0" (stack),
74 "D"(func)
75 : "memory", "cc", "edx", "ecx", "eax");
Andi Kleen04b361a2008-05-05 12:36:38 +020076}
77
Thomas Gleixnerde9b10a2008-05-05 15:58:15 +020078static inline int
79execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
80{
Linus Torvalds1da177e2005-04-16 15:20:36 -070081 union irq_ctx *curctx, *irqctx;
Thomas Gleixner403d8ef2008-05-05 18:13:50 +020082 u32 *isp, arg1, arg2;
Linus Torvalds1da177e2005-04-16 15:20:36 -070083
84 curctx = (union irq_ctx *) current_thread_info();
Lai Jiangshan42f8fae2009-02-17 11:46:42 +080085 irqctx = __get_cpu_var(hardirq_ctx);
Linus Torvalds1da177e2005-04-16 15:20:36 -070086
87 /*
88 * this is where we switch to the IRQ stack. However, if we are
89 * already using the IRQ stack (because we interrupted a hardirq
90 * handler) we can't do that and just have to keep using the
91 * current stack (which is the irq stack already after all)
92 */
Thomas Gleixnerde9b10a2008-05-05 15:58:15 +020093 if (unlikely(curctx == irqctx))
94 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070095
Thomas Gleixnerde9b10a2008-05-05 15:58:15 +020096 /* build the stack frame on the IRQ stack */
Jaswinder Singh Rajput72ade5f2009-01-04 16:32:36 +053097 isp = (u32 *) ((char *)irqctx + sizeof(*irqctx));
Thomas Gleixnerde9b10a2008-05-05 15:58:15 +020098 irqctx->tinfo.task = curctx->tinfo.task;
99 irqctx->tinfo.previous_esp = current_stack_pointer;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100
Thomas Gleixnerde9b10a2008-05-05 15:58:15 +0200101 /*
102 * Copy the softirq bits in preempt_count so that the
103 * softirq checks work in the hardirq context.
104 */
105 irqctx->tinfo.preempt_count =
106 (irqctx->tinfo.preempt_count & ~SOFTIRQ_MASK) |
107 (curctx->tinfo.preempt_count & SOFTIRQ_MASK);
Björn Steinbrinka5d157e2006-06-25 16:24:40 +0200108
Thomas Gleixnerde9b10a2008-05-05 15:58:15 +0200109 if (unlikely(overflow))
Thomas Gleixner403d8ef2008-05-05 18:13:50 +0200110 call_on_stack(print_stack_overflow, isp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111
Thomas Gleixner403d8ef2008-05-05 18:13:50 +0200112 asm volatile("xchgl %%ebx,%%esp \n"
113 "call *%%edi \n"
114 "movl %%ebx,%%esp \n"
115 : "=a" (arg1), "=d" (arg2), "=b" (isp)
116 : "0" (irq), "1" (desc), "2" (isp),
117 "D" (desc->handle_irq)
118 : "memory", "cc", "ecx");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119 return 1;
120}
121
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122/*
123 * allocate per-cpu stacks for hardirq and for softirq processing
124 */
Thomas Gleixner403d8ef2008-05-05 18:13:50 +0200125void __cpuinit irq_ctx_init(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126{
127 union irq_ctx *irqctx;
128
Lai Jiangshan42f8fae2009-02-17 11:46:42 +0800129 if (per_cpu(hardirq_ctx, cpu))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130 return;
131
Lai Jiangshan42f8fae2009-02-17 11:46:42 +0800132 irqctx = &per_cpu(hardirq_stack, cpu);
Thomas Gleixner403d8ef2008-05-05 18:13:50 +0200133 irqctx->tinfo.task = NULL;
134 irqctx->tinfo.exec_domain = NULL;
135 irqctx->tinfo.cpu = cpu;
136 irqctx->tinfo.preempt_count = HARDIRQ_OFFSET;
137 irqctx->tinfo.addr_limit = MAKE_MM_SEG(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138
Lai Jiangshan42f8fae2009-02-17 11:46:42 +0800139 per_cpu(hardirq_ctx, cpu) = irqctx;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140
Lai Jiangshan42f8fae2009-02-17 11:46:42 +0800141 irqctx = &per_cpu(softirq_stack, cpu);
Thomas Gleixner403d8ef2008-05-05 18:13:50 +0200142 irqctx->tinfo.task = NULL;
143 irqctx->tinfo.exec_domain = NULL;
144 irqctx->tinfo.cpu = cpu;
145 irqctx->tinfo.preempt_count = 0;
146 irqctx->tinfo.addr_limit = MAKE_MM_SEG(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147
Lai Jiangshan42f8fae2009-02-17 11:46:42 +0800148 per_cpu(softirq_ctx, cpu) = irqctx;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149
Thomas Gleixner403d8ef2008-05-05 18:13:50 +0200150 printk(KERN_DEBUG "CPU %u irqstacks, hard=%p soft=%p\n",
Lai Jiangshan42f8fae2009-02-17 11:46:42 +0800151 cpu, per_cpu(hardirq_ctx, cpu), per_cpu(softirq_ctx, cpu));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152}
153
Li Shaohuae1367da2005-06-25 14:54:56 -0700154void irq_ctx_exit(int cpu)
155{
Lai Jiangshan42f8fae2009-02-17 11:46:42 +0800156 per_cpu(hardirq_ctx, cpu) = NULL;
Li Shaohuae1367da2005-06-25 14:54:56 -0700157}
158
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159asmlinkage void do_softirq(void)
160{
161 unsigned long flags;
162 struct thread_info *curctx;
163 union irq_ctx *irqctx;
164 u32 *isp;
165
166 if (in_interrupt())
167 return;
168
169 local_irq_save(flags);
170
171 if (local_softirq_pending()) {
172 curctx = current_thread_info();
Lai Jiangshan42f8fae2009-02-17 11:46:42 +0800173 irqctx = __get_cpu_var(softirq_ctx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174 irqctx->tinfo.task = curctx->task;
175 irqctx->tinfo.previous_esp = current_stack_pointer;
176
177 /* build the stack frame on the softirq stack */
Jaswinder Singh Rajput72ade5f2009-01-04 16:32:36 +0530178 isp = (u32 *) ((char *)irqctx + sizeof(*irqctx));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179
Thomas Gleixner403d8ef2008-05-05 18:13:50 +0200180 call_on_stack(__do_softirq, isp);
Ingo Molnar55f327f2006-07-03 00:24:43 -0700181 /*
182 * Shouldnt happen, we returned above if in_interrupt():
Thomas Gleixner403d8ef2008-05-05 18:13:50 +0200183 */
Ingo Molnar55f327f2006-07-03 00:24:43 -0700184 WARN_ON_ONCE(softirq_count());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 }
186
187 local_irq_restore(flags);
188}
Thomas Gleixner403d8ef2008-05-05 18:13:50 +0200189
Thomas Gleixnerde9b10a2008-05-05 15:58:15 +0200190#else
191static inline int
192execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq) { return 0; }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193#endif
194
Jeremy Fitzhardinge9b2b76a2009-02-06 14:09:40 -0800195bool handle_irq(unsigned irq, struct pt_regs *regs)
196{
197 struct irq_desc *desc;
198 int overflow;
199
200 overflow = check_stack_overflow();
201
202 desc = irq_to_desc(irq);
203 if (unlikely(!desc))
204 return false;
205
206 if (!execute_on_irq_stack(overflow, desc, irq)) {
207 if (unlikely(overflow))
208 print_stack_overflow();
209 desc->handle_irq(irq, desc);
210 }
211
212 return true;
213}
214
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700215#ifdef CONFIG_HOTPLUG_CPU
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700216
Mike Travisd7b381b2008-12-16 17:33:58 -0800217/* A cpu has been removed from cpu_online_mask. Reset irq affinities. */
218void fixup_irqs(void)
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700219{
220 unsigned int irq;
221 static int warned;
Yinghai Lu199751d2008-08-19 20:50:27 -0700222 struct irq_desc *desc;
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700223
Yinghai Lu199751d2008-08-19 20:50:27 -0700224 for_each_irq_desc(irq, desc) {
Mike Travisd7b381b2008-12-16 17:33:58 -0800225 const struct cpumask *affinity;
Yinghai Lu08678b02008-08-19 20:50:05 -0700226
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800227 if (!desc)
228 continue;
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700229 if (irq == 2)
230 continue;
231
Mike Travis7f7ace02009-01-10 21:58:08 -0800232 affinity = desc->affinity;
Mike Travisd7b381b2008-12-16 17:33:58 -0800233 if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700234 printk("Breaking affinity for irq %i\n", irq);
Mike Travisd7b381b2008-12-16 17:33:58 -0800235 affinity = cpu_all_mask;
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700236 }
Yinghai Lu08678b02008-08-19 20:50:05 -0700237 if (desc->chip->set_affinity)
Mike Travisd7b381b2008-12-16 17:33:58 -0800238 desc->chip->set_affinity(irq, affinity);
Yinghai Lu08678b02008-08-19 20:50:05 -0700239 else if (desc->action && !(warned++))
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700240 printk("Cannot set affinity for irq %i\n", irq);
241 }
242
243#if 0
244 barrier();
245 /* Ingo Molnar says: "after the IO-APIC masks have been redirected
246 [note the nop - the interrupt-enable boundary on x86 is two
247 instructions from sti] - to flush out pending hardirqs and
248 IPIs. After this point nothing is supposed to reach this CPU." */
249 __asm__ __volatile__("sti; nop; cli");
250 barrier();
251#else
252 /* That doesn't seem sufficient. Give it 1ms. */
253 local_irq_enable();
254 mdelay(1);
255 local_irq_disable();
256#endif
257}
258#endif
259