blob: 49e8cf2e06f8eca9d016386d8e91ec95c3de26b4 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/x86-64/mm/fault.c
3 *
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
6 */
7
Linus Torvalds1da177e2005-04-16 15:20:36 -07008#include <linux/signal.h>
9#include <linux/sched.h>
10#include <linux/kernel.h>
11#include <linux/errno.h>
12#include <linux/string.h>
13#include <linux/types.h>
14#include <linux/ptrace.h>
15#include <linux/mman.h>
16#include <linux/mm.h>
17#include <linux/smp.h>
18#include <linux/smp_lock.h>
19#include <linux/interrupt.h>
20#include <linux/init.h>
21#include <linux/tty.h>
22#include <linux/vt_kern.h> /* For unblank_screen() */
23#include <linux/compiler.h>
24#include <linux/module.h>
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -070025#include <linux/kprobes.h>
Andi Kleenab2bf0c2006-12-07 02:14:06 +010026#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
28#include <asm/system.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <asm/pgalloc.h>
30#include <asm/smp.h>
31#include <asm/tlbflush.h>
32#include <asm/proto.h>
33#include <asm/kdebug.h>
34#include <asm-generic/sections.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035
Andi Kleen66c58152006-01-11 22:44:09 +010036/* Page fault error code bits */
37#define PF_PROT (1<<0) /* or no page found */
38#define PF_WRITE (1<<1)
39#define PF_USER (1<<2)
40#define PF_RSVD (1<<3)
41#define PF_INSTR (1<<4)
42
Andi Kleen273819a2006-09-26 10:52:35 +020043static ATOMIC_NOTIFIER_HEAD(notify_page_fault_chain);
Anil S Keshavamurthy1bd858a2006-06-26 00:25:25 -070044
45/* Hook to register for page fault notifications */
46int register_page_fault_notifier(struct notifier_block *nb)
47{
48 vmalloc_sync_all();
49 return atomic_notifier_chain_register(&notify_page_fault_chain, nb);
50}
Andi Kleen273819a2006-09-26 10:52:35 +020051EXPORT_SYMBOL_GPL(register_page_fault_notifier);
Anil S Keshavamurthy1bd858a2006-06-26 00:25:25 -070052
53int unregister_page_fault_notifier(struct notifier_block *nb)
54{
55 return atomic_notifier_chain_unregister(&notify_page_fault_chain, nb);
56}
Andi Kleen273819a2006-09-26 10:52:35 +020057EXPORT_SYMBOL_GPL(unregister_page_fault_notifier);
Anil S Keshavamurthy1bd858a2006-06-26 00:25:25 -070058
59static inline int notify_page_fault(enum die_val val, const char *str,
60 struct pt_regs *regs, long err, int trap, int sig)
61{
62 struct die_args args = {
63 .regs = regs,
64 .str = str,
65 .err = err,
66 .trapnr = trap,
67 .signr = sig
68 };
69 return atomic_notifier_call_chain(&notify_page_fault_chain, val, &args);
70}
Anil S Keshavamurthy1bd858a2006-06-26 00:25:25 -070071
Linus Torvalds1da177e2005-04-16 15:20:36 -070072/* Sometimes the CPU reports invalid exceptions on prefetch.
73 Check that here and ignore.
74 Opcode checker based on code by Richard Brunner */
75static noinline int is_prefetch(struct pt_regs *regs, unsigned long addr,
76 unsigned long error_code)
77{
Andi Kleenab2bf0c2006-12-07 02:14:06 +010078 unsigned char *instr;
Linus Torvalds1da177e2005-04-16 15:20:36 -070079 int scan_more = 1;
80 int prefetch = 0;
Andi Kleenf1290ec2005-04-16 15:24:59 -070081 unsigned char *max_instr;
Linus Torvalds1da177e2005-04-16 15:20:36 -070082
83 /* If it was a exec fault ignore */
Andi Kleen66c58152006-01-11 22:44:09 +010084 if (error_code & PF_INSTR)
Linus Torvalds1da177e2005-04-16 15:20:36 -070085 return 0;
86
Andi Kleendd2994f2006-09-26 10:52:33 +020087 instr = (unsigned char __user *)convert_rip_to_linear(current, regs);
Andi Kleenf1290ec2005-04-16 15:24:59 -070088 max_instr = instr + 15;
Linus Torvalds1da177e2005-04-16 15:20:36 -070089
Vincent Hanquez76381fe2005-06-23 00:08:46 -070090 if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -070091 return 0;
92
93 while (scan_more && instr < max_instr) {
94 unsigned char opcode;
95 unsigned char instr_hi;
96 unsigned char instr_lo;
97
Andi Kleenab2bf0c2006-12-07 02:14:06 +010098 if (probe_kernel_address(instr, opcode))
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 break;
100
101 instr_hi = opcode & 0xf0;
102 instr_lo = opcode & 0x0f;
103 instr++;
104
105 switch (instr_hi) {
106 case 0x20:
107 case 0x30:
108 /* Values 0x26,0x2E,0x36,0x3E are valid x86
109 prefixes. In long mode, the CPU will signal
110 invalid opcode if some of these prefixes are
111 present so we will never get here anyway */
112 scan_more = ((instr_lo & 7) == 0x6);
113 break;
114
115 case 0x40:
116 /* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
117 Need to figure out under what instruction mode the
118 instruction was issued ... */
119 /* Could check the LDT for lm, but for now it's good
120 enough to assume that long mode only uses well known
121 segments or kernel. */
Vincent Hanquez76381fe2005-06-23 00:08:46 -0700122 scan_more = (!user_mode(regs)) || (regs->cs == __USER_CS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123 break;
124
125 case 0x60:
126 /* 0x64 thru 0x67 are valid prefixes in all modes. */
127 scan_more = (instr_lo & 0xC) == 0x4;
128 break;
129 case 0xF0:
130 /* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
131 scan_more = !instr_lo || (instr_lo>>1) == 1;
132 break;
133 case 0x00:
134 /* Prefetch instruction is 0x0F0D or 0x0F18 */
135 scan_more = 0;
Andi Kleenab2bf0c2006-12-07 02:14:06 +0100136 if (probe_kernel_address(instr, opcode))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137 break;
138 prefetch = (instr_lo == 0xF) &&
139 (opcode == 0x0D || opcode == 0x18);
140 break;
141 default:
142 scan_more = 0;
143 break;
144 }
145 }
146 return prefetch;
147}
148
149static int bad_address(void *p)
150{
151 unsigned long dummy;
Andi Kleenab2bf0c2006-12-07 02:14:06 +0100152 return probe_kernel_address((unsigned long *)p, dummy);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153}
154
155void dump_pagetable(unsigned long address)
156{
157 pgd_t *pgd;
158 pud_t *pud;
159 pmd_t *pmd;
160 pte_t *pte;
161
162 asm("movq %%cr3,%0" : "=r" (pgd));
163
164 pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
165 pgd += pgd_index(address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166 if (bad_address(pgd)) goto bad;
Jan Beulichd646bce2006-02-03 21:51:47 +0100167 printk("PGD %lx ", pgd_val(*pgd));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168 if (!pgd_present(*pgd)) goto ret;
169
Andi Kleend2ae5b52006-06-26 13:57:56 +0200170 pud = pud_offset(pgd, address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171 if (bad_address(pud)) goto bad;
172 printk("PUD %lx ", pud_val(*pud));
173 if (!pud_present(*pud)) goto ret;
174
175 pmd = pmd_offset(pud, address);
176 if (bad_address(pmd)) goto bad;
177 printk("PMD %lx ", pmd_val(*pmd));
178 if (!pmd_present(*pmd)) goto ret;
179
180 pte = pte_offset_kernel(pmd, address);
181 if (bad_address(pte)) goto bad;
182 printk("PTE %lx", pte_val(*pte));
183ret:
184 printk("\n");
185 return;
186bad:
187 printk("BAD\n");
188}
189
190static const char errata93_warning[] =
191KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
192KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
193KERN_ERR "******* Please consider a BIOS update.\n"
194KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
195
196/* Workaround for K8 erratum #93 & buggy BIOS.
197 BIOS SMM functions are required to use a specific workaround
198 to avoid corruption of the 64bit RIP register on C stepping K8.
199 A lot of BIOS that didn't get tested properly miss this.
200 The OS sees this as a page fault with the upper 32bits of RIP cleared.
201 Try to work around it here.
202 Note we only handle faults in kernel here. */
203
204static int is_errata93(struct pt_regs *regs, unsigned long address)
205{
206 static int warned;
207 if (address != regs->rip)
208 return 0;
209 if ((address >> 32) != 0)
210 return 0;
211 address |= 0xffffffffUL << 32;
212 if ((address >= (u64)_stext && address <= (u64)_etext) ||
213 (address >= MODULES_VADDR && address <= MODULES_END)) {
214 if (!warned) {
215 printk(errata93_warning);
216 warned = 1;
217 }
218 regs->rip = address;
219 return 1;
220 }
221 return 0;
222}
223
224int unhandled_signal(struct task_struct *tsk, int sig)
225{
Sukadev Bhattiproluf400e192006-09-29 02:00:07 -0700226 if (is_init(tsk))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227 return 1;
Andi Kleen5e5ec102005-08-19 06:56:04 +0200228 if (tsk->ptrace & PT_PTRACED)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229 return 0;
230 return (tsk->sighand->action[sig-1].sa.sa_handler == SIG_IGN) ||
231 (tsk->sighand->action[sig-1].sa.sa_handler == SIG_DFL);
232}
233
234static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
235 unsigned long error_code)
236{
Jan Beulich12091402005-09-12 18:49:24 +0200237 unsigned long flags = oops_begin();
Jan Beulich6e3f3612006-01-11 22:42:14 +0100238 struct task_struct *tsk;
Jan Beulich12091402005-09-12 18:49:24 +0200239
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240 printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
241 current->comm, address);
242 dump_pagetable(address);
Jan Beulich6e3f3612006-01-11 22:42:14 +0100243 tsk = current;
244 tsk->thread.cr2 = address;
245 tsk->thread.trap_no = 14;
246 tsk->thread.error_code = error_code;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247 __die("Bad pagetable", regs, error_code);
Jan Beulich12091402005-09-12 18:49:24 +0200248 oops_end(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249 do_exit(SIGKILL);
250}
251
252/*
Andi Kleenf95190b2006-01-11 22:44:00 +0100253 * Handle a fault on the vmalloc area
Andi Kleen3b9ba4d2005-05-16 21:53:31 -0700254 *
255 * This assumes no large pages in there.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256 */
257static int vmalloc_fault(unsigned long address)
258{
259 pgd_t *pgd, *pgd_ref;
260 pud_t *pud, *pud_ref;
261 pmd_t *pmd, *pmd_ref;
262 pte_t *pte, *pte_ref;
263
264 /* Copy kernel mappings over when needed. This can also
265 happen within a race in page table update. In the later
266 case just flush. */
267
268 pgd = pgd_offset(current->mm ?: &init_mm, address);
269 pgd_ref = pgd_offset_k(address);
270 if (pgd_none(*pgd_ref))
271 return -1;
272 if (pgd_none(*pgd))
273 set_pgd(pgd, *pgd_ref);
Jan Beulich8c914cb2006-03-25 16:29:40 +0100274 else
Dave McCracken46a82b22006-09-25 23:31:48 -0700275 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276
277 /* Below here mismatches are bugs because these lower tables
278 are shared */
279
280 pud = pud_offset(pgd, address);
281 pud_ref = pud_offset(pgd_ref, address);
282 if (pud_none(*pud_ref))
283 return -1;
Dave McCracken46a82b22006-09-25 23:31:48 -0700284 if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285 BUG();
286 pmd = pmd_offset(pud, address);
287 pmd_ref = pmd_offset(pud_ref, address);
288 if (pmd_none(*pmd_ref))
289 return -1;
290 if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
291 BUG();
292 pte_ref = pte_offset_kernel(pmd_ref, address);
293 if (!pte_present(*pte_ref))
294 return -1;
295 pte = pte_offset_kernel(pmd, address);
Andi Kleen3b9ba4d2005-05-16 21:53:31 -0700296 /* Don't use pte_page here, because the mappings can point
297 outside mem_map, and the NUMA hash lookup cannot handle
298 that. */
299 if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 return 0;
302}
303
304int page_fault_trace = 0;
305int exception_trace = 1;
306
307/*
308 * This routine handles page faults. It determines the address,
309 * and the problem, and then passes it off to one of the appropriate
310 * routines.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 */
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -0700312asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
313 unsigned long error_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314{
315 struct task_struct *tsk;
316 struct mm_struct *mm;
317 struct vm_area_struct * vma;
318 unsigned long address;
319 const struct exception_table_entry *fixup;
320 int write;
Jan Beulich12091402005-09-12 18:49:24 +0200321 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322 siginfo_t info;
323
Arjan van de Vena9ba9a32006-03-25 16:30:10 +0100324 tsk = current;
325 mm = tsk->mm;
326 prefetchw(&mm->mmap_sem);
327
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328 /* get the address */
329 __asm__("movq %%cr2,%0":"=r" (address));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331 info.si_code = SEGV_MAPERR;
332
333
334 /*
335 * We fault-in kernel-space virtual memory on-demand. The
336 * 'reference' page table is init_mm.pgd.
337 *
338 * NOTE! We MUST NOT take any locks for this case. We may
339 * be in an interrupt or a critical region, and should
340 * only copy the information from the master page table,
341 * nothing more.
342 *
343 * This verifies that the fault happens in kernel space
344 * (error_code & 4) == 0, and that the fault was not a
Jan Beulich8b1bde92006-01-11 22:42:23 +0100345 * protection error (error_code & 9) == 0.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346 */
Suresh Siddha84929802005-06-21 17:14:32 -0700347 if (unlikely(address >= TASK_SIZE64)) {
Andi Kleenf95190b2006-01-11 22:44:00 +0100348 /*
349 * Don't check for the module range here: its PML4
350 * is always initialized because it's shared with the main
351 * kernel text. Only vmalloc may need PML4 syncups.
352 */
Andi Kleen66c58152006-01-11 22:44:09 +0100353 if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) &&
Andi Kleenf95190b2006-01-11 22:44:00 +0100354 ((address >= VMALLOC_START && address < VMALLOC_END))) {
Jan Beulich8c914cb2006-03-25 16:29:40 +0100355 if (vmalloc_fault(address) >= 0)
356 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357 }
Anil S Keshavamurthy1bd858a2006-06-26 00:25:25 -0700358 if (notify_page_fault(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
Jan Beulich8c914cb2006-03-25 16:29:40 +0100359 SIGSEGV) == NOTIFY_STOP)
360 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 /*
362 * Don't take the mm semaphore here. If we fixup a prefetch
363 * fault we could otherwise deadlock.
364 */
365 goto bad_area_nosemaphore;
366 }
367
Anil S Keshavamurthy1bd858a2006-06-26 00:25:25 -0700368 if (notify_page_fault(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
Jan Beulich8c914cb2006-03-25 16:29:40 +0100369 SIGSEGV) == NOTIFY_STOP)
370 return;
371
372 if (likely(regs->eflags & X86_EFLAGS_IF))
373 local_irq_enable();
374
375 if (unlikely(page_fault_trace))
376 printk("pagefault rip:%lx rsp:%lx cs:%lu ss:%lu address %lx error %lx\n",
377 regs->rip,regs->rsp,regs->cs,regs->ss,address,error_code);
378
Andi Kleen66c58152006-01-11 22:44:09 +0100379 if (unlikely(error_code & PF_RSVD))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380 pgtable_bad(address, regs, error_code);
381
382 /*
383 * If we're in an interrupt or have no user
384 * context, we must not take the fault..
385 */
386 if (unlikely(in_atomic() || !mm))
387 goto bad_area_nosemaphore;
388
389 again:
390 /* When running in the kernel we expect faults to occur only to
391 * addresses in user space. All other faults represent errors in the
392 * kernel and should generate an OOPS. Unfortunatly, in the case of an
Adrian Bunk80f72282006-06-30 18:27:16 +0200393 * erroneous fault occurring in a code path which already holds mmap_sem
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394 * we will deadlock attempting to validate the fault against the
395 * address space. Luckily the kernel only validly references user
396 * space from well defined areas of code, which are listed in the
397 * exceptions table.
398 *
399 * As the vast majority of faults will be valid we will only perform
400 * the source reference check when there is a possibilty of a deadlock.
401 * Attempt to lock the address space, if we cannot we then validate the
402 * source. If this is invalid we can skip the address space check,
403 * thus avoiding the deadlock.
404 */
405 if (!down_read_trylock(&mm->mmap_sem)) {
Andi Kleen66c58152006-01-11 22:44:09 +0100406 if ((error_code & PF_USER) == 0 &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407 !search_exception_tables(regs->rip))
408 goto bad_area_nosemaphore;
409 down_read(&mm->mmap_sem);
410 }
411
412 vma = find_vma(mm, address);
413 if (!vma)
414 goto bad_area;
415 if (likely(vma->vm_start <= address))
416 goto good_area;
417 if (!(vma->vm_flags & VM_GROWSDOWN))
418 goto bad_area;
419 if (error_code & 4) {
Chuck Ebbert03fdc2c2006-06-26 13:59:50 +0200420 /* Allow userspace just enough access below the stack pointer
421 * to let the 'enter' instruction work.
422 */
423 if (address + 65536 + 32 * sizeof(unsigned long) < regs->rsp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700424 goto bad_area;
425 }
426 if (expand_stack(vma, address))
427 goto bad_area;
428/*
429 * Ok, we have a good vm_area for this memory access, so
430 * we can handle it..
431 */
432good_area:
433 info.si_code = SEGV_ACCERR;
434 write = 0;
Andi Kleen66c58152006-01-11 22:44:09 +0100435 switch (error_code & (PF_PROT|PF_WRITE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436 default: /* 3: write, present */
437 /* fall through */
Andi Kleen66c58152006-01-11 22:44:09 +0100438 case PF_WRITE: /* write, not present */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439 if (!(vma->vm_flags & VM_WRITE))
440 goto bad_area;
441 write++;
442 break;
Andi Kleen66c58152006-01-11 22:44:09 +0100443 case PF_PROT: /* read, present */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444 goto bad_area;
Andi Kleen66c58152006-01-11 22:44:09 +0100445 case 0: /* read, not present */
Jason Barondf67b3d2006-09-29 01:58:58 -0700446 if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 goto bad_area;
448 }
449
450 /*
451 * If for any reason at all we couldn't handle the fault,
452 * make sure we exit gracefully rather than endlessly redo
453 * the fault.
454 */
455 switch (handle_mm_fault(mm, vma, address, write)) {
Alexander Nyberg96800212005-08-04 16:14:57 +0200456 case VM_FAULT_MINOR:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457 tsk->min_flt++;
458 break;
Alexander Nyberg96800212005-08-04 16:14:57 +0200459 case VM_FAULT_MAJOR:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460 tsk->maj_flt++;
461 break;
Alexander Nyberg96800212005-08-04 16:14:57 +0200462 case VM_FAULT_SIGBUS:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463 goto do_sigbus;
464 default:
465 goto out_of_memory;
466 }
467
468 up_read(&mm->mmap_sem);
469 return;
470
471/*
472 * Something tried to access memory that isn't in our memory map..
473 * Fix it, but check if it's kernel or user first..
474 */
475bad_area:
476 up_read(&mm->mmap_sem);
477
478bad_area_nosemaphore:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479 /* User mode accesses just cause a SIGSEGV */
Andi Kleen66c58152006-01-11 22:44:09 +0100480 if (error_code & PF_USER) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481 if (is_prefetch(regs, address, error_code))
482 return;
483
484 /* Work around K8 erratum #100 K8 in compat mode
485 occasionally jumps to illegal addresses >4GB. We
486 catch this here in the page fault handler because
487 these addresses are not reachable. Just detect this
488 case and return. Any code segment in LDT is
489 compatibility mode. */
490 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
491 (address >> 32))
492 return;
493
494 if (exception_trace && unhandled_signal(tsk, SIGSEGV)) {
495 printk(
496 "%s%s[%d]: segfault at %016lx rip %016lx rsp %016lx error %lx\n",
497 tsk->pid > 1 ? KERN_INFO : KERN_EMERG,
498 tsk->comm, tsk->pid, address, regs->rip,
499 regs->rsp, error_code);
500 }
501
502 tsk->thread.cr2 = address;
503 /* Kernel addresses are always protection faults */
504 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
505 tsk->thread.trap_no = 14;
506 info.si_signo = SIGSEGV;
507 info.si_errno = 0;
508 /* info.si_code has been set above */
509 info.si_addr = (void __user *)address;
510 force_sig_info(SIGSEGV, &info, tsk);
511 return;
512 }
513
514no_context:
515
516 /* Are we prepared to handle this kernel fault? */
517 fixup = search_exception_tables(regs->rip);
518 if (fixup) {
519 regs->rip = fixup->fixup;
520 return;
521 }
522
523 /*
524 * Hall of shame of CPU/BIOS bugs.
525 */
526
527 if (is_prefetch(regs, address, error_code))
528 return;
529
530 if (is_errata93(regs, address))
531 return;
532
533/*
534 * Oops. The kernel tried to access some bad page. We'll have to
535 * terminate things with extreme prejudice.
536 */
537
Jan Beulich12091402005-09-12 18:49:24 +0200538 flags = oops_begin();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539
540 if (address < PAGE_SIZE)
541 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
542 else
543 printk(KERN_ALERT "Unable to handle kernel paging request");
544 printk(" at %016lx RIP: \n" KERN_ALERT,address);
545 printk_address(regs->rip);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546 dump_pagetable(address);
Jan Beulich6e3f3612006-01-11 22:42:14 +0100547 tsk->thread.cr2 = address;
548 tsk->thread.trap_no = 14;
549 tsk->thread.error_code = error_code;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550 __die("Oops", regs, error_code);
551 /* Executive summary in case the body of the oops scrolled away */
552 printk(KERN_EMERG "CR2: %016lx\n", address);
Jan Beulich12091402005-09-12 18:49:24 +0200553 oops_end(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554 do_exit(SIGKILL);
555
556/*
557 * We ran out of memory, or some other thing happened to us that made
558 * us unable to handle the page fault gracefully.
559 */
560out_of_memory:
561 up_read(&mm->mmap_sem);
Sukadev Bhattiproluf400e192006-09-29 02:00:07 -0700562 if (is_init(current)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563 yield();
564 goto again;
565 }
566 printk("VM: killing process %s\n", tsk->comm);
567 if (error_code & 4)
568 do_exit(SIGKILL);
569 goto no_context;
570
571do_sigbus:
572 up_read(&mm->mmap_sem);
573
574 /* Kernel mode? Handle exceptions or die */
Andi Kleen66c58152006-01-11 22:44:09 +0100575 if (!(error_code & PF_USER))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576 goto no_context;
577
578 tsk->thread.cr2 = address;
579 tsk->thread.error_code = error_code;
580 tsk->thread.trap_no = 14;
581 info.si_signo = SIGBUS;
582 info.si_errno = 0;
583 info.si_code = BUS_ADRERR;
584 info.si_addr = (void __user *)address;
585 force_sig_info(SIGBUS, &info, tsk);
586 return;
587}
Andi Kleen9e43e1b2005-11-05 17:25:54 +0100588
Jan Beulich8c914cb2006-03-25 16:29:40 +0100589DEFINE_SPINLOCK(pgd_lock);
590struct page *pgd_list;
591
592void vmalloc_sync_all(void)
593{
594 /* Note that races in the updates of insync and start aren't
595 problematic:
596 insync can only get set bits added, and updates to start are only
597 improving performance (without affecting correctness if undone). */
598 static DECLARE_BITMAP(insync, PTRS_PER_PGD);
599 static unsigned long start = VMALLOC_START & PGDIR_MASK;
600 unsigned long address;
601
602 for (address = start; address <= VMALLOC_END; address += PGDIR_SIZE) {
603 if (!test_bit(pgd_index(address), insync)) {
604 const pgd_t *pgd_ref = pgd_offset_k(address);
605 struct page *page;
606
607 if (pgd_none(*pgd_ref))
608 continue;
609 spin_lock(&pgd_lock);
610 for (page = pgd_list; page;
611 page = (struct page *)page->index) {
612 pgd_t *pgd;
613 pgd = (pgd_t *)page_address(page) + pgd_index(address);
614 if (pgd_none(*pgd))
615 set_pgd(pgd, *pgd_ref);
616 else
Dave McCracken46a82b22006-09-25 23:31:48 -0700617 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
Jan Beulich8c914cb2006-03-25 16:29:40 +0100618 }
619 spin_unlock(&pgd_lock);
620 set_bit(pgd_index(address), insync);
621 }
622 if (address == start)
623 start = address + PGDIR_SIZE;
624 }
625 /* Check that there is no need to do the same for the modules area. */
626 BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
627 BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
628 (__START_KERNEL & PGDIR_MASK)));
629}
630
Andi Kleen9e43e1b2005-11-05 17:25:54 +0100631static int __init enable_pagefaulttrace(char *str)
632{
633 page_fault_trace = 1;
OGAWA Hirofumi9b410462006-03-31 02:30:33 -0800634 return 1;
Andi Kleen9e43e1b2005-11-05 17:25:54 +0100635}
636__setup("pagefaulttrace", enable_pagefaulttrace);