2 * linux/arch/x86-64/mm/fault.c
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
8 #include <linux/signal.h>
9 #include <linux/sched.h>
10 #include <linux/kernel.h>
11 #include <linux/errno.h>
12 #include <linux/string.h>
13 #include <linux/types.h>
14 #include <linux/ptrace.h>
15 #include <linux/mman.h>
17 #include <linux/smp.h>
18 #include <linux/interrupt.h>
19 #include <linux/init.h>
20 #include <linux/tty.h>
21 #include <linux/vt_kern.h> /* For unblank_screen() */
22 #include <linux/compiler.h>
23 #include <linux/vmalloc.h>
24 #include <linux/module.h>
25 #include <linux/kprobes.h>
26 #include <linux/uaccess.h>
27 #include <linux/kdebug.h>
28 #include <linux/kprobes.h>
30 #include <asm/system.h>
31 #include <asm/pgalloc.h>
33 #include <asm/tlbflush.h>
34 #include <asm/proto.h>
35 #include <asm-generic/sections.h>
37 /* Page fault error code bits */
38 #define PF_PROT (1<<0) /* or no page found */
39 #define PF_WRITE (1<<1)
40 #define PF_USER (1<<2)
41 #define PF_RSVD (1<<3)
42 #define PF_INSTR (1<<4)
45 static inline int notify_page_fault(struct pt_regs
*regs
)
49 /* kprobe_running() needs smp_processor_id() */
50 if (!user_mode(regs
)) {
52 if (kprobe_running() && kprobe_fault_handler(regs
, 14))
60 static inline int notify_page_fault(struct pt_regs
*regs
)
66 /* Sometimes the CPU reports invalid exceptions on prefetch.
67 Check that here and ignore.
68 Opcode checker based on code by Richard Brunner */
69 static noinline
int is_prefetch(struct pt_regs
*regs
, unsigned long addr
,
70 unsigned long error_code
)
75 unsigned char *max_instr
;
77 /* If it was a exec fault ignore */
78 if (error_code
& PF_INSTR
)
81 instr
= (unsigned char __user
*)convert_rip_to_linear(current
, regs
);
82 max_instr
= instr
+ 15;
84 if (user_mode(regs
) && instr
>= (unsigned char *)TASK_SIZE
)
87 while (scan_more
&& instr
< max_instr
) {
89 unsigned char instr_hi
;
90 unsigned char instr_lo
;
92 if (probe_kernel_address(instr
, opcode
))
95 instr_hi
= opcode
& 0xf0;
96 instr_lo
= opcode
& 0x0f;
102 /* Values 0x26,0x2E,0x36,0x3E are valid x86
103 prefixes. In long mode, the CPU will signal
104 invalid opcode if some of these prefixes are
105 present so we will never get here anyway */
106 scan_more
= ((instr_lo
& 7) == 0x6);
110 /* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
111 Need to figure out under what instruction mode the
112 instruction was issued ... */
113 /* Could check the LDT for lm, but for now it's good
114 enough to assume that long mode only uses well known
115 segments or kernel. */
116 scan_more
= (!user_mode(regs
)) || (regs
->cs
== __USER_CS
);
120 /* 0x64 thru 0x67 are valid prefixes in all modes. */
121 scan_more
= (instr_lo
& 0xC) == 0x4;
124 /* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
125 scan_more
= !instr_lo
|| (instr_lo
>>1) == 1;
128 /* Prefetch instruction is 0x0F0D or 0x0F18 */
130 if (probe_kernel_address(instr
, opcode
))
132 prefetch
= (instr_lo
== 0xF) &&
133 (opcode
== 0x0D || opcode
== 0x18);
143 static int bad_address(void *p
)
146 return probe_kernel_address((unsigned long *)p
, dummy
);
149 void dump_pagetable(unsigned long address
)
156 pgd
= (pgd_t
*)read_cr3();
158 pgd
= __va((unsigned long)pgd
& PHYSICAL_PAGE_MASK
);
159 pgd
+= pgd_index(address
);
160 if (bad_address(pgd
)) goto bad
;
161 printk("PGD %lx ", pgd_val(*pgd
));
162 if (!pgd_present(*pgd
)) goto ret
;
164 pud
= pud_offset(pgd
, address
);
165 if (bad_address(pud
)) goto bad
;
166 printk("PUD %lx ", pud_val(*pud
));
167 if (!pud_present(*pud
)) goto ret
;
169 pmd
= pmd_offset(pud
, address
);
170 if (bad_address(pmd
)) goto bad
;
171 printk("PMD %lx ", pmd_val(*pmd
));
172 if (!pmd_present(*pmd
)) goto ret
;
174 pte
= pte_offset_kernel(pmd
, address
);
175 if (bad_address(pte
)) goto bad
;
176 printk("PTE %lx", pte_val(*pte
));
184 static const char errata93_warning
[] =
185 KERN_ERR
"******* Your BIOS seems to not contain a fix for K8 errata #93\n"
186 KERN_ERR
"******* Working around it, but it may cause SEGVs or burn power.\n"
187 KERN_ERR
"******* Please consider a BIOS update.\n"
188 KERN_ERR
"******* Disabling USB legacy in the BIOS may also help.\n";
190 /* Workaround for K8 erratum #93 & buggy BIOS.
191 BIOS SMM functions are required to use a specific workaround
192 to avoid corruption of the 64bit RIP register on C stepping K8.
193 A lot of BIOS that didn't get tested properly miss this.
194 The OS sees this as a page fault with the upper 32bits of RIP cleared.
195 Try to work around it here.
196 Note we only handle faults in kernel here. */
198 static int is_errata93(struct pt_regs
*regs
, unsigned long address
)
201 if (address
!= regs
->rip
)
203 if ((address
>> 32) != 0)
205 address
|= 0xffffffffUL
<< 32;
206 if ((address
>= (u64
)_stext
&& address
<= (u64
)_etext
) ||
207 (address
>= MODULES_VADDR
&& address
<= MODULES_END
)) {
209 printk(errata93_warning
);
218 static noinline
void pgtable_bad(unsigned long address
, struct pt_regs
*regs
,
219 unsigned long error_code
)
221 unsigned long flags
= oops_begin();
222 struct task_struct
*tsk
;
224 printk(KERN_ALERT
"%s: Corrupted page table at address %lx\n",
225 current
->comm
, address
);
226 dump_pagetable(address
);
228 tsk
->thread
.cr2
= address
;
229 tsk
->thread
.trap_no
= 14;
230 tsk
->thread
.error_code
= error_code
;
231 __die("Bad pagetable", regs
, error_code
);
237 * Handle a fault on the vmalloc area
239 * This assumes no large pages in there.
241 static int vmalloc_fault(unsigned long address
)
243 pgd_t
*pgd
, *pgd_ref
;
244 pud_t
*pud
, *pud_ref
;
245 pmd_t
*pmd
, *pmd_ref
;
246 pte_t
*pte
, *pte_ref
;
248 /* Copy kernel mappings over when needed. This can also
249 happen within a race in page table update. In the later
252 pgd
= pgd_offset(current
->mm
?: &init_mm
, address
);
253 pgd_ref
= pgd_offset_k(address
);
254 if (pgd_none(*pgd_ref
))
257 set_pgd(pgd
, *pgd_ref
);
259 BUG_ON(pgd_page_vaddr(*pgd
) != pgd_page_vaddr(*pgd_ref
));
261 /* Below here mismatches are bugs because these lower tables
264 pud
= pud_offset(pgd
, address
);
265 pud_ref
= pud_offset(pgd_ref
, address
);
266 if (pud_none(*pud_ref
))
268 if (pud_none(*pud
) || pud_page_vaddr(*pud
) != pud_page_vaddr(*pud_ref
))
270 pmd
= pmd_offset(pud
, address
);
271 pmd_ref
= pmd_offset(pud_ref
, address
);
272 if (pmd_none(*pmd_ref
))
274 if (pmd_none(*pmd
) || pmd_page(*pmd
) != pmd_page(*pmd_ref
))
276 pte_ref
= pte_offset_kernel(pmd_ref
, address
);
277 if (!pte_present(*pte_ref
))
279 pte
= pte_offset_kernel(pmd
, address
);
280 /* Don't use pte_page here, because the mappings can point
281 outside mem_map, and the NUMA hash lookup cannot handle
283 if (!pte_present(*pte
) || pte_pfn(*pte
) != pte_pfn(*pte_ref
))
288 static int page_fault_trace
;
289 int show_unhandled_signals
= 1;
292 * This routine handles page faults. It determines the address,
293 * and the problem, and then passes it off to one of the appropriate
296 asmlinkage
void __kprobes
do_page_fault(struct pt_regs
*regs
,
297 unsigned long error_code
)
299 struct task_struct
*tsk
;
300 struct mm_struct
*mm
;
301 struct vm_area_struct
* vma
;
302 unsigned long address
;
303 const struct exception_table_entry
*fixup
;
310 prefetchw(&mm
->mmap_sem
);
312 /* get the address */
313 address
= read_cr2();
315 info
.si_code
= SEGV_MAPERR
;
319 * We fault-in kernel-space virtual memory on-demand. The
320 * 'reference' page table is init_mm.pgd.
322 * NOTE! We MUST NOT take any locks for this case. We may
323 * be in an interrupt or a critical region, and should
324 * only copy the information from the master page table,
327 * This verifies that the fault happens in kernel space
328 * (error_code & 4) == 0, and that the fault was not a
329 * protection error (error_code & 9) == 0.
331 if (unlikely(address
>= TASK_SIZE64
)) {
333 * Don't check for the module range here: its PML4
334 * is always initialized because it's shared with the main
335 * kernel text. Only vmalloc may need PML4 syncups.
337 if (!(error_code
& (PF_RSVD
|PF_USER
|PF_PROT
)) &&
338 ((address
>= VMALLOC_START
&& address
< VMALLOC_END
))) {
339 if (vmalloc_fault(address
) >= 0)
342 if (notify_page_fault(regs
))
345 * Don't take the mm semaphore here. If we fixup a prefetch
346 * fault we could otherwise deadlock.
348 goto bad_area_nosemaphore
;
351 if (notify_page_fault(regs
))
354 if (likely(regs
->eflags
& X86_EFLAGS_IF
))
357 if (unlikely(page_fault_trace
))
358 printk("pagefault rip:%lx rsp:%lx cs:%lu ss:%lu address %lx error %lx\n",
359 regs
->rip
,regs
->rsp
,regs
->cs
,regs
->ss
,address
,error_code
);
361 if (unlikely(error_code
& PF_RSVD
))
362 pgtable_bad(address
, regs
, error_code
);
365 * If we're in an interrupt or have no user
366 * context, we must not take the fault..
368 if (unlikely(in_atomic() || !mm
))
369 goto bad_area_nosemaphore
;
372 * User-mode registers count as a user access even for any
373 * potential system fault or CPU buglet.
375 if (user_mode_vm(regs
))
376 error_code
|= PF_USER
;
379 /* When running in the kernel we expect faults to occur only to
380 * addresses in user space. All other faults represent errors in the
381 * kernel and should generate an OOPS. Unfortunatly, in the case of an
382 * erroneous fault occurring in a code path which already holds mmap_sem
383 * we will deadlock attempting to validate the fault against the
384 * address space. Luckily the kernel only validly references user
385 * space from well defined areas of code, which are listed in the
388 * As the vast majority of faults will be valid we will only perform
389 * the source reference check when there is a possibilty of a deadlock.
390 * Attempt to lock the address space, if we cannot we then validate the
391 * source. If this is invalid we can skip the address space check,
392 * thus avoiding the deadlock.
394 if (!down_read_trylock(&mm
->mmap_sem
)) {
395 if ((error_code
& PF_USER
) == 0 &&
396 !search_exception_tables(regs
->rip
))
397 goto bad_area_nosemaphore
;
398 down_read(&mm
->mmap_sem
);
401 vma
= find_vma(mm
, address
);
404 if (likely(vma
->vm_start
<= address
))
406 if (!(vma
->vm_flags
& VM_GROWSDOWN
))
408 if (error_code
& 4) {
409 /* Allow userspace just enough access below the stack pointer
410 * to let the 'enter' instruction work.
412 if (address
+ 65536 + 32 * sizeof(unsigned long) < regs
->rsp
)
415 if (expand_stack(vma
, address
))
418 * Ok, we have a good vm_area for this memory access, so
422 info
.si_code
= SEGV_ACCERR
;
424 switch (error_code
& (PF_PROT
|PF_WRITE
)) {
425 default: /* 3: write, present */
427 case PF_WRITE
: /* write, not present */
428 if (!(vma
->vm_flags
& VM_WRITE
))
432 case PF_PROT
: /* read, present */
434 case 0: /* read, not present */
435 if (!(vma
->vm_flags
& (VM_READ
| VM_EXEC
| VM_WRITE
)))
440 * If for any reason at all we couldn't handle the fault,
441 * make sure we exit gracefully rather than endlessly redo
444 fault
= handle_mm_fault(mm
, vma
, address
, write
);
445 if (unlikely(fault
& VM_FAULT_ERROR
)) {
446 if (fault
& VM_FAULT_OOM
)
448 else if (fault
& VM_FAULT_SIGBUS
)
452 if (fault
& VM_FAULT_MAJOR
)
456 up_read(&mm
->mmap_sem
);
460 * Something tried to access memory that isn't in our memory map..
461 * Fix it, but check if it's kernel or user first..
464 up_read(&mm
->mmap_sem
);
466 bad_area_nosemaphore
:
467 /* User mode accesses just cause a SIGSEGV */
468 if (error_code
& PF_USER
) {
471 * It's possible to have interrupts off here.
475 if (is_prefetch(regs
, address
, error_code
))
478 /* Work around K8 erratum #100 K8 in compat mode
479 occasionally jumps to illegal addresses >4GB. We
480 catch this here in the page fault handler because
481 these addresses are not reachable. Just detect this
482 case and return. Any code segment in LDT is
483 compatibility mode. */
484 if ((regs
->cs
== __USER32_CS
|| (regs
->cs
& (1<<2))) &&
488 if (show_unhandled_signals
&& unhandled_signal(tsk
, SIGSEGV
) &&
489 printk_ratelimit()) {
491 "%s%s[%d]: segfault at %016lx rip %016lx rsp %016lx error %lx\n",
492 tsk
->pid
> 1 ? KERN_INFO
: KERN_EMERG
,
493 tsk
->comm
, tsk
->pid
, address
, regs
->rip
,
494 regs
->rsp
, error_code
);
497 tsk
->thread
.cr2
= address
;
498 /* Kernel addresses are always protection faults */
499 tsk
->thread
.error_code
= error_code
| (address
>= TASK_SIZE
);
500 tsk
->thread
.trap_no
= 14;
501 info
.si_signo
= SIGSEGV
;
503 /* info.si_code has been set above */
504 info
.si_addr
= (void __user
*)address
;
505 force_sig_info(SIGSEGV
, &info
, tsk
);
511 /* Are we prepared to handle this kernel fault? */
512 fixup
= search_exception_tables(regs
->rip
);
514 regs
->rip
= fixup
->fixup
;
519 * Hall of shame of CPU/BIOS bugs.
522 if (is_prefetch(regs
, address
, error_code
))
525 if (is_errata93(regs
, address
))
529 * Oops. The kernel tried to access some bad page. We'll have to
530 * terminate things with extreme prejudice.
533 flags
= oops_begin();
535 if (address
< PAGE_SIZE
)
536 printk(KERN_ALERT
"Unable to handle kernel NULL pointer dereference");
538 printk(KERN_ALERT
"Unable to handle kernel paging request");
539 printk(" at %016lx RIP: \n" KERN_ALERT
,address
);
540 printk_address(regs
->rip
);
541 dump_pagetable(address
);
542 tsk
->thread
.cr2
= address
;
543 tsk
->thread
.trap_no
= 14;
544 tsk
->thread
.error_code
= error_code
;
545 __die("Oops", regs
, error_code
);
546 /* Executive summary in case the body of the oops scrolled away */
547 printk(KERN_EMERG
"CR2: %016lx\n", address
);
552 * We ran out of memory, or some other thing happened to us that made
553 * us unable to handle the page fault gracefully.
556 up_read(&mm
->mmap_sem
);
557 if (is_init(current
)) {
561 printk("VM: killing process %s\n", tsk
->comm
);
563 do_group_exit(SIGKILL
);
567 up_read(&mm
->mmap_sem
);
569 /* Kernel mode? Handle exceptions or die */
570 if (!(error_code
& PF_USER
))
573 tsk
->thread
.cr2
= address
;
574 tsk
->thread
.error_code
= error_code
;
575 tsk
->thread
.trap_no
= 14;
576 info
.si_signo
= SIGBUS
;
578 info
.si_code
= BUS_ADRERR
;
579 info
.si_addr
= (void __user
*)address
;
580 force_sig_info(SIGBUS
, &info
, tsk
);
584 DEFINE_SPINLOCK(pgd_lock
);
587 void vmalloc_sync_all(void)
589 /* Note that races in the updates of insync and start aren't
591 insync can only get set bits added, and updates to start are only
592 improving performance (without affecting correctness if undone). */
593 static DECLARE_BITMAP(insync
, PTRS_PER_PGD
);
594 static unsigned long start
= VMALLOC_START
& PGDIR_MASK
;
595 unsigned long address
;
597 for (address
= start
; address
<= VMALLOC_END
; address
+= PGDIR_SIZE
) {
598 if (!test_bit(pgd_index(address
), insync
)) {
599 const pgd_t
*pgd_ref
= pgd_offset_k(address
);
602 if (pgd_none(*pgd_ref
))
604 spin_lock(&pgd_lock
);
605 list_for_each_entry(page
, &pgd_list
, lru
) {
607 pgd
= (pgd_t
*)page_address(page
) + pgd_index(address
);
609 set_pgd(pgd
, *pgd_ref
);
611 BUG_ON(pgd_page_vaddr(*pgd
) != pgd_page_vaddr(*pgd_ref
));
613 spin_unlock(&pgd_lock
);
614 set_bit(pgd_index(address
), insync
);
616 if (address
== start
)
617 start
= address
+ PGDIR_SIZE
;
619 /* Check that there is no need to do the same for the modules area. */
620 BUILD_BUG_ON(!(MODULES_VADDR
> __START_KERNEL
));
621 BUILD_BUG_ON(!(((MODULES_END
- 1) & PGDIR_MASK
) ==
622 (__START_KERNEL
& PGDIR_MASK
)));
625 static int __init
enable_pagefaulttrace(char *str
)
627 page_fault_trace
= 1;
630 __setup("pagefaulttrace", enable_pagefaulttrace
);