arch/x86_64/mm/fault.c

   1 /*
   2  *  linux/arch/x86-64/mm/fault.c
   3  *
   4  *  Copyright (C) 1995  Linus Torvalds
   5  *  Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
   6  */
   7
   8 #include <linux/config.h>
   9 #include <linux/signal.h>
  10 #include <linux/sched.h>
  11 #include <linux/kernel.h>
  12 #include <linux/errno.h>
  13 #include <linux/string.h>
  14 #include <linux/types.h>
  15 #include <linux/ptrace.h>
  16 #include <linux/mman.h>
  17 #include <linux/mm.h>
  18 #include <linux/smp.h>
  19 #include <linux/smp_lock.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/init.h>
  22 #include <linux/tty.h>
  23 #include <linux/vt_kern.h>              /* For unblank_screen() */
  24 #include <linux/compiler.h>
  25 #include <linux/module.h>
  26 #include <linux/kprobes.h>
  27
  28 #include <asm/system.h>
  29 #include <asm/uaccess.h>
  30 #include <asm/pgalloc.h>
  31 #include <asm/smp.h>
  32 #include <asm/tlbflush.h>
  33 #include <asm/proto.h>
  34 #include <asm/kdebug.h>
  35 #include <asm-generic/sections.h>
  36
  37 /* Page fault error code bits */
  38 #define PF_PROT (1<<0)          /* or no page found */
  39 #define PF_WRITE        (1<<1)
  40 #define PF_USER (1<<2)
  41 #define PF_RSVD (1<<3)
  42 #define PF_INSTR        (1<<4)
  43
  44 void bust_spinlocks(int yes)
  45 {
  46         int loglevel_save = console_loglevel;
  47         if (yes) {
  48                 oops_in_progress = 1;
  49         } else {
  50 #ifdef CONFIG_VT
  51                 unblank_screen();
  52 #endif
  53                 oops_in_progress = 0;
  54                 /*
  55                  * OK, the message is on the console.  Now we call printk()
  56                  * without oops_in_progress set so that printk will give klogd
  57                  * a poke.  Hold onto your hats...
  58                  */
  59                 console_loglevel = 15;          /* NMI oopser may have shut the console up */
  60                 printk(" ");
  61                 console_loglevel = loglevel_save;
  62         }
  63 }
  64
  65 /* Sometimes the CPU reports invalid exceptions on prefetch.
  66    Check that here and ignore.
  67    Opcode checker based on code by Richard Brunner */
  68 static noinline int is_prefetch(struct pt_regs *regs, unsigned long addr,
  69                                 unsigned long error_code)
  70 {
  71         unsigned char *instr;
  72         int scan_more = 1;
  73         int prefetch = 0;
  74         unsigned char *max_instr;
  75
  76         /* If it was a exec fault ignore */
  77         if (error_code & PF_INSTR)
  78                 return 0;
  79
  80         instr = (unsigned char *)convert_rip_to_linear(current, regs);
  81         max_instr = instr + 15;
  82
  83         if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
  84                 return 0;
  85
  86         while (scan_more && instr < max_instr) {
  87                 unsigned char opcode;
  88                 unsigned char instr_hi;
  89                 unsigned char instr_lo;
  90
  91                 if (__get_user(opcode, instr))
  92                         break;
  93
  94                 instr_hi = opcode & 0xf0;
  95                 instr_lo = opcode & 0x0f;
  96                 instr++;
  97
  98                 switch (instr_hi) {
  99                 case 0x20:
 100                 case 0x30:
 101                         /* Values 0x26,0x2E,0x36,0x3E are valid x86
 102                            prefixes.  In long mode, the CPU will signal
 103                            invalid opcode if some of these prefixes are
 104                            present so we will never get here anyway */
 105                         scan_more = ((instr_lo & 7) == 0x6);
 106                         break;
 107
 108                 case 0x40:
 109                         /* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
 110                            Need to figure out under what instruction mode the
 111                            instruction was issued ... */
 112                         /* Could check the LDT for lm, but for now it's good
 113                            enough to assume that long mode only uses well known
 114                            segments or kernel. */
 115                         scan_more = (!user_mode(regs)) || (regs->cs == __USER_CS);
 116                         break;
 117
 118                 case 0x60:
 119                         /* 0x64 thru 0x67 are valid prefixes in all modes. */
 120                         scan_more = (instr_lo & 0xC) == 0x4;
 121                         break;
 122                 case 0xF0:
 123                         /* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
 124                         scan_more = !instr_lo || (instr_lo>>1) == 1;
 125                         break;
 126                 case 0x00:
 127                         /* Prefetch instruction is 0x0F0D or 0x0F18 */
 128                         scan_more = 0;
 129                         if (__get_user(opcode, instr))
 130                                 break;
 131                         prefetch = (instr_lo == 0xF) &&
 132                                 (opcode == 0x0D || opcode == 0x18);
 133                         break;
 134                 default:
 135                         scan_more = 0;
 136                         break;
 137                 }
 138         }
 139         return prefetch;
 140 }
 141
 142 static int bad_address(void *p)
 143 {
 144         unsigned long dummy;
 145         return __get_user(dummy, (unsigned long *)p);
 146 }
 147
 148 void dump_pagetable(unsigned long address)
 149 {
 150         pgd_t *pgd;
 151         pud_t *pud;
 152         pmd_t *pmd;
 153         pte_t *pte;
 154
 155         asm("movq %%cr3,%0" : "=r" (pgd));
 156
 157         pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
 158         pgd += pgd_index(address);
 159         if (bad_address(pgd)) goto bad;
 160         printk("PGD %lx ", pgd_val(*pgd));
 161         if (!pgd_present(*pgd)) goto ret;
 162
 163         pud = __pud_offset_k((pud_t *)pgd_page(*pgd), address);
 164         if (bad_address(pud)) goto bad;
 165         printk("PUD %lx ", pud_val(*pud));
 166         if (!pud_present(*pud)) goto ret;
 167
 168         pmd = pmd_offset(pud, address);
 169         if (bad_address(pmd)) goto bad;
 170         printk("PMD %lx ", pmd_val(*pmd));
 171         if (!pmd_present(*pmd)) goto ret;
 172
 173         pte = pte_offset_kernel(pmd, address);
 174         if (bad_address(pte)) goto bad;
 175         printk("PTE %lx", pte_val(*pte));
 176 ret:
 177         printk("\n");
 178         return;
 179 bad:
 180         printk("BAD\n");
 181 }
 182
 183 static const char errata93_warning[] =
 184 KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
 185 KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
 186 KERN_ERR "******* Please consider a BIOS update.\n"
 187 KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
 188
 189 /* Workaround for K8 erratum #93 & buggy BIOS.
 190    BIOS SMM functions are required to use a specific workaround
 191    to avoid corruption of the 64bit RIP register on C stepping K8.
 192    A lot of BIOS that didn't get tested properly miss this.
 193    The OS sees this as a page fault with the upper 32bits of RIP cleared.
 194    Try to work around it here.
 195    Note we only handle faults in kernel here. */
 196
 197 static int is_errata93(struct pt_regs *regs, unsigned long address)
 198 {
 199         static int warned;
 200         if (address != regs->rip)
 201                 return 0;
 202         if ((address >> 32) != 0)
 203                 return 0;
 204         address |= 0xffffffffUL << 32;
 205         if ((address >= (u64)_stext && address <= (u64)_etext) ||
 206             (address >= MODULES_VADDR && address <= MODULES_END)) {
 207                 if (!warned) {
 208                         printk(errata93_warning);
 209                         warned = 1;
 210                 }
 211                 regs->rip = address;
 212                 return 1;
 213         }
 214         return 0;
 215 }
 216
 217 int unhandled_signal(struct task_struct *tsk, int sig)
 218 {
 219         if (tsk->pid == 1)
 220                 return 1;
 221         if (tsk->ptrace & PT_PTRACED)
 222                 return 0;
 223         return (tsk->sighand->action[sig-1].sa.sa_handler == SIG_IGN) ||
 224                 (tsk->sighand->action[sig-1].sa.sa_handler == SIG_DFL);
 225 }
 226
 227 static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
 228                                  unsigned long error_code)
 229 {
 230         unsigned long flags = oops_begin();
 231         struct task_struct *tsk;
 232
 233         printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
 234                current->comm, address);
 235         dump_pagetable(address);
 236         tsk = current;
 237         tsk->thread.cr2 = address;
 238         tsk->thread.trap_no = 14;
 239         tsk->thread.error_code = error_code;
 240         __die("Bad pagetable", regs, error_code);
 241         oops_end(flags);
 242         do_exit(SIGKILL);
 243 }
 244
 245 /*
 246  * Handle a fault on the vmalloc area
 247  *
 248  * This assumes no large pages in there.
 249  */
 250 static int vmalloc_fault(unsigned long address)
 251 {
 252         pgd_t *pgd, *pgd_ref;
 253         pud_t *pud, *pud_ref;
 254         pmd_t *pmd, *pmd_ref;
 255         pte_t *pte, *pte_ref;
 256
 257         /* Copy kernel mappings over when needed. This can also
 258            happen within a race in page table update. In the later
 259            case just flush. */
 260
 261         pgd = pgd_offset(current->mm ?: &init_mm, address);
 262         pgd_ref = pgd_offset_k(address);
 263         if (pgd_none(*pgd_ref))
 264                 return -1;
 265         if (pgd_none(*pgd))
 266                 set_pgd(pgd, *pgd_ref);
 267
 268         /* Below here mismatches are bugs because these lower tables
 269            are shared */
 270
 271         pud = pud_offset(pgd, address);
 272         pud_ref = pud_offset(pgd_ref, address);
 273         if (pud_none(*pud_ref))
 274                 return -1;
 275         if (pud_none(*pud) || pud_page(*pud) != pud_page(*pud_ref))
 276                 BUG();
 277         pmd = pmd_offset(pud, address);
 278         pmd_ref = pmd_offset(pud_ref, address);
 279         if (pmd_none(*pmd_ref))
 280                 return -1;
 281         if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
 282                 BUG();
 283         pte_ref = pte_offset_kernel(pmd_ref, address);
 284         if (!pte_present(*pte_ref))
 285                 return -1;
 286         pte = pte_offset_kernel(pmd, address);
 287         /* Don't use pte_page here, because the mappings can point
 288            outside mem_map, and the NUMA hash lookup cannot handle
 289            that. */
 290         if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
 291                 BUG();
 292         return 0;
 293 }
 294
 295 int page_fault_trace = 0;
 296 int exception_trace = 1;
 297
 298 /*
 299  * This routine handles page faults.  It determines the address,
 300  * and the problem, and then passes it off to one of the appropriate
 301  * routines.
 302  */
 303 asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
 304                                         unsigned long error_code)
 305 {
 306         struct task_struct *tsk;
 307         struct mm_struct *mm;
 308         struct vm_area_struct * vma;
 309         unsigned long address;
 310         const struct exception_table_entry *fixup;
 311         int write;
 312         unsigned long flags;
 313         siginfo_t info;
 314
 315         /* get the address */
 316         __asm__("movq %%cr2,%0":"=r" (address));
 317         if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
 318                                         SIGSEGV) == NOTIFY_STOP)
 319                 return;
 320
 321         if (likely(regs->eflags & X86_EFLAGS_IF))
 322                 local_irq_enable();
 323
 324         if (unlikely(page_fault_trace))
 325                 printk("pagefault rip:%lx rsp:%lx cs:%lu ss:%lu address %lx error %lx\n",
 326                        regs->rip,regs->rsp,regs->cs,regs->ss,address,error_code);
 327
 328         tsk = current;
 329         mm = tsk->mm;
 330         info.si_code = SEGV_MAPERR;
 331
 332
 333         /*
 334          * We fault-in kernel-space virtual memory on-demand. The
 335          * 'reference' page table is init_mm.pgd.
 336          *
 337          * NOTE! We MUST NOT take any locks for this case. We may
 338          * be in an interrupt or a critical region, and should
 339          * only copy the information from the master page table,
 340          * nothing more.
 341          *
 342          * This verifies that the fault happens in kernel space
 343          * (error_code & 4) == 0, and that the fault was not a
 344          * protection error (error_code & 9) == 0.
 345          */
 346         if (unlikely(address >= TASK_SIZE64)) {
 347                 /*
 348                  * Don't check for the module range here: its PML4
 349                  * is always initialized because it's shared with the main
 350                  * kernel text. Only vmalloc may need PML4 syncups.
 351                  */
 352                 if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) &&
 353                       ((address >= VMALLOC_START && address < VMALLOC_END))) {
 354                         if (vmalloc_fault(address) < 0)
 355                                 goto bad_area_nosemaphore;
 356                         return;
 357                 }
 358                 /*
 359                  * Don't take the mm semaphore here. If we fixup a prefetch
 360                  * fault we could otherwise deadlock.
 361                  */
 362                 goto bad_area_nosemaphore;
 363         }
 364
 365         if (unlikely(error_code & PF_RSVD))
 366                 pgtable_bad(address, regs, error_code);
 367
 368         /*
 369          * If we're in an interrupt or have no user
 370          * context, we must not take the fault..
 371          */
 372         if (unlikely(in_atomic() || !mm))
 373                 goto bad_area_nosemaphore;
 374
 375  again:
 376         /* When running in the kernel we expect faults to occur only to
 377          * addresses in user space.  All other faults represent errors in the
 378          * kernel and should generate an OOPS.  Unfortunatly, in the case of an
 379          * erroneous fault occuring in a code path which already holds mmap_sem
 380          * we will deadlock attempting to validate the fault against the
 381          * address space.  Luckily the kernel only validly references user
 382          * space from well defined areas of code, which are listed in the
 383          * exceptions table.
 384          *
 385          * As the vast majority of faults will be valid we will only perform
 386          * the source reference check when there is a possibilty of a deadlock.
 387          * Attempt to lock the address space, if we cannot we then validate the
 388          * source.  If this is invalid we can skip the address space check,
 389          * thus avoiding the deadlock.
 390          */
 391         if (!down_read_trylock(&mm->mmap_sem)) {
 392                 if ((error_code & PF_USER) == 0 &&
 393                     !search_exception_tables(regs->rip))
 394                         goto bad_area_nosemaphore;
 395                 down_read(&mm->mmap_sem);
 396         }
 397
 398         vma = find_vma(mm, address);
 399         if (!vma)
 400                 goto bad_area;
 401         if (likely(vma->vm_start <= address))
 402                 goto good_area;
 403         if (!(vma->vm_flags & VM_GROWSDOWN))
 404                 goto bad_area;
 405         if (error_code & 4) {
 406                 // XXX: align red zone size with ABI
 407                 if (address + 128 < regs->rsp)
 408                         goto bad_area;
 409         }
 410         if (expand_stack(vma, address))
 411                 goto bad_area;
 412 /*
 413  * Ok, we have a good vm_area for this memory access, so
 414  * we can handle it..
 415  */
 416 good_area:
 417         info.si_code = SEGV_ACCERR;
 418         write = 0;
 419         switch (error_code & (PF_PROT|PF_WRITE)) {
 420                 default:        /* 3: write, present */
 421                         /* fall through */
 422                 case PF_WRITE:          /* write, not present */
 423                         if (!(vma->vm_flags & VM_WRITE))
 424                                 goto bad_area;
 425                         write++;
 426                         break;
 427                 case PF_PROT:           /* read, present */
 428                         goto bad_area;
 429                 case 0:                 /* read, not present */
 430                         if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
 431                                 goto bad_area;
 432         }
 433
 434         /*
 435          * If for any reason at all we couldn't handle the fault,
 436          * make sure we exit gracefully rather than endlessly redo
 437          * the fault.
 438          */
 439         switch (handle_mm_fault(mm, vma, address, write)) {
 440         case VM_FAULT_MINOR:
 441                 tsk->min_flt++;
 442                 break;
 443         case VM_FAULT_MAJOR:
 444                 tsk->maj_flt++;
 445                 break;
 446         case VM_FAULT_SIGBUS:
 447                 goto do_sigbus;
 448         default:
 449                 goto out_of_memory;
 450         }
 451
 452         up_read(&mm->mmap_sem);
 453         return;
 454
 455 /*
 456  * Something tried to access memory that isn't in our memory map..
 457  * Fix it, but check if it's kernel or user first..
 458  */
 459 bad_area:
 460         up_read(&mm->mmap_sem);
 461
 462 bad_area_nosemaphore:
 463         /* User mode accesses just cause a SIGSEGV */
 464         if (error_code & PF_USER) {
 465                 if (is_prefetch(regs, address, error_code))
 466                         return;
 467
 468                 /* Work around K8 erratum #100 K8 in compat mode
 469                    occasionally jumps to illegal addresses >4GB.  We
 470                    catch this here in the page fault handler because
 471                    these addresses are not reachable. Just detect this
 472                    case and return.  Any code segment in LDT is
 473                    compatibility mode. */
 474                 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
 475                     (address >> 32))
 476                         return;
 477
 478                 if (exception_trace && unhandled_signal(tsk, SIGSEGV)) {
 479                         printk(
 480                        "%s%s[%d]: segfault at %016lx rip %016lx rsp %016lx error %lx\n",
 481                                         tsk->pid > 1 ? KERN_INFO : KERN_EMERG,
 482                                         tsk->comm, tsk->pid, address, regs->rip,
 483                                         regs->rsp, error_code);
 484                 }
 485
 486                 tsk->thread.cr2 = address;
 487                 /* Kernel addresses are always protection faults */
 488                 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
 489                 tsk->thread.trap_no = 14;
 490                 info.si_signo = SIGSEGV;
 491                 info.si_errno = 0;
 492                 /* info.si_code has been set above */
 493                 info.si_addr = (void __user *)address;
 494                 force_sig_info(SIGSEGV, &info, tsk);
 495                 return;
 496         }
 497
 498 no_context:
 499
 500         /* Are we prepared to handle this kernel fault?  */
 501         fixup = search_exception_tables(regs->rip);
 502         if (fixup) {
 503                 regs->rip = fixup->fixup;
 504                 return;
 505         }
 506
 507         /*
 508          * Hall of shame of CPU/BIOS bugs.
 509          */
 510
 511         if (is_prefetch(regs, address, error_code))
 512                 return;
 513
 514         if (is_errata93(regs, address))
 515                 return;
 516
 517 /*
 518  * Oops. The kernel tried to access some bad page. We'll have to
 519  * terminate things with extreme prejudice.
 520  */
 521
 522         flags = oops_begin();
 523
 524         if (address < PAGE_SIZE)
 525                 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
 526         else
 527                 printk(KERN_ALERT "Unable to handle kernel paging request");
 528         printk(" at %016lx RIP: \n" KERN_ALERT,address);
 529         printk_address(regs->rip);
 530         printk("\n");
 531         dump_pagetable(address);
 532         tsk->thread.cr2 = address;
 533         tsk->thread.trap_no = 14;
 534         tsk->thread.error_code = error_code;
 535         __die("Oops", regs, error_code);
 536         /* Executive summary in case the body of the oops scrolled away */
 537         printk(KERN_EMERG "CR2: %016lx\n", address);
 538         oops_end(flags);
 539         do_exit(SIGKILL);
 540
 541 /*
 542  * We ran out of memory, or some other thing happened to us that made
 543  * us unable to handle the page fault gracefully.
 544  */
 545 out_of_memory:
 546         up_read(&mm->mmap_sem);
 547         if (current->pid == 1) {
 548                 yield();
 549                 goto again;
 550         }
 551         printk("VM: killing process %s\n", tsk->comm);
 552         if (error_code & 4)
 553                 do_exit(SIGKILL);
 554         goto no_context;
 555
 556 do_sigbus:
 557         up_read(&mm->mmap_sem);
 558
 559         /* Kernel mode? Handle exceptions or die */
 560         if (!(error_code & PF_USER))
 561                 goto no_context;
 562
 563         tsk->thread.cr2 = address;
 564         tsk->thread.error_code = error_code;
 565         tsk->thread.trap_no = 14;
 566         info.si_signo = SIGBUS;
 567         info.si_errno = 0;
 568         info.si_code = BUS_ADRERR;
 569         info.si_addr = (void __user *)address;
 570         force_sig_info(SIGBUS, &info, tsk);
 571         return;
 572 }
 573
 574 static int __init enable_pagefaulttrace(char *str)
 575 {
 576         page_fault_trace = 1;
 577         return 0;
 578 }
 579 __setup("pagefaulttrace", enable_pagefaulttrace);