arch/sh/mm/fault_64.c

   1 /*
   2  * The SH64 TLB miss.
   3  *
   4  * Original code from fault.c
   5  * Copyright (C) 2000, 2001  Paolo Alberelli
   6  *
   7  * Fast PTE->TLB refill path
   8  * Copyright (C) 2003 Richard.Curnow@superh.com
   9  *
  10  * IMPORTANT NOTES :
  11  * The do_fast_page_fault function is called from a context in entry.S
  12  * where very few registers have been saved.  In particular, the code in
  13  * this file must be compiled not to use ANY caller-save registers that
  14  * are not part of the restricted save set.  Also, it means that code in
  15  * this file must not make calls to functions elsewhere in the kernel, or
  16  * else the excepting context will see corruption in its caller-save
  17  * registers.  Plus, the entry.S save area is non-reentrant, so this code
  18  * has to run with SR.BL==1, i.e. no interrupts taken inside it and panic
  19  * on any exception.
  20  *
  21  * This file is subject to the terms and conditions of the GNU General Public
  22  * License.  See the file "COPYING" in the main directory of this archive
  23  * for more details.
  24  */
  25 #include <linux/signal.h>
  26 #include <linux/sched.h>
  27 #include <linux/kernel.h>
  28 #include <linux/errno.h>
  29 #include <linux/string.h>
  30 #include <linux/types.h>
  31 #include <linux/ptrace.h>
  32 #include <linux/mman.h>
  33 #include <linux/mm.h>
  34 #include <linux/smp.h>
  35 #include <linux/interrupt.h>
  36 #include <asm/system.h>
  37 #include <asm/tlb.h>
  38 #include <asm/io.h>
  39 #include <asm/uaccess.h>
  40 #include <asm/pgalloc.h>
  41 #include <asm/mmu_context.h>
  42 #include <cpu/registers.h>
  43
  44 /* Callable from fault.c, so not static */
  45 inline void __do_tlb_refill(unsigned long address,
  46                             unsigned long long is_text_not_data, pte_t *pte)
  47 {
  48         unsigned long long ptel;
  49         unsigned long long pteh=0;
  50         struct tlb_info *tlbp;
  51         unsigned long long next;
  52
  53         /* Get PTEL first */
  54         ptel = pte_val(*pte);
  55
  56         /*
  57          * Set PTEH register
  58          */
  59         pteh = address & MMU_VPN_MASK;
  60
  61         /* Sign extend based on neff. */
  62 #if (NEFF == 32)
  63         /* Faster sign extension */
  64         pteh = (unsigned long long)(signed long long)(signed long)pteh;
  65 #else
  66         /* General case */
  67         pteh = (pteh & NEFF_SIGN) ? (pteh | NEFF_MASK) : pteh;
  68 #endif
  69
  70         /* Set the ASID. */
  71         pteh |= get_asid() << PTEH_ASID_SHIFT;
  72         pteh |= PTEH_VALID;
  73
  74         /* Set PTEL register, set_pte has performed the sign extension */
  75         ptel &= _PAGE_FLAGS_HARDWARE_MASK; /* drop software flags */
  76
  77         tlbp = is_text_not_data ? &(cpu_data->itlb) : &(cpu_data->dtlb);
  78         next = tlbp->next;
  79         __flush_tlb_slot(next);
  80         asm volatile ("putcfg %0,1,%2\n\n\t"
  81                       "putcfg %0,0,%1\n"
  82                       :  : "r" (next), "r" (pteh), "r" (ptel) );
  83
  84         next += TLB_STEP;
  85         if (next > tlbp->last) next = tlbp->first;
  86         tlbp->next = next;
  87
  88 }
  89
  90 static int handle_vmalloc_fault(struct mm_struct *mm,
  91                                 unsigned long protection_flags,
  92                                 unsigned long long textaccess,
  93                                 unsigned long address)
  94 {
  95         pgd_t *dir;
  96         pud_t *pud;
  97         pmd_t *pmd;
  98         static pte_t *pte;
  99         pte_t entry;
 100
 101         dir = pgd_offset_k(address);
 102
 103         pud = pud_offset(dir, address);
 104         if (pud_none_or_clear_bad(pud))
 105                 return 0;
 106
 107         pmd = pmd_offset(pud, address);
 108         if (pmd_none_or_clear_bad(pmd))
 109                 return 0;
 110
 111         pte = pte_offset_kernel(pmd, address);
 112         entry = *pte;
 113
 114         if (pte_none(entry) || !pte_present(entry))
 115                 return 0;
 116         if ((pte_val(entry) & protection_flags) != protection_flags)
 117                 return 0;
 118
 119         __do_tlb_refill(address, textaccess, pte);
 120
 121         return 1;
 122 }
 123
 124 static int handle_tlbmiss(struct mm_struct *mm,
 125                           unsigned long long protection_flags,
 126                           unsigned long long textaccess,
 127                           unsigned long address)
 128 {
 129         pgd_t *dir;
 130         pud_t *pud;
 131         pmd_t *pmd;
 132         pte_t *pte;
 133         pte_t entry;
 134
 135         /* NB. The PGD currently only contains a single entry - there is no
 136            page table tree stored for the top half of the address space since
 137            virtual pages in that region should never be mapped in user mode.
 138            (In kernel mode, the only things in that region are the 512Mb super
 139            page (locked in), and vmalloc (modules) +  I/O device pages (handled
 140            by handle_vmalloc_fault), so no PGD for the upper half is required
 141            by kernel mode either).
 142
 143            See how mm->pgd is allocated and initialised in pgd_alloc to see why
 144            the next test is necessary.  - RPC */
 145         if (address >= (unsigned long) TASK_SIZE)
 146                 /* upper half - never has page table entries. */
 147                 return 0;
 148
 149         dir = pgd_offset(mm, address);
 150         if (pgd_none(*dir) || !pgd_present(*dir))
 151                 return 0;
 152         if (!pgd_present(*dir))
 153                 return 0;
 154
 155         pud = pud_offset(dir, address);
 156         if (pud_none(*pud) || !pud_present(*pud))
 157                 return 0;
 158
 159         pmd = pmd_offset(pud, address);
 160         if (pmd_none(*pmd) || !pmd_present(*pmd))
 161                 return 0;
 162
 163         pte = pte_offset_kernel(pmd, address);
 164         entry = *pte;
 165
 166         if (pte_none(entry) || !pte_present(entry))
 167                 return 0;
 168
 169         /*
 170          * If the page doesn't have sufficient protection bits set to
 171          * service the kind of fault being handled, there's not much
 172          * point doing the TLB refill.  Punt the fault to the general
 173          * handler.
 174          */
 175         if ((pte_val(entry) & protection_flags) != protection_flags)
 176                 return 0;
 177
 178         __do_tlb_refill(address, textaccess, pte);
 179
 180         return 1;
 181 }
 182
 183 /*
 184  * Put all this information into one structure so that everything is just
 185  * arithmetic relative to a single base address.  This reduces the number
 186  * of movi/shori pairs needed just to load addresses of static data.
 187  */
 188 struct expevt_lookup {
 189         unsigned short protection_flags[8];
 190         unsigned char  is_text_access[8];
 191         unsigned char  is_write_access[8];
 192 };
 193
 194 #define PRU (1<<9)
 195 #define PRW (1<<8)
 196 #define PRX (1<<7)
 197 #define PRR (1<<6)
 198
 199 #define DIRTY (_PAGE_DIRTY | _PAGE_ACCESSED)
 200 #define YOUNG (_PAGE_ACCESSED)
 201
 202 /* Sized as 8 rather than 4 to allow checking the PTE's PRU bit against whether
 203    the fault happened in user mode or privileged mode. */
 204 static struct expevt_lookup expevt_lookup_table = {
 205         .protection_flags = {PRX, PRX, 0, 0, PRR, PRR, PRW, PRW},
 206         .is_text_access   = {1,   1,   0, 0, 0,   0,   0,   0}
 207 };
 208
 209 /*
 210    This routine handles page faults that can be serviced just by refilling a
 211    TLB entry from an existing page table entry.  (This case represents a very
 212    large majority of page faults.) Return 1 if the fault was successfully
 213    handled.  Return 0 if the fault could not be handled.  (This leads into the
 214    general fault handling in fault.c which deals with mapping file-backed
 215    pages, stack growth, segmentation faults, swapping etc etc)
 216  */
 217 asmlinkage int do_fast_page_fault(unsigned long long ssr_md,
 218                                   unsigned long long expevt,
 219                                   unsigned long address)
 220 {
 221         struct task_struct *tsk;
 222         struct mm_struct *mm;
 223         unsigned long long textaccess;
 224         unsigned long long protection_flags;
 225         unsigned long long index;
 226         unsigned long long expevt4;
 227
 228         /* The next few lines implement a way of hashing EXPEVT into a
 229          * small array index which can be used to lookup parameters
 230          * specific to the type of TLBMISS being handled.
 231          *
 232          * Note:
 233          *      ITLBMISS has EXPEVT==0xa40
 234          *      RTLBMISS has EXPEVT==0x040
 235          *      WTLBMISS has EXPEVT==0x060
 236          */
 237         expevt4 = (expevt >> 4);
 238         /* TODO : xor ssr_md into this expression too. Then we can check
 239          * that PRU is set when it needs to be. */
 240         index = expevt4 ^ (expevt4 >> 5);
 241         index &= 7;
 242         protection_flags = expevt_lookup_table.protection_flags[index];
 243         textaccess       = expevt_lookup_table.is_text_access[index];
 244
 245         /* SIM
 246          * Note this is now called with interrupts still disabled
 247          * This is to cope with being called for a missing IO port
 248          * address with interrupts disabled. This should be fixed as
 249          * soon as we have a better 'fast path' miss handler.
 250          *
 251          * Plus take care how you try and debug this stuff.
 252          * For example, writing debug data to a port which you
 253          * have just faulted on is not going to work.
 254          */
 255
 256         tsk = current;
 257         mm = tsk->mm;
 258
 259         if ((address >= VMALLOC_START && address < VMALLOC_END) ||
 260             (address >= IOBASE_VADDR  && address < IOBASE_END)) {
 261                 if (ssr_md)
 262                         /*
 263                          * Process-contexts can never have this address
 264                          * range mapped
 265                          */
 266                         if (handle_vmalloc_fault(mm, protection_flags,
 267                                                  textaccess, address))
 268                                 return 1;
 269         } else if (!in_interrupt() && mm) {
 270                 if (handle_tlbmiss(mm, protection_flags, textaccess, address))
 271                         return 1;
 272         }
 273
 274         return 0;
 275 }