arch/ppc32/src/mm/tlb.c

   1 /*
   2  * Copyright (C) 2006 Martin Decky
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  *
   9  * - Redistributions of source code must retain the above copyright
  10  *   notice, this list of conditions and the following disclaimer.
  11  * - Redistributions in binary form must reproduce the above copyright
  12  *   notice, this list of conditions and the following disclaimer in the
  13  *   documentation and/or other materials provided with the distribution.
  14  * - The name of the author may not be used to endorse or promote products
  15  *   derived from this software without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 /** @addtogroup ppc32mm
  30  * @{
  31  */
  32 /** @file
  33  */
  34
  35 #include <mm/tlb.h>
  36 #include <arch/mm/tlb.h>
  37 #include <arch/interrupt.h>
  38 #include <mm/as.h>
  39 #include <arch.h>
  40 #include <print.h>
  41 #include <symtab.h>
  42
  43
  44 /** Try to find PTE for faulting address
  45  *
  46  * Try to find PTE for faulting address.
  47  * The as->lock must be held on entry to this function
  48  * if lock is true.
  49  *
  50  * @param as       Address space.
  51  * @param lock     Lock/unlock the address space.
  52  * @param badvaddr Faulting virtual address.
  53  * @param access   Access mode that caused the fault.
  54  * @param istate   Pointer to interrupted state.
  55  * @param pfrc     Pointer to variable where as_page_fault() return code will be stored.
  56  * @return         PTE on success, NULL otherwise.
  57  *
  58  */
  59 static pte_t *find_mapping_and_check(as_t *as, bool lock, __address badvaddr, int access, istate_t *istate, int *pfrc)
  60 {
  61         /*
  62          * Check if the mapping exists in page tables.
  63          */
  64         pte_t *pte = page_mapping_find(as, badvaddr);
  65         if ((pte) && (pte->p)) {
  66                 /*
  67                  * Mapping found in page tables.
  68                  * Immediately succeed.
  69                  */
  70                 return pte;
  71         } else {
  72                 int rc;
  73
  74                 /*
  75                  * Mapping not found in page tables.
  76                  * Resort to higher-level page fault handler.
  77                  */
  78                 page_table_unlock(as, lock);
  79                 switch (rc = as_page_fault(badvaddr, access, istate)) {
  80                         case AS_PF_OK:
  81                                 /*
  82                                  * The higher-level page fault handler succeeded,
  83                                  * The mapping ought to be in place.
  84                                  */
  85                                 page_table_lock(as, lock);
  86                                 pte = page_mapping_find(as, badvaddr);
  87                                 ASSERT((pte) && (pte->p));
  88                                 *pfrc = 0;
  89                                 return pte;
  90                         case AS_PF_DEFER:
  91                                 page_table_lock(as, lock);
  92                                 *pfrc = rc;
  93                                 return NULL;
  94                         case AS_PF_FAULT:
  95                                 page_table_lock(as, lock);
  96                                 printf("Page fault.\n");
  97                                 *pfrc = rc;
  98                                 return NULL;
  99                         default:
 100                                 panic("unexpected rc (%d)\n", rc);
 101                 }
 102         }
 103 }
 104
 105
 106 static void pht_refill_fail(__address badvaddr, istate_t *istate)
 107 {
 108         char *symbol = "";
 109         char *sym2 = "";
 110
 111         char *s = get_symtab_entry(istate->pc);
 112         if (s)
 113                 symbol = s;
 114         s = get_symtab_entry(istate->lr);
 115         if (s)
 116                 sym2 = s;
 117         panic("%p: PHT Refill Exception at %p (%s<-%s)\n", badvaddr, istate->pc, symbol, sym2);
 118 }
 119
 120
 121 static void pht_insert(const __address vaddr, const pfn_t pfn)
 122 {
 123         __u32 page = (vaddr >> 12) & 0xffff;
 124         __u32 api = (vaddr >> 22) & 0x3f;
 125
 126         __u32 vsid;
 127         asm volatile (
 128                 "mfsrin %0, %1\n"
 129                 : "=r" (vsid)
 130                 : "r" (vaddr)
 131         );
 132
 133         __u32 sdr1;
 134         asm volatile (
 135                 "mfsdr1 %0\n"
 136                 : "=r" (sdr1)
 137         );
 138         phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
 139
 140         /* Primary hash (xor) */
 141         __u32 h = 0;
 142         __u32 hash = vsid ^ page;
 143         __u32 base = (hash & 0x3ff) << 3;
 144         __u32 i;
 145         bool found = false;
 146
 147         /* Find unused or colliding
 148            PTE in PTEG */
 149         for (i = 0; i < 8; i++) {
 150                 if ((!phte[base + i].v) || ((phte[base + i].vsid == vsid) && (phte[base + i].api == api))) {
 151                         found = true;
 152                         break;
 153                 }
 154         }
 155
 156         if (!found) {
 157                 /* Secondary hash (not) */
 158                 __u32 base2 = (~hash & 0x3ff) << 3;
 159
 160                 /* Find unused or colliding
 161                    PTE in PTEG */
 162                 for (i = 0; i < 8; i++) {
 163                         if ((!phte[base2 + i].v) || ((phte[base2 + i].vsid == vsid) && (phte[base2 + i].api == api))) {
 164                                 found = true;
 165                                 base = base2;
 166                                 h = 1;
 167                                 break;
 168                         }
 169                 }
 170
 171                 if (!found) {
 172                         // TODO: A/C precedence groups
 173                         i = page % 8;
 174                 }
 175         }
 176
 177         phte[base + i].v = 1;
 178         phte[base + i].vsid = vsid;
 179         phte[base + i].h = h;
 180         phte[base + i].api = api;
 181         phte[base + i].rpn = pfn;
 182         phte[base + i].r = 0;
 183         phte[base + i].c = 0;
 184         phte[base + i].pp = 2; // FIXME
 185 }
 186
 187
 188 static void pht_real_insert(const __address vaddr, const pfn_t pfn)
 189 {
 190         __u32 page = (vaddr >> 12) & 0xffff;
 191         __u32 api = (vaddr >> 22) & 0x3f;
 192
 193         __u32 vsid;
 194         asm volatile (
 195                 "mfsrin %0, %1\n"
 196                 : "=r" (vsid)
 197                 : "r" (vaddr)
 198         );
 199
 200         __u32 sdr1;
 201         asm volatile (
 202                 "mfsdr1 %0\n"
 203                 : "=r" (sdr1)
 204         );
 205         phte_t *phte_physical = (phte_t *) (sdr1 & 0xffff0000);
 206
 207         /* Primary hash (xor) */
 208         __u32 h = 0;
 209         __u32 hash = vsid ^ page;
 210         __u32 base = (hash & 0x3ff) << 3;
 211         __u32 i;
 212         bool found = false;
 213
 214         /* Find unused or colliding
 215            PTE in PTEG */
 216         for (i = 0; i < 8; i++) {
 217                 if ((!phte_physical[base + i].v) || ((phte_physical[base + i].vsid == vsid) && (phte_physical[base + i].api == api))) {
 218                         found = true;
 219                         break;
 220                 }
 221         }
 222
 223         if (!found) {
 224                 /* Secondary hash (not) */
 225                 __u32 base2 = (~hash & 0x3ff) << 3;
 226
 227                 /* Find unused or colliding
 228                    PTE in PTEG */
 229                 for (i = 0; i < 8; i++) {
 230                         if ((!phte_physical[base2 + i].v) || ((phte_physical[base2 + i].vsid == vsid) && (phte_physical[base2 + i].api == api))) {
 231                                 found = true;
 232                                 base = base2;
 233                                 h = 1;
 234                                 break;
 235                         }
 236                 }
 237
 238                 if (!found) {
 239                         // TODO: A/C precedence groups
 240                         i = page % 8;
 241                 }
 242         }
 243
 244         phte_physical[base + i].v = 1;
 245         phte_physical[base + i].vsid = vsid;
 246         phte_physical[base + i].h = h;
 247         phte_physical[base + i].api = api;
 248         phte_physical[base + i].rpn = pfn;
 249         phte_physical[base + i].r = 0;
 250         phte_physical[base + i].c = 0;
 251         phte_physical[base + i].pp = 2; // FIXME
 252 }
 253
 254
 255 /** Process Instruction/Data Storage Interrupt
 256  *
 257  * @param n Interrupt vector number.
 258  * @param istate Interrupted register context.
 259  *
 260  */
 261 void pht_refill(int n, istate_t *istate)
 262 {
 263         __address badvaddr;
 264         pte_t *pte;
 265         int pfrc;
 266         as_t *as;
 267         bool lock;
 268
 269         if (AS == NULL) {
 270                 as = AS_KERNEL;
 271                 lock = false;
 272         } else {
 273                 as = AS;
 274                 lock = true;
 275         }
 276
 277         if (n == VECTOR_DATA_STORAGE) {
 278                 asm volatile (
 279                         "mfdar %0\n"
 280                         : "=r" (badvaddr)
 281                 );
 282         } else
 283                 badvaddr = istate->pc;
 284
 285         page_table_lock(as, lock);
 286
 287         pte = find_mapping_and_check(as, lock, badvaddr, PF_ACCESS_READ /* FIXME */, istate, &pfrc);
 288         if (!pte) {
 289                 switch (pfrc) {
 290                         case AS_PF_FAULT:
 291                                 goto fail;
 292                                 break;
 293                         case AS_PF_DEFER:
 294                                 /*
 295                                  * The page fault came during copy_from_uspace()
 296                                  * or copy_to_uspace().
 297                                  */
 298                                 page_table_unlock(as, lock);
 299                                 return;
 300                         default:
 301                                 panic("Unexpected pfrc (%d)\n", pfrc);
 302                 }
 303         }
 304
 305         pte->a = 1; /* Record access to PTE */
 306         pht_insert(badvaddr, pte->pfn);
 307
 308         page_table_unlock(as, lock);
 309         return;
 310
 311 fail:
 312         page_table_unlock(as, lock);
 313         pht_refill_fail(badvaddr, istate);
 314 }
 315
 316
 317 /** Process Instruction/Data Storage Interrupt in Real Mode
 318  *
 319  * @param n Interrupt vector number.
 320  * @param istate Interrupted register context.
 321  *
 322  */
 323 bool pht_real_refill(int n, istate_t *istate)
 324 {
 325         __address badvaddr;
 326
 327         if (n == VECTOR_DATA_STORAGE) {
 328                 asm volatile (
 329                         "mfdar %0\n"
 330                         : "=r" (badvaddr)
 331                 );
 332         } else
 333                 badvaddr = istate->pc;
 334
 335         __u32 physmem;
 336         asm volatile (
 337                 "mfsprg3 %0\n"
 338                 : "=r" (physmem)
 339         );
 340
 341         if ((badvaddr >= PA2KA(0)) && (badvaddr < PA2KA(physmem))) {
 342                 pht_real_insert(badvaddr, KA2PA(badvaddr) >> 12);
 343                 return true;
 344         }
 345
 346         return false;
 347 }
 348
 349
 350 void tlb_arch_init(void)
 351 {
 352         tlb_invalidate_all();
 353 }
 354
 355
 356 void tlb_invalidate_all(void)
 357 {
 358         asm volatile (
 359                 "tlbia\n"
 360                 "tlbsync\n"
 361         );
 362 }
 363
 364
 365 void tlb_invalidate_asid(asid_t asid)
 366 {
 367         // TODO
 368         tlb_invalidate_all();
 369 }
 370
 371
 372 void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
 373 {
 374         // TODO
 375         tlb_invalidate_all();
 376 }
 377
 378
 379 #define PRINT_BAT(name, ureg, lreg) \
 380         asm volatile ( \
 381                 "mfspr %0," #ureg "\n" \
 382                 "mfspr %1," #lreg "\n" \
 383                 : "=r" (upper), "=r" (lower) \
 384         ); \
 385         mask = (upper & 0x1ffc) >> 2; \
 386         if (upper & 3) { \
 387                 __u32 tmp = mask; \
 388                 length = 128; \
 389                 while (tmp) { \
 390                         if ((tmp & 1) == 0) { \
 391                                 printf("ibat[0]: error in mask\n"); \
 392                                 break; \
 393                         } \
 394                         length <<= 1; \
 395                         tmp >>= 1; \
 396                 } \
 397         } else \
 398                 length = 0; \
 399         printf(name ": page=%.*p frame=%.*p length=%d KB (mask=%#x)%s%s\n", sizeof(upper) * 2, upper & 0xffff0000, sizeof(lower) * 2, lower & 0xffff0000, length, mask, ((upper >> 1) & 1) ? " supervisor" : "", (upper & 1) ? " user" : "");
 400
 401
 402 void tlb_print(void)
 403 {
 404         __u32 sr;
 405
 406         for (sr = 0; sr < 16; sr++) {
 407                 __u32 vsid;
 408                 asm volatile (
 409                         "mfsrin %0, %1\n"
 410                         : "=r" (vsid)
 411                         : "r" (sr << 28)
 412                 );
 413                 printf("vsid[%d]: VSID=%.*p (ASID=%d)%s%s\n", sr, sizeof(vsid) * 2, vsid & 0xffffff, (vsid & 0xffffff) >> 4, ((vsid >> 30) & 1) ? " supervisor" : "", ((vsid >> 29) & 1) ? " user" : "");
 414         }
 415
 416         __u32 upper;
 417         __u32 lower;
 418         __u32 mask;
 419         __u32 length;
 420
 421         PRINT_BAT("ibat[0]", 528, 529);
 422         PRINT_BAT("ibat[1]", 530, 531);
 423         PRINT_BAT("ibat[2]", 532, 533);
 424         PRINT_BAT("ibat[3]", 534, 535);
 425
 426         PRINT_BAT("dbat[0]", 536, 537);
 427         PRINT_BAT("dbat[1]", 538, 539);
 428         PRINT_BAT("dbat[2]", 540, 541);
 429         PRINT_BAT("dbat[3]", 542, 543);
 430 }
 431
 432 /** @}
 433  */