kernel/arch/sparc64/src/mm/tlb.c

   1 /*
   2  * Copyright (C) 2005 Jakub Jermar
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  *
   9  * - Redistributions of source code must retain the above copyright
  10  *   notice, this list of conditions and the following disclaimer.
  11  * - Redistributions in binary form must reproduce the above copyright
  12  *   notice, this list of conditions and the following disclaimer in the
  13  *   documentation and/or other materials provided with the distribution.
  14  * - The name of the author may not be used to endorse or promote products
  15  *   derived from this software without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 /** @addtogroup sparc64mm
  30  * @{
  31  */
  32 /** @file
  33  */
  34
  35 #include <arch/mm/tlb.h>
  36 #include <mm/tlb.h>
  37 #include <mm/as.h>
  38 #include <mm/asid.h>
  39 #include <arch/mm/frame.h>
  40 #include <arch/mm/page.h>
  41 #include <arch/mm/mmu.h>
  42 #include <arch/interrupt.h>
  43 #include <interrupt.h>
  44 #include <arch.h>
  45 #include <print.h>
  46 #include <arch/types.h>
  47 #include <typedefs.h>
  48 #include <config.h>
  49 #include <arch/trap/trap.h>
  50 #include <arch/trap/exception.h>
  51 #include <panic.h>
  52 #include <arch/asm.h>
  53
  54 #ifdef CONFIG_TSB
  55 #include <arch/mm/tsb.h>
  56 #endif
  57
  58 static void dtlb_pte_copy(pte_t *t, bool ro);
  59 static void itlb_pte_copy(pte_t *t);
  60 static void do_fast_instruction_access_mmu_miss_fault(istate_t *istate, const char *str);
  61 static void do_fast_data_access_mmu_miss_fault(istate_t *istate, tlb_tag_access_reg_t tag, const char *str);
  62 static void do_fast_data_access_protection_fault(istate_t *istate, tlb_tag_access_reg_t tag, const char *str);
  63
  64 char *context_encoding[] = {
  65         "Primary",
  66         "Secondary",
  67         "Nucleus",
  68         "Reserved"
  69 };
  70
  71 void tlb_arch_init(void)
  72 {
  73         /*
  74          * Invalidate all non-locked DTLB and ITLB entries.
  75          */
  76         tlb_invalidate_all();
  77
  78         /*
  79          * Clear both SFSRs.
  80          */
  81         dtlb_sfsr_write(0);
  82         itlb_sfsr_write(0);
  83 }
  84
  85 /** Insert privileged mapping into DMMU TLB.
  86  *
  87  * @param page Virtual page address.
  88  * @param frame Physical frame address.
  89  * @param pagesize Page size.
  90  * @param locked True for permanent mappings, false otherwise.
  91  * @param cacheable True if the mapping is cacheable, false otherwise.
  92  */
  93 void dtlb_insert_mapping(uintptr_t page, uintptr_t frame, int pagesize, bool locked, bool cacheable)
  94 {
  95         tlb_tag_access_reg_t tag;
  96         tlb_data_t data;
  97         page_address_t pg;
  98         frame_address_t fr;
  99
 100         pg.address = page;
 101         fr.address = frame;
 102
 103         tag.value = ASID_KERNEL;
 104         tag.vpn = pg.vpn;
 105
 106         dtlb_tag_access_write(tag.value);
 107
 108         data.value = 0;
 109         data.v = true;
 110         data.size = pagesize;
 111         data.pfn = fr.pfn;
 112         data.l = locked;
 113         data.cp = cacheable;
 114 #ifdef CONFIG_VIRT_IDX_DCACHE
 115         data.cv = cacheable;
 116 #endif /* CONFIG_VIRT_IDX_DCACHE */
 117         data.p = true;
 118         data.w = true;
 119         data.g = false;
 120
 121         dtlb_data_in_write(data.value);
 122 }
 123
 124 /** Copy PTE to TLB.
 125  *
 126  * @param t Page Table Entry to be copied.
 127  * @param ro If true, the entry will be created read-only, regardless of its w field.
 128  */
 129 void dtlb_pte_copy(pte_t *t, bool ro)
 130 {
 131         tlb_tag_access_reg_t tag;
 132         tlb_data_t data;
 133         page_address_t pg;
 134         frame_address_t fr;
 135
 136         pg.address = t->page;
 137         fr.address = t->frame;
 138
 139         tag.value = 0;
 140         tag.context = t->as->asid;
 141         tag.vpn = pg.vpn;
 142
 143         dtlb_tag_access_write(tag.value);
 144
 145         data.value = 0;
 146         data.v = true;
 147         data.size = PAGESIZE_8K;
 148         data.pfn = fr.pfn;
 149         data.l = false;
 150         data.cp = t->c;
 151 #ifdef CONFIG_VIRT_IDX_DCACHE
 152         data.cv = t->c;
 153 #endif /* CONFIG_VIRT_IDX_DCACHE */
 154         data.p = t->k;          /* p like privileged */
 155         data.w = ro ? false : t->w;
 156         data.g = t->g;
 157
 158         dtlb_data_in_write(data.value);
 159 }
 160
 161 /** Copy PTE to ITLB.
 162  *
 163  * @param t Page Table Entry to be copied.
 164  */
 165 void itlb_pte_copy(pte_t *t)
 166 {
 167         tlb_tag_access_reg_t tag;
 168         tlb_data_t data;
 169         page_address_t pg;
 170         frame_address_t fr;
 171
 172         pg.address = t->page;
 173         fr.address = t->frame;
 174
 175         tag.value = 0;
 176         tag.context = t->as->asid;
 177         tag.vpn = pg.vpn;
 178
 179         itlb_tag_access_write(tag.value);
 180
 181         data.value = 0;
 182         data.v = true;
 183         data.size = PAGESIZE_8K;
 184         data.pfn = fr.pfn;
 185         data.l = false;
 186         data.cp = t->c;
 187         data.p = t->k;          /* p like privileged */
 188         data.w = false;
 189         data.g = t->g;
 190
 191         itlb_data_in_write(data.value);
 192 }
 193
 194 /** ITLB miss handler. */
 195 void fast_instruction_access_mmu_miss(int n, istate_t *istate)
 196 {
 197         uintptr_t va = ALIGN_DOWN(istate->tpc, PAGE_SIZE);
 198         pte_t *t;
 199
 200         page_table_lock(AS, true);
 201         t = page_mapping_find(AS, va);
 202         if (t && PTE_EXECUTABLE(t)) {
 203                 /*
 204                  * The mapping was found in the software page hash table.
 205                  * Insert it into ITLB.
 206                  */
 207                 t->a = true;
 208                 itlb_pte_copy(t);
 209 #ifdef CONFIG_TSB
 210                 itsb_pte_copy(t);
 211 #endif
 212                 page_table_unlock(AS, true);
 213         } else {
 214                 /*
 215                  * Forward the page fault to the address space page fault handler.
 216                  */
 217                 page_table_unlock(AS, true);
 218                 if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
 219                         do_fast_instruction_access_mmu_miss_fault(istate, __FUNCTION__);
 220                 }
 221         }
 222 }
 223
 224 /** DTLB miss handler.
 225  *
 226  * Note that some faults (e.g. kernel faults) were already resolved
 227  * by the low-level, assembly language part of the fast_data_access_mmu_miss
 228  * handler.
 229  */
 230 void fast_data_access_mmu_miss(int n, istate_t *istate)
 231 {
 232         tlb_tag_access_reg_t tag;
 233         uintptr_t va;
 234         pte_t *t;
 235
 236         tag.value = dtlb_tag_access_read();
 237         va = tag.vpn << PAGE_WIDTH;
 238
 239         if (tag.context == ASID_KERNEL) {
 240                 if (!tag.vpn) {
 241                         /* NULL access in kernel */
 242                         do_fast_data_access_mmu_miss_fault(istate, tag, __FUNCTION__);
 243                 }
 244                 do_fast_data_access_mmu_miss_fault(istate, tag, "Unexpected kernel page fault.");
 245         }
 246
 247         page_table_lock(AS, true);
 248         t = page_mapping_find(AS, va);
 249         if (t) {
 250                 /*
 251                  * The mapping was found in the software page hash table.
 252                  * Insert it into DTLB.
 253                  */
 254                 t->a = true;
 255                 dtlb_pte_copy(t, true);
 256 #ifdef CONFIG_TSB
 257                 dtsb_pte_copy(t, true);
 258 #endif
 259                 page_table_unlock(AS, true);
 260         } else {
 261                 /*
 262                  * Forward the page fault to the address space page fault handler.
 263                  */
 264                 page_table_unlock(AS, true);
 265                 if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
 266                         do_fast_data_access_mmu_miss_fault(istate, tag, __FUNCTION__);
 267                 }
 268         }
 269 }
 270
 271 /** DTLB protection fault handler. */
 272 void fast_data_access_protection(int n, istate_t *istate)
 273 {
 274         tlb_tag_access_reg_t tag;
 275         uintptr_t va;
 276         pte_t *t;
 277
 278         tag.value = dtlb_tag_access_read();
 279         va = tag.vpn << PAGE_WIDTH;
 280
 281         page_table_lock(AS, true);
 282         t = page_mapping_find(AS, va);
 283         if (t && PTE_WRITABLE(t)) {
 284                 /*
 285                  * The mapping was found in the software page hash table and is writable.
 286                  * Demap the old mapping and insert an updated mapping into DTLB.
 287                  */
 288                 t->a = true;
 289                 t->d = true;
 290                 dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_SECONDARY, va);
 291                 dtlb_pte_copy(t, false);
 292 #ifdef CONFIG_TSB
 293                 dtsb_pte_copy(t, false);
 294 #endif
 295                 page_table_unlock(AS, true);
 296         } else {
 297                 /*
 298                  * Forward the page fault to the address space page fault handler.
 299                  */
 300                 page_table_unlock(AS, true);
 301                 if (as_page_fault(va, PF_ACCESS_WRITE, istate) == AS_PF_FAULT) {
 302                         do_fast_data_access_protection_fault(istate, tag, __FUNCTION__);
 303                 }
 304         }
 305 }
 306
 307 /** Print contents of both TLBs. */
 308 void tlb_print(void)
 309 {
 310         int i;
 311         tlb_data_t d;
 312         tlb_tag_read_reg_t t;
 313
 314         printf("I-TLB contents:\n");
 315         for (i = 0; i < ITLB_ENTRY_COUNT; i++) {
 316                 d.value = itlb_data_access_read(i);
 317                 t.value = itlb_tag_read_read(i);
 318
 319                 printf("%d: vpn=%#llx, context=%d, v=%d, size=%d, nfo=%d, ie=%d, soft2=%#x, diag=%#x, pfn=%#x, soft=%#x, l=%d, cp=%d, cv=%d, e=%d, p=%d, w=%d, g=%d\n",
 320                         i, t.vpn, t.context, d.v, d.size, d.nfo, d.ie, d.soft2, d.diag, d.pfn, d.soft, d.l, d.cp, d.cv, d.e, d.p, d.w, d.g);
 321         }
 322
 323         printf("D-TLB contents:\n");
 324         for (i = 0; i < DTLB_ENTRY_COUNT; i++) {
 325                 d.value = dtlb_data_access_read(i);
 326                 t.value = dtlb_tag_read_read(i);
 327
 328                 printf("%d: vpn=%#llx, context=%d, v=%d, size=%d, nfo=%d, ie=%d, soft2=%#x, diag=%#x, pfn=%#x, soft=%#x, l=%d, cp=%d, cv=%d, e=%d, p=%d, w=%d, g=%d\n",
 329                         i, t.vpn, t.context, d.v, d.size, d.nfo, d.ie, d.soft2, d.diag, d.pfn, d.soft, d.l, d.cp, d.cv, d.e, d.p, d.w, d.g);
 330         }
 331
 332 }
 333
 334 void do_fast_instruction_access_mmu_miss_fault(istate_t *istate, const char *str)
 335 {
 336         fault_if_from_uspace(istate, "%s\n", str);
 337         dump_istate(istate);
 338         panic("%s\n", str);
 339 }
 340
 341 void do_fast_data_access_mmu_miss_fault(istate_t *istate, tlb_tag_access_reg_t tag, const char *str)
 342 {
 343         uintptr_t va;
 344
 345         va = tag.vpn << PAGE_WIDTH;
 346
 347         fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d)\n", str, va, tag.context);
 348         dump_istate(istate);
 349         printf("Faulting page: %p, ASID=%d\n", va, tag.context);
 350         panic("%s\n", str);
 351 }
 352
 353 void do_fast_data_access_protection_fault(istate_t *istate, tlb_tag_access_reg_t tag, const char *str)
 354 {
 355         uintptr_t va;
 356
 357         va = tag.vpn << PAGE_WIDTH;
 358
 359         fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d)\n", str, va, tag.context);
 360         printf("Faulting page: %p, ASID=%d\n", va, tag.context);
 361         dump_istate(istate);
 362         panic("%s\n", str);
 363 }
 364
 365 void dump_sfsr_and_sfar(void)
 366 {
 367         tlb_sfsr_reg_t sfsr;
 368         uintptr_t sfar;
 369
 370         sfsr.value = dtlb_sfsr_read();
 371         sfar = dtlb_sfar_read();
 372
 373         printf("DTLB SFSR: asi=%#x, ft=%#x, e=%d, ct=%d, pr=%d, w=%d, ow=%d, fv=%d\n",
 374                 sfsr.asi, sfsr.ft, sfsr.e, sfsr.ct, sfsr.pr, sfsr.w, sfsr.ow, sfsr.fv);
 375         printf("DTLB SFAR: address=%p\n", sfar);
 376
 377         dtlb_sfsr_write(0);
 378 }
 379
 380 /** Invalidate all unlocked ITLB and DTLB entries. */
 381 void tlb_invalidate_all(void)
 382 {
 383         int i;
 384         tlb_data_t d;
 385         tlb_tag_read_reg_t t;
 386
 387         for (i = 0; i < ITLB_ENTRY_COUNT; i++) {
 388                 d.value = itlb_data_access_read(i);
 389                 if (!d.l) {
 390                         t.value = itlb_tag_read_read(i);
 391                         d.v = false;
 392                         itlb_tag_access_write(t.value);
 393                         itlb_data_access_write(i, d.value);
 394                 }
 395         }
 396
 397         for (i = 0; i < DTLB_ENTRY_COUNT; i++) {
 398                 d.value = dtlb_data_access_read(i);
 399                 if (!d.l) {
 400                         t.value = dtlb_tag_read_read(i);
 401                         d.v = false;
 402                         dtlb_tag_access_write(t.value);
 403                         dtlb_data_access_write(i, d.value);
 404                 }
 405         }
 406
 407 }
 408
 409 /** Invalidate all ITLB and DTLB entries that belong to specified ASID (Context).
 410  *
 411  * @param asid Address Space ID.
 412  */
 413 void tlb_invalidate_asid(asid_t asid)
 414 {
 415         tlb_context_reg_t pc_save, ctx;
 416
 417         /* switch to nucleus because we are mapped by the primary context */
 418         nucleus_enter();
 419
 420         ctx.v = pc_save.v = mmu_primary_context_read();
 421         ctx.context = asid;
 422         mmu_primary_context_write(ctx.v);
 423
 424         itlb_demap(TLB_DEMAP_CONTEXT, TLB_DEMAP_PRIMARY, 0);
 425         dtlb_demap(TLB_DEMAP_CONTEXT, TLB_DEMAP_PRIMARY, 0);
 426
 427         mmu_primary_context_write(pc_save.v);
 428
 429         nucleus_leave();
 430 }
 431
 432 /** Invalidate all ITLB and DTLB entries for specified page range in specified address space.
 433  *
 434  * @param asid Address Space ID.
 435  * @param page First page which to sweep out from ITLB and DTLB.
 436  * @param cnt Number of ITLB and DTLB entries to invalidate.
 437  */
 438 void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
 439 {
 440         int i;
 441         tlb_context_reg_t pc_save, ctx;
 442
 443         /* switch to nucleus because we are mapped by the primary context */
 444         nucleus_enter();
 445
 446         ctx.v = pc_save.v = mmu_primary_context_read();
 447         ctx.context = asid;
 448         mmu_primary_context_write(ctx.v);
 449
 450         for (i = 0; i < cnt; i++) {
 451                 itlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY, page + i * PAGE_SIZE);
 452                 dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY, page + i * PAGE_SIZE);
 453         }
 454
 455         mmu_primary_context_write(pc_save.v);
 456
 457         nucleus_leave();
 458 }
 459
 460 /** @}
 461  */