arch/x86/kernel/platform/i86pc/vm/kboot_mmu.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright 2018 Joyent, Inc.
  27  */
  28
  29 #include <sys/types.h>
  30 #include <sys/systm.h>
  31 #include <sys/archsystm.h>
  32 #include <sys/debug.h>
  33 #include <sys/bootconf.h>
  34 #include <sys/bootsvcs.h>
  35 #include <sys/bootinfo.h>
  36 #include <sys/mman.h>
  37 #include <sys/cmn_err.h>
  38 #include <sys/param.h>
  39 #include <sys/machparam.h>
  40 #include <sys/machsystm.h>
  41 #include <sys/promif.h>
  42 #include <sys/kobj.h>
  43 #include <vm/kboot_mmu.h>
  44 #include <vm/hat_pte.h>
  45 #include <vm/hat_i86.h>
  46 #include <vm/seg_kmem.h>
  47
  48 #if 0
  49 /*
  50  * Joe's debug printing
  51  */
  52 #define DBG(x)    \
  53         bop_printf(NULL, "kboot_mmu.c: %s is %" PRIx64 "\n", #x, (uint64_t)(x));
  54 #else
  55 #define DBG(x)  /* naught */
  56 #endif
  57
  58 /*
  59  * Page table and memory stuff.
  60  */
  61 static caddr_t window;
  62 static caddr_t pte_to_window;
  63
  64 /*
  65  * this are needed by mmu_init()
  66  */
  67 int kbm_nx_support = 0;         /* NX bit in PTEs is in use */
  68 int kbm_pae_support = 0;        /* PAE is 64 bit Page table entries */
  69 int kbm_pge_support = 0;        /* PGE is Page table global bit enabled */
  70 int kbm_largepage_support = 0;
  71 uint_t kbm_nucleus_size = 0;
  72
  73 #define BOOT_SHIFT(l)   (shift_amt[l])
  74 #define BOOT_SZ(l)      ((size_t)1 << BOOT_SHIFT(l))
  75 #define BOOT_OFFSET(l)  (BOOT_SZ(l) - 1)
  76 #define BOOT_MASK(l)    (~BOOT_OFFSET(l))
  77
  78 /*
  79  * Initialize memory management parameters for boot time page table management
  80  */
  81 void
  82 kbm_init(struct xboot_info *bi)
  83 {
  84         /*
  85          * configure mmu information
  86          */
  87         kbm_nucleus_size = (uintptr_t)bi->bi_kseg_size;
  88         kbm_largepage_support = bi->bi_use_largepage;
  89         kbm_nx_support = bi->bi_use_nx;
  90         kbm_pae_support = bi->bi_use_pae;
  91         kbm_pge_support = bi->bi_use_pge;
  92         window = bi->bi_pt_window;
  93         DBG(window);
  94         pte_to_window = bi->bi_pte_to_pt_window;
  95         DBG(pte_to_window);
  96         if (kbm_pae_support) {
  97                 shift_amt = shift_amt_pae;
  98                 ptes_per_table = 512;
  99                 pte_size = 8;
 100                 lpagesize = TWO_MEG;
 101 #ifdef __amd64
 102                 top_level = 3;
 103 #else
 104                 top_level = 2;
 105 #endif
 106         } else {
 107                 shift_amt = shift_amt_nopae;
 108                 ptes_per_table = 1024;
 109                 pte_size = 4;
 110                 lpagesize = FOUR_MEG;
 111                 top_level = 1;
 112         }
 113
 114         top_page_table = bi->bi_top_page_table;
 115         DBG(top_page_table);
 116 }
 117
 118 /*
 119  * Change the addressible page table window to point at a given page
 120  */
 121 /*ARGSUSED*/
 122 void *
 123 kbm_remap_window(paddr_t physaddr, int writeable)
 124 {
 125         x86pte_t pt_bits = PT_NOCONSIST | PT_VALID | PT_WRITABLE;
 126
 127         DBG(physaddr);
 128
 129         if (kbm_pae_support)
 130                 *((x86pte_t *)pte_to_window) = physaddr | pt_bits;
 131         else
 132                 *((x86pte32_t *)pte_to_window) = physaddr | pt_bits;
 133         mmu_invlpg(window);
 134         DBG(window);
 135         return (window);
 136 }
 137
 138 /*
 139  * Add a mapping for the physical page at the given virtual address.
 140  */
 141 void
 142 kbm_map(uintptr_t va, paddr_t pa, uint_t level, uint_t is_kernel)
 143 {
 144         x86pte_t *ptep;
 145         paddr_t pte_physaddr;
 146         x86pte_t pteval;
 147
 148         if (khat_running)
 149                 panic("kbm_map() called too late");
 150
 151         pteval = pa_to_ma(pa) | PT_NOCONSIST | PT_VALID | PT_WRITABLE;
 152         if (level >= 1)
 153                 pteval |= PT_PAGESIZE;
 154         if (kbm_pge_support && is_kernel)
 155                 pteval |= PT_GLOBAL;
 156
 157
 158         /*
 159          * Find the pte that will map this address. This creates any
 160          * missing intermediate level page tables.
 161          */
 162         ptep = find_pte(va, &pte_physaddr, level, 0);
 163         if (ptep == NULL)
 164                 bop_panic("kbm_map: find_pte returned NULL");
 165
 166         if (kbm_pae_support)
 167                 *ptep = pteval;
 168         else
 169                 *((x86pte32_t *)ptep) = pteval;
 170         mmu_invlpg((caddr_t)va);
 171 }
 172
 173
 174
 175 /*
 176  * Probe the boot time page tables to find the first mapping
 177  * including va (or higher) and return non-zero if one is found.
 178  * va is updated to the starting address and len to the pagesize.
 179  * pp will be set to point to the 1st page_t of the mapped page(s).
 180  *
 181  * Note that if va is in the middle of a large page, the returned va
 182  * will be less than what was asked for.
 183  */
 184 int
 185 kbm_probe(uintptr_t *va, size_t *len, pfn_t *pfn, uint_t *prot)
 186 {
 187         uintptr_t       probe_va;
 188         x86pte_t        *ptep;
 189         paddr_t         pte_physaddr;
 190         x86pte_t        pte_val;
 191         level_t         l;
 192
 193         if (khat_running)
 194                 panic("kbm_probe() called too late");
 195         *len = 0;
 196         *pfn = PFN_INVALID;
 197         *prot = 0;
 198         probe_va = *va;
 199 restart_new_va:
 200         l = top_level;
 201         for (;;) {
 202                 if (IN_VA_HOLE(probe_va))
 203                         probe_va = mmu.hole_end;
 204
 205                 if (IN_HYPERVISOR_VA(probe_va))
 206 #if defined(__amd64) && defined(__xpv)
 207                         probe_va = HYPERVISOR_VIRT_END;
 208 #else
 209                         return (0);
 210 #endif
 211
 212                 /*
 213                  * If we don't have a valid PTP/PTE at this level
 214                  * then we can bump VA by this level's pagesize and try again.
 215                  * When the probe_va wraps around, we are done.
 216                  */
 217                 ptep = find_pte(probe_va, &pte_physaddr, l, 1);
 218                 if (ptep == NULL)
 219                         bop_panic("kbm_probe: find_pte returned NULL");
 220                 if (kbm_pae_support)
 221                         pte_val = *ptep;
 222                 else
 223                         pte_val = *((x86pte32_t *)ptep);
 224                 if (!PTE_ISVALID(pte_val)) {
 225                         probe_va = (probe_va & BOOT_MASK(l)) + BOOT_SZ(l);
 226                         if (probe_va <= *va)
 227                                 return (0);
 228                         goto restart_new_va;
 229                 }
 230
 231                 /*
 232                  * If this entry is a pointer to a lower level page table
 233                  * go down to it.
 234                  */
 235                 if (!PTE_ISPAGE(pte_val, l)) {
 236                         ASSERT(l > 0);
 237                         --l;
 238                         continue;
 239                 }
 240
 241                 /*
 242                  * We found a boot level page table entry
 243                  */
 244                 *len = BOOT_SZ(l);
 245                 *va = probe_va & ~(*len - 1);
 246                 *pfn = PTE2PFN(pte_val, l);
 247
 248
 249                 *prot = PROT_READ | PROT_EXEC;
 250                 if (PTE_GET(pte_val, PT_WRITABLE))
 251                         *prot |= PROT_WRITE;
 252
 253                 /*
 254                  * pt_nx is cleared if processor doesn't support NX bit
 255                  */
 256                 if (PTE_GET(pte_val, mmu.pt_nx))
 257                         *prot &= ~PROT_EXEC;
 258
 259                 return (1);
 260         }
 261 }
 262
 263
 264 /*
 265  * Destroy a boot loader page table 4K mapping.
 266  */
 267 void
 268 kbm_unmap(uintptr_t va)
 269 {
 270         if (khat_running)
 271                 panic("kbm_unmap() called too late");
 272         else {
 273                 x86pte_t *ptep;
 274                 level_t level = 0;
 275                 uint_t  probe_only = 1;
 276
 277                 ptep = find_pte(va, NULL, level, probe_only);
 278                 if (ptep == NULL)
 279                         return;
 280
 281                 if (kbm_pae_support)
 282                         *ptep = 0;
 283                 else
 284                         *((x86pte32_t *)ptep) = 0;
 285                 mmu_invlpg((caddr_t)va);
 286         }
 287 }
 288
 289
 290 /*
 291  * Change a boot loader page table 4K mapping.
 292  * Returns the pfn of the old mapping.
 293  */
 294 pfn_t
 295 kbm_remap(uintptr_t va, pfn_t pfn)
 296 {
 297         x86pte_t *ptep;
 298         level_t level = 0;
 299         uint_t  probe_only = 1;
 300         x86pte_t pte_val = pa_to_ma(pfn_to_pa(pfn)) | PT_WRITABLE |
 301             PT_NOCONSIST | PT_VALID;
 302         x86pte_t old_pte;
 303
 304         if (khat_running)
 305                 panic("kbm_remap() called too late");
 306         ptep = find_pte(va, NULL, level, probe_only);
 307         if (ptep == NULL)
 308                 bop_panic("kbm_remap: find_pte returned NULL");
 309
 310         if (kbm_pae_support)
 311                 old_pte = *ptep;
 312         else
 313                 old_pte = *((x86pte32_t *)ptep);
 314
 315         if (kbm_pae_support)
 316                 *((x86pte_t *)ptep) = pte_val;
 317         else
 318                 *((x86pte32_t *)ptep) = pte_val;
 319         mmu_invlpg((caddr_t)va);
 320
 321         if (!(old_pte & PT_VALID) || ma_to_pa(old_pte) == -1)
 322                 return (PFN_INVALID);
 323         return (mmu_btop(ma_to_pa(old_pte)));
 324 }
 325
 326
 327 /*
 328  * Change a boot loader page table 4K mapping to read only.
 329  */
 330 void
 331 kbm_read_only(uintptr_t va, paddr_t pa)
 332 {
 333         x86pte_t pte_val = pa_to_ma(pa) |
 334             PT_NOCONSIST | PT_REF | PT_MOD | PT_VALID;
 335
 336         x86pte_t *ptep;
 337         level_t level = 0;
 338
 339         ptep = find_pte(va, NULL, level, 0);
 340         if (ptep == NULL)
 341                 bop_panic("kbm_read_only: find_pte returned NULL");
 342
 343         if (kbm_pae_support)
 344                 *ptep = pte_val;
 345         else
 346                 *((x86pte32_t *)ptep) = pte_val;
 347         mmu_invlpg((caddr_t)va);
 348 }
 349
 350 /*
 351  * interfaces for kernel debugger to access physical memory
 352  */
 353 static x86pte_t save_pte;
 354
 355 void *
 356 kbm_push(paddr_t pa)
 357 {
 358         static int first_time = 1;
 359
 360         if (first_time) {
 361                 first_time = 0;
 362                 return (window);
 363         }
 364
 365         if (kbm_pae_support)
 366                 save_pte = *((x86pte_t *)pte_to_window);
 367         else
 368                 save_pte = *((x86pte32_t *)pte_to_window);
 369         return (kbm_remap_window(pa, 0));
 370 }
 371
 372 void
 373 kbm_pop(void)
 374 {
 375         if (kbm_pae_support)
 376                 *((x86pte_t *)pte_to_window) = save_pte;
 377         else
 378                 *((x86pte32_t *)pte_to_window) = save_pte;
 379         mmu_invlpg(window);
 380 }
 381
 382 x86pte_t
 383 get_pteval(paddr_t table, uint_t index)
 384 {
 385         void *table_ptr = kbm_remap_window(table, 0);
 386
 387         if (kbm_pae_support)
 388                 return (((x86pte_t *)table_ptr)[index]);
 389         return (((x86pte32_t *)table_ptr)[index]);
 390 }
 391
 392 void
 393 set_pteval(paddr_t table, uint_t index, uint_t level, x86pte_t pteval)
 394 {
 395         void *table_ptr = kbm_remap_window(table, 0);
 396         if (kbm_pae_support)
 397                 ((x86pte_t *)table_ptr)[index] = pteval;
 398         else
 399                 ((x86pte32_t *)table_ptr)[index] = pteval;
 400         if (level == top_level && level == 2)
 401                 reload_cr3();
 402 }
 403
 404 paddr_t
 405 make_ptable(x86pte_t *pteval, uint_t level)
 406 {
 407         paddr_t new_table;
 408         void *table_ptr;
 409
 410         new_table = do_bop_phys_alloc(MMU_PAGESIZE, MMU_PAGESIZE);
 411         table_ptr = kbm_remap_window(new_table, 1);
 412         bzero(table_ptr, MMU_PAGESIZE);
 413
 414         if (level == top_level && level == 2)
 415                 *pteval = pa_to_ma(new_table) | PT_VALID;
 416         else
 417                 *pteval = pa_to_ma(new_table) |
 418                     PT_VALID | PT_REF | PT_USER | PT_WRITABLE;
 419
 420         return (new_table);
 421 }
 422
 423 x86pte_t *
 424 map_pte(paddr_t table, uint_t index)
 425 {
 426         void *table_ptr = kbm_remap_window(table, 0);
 427         return ((x86pte_t *)((caddr_t)table_ptr + index * pte_size));
 428 }