include/asm-x86_64/pgtable.h

   1 #ifndef _X86_64_PGTABLE_H
   2 #define _X86_64_PGTABLE_H
   3
   4 /*
   5  * This file contains the functions and defines necessary to modify and use
   6  * the x86-64 page table tree.
   7  *
   8  * x86-64 has a 4 level table setup. Generic linux MM only supports
   9  * three levels. The fourth level is currently a single static page that
  10  * is shared by everybody and just contains a pointer to the current
  11  * three level page setup on the beginning and some kernel mappings at
  12  * the end. For more details see Documentation/x86_64/mm.txt
  13  */
  14 #include <asm/processor.h>
  15 #include <asm/fixmap.h>
  16 #include <asm/bitops.h>
  17 #include <linux/threads.h>
  18 #include <asm/pda.h>
  19
  20 extern pgd_t level3_kernel_pgt[512];
  21 extern pgd_t level3_physmem_pgt[512];
  22 extern pgd_t level3_ident_pgt[512];
  23 extern pmd_t level2_kernel_pgt[512];
  24 extern pml4_t init_level4_pgt[];
  25 extern pgd_t boot_vmalloc_pgt[];
  26 extern unsigned long __supported_pte_mask;
  27
  28 #define swapper_pg_dir NULL
  29
  30 extern void paging_init(void);
  31 extern void clear_kernel_mapping(unsigned long addr, unsigned long size);
  32
  33 extern unsigned long pgkern_mask;
  34
  35 /*
  36  * ZERO_PAGE is a global shared page that is always zero: used
  37  * for zero-mapped memory areas etc..
  38  */
  39 extern unsigned long empty_zero_page[1024];
  40 #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
  41
  42 #define PML4_SHIFT      39
  43 #define PTRS_PER_PML4   512
  44
  45 /*
  46  * PGDIR_SHIFT determines what a top-level page table entry can map
  47  */
  48 #define PGDIR_SHIFT     30
  49 #define PTRS_PER_PGD    512
  50
  51 /*
  52  * PMD_SHIFT determines the size of the area a middle-level
  53  * page table can map
  54  */
  55 #define PMD_SHIFT       21
  56 #define PTRS_PER_PMD    512
  57
  58 /*
  59  * entries per page directory level
  60  */
  61 #define PTRS_PER_PTE    512
  62
  63 #define pte_ERROR(e) \
  64         printk("%s:%d: bad pte %p(%016lx).\n", __FILE__, __LINE__, &(e), pte_val(e))
  65 #define pmd_ERROR(e) \
  66         printk("%s:%d: bad pmd %p(%016lx).\n", __FILE__, __LINE__, &(e), pmd_val(e))
  67 #define pgd_ERROR(e) \
  68         printk("%s:%d: bad pgd %p(%016lx).\n", __FILE__, __LINE__, &(e), pgd_val(e))
  69
  70
  71 #define pml4_none(x)    (!pml4_val(x))
  72 #define pgd_none(x)     (!pgd_val(x))
  73
  74 extern inline int pgd_present(pgd_t pgd)        { return !pgd_none(pgd); }
  75
  76 static inline void set_pte(pte_t *dst, pte_t val)
  77 {
  78         pte_val(*dst) = pte_val(val);
  79 }
  80
  81 static inline void set_pmd(pmd_t *dst, pmd_t val)
  82 {
  83         pmd_val(*dst) = pmd_val(val);
  84 }
  85
  86 static inline void set_pgd(pgd_t *dst, pgd_t val)
  87 {
  88         pgd_val(*dst) = pgd_val(val);
  89 }
  90
  91 extern inline void pgd_clear (pgd_t * pgd)
  92 {
  93         set_pgd(pgd, __pgd(0));
  94 }
  95
  96 static inline void set_pml4(pml4_t *dst, pml4_t val)
  97 {
  98         pml4_val(*dst) = pml4_val(val);
  99 }
 100
 101 #define pgd_page(pgd) \
 102 ((unsigned long) __va(pgd_val(pgd) & PHYSICAL_PAGE_MASK))
 103
 104 #define ptep_get_and_clear(xp)  __pte(xchg(&(xp)->pte, 0))
 105 #define pte_same(a, b)          ((a).pte == (b).pte)
 106
 107 #define PML4_SIZE       (1UL << PML4_SHIFT)
 108 #define PML4_MASK       (~(PML4_SIZE-1))
 109 #define PMD_SIZE        (1UL << PMD_SHIFT)
 110 #define PMD_MASK        (~(PMD_SIZE-1))
 111 #define PGDIR_SIZE      (1UL << PGDIR_SHIFT)
 112 #define PGDIR_MASK      (~(PGDIR_SIZE-1))
 113
 114 #define USER_PTRS_PER_PGD       (TASK_SIZE/PGDIR_SIZE)
 115 #define FIRST_USER_PGD_NR       0
 116
 117 #define USER_PGD_PTRS (PAGE_OFFSET >> PGDIR_SHIFT)
 118 #define KERNEL_PGD_PTRS (PTRS_PER_PGD-USER_PGD_PTRS)
 119
 120 #define TWOLEVEL_PGDIR_SHIFT    20
 121 #define BOOT_USER_L4_PTRS 1
 122 #define BOOT_KERNEL_L4_PTRS 511 /* But we will do it in 4rd level */
 123
 124
 125
 126 #ifndef __ASSEMBLY__
 127 #define VMALLOC_START    0xffffff0000000000
 128 #define VMALLOC_END      0xffffff7fffffffff
 129 #define VMALLOC_VMADDR(x) ((unsigned long)(x))
 130 #define MODULES_VADDR    0xffffffffa0000000
 131 #define MODULES_END      0xffffffffafffffff
 132 #define MODULES_LEN   (MODULES_END - MODULES_VADDR)
 133
 134 #define IOMAP_START      0xfffffe8000000000
 135
 136 #define _PAGE_BIT_PRESENT       0
 137 #define _PAGE_BIT_RW            1
 138 #define _PAGE_BIT_USER          2
 139 #define _PAGE_BIT_PWT           3
 140 #define _PAGE_BIT_PCD           4
 141 #define _PAGE_BIT_ACCESSED      5
 142 #define _PAGE_BIT_DIRTY         6
 143 #define _PAGE_BIT_PSE           7       /* 4 MB (or 2MB) page */
 144 #define _PAGE_BIT_GLOBAL        8       /* Global TLB entry PPro+ */
 145 #define _PAGE_BIT_NX           63       /* No execute: only valid after cpuid check */
 146
 147 #define _PAGE_PRESENT   0x001
 148 #define _PAGE_RW        0x002
 149 #define _PAGE_USER      0x004
 150 #define _PAGE_PWT       0x008
 151 #define _PAGE_PCD       0x010
 152 #define _PAGE_ACCESSED  0x020
 153 #define _PAGE_DIRTY     0x040
 154 #define _PAGE_PSE       0x080   /* 2MB page */
 155 #define _PAGE_FILE      0x040   /* set:pagecache, unset:swap */
 156 #define _PAGE_GLOBAL    0x100   /* Global TLB entry */
 157
 158 #define _PAGE_PROTNONE  0x080   /* If not present */
 159 #define _PAGE_NX        (1UL<<_PAGE_BIT_NX)
 160
 161 #define _PAGE_TABLE     (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | _PAGE_ACCESSED | _PAGE_DIRTY)
 162 #define _KERNPG_TABLE   (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY)
 163
 164 #define _PAGE_CHG_MASK  (PTE_MASK | _PAGE_ACCESSED | _PAGE_DIRTY)
 165
 166 #define PAGE_NONE       __pgprot(_PAGE_PROTNONE | _PAGE_ACCESSED)
 167 #define PAGE_SHARED     __pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | _PAGE_ACCESSED | _PAGE_NX)
 168 #define PAGE_SHARED_EXEC __pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | _PAGE_ACCESSED)
 169 #define PAGE_COPY_NOEXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_ACCESSED | _PAGE_NX)
 170 #define PAGE_COPY PAGE_COPY_NOEXEC
 171 #define PAGE_COPY_EXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_ACCESSED)
 172 #define PAGE_READONLY   __pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_ACCESSED | _PAGE_NX)
 173 #define PAGE_READONLY_EXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_ACCESSED)
 174 #define __PAGE_KERNEL \
 175         (_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_NX)
 176 #define __PAGE_KERNEL_EXECUTABLE \
 177         (_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | _PAGE_ACCESSED)
 178 #define __PAGE_KERNEL_NOCACHE \
 179         (_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | _PAGE_PCD | _PAGE_ACCESSED | _PAGE_NX)
 180 #define __PAGE_KERNEL_RO \
 181         (_PAGE_PRESENT | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_NX)
 182 #define __PAGE_KERNEL_VSYSCALL \
 183         (_PAGE_PRESENT | _PAGE_USER | _PAGE_ACCESSED)
 184 #define __PAGE_KERNEL_LARGE \
 185         (__PAGE_KERNEL | _PAGE_PSE)
 186
 187 #define MAKE_GLOBAL(x) __pgprot((x) | _PAGE_GLOBAL)
 188
 189 #define PAGE_KERNEL MAKE_GLOBAL(__PAGE_KERNEL)
 190 #define PAGE_KERNEL_EXECUTABLE MAKE_GLOBAL(__PAGE_KERNEL_EXECUTABLE)
 191 #define PAGE_KERNEL_RO MAKE_GLOBAL(__PAGE_KERNEL_RO)
 192 #define PAGE_KERNEL_NOCACHE MAKE_GLOBAL(__PAGE_KERNEL_NOCACHE)
 193 #define PAGE_KERNEL_VSYSCALL MAKE_GLOBAL(__PAGE_KERNEL_VSYSCALL)
 194 #define PAGE_KERNEL_LARGE MAKE_GLOBAL(__PAGE_KERNEL_LARGE)
 195
 196 /*         xwr */
 197 #define __P000  PAGE_NONE
 198 #define __P001  PAGE_READONLY
 199 #define __P010  PAGE_COPY
 200 #define __P011  PAGE_COPY
 201 #define __P100  PAGE_READONLY_EXEC
 202 #define __P101  PAGE_READONLY_EXEC
 203 #define __P110  PAGE_COPY_EXEC
 204 #define __P111  PAGE_COPY_EXEC
 205
 206 #define __S000  PAGE_NONE
 207 #define __S001  PAGE_READONLY
 208 #define __S010  PAGE_SHARED
 209 #define __S011  PAGE_SHARED
 210 #define __S100  PAGE_READONLY_EXEC
 211 #define __S101  PAGE_READONLY_EXEC
 212 #define __S110  PAGE_SHARED_EXEC
 213 #define __S111  PAGE_SHARED_EXEC
 214
 215 static inline unsigned long pgd_bad(pgd_t pgd)
 216 {
 217        unsigned long val = pgd_val(pgd);
 218        val &= ~PTE_MASK;
 219        val &= ~(_PAGE_USER | _PAGE_DIRTY);
 220        return val & ~(_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED);
 221 }
 222
 223 #define pte_none(x)     (!pte_val(x))
 224 #define pte_present(x)  (pte_val(x) & (_PAGE_PRESENT | _PAGE_PROTNONE))
 225 #define pte_clear(xp)   do { set_pte(xp, __pte(0)); } while (0)
 226
 227 #define pages_to_mb(x) ((x) >> (20-PAGE_SHIFT)) /* FIXME: is this
 228                                                    right? */
 229 #define pte_page(x)     pfn_to_page(pte_pfn(x))
 230 #define pte_pfn(x)  ((pte_val(x) >> PAGE_SHIFT) & __PHYSICAL_MASK)
 231
 232 static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
 233 {
 234         pte_t pte;
 235         pte_val(pte) = (page_nr << PAGE_SHIFT);
 236         pte_val(pte) |= pgprot_val(pgprot);
 237         pte_val(pte) &= __supported_pte_mask;
 238         return pte;
 239 }
 240
 241 /*
 242  * The following only work if pte_present() is true.
 243  * Undefined behaviour if not..
 244  */
 245 static inline int pte_user(pte_t pte)           { return pte_val(pte) & _PAGE_USER; }
 246 extern inline int pte_read(pte_t pte)           { return pte_val(pte) & _PAGE_USER; }
 247 extern inline int pte_exec(pte_t pte)           { return pte_val(pte) & _PAGE_USER; }
 248 extern inline int pte_dirty(pte_t pte)          { return pte_val(pte) & _PAGE_DIRTY; }
 249 extern inline int pte_young(pte_t pte)          { return pte_val(pte) & _PAGE_ACCESSED; }
 250 extern inline int pte_write(pte_t pte)          { return pte_val(pte) & _PAGE_RW; }
 251 static inline int pte_file(pte_t pte)           { return pte_val(pte) & _PAGE_FILE; }
 252
 253 extern inline pte_t pte_rdprotect(pte_t pte)    { set_pte(&pte, __pte(pte_val(pte) & ~_PAGE_USER)); return pte; }
 254 extern inline pte_t pte_exprotect(pte_t pte)    { set_pte(&pte, __pte(pte_val(pte) & ~_PAGE_USER)); return pte; }
 255 extern inline pte_t pte_mkclean(pte_t pte)      { set_pte(&pte, __pte(pte_val(pte) & ~_PAGE_DIRTY)); return pte; }
 256 extern inline pte_t pte_mkold(pte_t pte)        { set_pte(&pte, __pte(pte_val(pte) & ~_PAGE_ACCESSED)); return pte; }
 257 extern inline pte_t pte_wrprotect(pte_t pte)    { set_pte(&pte, __pte(pte_val(pte) & ~_PAGE_RW)); return pte; }
 258 extern inline pte_t pte_mkread(pte_t pte)       { set_pte(&pte, __pte(pte_val(pte) | _PAGE_USER)); return pte; }
 259 extern inline pte_t pte_mkexec(pte_t pte)       { set_pte(&pte, __pte(pte_val(pte) | _PAGE_USER)); return pte; }
 260 extern inline pte_t pte_mkdirty(pte_t pte)      { set_pte(&pte, __pte(pte_val(pte) | _PAGE_DIRTY)); return pte; }
 261 extern inline pte_t pte_mkyoung(pte_t pte)      { set_pte(&pte, __pte(pte_val(pte) | _PAGE_ACCESSED)); return pte; }
 262 extern inline pte_t pte_mkwrite(pte_t pte)      { set_pte(&pte, __pte(pte_val(pte) | _PAGE_RW)); return pte; }
 263 static inline  int ptep_test_and_clear_dirty(pte_t *ptep)       { return test_and_clear_bit(_PAGE_BIT_DIRTY, ptep); }
 264 static inline  int ptep_test_and_clear_young(pte_t *ptep)       { return test_and_clear_bit(_PAGE_BIT_ACCESSED, ptep); }
 265 static inline void ptep_set_wrprotect(pte_t *ptep)              { clear_bit(_PAGE_BIT_RW, ptep); }
 266 static inline void ptep_mkdirty(pte_t *ptep)                    { set_bit(_PAGE_BIT_DIRTY, ptep); }
 267
 268 /*
 269  * Macro to mark a page protection value as "uncacheable".
 270  */
 271 #define pgprot_noncached(prot)  (__pgprot(pgprot_val(prot) | _PAGE_PCD | _PAGE_PWT))
 272
 273 #define __LARGE_PTE (_PAGE_PSE|_PAGE_PRESENT)
 274 static inline int pmd_large(pmd_t pte) {
 275         return (pmd_val(pte) & __LARGE_PTE) == __LARGE_PTE;
 276 }
 277
 278
 279 /*
 280  * Conversion functions: convert a page and protection to a page entry,
 281  * and a page entry and page directory to the page they refer to.
 282  */
 283
 284 #define page_pte(page) page_pte_prot(page, __pgprot(0))
 285
 286 /*
 287  * Level 4 access.
 288  * Never use these in the common code.
 289  */
 290 #define pml4_page(pml4) ((unsigned long) __va(pml4_val(pml4) & PTE_MASK))
 291 #define pml4_index(address) ((address >> PML4_SHIFT) & (PTRS_PER_PML4-1))
 292 #define pml4_offset_k(address) (init_level4_pgt + pml4_index(address))
 293 #define pml4_present(pml4) (pml4_val(pml4) & _PAGE_PRESENT)
 294 #define mk_kernel_pml4(address) ((pml4_t){ (address) | _KERNPG_TABLE })
 295 #define level3_offset_k(dir, address) ((pgd_t *) pml4_page(*(dir)) + pgd_index(address))
 296
 297 /* PGD - Level3 access */
 298 /* to find an entry in a page-table-directory. */
 299 #define pgd_index(address) ((address >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
 300 static inline pgd_t *__pgd_offset_k(pgd_t *pgd, unsigned long address)
 301 {
 302         return pgd + pgd_index(address);
 303 }
 304
 305 /* Find correct pgd via the hidden fourth level page level: */
 306
 307 /* This accesses the reference page table of the boot cpu.
 308    Other CPUs get synced lazily via the page fault handler. */
 309 static inline pgd_t *pgd_offset_k(unsigned long address)
 310 {
 311         unsigned long addr;
 312
 313         addr = pml4_val(init_level4_pgt[pml4_index(address)]);
 314         addr &= PHYSICAL_PAGE_MASK;
 315         return __pgd_offset_k((pgd_t *)__va(addr), address);
 316 }
 317
 318 /* Access the pgd of the page table as seen by the current CPU. */
 319 static inline pgd_t *current_pgd_offset_k(unsigned long address)
 320 {
 321         unsigned long addr;
 322
 323         addr = read_pda(level4_pgt)[pml4_index(address)];
 324         addr &= PHYSICAL_PAGE_MASK;
 325         return __pgd_offset_k((pgd_t *)__va(addr), address);
 326 }
 327
 328 #define pgd_offset(mm, address) ((mm)->pgd+pgd_index(address))
 329
 330 /* PMD  - Level 2 access */
 331 #define pmd_page_kernel(pmd) ((unsigned long) __va(pmd_val(pmd) & PTE_MASK))
 332 #define pmd_page(pmd)           (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT))
 333
 334 #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
 335 #define pmd_offset(dir, address) ((pmd_t *) pgd_page(*(dir)) + \
 336                         pmd_index(address))
 337 #define pmd_none(x)     (!pmd_val(x))
 338 #define pmd_present(x)  (pmd_val(x) & _PAGE_PRESENT)
 339 #define pmd_clear(xp)   do { set_pmd(xp, __pmd(0)); } while (0)
 340 #define pmd_bad(x)      ((pmd_val(x) & (~PTE_MASK & ~_PAGE_USER)) != _KERNPG_TABLE )
 341 #define pfn_pmd(nr,prot) (__pmd(((nr) << PAGE_SHIFT) | pgprot_val(prot)))
 342 #define pmd_pfn(x)  ((pmd_val(x) >> PAGE_SHIFT) & __PHYSICAL_MASK)
 343
 344 #define pte_to_pgoff(pte) ((pte_val(pte) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT)
 345 #define pgoff_to_pte(off) ((pte_t) { ((off) << PAGE_SHIFT) | _PAGE_FILE })
 346 #define PTE_FILE_MAX_BITS __PHYSICAL_MASK_SHIFT
 347
 348 /* PTE - Level 1 access. */
 349
 350 /* page, protection -> pte */
 351 #define mk_pte(page, pgprot)    pfn_pte(page_to_pfn(page), (pgprot))
 352 #define mk_pte_huge(entry) (pte_val(entry) |= _PAGE_PRESENT | _PAGE_PSE)
 353
 354 /* physical address -> PTE */
 355 static inline pte_t mk_pte_phys(unsigned long physpage, pgprot_t pgprot)
 356 {
 357         pte_t pte;
 358         pte_val(pte) = physpage | pgprot_val(pgprot);
 359         return pte;
 360 }
 361
 362 /* Change flags of a PTE */
 363 extern inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 364 {
 365         pte_val(pte) &= _PAGE_CHG_MASK;
 366         pte_val(pte) |= pgprot_val(newprot);
 367         pte_val(pte) &= __supported_pte_mask;
 368        return pte;
 369 }
 370
 371 #define pte_index(address) \
 372                 ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
 373 #define pte_offset_kernel(dir, address) ((pte_t *) pmd_page_kernel(*(dir)) + \
 374                         pte_index(address))
 375
 376 /* x86-64 always has all page tables mapped. */
 377 #define pte_offset_map(dir,address) pte_offset_kernel(dir,address)
 378 #define pte_offset_map_nested(dir,address) pte_offset_kernel(dir,address)
 379 #define pte_unmap(pte) /* NOP */
 380 #define pte_unmap_nested(pte) /* NOP */
 381
 382 #define update_mmu_cache(vma,address,pte) do { } while (0)
 383
 384 /* Encode and de-code a swap entry */
 385 #define __swp_type(x)                   (((x).val >> 1) & 0x3f)
 386 #define __swp_offset(x)                 ((x).val >> 8)
 387 #define __swp_entry(type, offset)       ((swp_entry_t) { ((type) << 1) | ((offset) << 8) })
 388 #define __pte_to_swp_entry(pte)         ((swp_entry_t) { pte_val(pte) })
 389 #define __swp_entry_to_pte(x)           ((pte_t) { (x).val })
 390
 391 typedef pte_t *pte_addr_t;
 392
 393 #endif /* !__ASSEMBLY__ */
 394
 395 extern int kern_addr_valid(unsigned long addr);
 396
 397 #define io_remap_page_range remap_page_range
 398
 399 #define HAVE_ARCH_UNMAPPED_AREA
 400
 401 #define pgtable_cache_init()   do { } while (0)
 402 #define check_pgt_cache()      do { } while (0)
 403
 404 #define PAGE_AGP    PAGE_KERNEL_NOCACHE
 405 #define HAVE_PAGE_AGP 1
 406
 407 /* fs/proc/kcore.c */
 408 #define kc_vaddr_to_offset(v) ((v) & __VIRTUAL_MASK)
 409 #define kc_offset_to_vaddr(o) \
 410    (((o) & (1UL << (__VIRTUAL_MASK_SHIFT-1))) ? ((o) | (~__VIRTUAL_MASK)) : (o))
 411
 412 #endif /* _X86_64_PGTABLE_H */