2 #include <asm/pgalloc.h>
3 #include <asm/pgtable.h>
5 #include <asm/fixmap.h>
7 #define PGALLOC_GFP GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO
10 #define PGALLOC_USER_GFP __GFP_HIGHMEM
12 #define PGALLOC_USER_GFP 0
15 gfp_t __userpte_alloc_gfp
= PGALLOC_GFP
| PGALLOC_USER_GFP
;
17 pte_t
*pte_alloc_one_kernel(struct mm_struct
*mm
, unsigned long address
)
19 return (pte_t
*)__get_free_page(PGALLOC_GFP
);
22 pgtable_t
pte_alloc_one(struct mm_struct
*mm
, unsigned long address
)
26 pte
= alloc_pages(__userpte_alloc_gfp
, 0);
28 pgtable_page_ctor(pte
);
32 static int __init
setup_userpte(char *arg
)
38 * "userpte=nohigh" disables allocation of user pagetables in
41 if (strcmp(arg
, "nohigh") == 0)
42 __userpte_alloc_gfp
&= ~__GFP_HIGHMEM
;
47 early_param("userpte", setup_userpte
);
49 void ___pte_free_tlb(struct mmu_gather
*tlb
, struct page
*pte
)
51 pgtable_page_dtor(pte
);
52 paravirt_release_pte(page_to_pfn(pte
));
53 tlb_remove_page(tlb
, pte
);
56 #if PAGETABLE_LEVELS > 2
57 void ___pmd_free_tlb(struct mmu_gather
*tlb
, pmd_t
*pmd
)
59 paravirt_release_pmd(__pa(pmd
) >> PAGE_SHIFT
);
60 tlb_remove_page(tlb
, virt_to_page(pmd
));
63 #if PAGETABLE_LEVELS > 3
64 void ___pud_free_tlb(struct mmu_gather
*tlb
, pud_t
*pud
)
66 paravirt_release_pud(__pa(pud
) >> PAGE_SHIFT
);
67 tlb_remove_page(tlb
, virt_to_page(pud
));
69 #endif /* PAGETABLE_LEVELS > 3 */
70 #endif /* PAGETABLE_LEVELS > 2 */
72 static inline void pgd_list_add(pgd_t
*pgd
)
74 struct page
*page
= virt_to_page(pgd
);
76 list_add(&page
->lru
, &pgd_list
);
79 static inline void pgd_list_del(pgd_t
*pgd
)
81 struct page
*page
= virt_to_page(pgd
);
86 #define UNSHARED_PTRS_PER_PGD \
87 (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)
89 static void pgd_ctor(pgd_t
*pgd
)
91 /* If the pgd points to a shared pagetable level (either the
92 ptes in non-PAE, or shared PMD in PAE), then just copy the
93 references from swapper_pg_dir. */
94 if (PAGETABLE_LEVELS
== 2 ||
95 (PAGETABLE_LEVELS
== 3 && SHARED_KERNEL_PMD
) ||
96 PAGETABLE_LEVELS
== 4) {
97 clone_pgd_range(pgd
+ KERNEL_PGD_BOUNDARY
,
98 swapper_pg_dir
+ KERNEL_PGD_BOUNDARY
,
100 paravirt_alloc_pmd_clone(__pa(pgd
) >> PAGE_SHIFT
,
101 __pa(swapper_pg_dir
) >> PAGE_SHIFT
,
106 /* list required to sync kernel mapping updates */
107 if (!SHARED_KERNEL_PMD
)
111 static void pgd_dtor(pgd_t
*pgd
)
113 unsigned long flags
; /* can be called from interrupt context */
115 if (SHARED_KERNEL_PMD
)
118 spin_lock_irqsave(&pgd_lock
, flags
);
120 spin_unlock_irqrestore(&pgd_lock
, flags
);
124 * List of all pgd's needed for non-PAE so it can invalidate entries
125 * in both cached and uncached pgd's; not needed for PAE since the
126 * kernel pmd is shared. If PAE were not to share the pmd a similar
127 * tactic would be needed. This is essentially codepath-based locking
128 * against pageattr.c; it is the unique case in which a valid change
129 * of kernel pagetables can't be lazily synchronized by vmalloc faults.
130 * vmalloc faults work because attached pagetables are never freed.
134 #ifdef CONFIG_X86_PAE
136 * In PAE mode, we need to do a cr3 reload (=tlb flush) when
137 * updating the top-level pagetable entries to guarantee the
138 * processor notices the update. Since this is expensive, and
139 * all 4 top-level entries are used almost immediately in a
140 * new process's life, we just pre-populate them here.
142 * Also, if we're in a paravirt environment where the kernel pmd is
143 * not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
144 * and initialize the kernel pmds here.
146 #define PREALLOCATED_PMDS UNSHARED_PTRS_PER_PGD
148 void pud_populate(struct mm_struct
*mm
, pud_t
*pudp
, pmd_t
*pmd
)
150 paravirt_alloc_pmd(mm
, __pa(pmd
) >> PAGE_SHIFT
);
152 /* Note: almost everything apart from _PAGE_PRESENT is
153 reserved at the pmd (PDPT) level. */
154 set_pud(pudp
, __pud(__pa(pmd
) | _PAGE_PRESENT
));
157 * According to Intel App note "TLBs, Paging-Structure Caches,
158 * and Their Invalidation", April 2007, document 317080-001,
159 * section 8.1: in PAE mode we explicitly have to flush the
160 * TLB via cr3 if the top-level pgd is changed...
162 if (mm
== current
->active_mm
)
163 write_cr3(read_cr3());
165 #else /* !CONFIG_X86_PAE */
167 /* No need to prepopulate any pagetable entries in non-PAE modes. */
168 #define PREALLOCATED_PMDS 0
170 #endif /* CONFIG_X86_PAE */
172 static void free_pmds(pmd_t
*pmds
[])
176 for(i
= 0; i
< PREALLOCATED_PMDS
; i
++)
178 free_page((unsigned long)pmds
[i
]);
181 static int preallocate_pmds(pmd_t
*pmds
[])
186 for(i
= 0; i
< PREALLOCATED_PMDS
; i
++) {
187 pmd_t
*pmd
= (pmd_t
*)__get_free_page(PGALLOC_GFP
);
202 * Mop up any pmd pages which may still be attached to the pgd.
203 * Normally they will be freed by munmap/exit_mmap, but any pmd we
204 * preallocate which never got a corresponding vma will need to be
207 static void pgd_mop_up_pmds(struct mm_struct
*mm
, pgd_t
*pgdp
)
211 for(i
= 0; i
< PREALLOCATED_PMDS
; i
++) {
214 if (pgd_val(pgd
) != 0) {
215 pmd_t
*pmd
= (pmd_t
*)pgd_page_vaddr(pgd
);
217 pgdp
[i
] = native_make_pgd(0);
219 paravirt_release_pmd(pgd_val(pgd
) >> PAGE_SHIFT
);
225 static void pgd_prepopulate_pmd(struct mm_struct
*mm
, pgd_t
*pgd
, pmd_t
*pmds
[])
231 if (PREALLOCATED_PMDS
== 0) /* Work around gcc-3.4.x bug */
234 pud
= pud_offset(pgd
, 0);
236 for (addr
= i
= 0; i
< PREALLOCATED_PMDS
;
237 i
++, pud
++, addr
+= PUD_SIZE
) {
238 pmd_t
*pmd
= pmds
[i
];
240 if (i
>= KERNEL_PGD_BOUNDARY
)
241 memcpy(pmd
, (pmd_t
*)pgd_page_vaddr(swapper_pg_dir
[i
]),
242 sizeof(pmd_t
) * PTRS_PER_PMD
);
244 pud_populate(mm
, pud
, pmd
);
248 pgd_t
*pgd_alloc(struct mm_struct
*mm
)
251 pmd_t
*pmds
[PREALLOCATED_PMDS
];
254 pgd
= (pgd_t
*)__get_free_page(PGALLOC_GFP
);
261 if (preallocate_pmds(pmds
) != 0)
264 if (paravirt_pgd_alloc(mm
) != 0)
268 * Make sure that pre-populating the pmds is atomic with
269 * respect to anything walking the pgd_list, so that they
270 * never see a partially populated pgd.
272 spin_lock_irqsave(&pgd_lock
, flags
);
275 pgd_prepopulate_pmd(mm
, pgd
, pmds
);
277 spin_unlock_irqrestore(&pgd_lock
, flags
);
284 free_page((unsigned long)pgd
);
289 void pgd_free(struct mm_struct
*mm
, pgd_t
*pgd
)
291 pgd_mop_up_pmds(mm
, pgd
);
293 paravirt_pgd_free(mm
, pgd
);
294 free_page((unsigned long)pgd
);
297 int ptep_set_access_flags(struct vm_area_struct
*vma
,
298 unsigned long address
, pte_t
*ptep
,
299 pte_t entry
, int dirty
)
301 int changed
= !pte_same(*ptep
, entry
);
303 if (changed
&& dirty
) {
305 pte_update_defer(vma
->vm_mm
, address
, ptep
);
306 flush_tlb_page(vma
, address
);
312 int ptep_test_and_clear_young(struct vm_area_struct
*vma
,
313 unsigned long addr
, pte_t
*ptep
)
317 if (pte_young(*ptep
))
318 ret
= test_and_clear_bit(_PAGE_BIT_ACCESSED
,
319 (unsigned long *) &ptep
->pte
);
322 pte_update(vma
->vm_mm
, addr
, ptep
);
327 int ptep_clear_flush_young(struct vm_area_struct
*vma
,
328 unsigned long address
, pte_t
*ptep
)
332 young
= ptep_test_and_clear_young(vma
, address
, ptep
);
334 flush_tlb_page(vma
, address
);
340 * reserve_top_address - reserves a hole in the top of kernel address space
341 * @reserve - size of hole to reserve
343 * Can be used to relocate the fixmap area and poke a hole in the top
344 * of kernel address space to make room for a hypervisor.
346 void __init
reserve_top_address(unsigned long reserve
)
349 BUG_ON(fixmaps_set
> 0);
350 printk(KERN_INFO
"Reserving virtual address space above 0x%08x\n",
352 __FIXADDR_TOP
= -reserve
- PAGE_SIZE
;
358 void __native_set_fixmap(enum fixed_addresses idx
, pte_t pte
)
360 unsigned long address
= __fix_to_virt(idx
);
362 if (idx
>= __end_of_fixed_addresses
) {
366 set_pte_vaddr(address
, pte
);
370 void native_set_fixmap(enum fixed_addresses idx
, phys_addr_t phys
,
373 __native_set_fixmap(idx
, pfn_pte(phys
>> PAGE_SHIFT
, flags
));