2 * Copyright 2002 Andi Kleen, SuSE Labs.
3 * Thanks to Ben LaHaise for precious feedback.
5 #include <linux/highmem.h>
6 #include <linux/bootmem.h>
7 #include <linux/module.h>
8 #include <linux/sched.h>
9 #include <linux/slab.h>
13 #include <asm/processor.h>
14 #include <asm/tlbflush.h>
15 #include <asm/sections.h>
16 #include <asm/uaccess.h>
17 #include <asm/pgalloc.h>
20 within(unsigned long addr
, unsigned long start
, unsigned long end
)
22 return addr
>= start
&& addr
< end
;
30 * clflush_cache_range - flush a cache range with clflush
31 * @addr: virtual start address
32 * @size: number of bytes to flush
34 * clflush is an unordered instruction which needs fencing with mfence
35 * to avoid ordering issues.
37 void clflush_cache_range(void *vaddr
, unsigned int size
)
39 void *vend
= vaddr
+ size
- 1;
43 for (; vaddr
< vend
; vaddr
+= boot_cpu_data
.x86_clflush_size
)
46 * Flush any possible final partial cacheline:
53 static void __cpa_flush_all(void *arg
)
56 * Flush all to work around Errata in early athlons regarding
57 * large page flushing.
61 if (boot_cpu_data
.x86_model
>= 4)
65 static void cpa_flush_all(void)
67 BUG_ON(irqs_disabled());
69 on_each_cpu(__cpa_flush_all
, NULL
, 1, 1);
72 static void __cpa_flush_range(void *arg
)
75 * We could optimize that further and do individual per page
76 * tlb invalidates for a low number of pages. Caveat: we must
77 * flush the high aliases on 64bit as well.
82 static void cpa_flush_range(unsigned long start
, int numpages
)
84 unsigned int i
, level
;
87 BUG_ON(irqs_disabled());
88 WARN_ON(PAGE_ALIGN(start
) != start
);
90 on_each_cpu(__cpa_flush_range
, NULL
, 1, 1);
93 * We only need to flush on one CPU,
94 * clflush is a MESI-coherent instruction that
95 * will cause all other CPUs to flush the same
98 for (i
= 0, addr
= start
; i
< numpages
; i
++, addr
+= PAGE_SIZE
) {
99 pte_t
*pte
= lookup_address(addr
, &level
);
102 * Only flush present addresses:
104 if (pte
&& pte_present(*pte
))
105 clflush_cache_range((void *) addr
, PAGE_SIZE
);
109 #define HIGH_MAP_START __START_KERNEL_map
110 #define HIGH_MAP_END (__START_KERNEL_map + KERNEL_TEXT_SIZE)
114 * Converts a virtual address to a X86-64 highmap address
116 static unsigned long virt_to_highmap(void *address
)
119 return __pa((unsigned long)address
) + HIGH_MAP_START
- phys_base
;
121 return (unsigned long)address
;
126 * Certain areas of memory on x86 require very specific protection flags,
127 * for example the BIOS area or kernel text. Callers don't always get this
128 * right (again, ioremap() on BIOS memory is not uncommon) so this function
129 * checks and fixes these known static required protection bits.
131 static inline pgprot_t
static_protections(pgprot_t prot
, unsigned long address
)
133 pgprot_t forbidden
= __pgprot(0);
136 * The BIOS area between 640k and 1Mb needs to be executable for
137 * PCI BIOS based config access (CONFIG_PCI_GOBIOS) support.
139 if (within(__pa(address
), BIOS_BEGIN
, BIOS_END
))
140 pgprot_val(forbidden
) |= _PAGE_NX
;
143 * The kernel text needs to be executable for obvious reasons
144 * Does not cover __inittext since that is gone later on
146 if (within(address
, (unsigned long)_text
, (unsigned long)_etext
))
147 pgprot_val(forbidden
) |= _PAGE_NX
;
149 * Do the same for the x86-64 high kernel mapping
151 if (within(address
, virt_to_highmap(_text
), virt_to_highmap(_etext
)))
152 pgprot_val(forbidden
) |= _PAGE_NX
;
155 #ifdef CONFIG_DEBUG_RODATA
156 /* The .rodata section needs to be read-only */
157 if (within(address
, (unsigned long)__start_rodata
,
158 (unsigned long)__end_rodata
))
159 pgprot_val(forbidden
) |= _PAGE_RW
;
161 * Do the same for the x86-64 high kernel mapping
163 if (within(address
, virt_to_highmap(__start_rodata
),
164 virt_to_highmap(__end_rodata
)))
165 pgprot_val(forbidden
) |= _PAGE_RW
;
168 prot
= __pgprot(pgprot_val(prot
) & ~pgprot_val(forbidden
));
173 pte_t
*lookup_address(unsigned long address
, int *level
)
175 pgd_t
*pgd
= pgd_offset_k(address
);
179 *level
= PG_LEVEL_NONE
;
183 pud
= pud_offset(pgd
, address
);
186 pmd
= pmd_offset(pud
, address
);
190 *level
= PG_LEVEL_2M
;
194 *level
= PG_LEVEL_4K
;
195 return pte_offset_kernel(pmd
, address
);
198 static void __set_pmd_pte(pte_t
*kpte
, unsigned long address
, pte_t pte
)
201 set_pte_atomic(kpte
, pte
);
203 if (!SHARED_KERNEL_PMD
) {
206 list_for_each_entry(page
, &pgd_list
, lru
) {
211 pgd
= (pgd_t
*)page_address(page
) + pgd_index(address
);
212 pud
= pud_offset(pgd
, address
);
213 pmd
= pmd_offset(pud
, address
);
214 set_pte_atomic((pte_t
*)pmd
, pte
);
220 static int split_large_page(pte_t
*kpte
, unsigned long address
)
222 pgprot_t ref_prot
= pte_pgprot(pte_clrhuge(*kpte
));
223 gfp_t gfp_flags
= GFP_KERNEL
;
228 unsigned int i
, level
;
230 #ifdef CONFIG_DEBUG_PAGEALLOC
231 gfp_flags
= __GFP_HIGH
| __GFP_NOFAIL
| __GFP_NOWARN
;
232 gfp_flags
= GFP_ATOMIC
| __GFP_NOWARN
;
234 base
= alloc_pages(gfp_flags
, 0);
238 spin_lock_irqsave(&pgd_lock
, flags
);
240 * Check for races, another CPU might have split this page
243 tmp
= lookup_address(address
, &level
);
249 address
= __pa(address
);
250 addr
= address
& LARGE_PAGE_MASK
;
251 pbase
= (pte_t
*)page_address(base
);
253 paravirt_alloc_pt(&init_mm
, page_to_pfn(base
));
256 for (i
= 0; i
< PTRS_PER_PTE
; i
++, addr
+= PAGE_SIZE
)
257 set_pte(&pbase
[i
], pfn_pte(addr
>> PAGE_SHIFT
, ref_prot
));
260 * Install the new, split up pagetable. Important detail here:
262 * On Intel the NX bit of all levels must be cleared to make a
263 * page executable. See section 4.13.2 of Intel 64 and IA-32
264 * Architectures Software Developer's Manual).
266 ref_prot
= pte_pgprot(pte_mkexec(pte_clrhuge(*kpte
)));
267 __set_pmd_pte(kpte
, address
, mk_pte(base
, ref_prot
));
271 spin_unlock_irqrestore(&pgd_lock
, flags
);
274 __free_pages(base
, 0);
280 __change_page_attr(unsigned long address
, unsigned long pfn
,
281 pgprot_t mask_set
, pgprot_t mask_clr
)
283 struct page
*kpte_page
;
288 BUG_ON(pfn
> max_low_pfn
);
292 kpte
= lookup_address(address
, &level
);
296 kpte_page
= virt_to_page(kpte
);
297 BUG_ON(PageLRU(kpte_page
));
298 BUG_ON(PageCompound(kpte_page
));
300 if (level
== PG_LEVEL_4K
) {
301 pgprot_t new_prot
= pte_pgprot(*kpte
);
302 pte_t new_pte
, old_pte
= *kpte
;
304 pgprot_val(new_prot
) &= ~pgprot_val(mask_clr
);
305 pgprot_val(new_prot
) |= pgprot_val(mask_set
);
307 new_prot
= static_protections(new_prot
, address
);
309 new_pte
= pfn_pte(pfn
, canon_pgprot(new_prot
));
310 BUG_ON(pte_pfn(new_pte
) != pte_pfn(old_pte
));
312 set_pte_atomic(kpte
, new_pte
);
314 err
= split_large_page(kpte
, address
);
322 * change_page_attr_addr - Change page table attributes in linear mapping
323 * @address: Virtual address in linear mapping.
324 * @prot: New page table attribute (PAGE_*)
326 * Change page attributes of a page in the direct mapping. This is a variant
327 * of change_page_attr() that also works on memory holes that do not have
328 * mem_map entry (pfn_valid() is false).
330 * See change_page_attr() documentation for more details.
332 * Modules and drivers should use the set_memory_* APIs instead.
337 change_page_attr_addr(unsigned long address
, pgprot_t mask_set
,
340 unsigned long phys_addr
= __pa(address
);
341 unsigned long pfn
= phys_addr
>> PAGE_SHIFT
;
346 * If we are inside the high mapped kernel range, then we
347 * fixup the low mapping first. __va() returns the virtual
348 * address in the linear mapping:
350 if (within(address
, HIGH_MAP_START
, HIGH_MAP_END
))
351 address
= (unsigned long) __va(phys_addr
);
354 err
= __change_page_attr(address
, pfn
, mask_set
, mask_clr
);
360 * If the physical address is inside the kernel map, we need
361 * to touch the high mapped kernel as well:
363 if (within(phys_addr
, 0, KERNEL_TEXT_SIZE
)) {
365 * Calc the high mapping address. See __phys_addr()
366 * for the non obvious details.
368 * Note that NX and other required permissions are
369 * checked in static_protections().
371 address
= phys_addr
+ HIGH_MAP_START
- phys_base
;
374 * Our high aliases are imprecise, because we check
375 * everything between 0 and KERNEL_TEXT_SIZE, so do
376 * not propagate lookup failures back to users:
378 __change_page_attr(address
, pfn
, mask_set
, mask_clr
);
384 static int __change_page_attr_set_clr(unsigned long addr
, int numpages
,
385 pgprot_t mask_set
, pgprot_t mask_clr
)
390 for (i
= 0; i
< numpages
; i
++, addr
+= PAGE_SIZE
) {
391 ret
= change_page_attr_addr(addr
, mask_set
, mask_clr
);
399 static int change_page_attr_set_clr(unsigned long addr
, int numpages
,
400 pgprot_t mask_set
, pgprot_t mask_clr
)
402 int ret
= __change_page_attr_set_clr(addr
, numpages
, mask_set
,
406 * On success we use clflush, when the CPU supports it to
407 * avoid the wbindv. If the CPU does not support it and in the
408 * error case we fall back to cpa_flush_all (which uses
411 if (!ret
&& cpu_has_clflush
)
412 cpa_flush_range(addr
, numpages
);
419 static inline int change_page_attr_set(unsigned long addr
, int numpages
,
422 return change_page_attr_set_clr(addr
, numpages
, mask
, __pgprot(0));
425 static inline int change_page_attr_clear(unsigned long addr
, int numpages
,
428 return change_page_attr_set_clr(addr
, numpages
, __pgprot(0), mask
);
431 int set_memory_uc(unsigned long addr
, int numpages
)
433 return change_page_attr_set(addr
, numpages
,
434 __pgprot(_PAGE_PCD
| _PAGE_PWT
));
436 EXPORT_SYMBOL(set_memory_uc
);
438 int set_memory_wb(unsigned long addr
, int numpages
)
440 return change_page_attr_clear(addr
, numpages
,
441 __pgprot(_PAGE_PCD
| _PAGE_PWT
));
443 EXPORT_SYMBOL(set_memory_wb
);
445 int set_memory_x(unsigned long addr
, int numpages
)
447 return change_page_attr_clear(addr
, numpages
, __pgprot(_PAGE_NX
));
449 EXPORT_SYMBOL(set_memory_x
);
451 int set_memory_nx(unsigned long addr
, int numpages
)
453 return change_page_attr_set(addr
, numpages
, __pgprot(_PAGE_NX
));
455 EXPORT_SYMBOL(set_memory_nx
);
457 int set_memory_ro(unsigned long addr
, int numpages
)
459 return change_page_attr_clear(addr
, numpages
, __pgprot(_PAGE_RW
));
462 int set_memory_rw(unsigned long addr
, int numpages
)
464 return change_page_attr_set(addr
, numpages
, __pgprot(_PAGE_RW
));
467 int set_memory_np(unsigned long addr
, int numpages
)
469 return change_page_attr_clear(addr
, numpages
, __pgprot(_PAGE_PRESENT
));
472 int set_pages_uc(struct page
*page
, int numpages
)
474 unsigned long addr
= (unsigned long)page_address(page
);
476 return set_memory_uc(addr
, numpages
);
478 EXPORT_SYMBOL(set_pages_uc
);
480 int set_pages_wb(struct page
*page
, int numpages
)
482 unsigned long addr
= (unsigned long)page_address(page
);
484 return set_memory_wb(addr
, numpages
);
486 EXPORT_SYMBOL(set_pages_wb
);
488 int set_pages_x(struct page
*page
, int numpages
)
490 unsigned long addr
= (unsigned long)page_address(page
);
492 return set_memory_x(addr
, numpages
);
494 EXPORT_SYMBOL(set_pages_x
);
496 int set_pages_nx(struct page
*page
, int numpages
)
498 unsigned long addr
= (unsigned long)page_address(page
);
500 return set_memory_nx(addr
, numpages
);
502 EXPORT_SYMBOL(set_pages_nx
);
504 int set_pages_ro(struct page
*page
, int numpages
)
506 unsigned long addr
= (unsigned long)page_address(page
);
508 return set_memory_ro(addr
, numpages
);
511 int set_pages_rw(struct page
*page
, int numpages
)
513 unsigned long addr
= (unsigned long)page_address(page
);
515 return set_memory_rw(addr
, numpages
);
519 #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_CPA_DEBUG)
520 static inline int __change_page_attr_set(unsigned long addr
, int numpages
,
523 return __change_page_attr_set_clr(addr
, numpages
, mask
, __pgprot(0));
526 static inline int __change_page_attr_clear(unsigned long addr
, int numpages
,
529 return __change_page_attr_set_clr(addr
, numpages
, __pgprot(0), mask
);
533 #ifdef CONFIG_DEBUG_PAGEALLOC
535 static int __set_pages_p(struct page
*page
, int numpages
)
537 unsigned long addr
= (unsigned long)page_address(page
);
539 return __change_page_attr_set(addr
, numpages
,
540 __pgprot(_PAGE_PRESENT
| _PAGE_RW
));
543 static int __set_pages_np(struct page
*page
, int numpages
)
545 unsigned long addr
= (unsigned long)page_address(page
);
547 return __change_page_attr_clear(addr
, numpages
,
548 __pgprot(_PAGE_PRESENT
));
551 void kernel_map_pages(struct page
*page
, int numpages
, int enable
)
553 if (PageHighMem(page
))
556 debug_check_no_locks_freed(page_address(page
),
557 numpages
* PAGE_SIZE
);
561 * If page allocator is not up yet then do not call c_p_a():
563 if (!debug_pagealloc_enabled
)
567 * The return value is ignored - the calls cannot fail,
568 * large pages are disabled at boot time:
571 __set_pages_p(page
, numpages
);
573 __set_pages_np(page
, numpages
);
576 * We should perform an IPI and flush all tlbs,
577 * but that can deadlock->flush only current cpu:
584 * The testcases use internal knowledge of the implementation that shouldn't
585 * be exposed to the rest of the kernel. Include these directly here.
587 #ifdef CONFIG_CPA_DEBUG
588 #include "pageattr-test.c"