2 * Based on arch/arm/mm/mmu.c
4 * Copyright (C) 1995-2005 Russell King
5 * Copyright (C) 2012 ARM Ltd.
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 #include <linux/cache.h>
21 #include <linux/export.h>
22 #include <linux/kernel.h>
23 #include <linux/errno.h>
24 #include <linux/init.h>
25 #include <linux/ioport.h>
26 #include <linux/kexec.h>
27 #include <linux/libfdt.h>
28 #include <linux/mman.h>
29 #include <linux/nodemask.h>
30 #include <linux/memblock.h>
34 #include <linux/vmalloc.h>
36 #include <asm/barrier.h>
37 #include <asm/cputype.h>
38 #include <asm/fixmap.h>
39 #include <asm/kasan.h>
40 #include <asm/kernel-pgtable.h>
41 #include <asm/sections.h>
42 #include <asm/setup.h>
43 #include <asm/sizes.h>
45 #include <asm/memblock.h>
46 #include <asm/mmu_context.h>
47 #include <asm/ptdump.h>
48 #include <asm/tlbflush.h>
50 #define NO_BLOCK_MAPPINGS BIT(0)
51 #define NO_CONT_MAPPINGS BIT(1)
53 u64 idmap_t0sz
= TCR_T0SZ(VA_BITS
);
54 u64 idmap_ptrs_per_pgd
= PTRS_PER_PGD
;
56 u64 kimage_voffset __ro_after_init
;
57 EXPORT_SYMBOL(kimage_voffset
);
60 * Empty_zero_page is a special page that is used for zero-initialized data
63 unsigned long empty_zero_page
[PAGE_SIZE
/ sizeof(unsigned long)] __page_aligned_bss
;
64 EXPORT_SYMBOL(empty_zero_page
);
66 static pte_t bm_pte
[PTRS_PER_PTE
] __page_aligned_bss
;
67 static pmd_t bm_pmd
[PTRS_PER_PMD
] __page_aligned_bss __maybe_unused
;
68 static pud_t bm_pud
[PTRS_PER_PUD
] __page_aligned_bss __maybe_unused
;
70 pgprot_t
phys_mem_access_prot(struct file
*file
, unsigned long pfn
,
71 unsigned long size
, pgprot_t vma_prot
)
74 return pgprot_noncached(vma_prot
);
75 else if (file
->f_flags
& O_SYNC
)
76 return pgprot_writecombine(vma_prot
);
79 EXPORT_SYMBOL(phys_mem_access_prot
);
81 static phys_addr_t __init
early_pgtable_alloc(void)
86 phys
= memblock_alloc(PAGE_SIZE
, PAGE_SIZE
);
89 * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE
90 * slot will be free, so we can (ab)use the FIX_PTE slot to initialise
93 ptr
= pte_set_fixmap(phys
);
95 memset(ptr
, 0, PAGE_SIZE
);
98 * Implicit barriers also ensure the zeroed page is visible to the page
106 static bool pgattr_change_is_safe(u64 old
, u64
new)
109 * The following mapping attributes may be updated in live
110 * kernel mappings without the need for break-before-make.
112 static const pteval_t mask
= PTE_PXN
| PTE_RDONLY
| PTE_WRITE
| PTE_NG
;
114 /* creating or taking down mappings is always safe */
115 if (old
== 0 || new == 0)
118 /* live contiguous mappings may not be manipulated at all */
119 if ((old
| new) & PTE_CONT
)
122 /* Transitioning from Non-Global to Global is unsafe */
123 if (old
& ~new & PTE_NG
)
126 return ((old
^ new) & ~mask
) == 0;
129 static void init_pte(pmd_t
*pmdp
, unsigned long addr
, unsigned long end
,
130 phys_addr_t phys
, pgprot_t prot
)
134 ptep
= pte_set_fixmap_offset(pmdp
, addr
);
136 pte_t old_pte
= READ_ONCE(*ptep
);
138 set_pte(ptep
, pfn_pte(__phys_to_pfn(phys
), prot
));
141 * After the PTE entry has been populated once, we
142 * only allow updates to the permission attributes.
144 BUG_ON(!pgattr_change_is_safe(pte_val(old_pte
),
145 READ_ONCE(pte_val(*ptep
))));
148 } while (ptep
++, addr
+= PAGE_SIZE
, addr
!= end
);
153 static void alloc_init_cont_pte(pmd_t
*pmdp
, unsigned long addr
,
154 unsigned long end
, phys_addr_t phys
,
156 phys_addr_t (*pgtable_alloc
)(void),
160 pmd_t pmd
= READ_ONCE(*pmdp
);
162 BUG_ON(pmd_sect(pmd
));
164 phys_addr_t pte_phys
;
165 BUG_ON(!pgtable_alloc
);
166 pte_phys
= pgtable_alloc();
167 __pmd_populate(pmdp
, pte_phys
, PMD_TYPE_TABLE
);
168 pmd
= READ_ONCE(*pmdp
);
170 BUG_ON(pmd_bad(pmd
));
173 pgprot_t __prot
= prot
;
175 next
= pte_cont_addr_end(addr
, end
);
177 /* use a contiguous mapping if the range is suitably aligned */
178 if ((((addr
| next
| phys
) & ~CONT_PTE_MASK
) == 0) &&
179 (flags
& NO_CONT_MAPPINGS
) == 0)
180 __prot
= __pgprot(pgprot_val(prot
) | PTE_CONT
);
182 init_pte(pmdp
, addr
, next
, phys
, __prot
);
185 } while (addr
= next
, addr
!= end
);
188 static void init_pmd(pud_t
*pudp
, unsigned long addr
, unsigned long end
,
189 phys_addr_t phys
, pgprot_t prot
,
190 phys_addr_t (*pgtable_alloc
)(void), int flags
)
195 pmdp
= pmd_set_fixmap_offset(pudp
, addr
);
197 pmd_t old_pmd
= READ_ONCE(*pmdp
);
199 next
= pmd_addr_end(addr
, end
);
201 /* try section mapping first */
202 if (((addr
| next
| phys
) & ~SECTION_MASK
) == 0 &&
203 (flags
& NO_BLOCK_MAPPINGS
) == 0) {
204 pmd_set_huge(pmdp
, phys
, prot
);
207 * After the PMD entry has been populated once, we
208 * only allow updates to the permission attributes.
210 BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd
),
211 READ_ONCE(pmd_val(*pmdp
))));
213 alloc_init_cont_pte(pmdp
, addr
, next
, phys
, prot
,
214 pgtable_alloc
, flags
);
216 BUG_ON(pmd_val(old_pmd
) != 0 &&
217 pmd_val(old_pmd
) != READ_ONCE(pmd_val(*pmdp
)));
220 } while (pmdp
++, addr
= next
, addr
!= end
);
225 static void alloc_init_cont_pmd(pud_t
*pudp
, unsigned long addr
,
226 unsigned long end
, phys_addr_t phys
,
228 phys_addr_t (*pgtable_alloc
)(void), int flags
)
231 pud_t pud
= READ_ONCE(*pudp
);
234 * Check for initial section mappings in the pgd/pud.
236 BUG_ON(pud_sect(pud
));
238 phys_addr_t pmd_phys
;
239 BUG_ON(!pgtable_alloc
);
240 pmd_phys
= pgtable_alloc();
241 __pud_populate(pudp
, pmd_phys
, PUD_TYPE_TABLE
);
242 pud
= READ_ONCE(*pudp
);
244 BUG_ON(pud_bad(pud
));
247 pgprot_t __prot
= prot
;
249 next
= pmd_cont_addr_end(addr
, end
);
251 /* use a contiguous mapping if the range is suitably aligned */
252 if ((((addr
| next
| phys
) & ~CONT_PMD_MASK
) == 0) &&
253 (flags
& NO_CONT_MAPPINGS
) == 0)
254 __prot
= __pgprot(pgprot_val(prot
) | PTE_CONT
);
256 init_pmd(pudp
, addr
, next
, phys
, __prot
, pgtable_alloc
, flags
);
259 } while (addr
= next
, addr
!= end
);
262 static inline bool use_1G_block(unsigned long addr
, unsigned long next
,
265 if (PAGE_SHIFT
!= 12)
268 if (((addr
| next
| phys
) & ~PUD_MASK
) != 0)
274 static void alloc_init_pud(pgd_t
*pgdp
, unsigned long addr
, unsigned long end
,
275 phys_addr_t phys
, pgprot_t prot
,
276 phys_addr_t (*pgtable_alloc
)(void),
281 pgd_t pgd
= READ_ONCE(*pgdp
);
284 phys_addr_t pud_phys
;
285 BUG_ON(!pgtable_alloc
);
286 pud_phys
= pgtable_alloc();
287 __pgd_populate(pgdp
, pud_phys
, PUD_TYPE_TABLE
);
288 pgd
= READ_ONCE(*pgdp
);
290 BUG_ON(pgd_bad(pgd
));
292 pudp
= pud_set_fixmap_offset(pgdp
, addr
);
294 pud_t old_pud
= READ_ONCE(*pudp
);
296 next
= pud_addr_end(addr
, end
);
299 * For 4K granule only, attempt to put down a 1GB block
301 if (use_1G_block(addr
, next
, phys
) &&
302 (flags
& NO_BLOCK_MAPPINGS
) == 0) {
303 pud_set_huge(pudp
, phys
, prot
);
306 * After the PUD entry has been populated once, we
307 * only allow updates to the permission attributes.
309 BUG_ON(!pgattr_change_is_safe(pud_val(old_pud
),
310 READ_ONCE(pud_val(*pudp
))));
312 alloc_init_cont_pmd(pudp
, addr
, next
, phys
, prot
,
313 pgtable_alloc
, flags
);
315 BUG_ON(pud_val(old_pud
) != 0 &&
316 pud_val(old_pud
) != READ_ONCE(pud_val(*pudp
)));
319 } while (pudp
++, addr
= next
, addr
!= end
);
324 static void __create_pgd_mapping(pgd_t
*pgdir
, phys_addr_t phys
,
325 unsigned long virt
, phys_addr_t size
,
327 phys_addr_t (*pgtable_alloc
)(void),
330 unsigned long addr
, length
, end
, next
;
331 pgd_t
*pgdp
= pgd_offset_raw(pgdir
, virt
);
334 * If the virtual and physical address don't have the same offset
335 * within a page, we cannot map the region as the caller expects.
337 if (WARN_ON((phys
^ virt
) & ~PAGE_MASK
))
341 addr
= virt
& PAGE_MASK
;
342 length
= PAGE_ALIGN(size
+ (virt
& ~PAGE_MASK
));
346 next
= pgd_addr_end(addr
, end
);
347 alloc_init_pud(pgdp
, addr
, next
, phys
, prot
, pgtable_alloc
,
350 } while (pgdp
++, addr
= next
, addr
!= end
);
353 static phys_addr_t
pgd_pgtable_alloc(void)
355 void *ptr
= (void *)__get_free_page(PGALLOC_GFP
);
356 if (!ptr
|| !pgtable_page_ctor(virt_to_page(ptr
)))
359 /* Ensure the zeroed page is visible to the page table walker */
365 * This function can only be used to modify existing table entries,
366 * without allocating new levels of table. Note that this permits the
367 * creation of new section or page entries.
369 static void __init
create_mapping_noalloc(phys_addr_t phys
, unsigned long virt
,
370 phys_addr_t size
, pgprot_t prot
)
372 if (virt
< VMALLOC_START
) {
373 pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
377 __create_pgd_mapping(init_mm
.pgd
, phys
, virt
, size
, prot
, NULL
,
381 void __init
create_pgd_mapping(struct mm_struct
*mm
, phys_addr_t phys
,
382 unsigned long virt
, phys_addr_t size
,
383 pgprot_t prot
, bool page_mappings_only
)
387 BUG_ON(mm
== &init_mm
);
389 if (page_mappings_only
)
390 flags
= NO_BLOCK_MAPPINGS
| NO_CONT_MAPPINGS
;
392 __create_pgd_mapping(mm
->pgd
, phys
, virt
, size
, prot
,
393 pgd_pgtable_alloc
, flags
);
396 static void update_mapping_prot(phys_addr_t phys
, unsigned long virt
,
397 phys_addr_t size
, pgprot_t prot
)
399 if (virt
< VMALLOC_START
) {
400 pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n",
405 __create_pgd_mapping(init_mm
.pgd
, phys
, virt
, size
, prot
, NULL
,
408 /* flush the TLBs after updating live kernel mappings */
409 flush_tlb_kernel_range(virt
, virt
+ size
);
412 static void __init
__map_memblock(pgd_t
*pgdp
, phys_addr_t start
,
413 phys_addr_t end
, pgprot_t prot
, int flags
)
415 __create_pgd_mapping(pgdp
, start
, __phys_to_virt(start
), end
- start
,
416 prot
, early_pgtable_alloc
, flags
);
419 void __init
mark_linear_text_alias_ro(void)
422 * Remove the write permissions from the linear alias of .text/.rodata
424 update_mapping_prot(__pa_symbol(_text
), (unsigned long)lm_alias(_text
),
425 (unsigned long)__init_begin
- (unsigned long)_text
,
429 static void __init
map_mem(pgd_t
*pgdp
)
431 phys_addr_t kernel_start
= __pa_symbol(_text
);
432 phys_addr_t kernel_end
= __pa_symbol(__init_begin
);
433 struct memblock_region
*reg
;
436 if (debug_pagealloc_enabled())
437 flags
= NO_BLOCK_MAPPINGS
| NO_CONT_MAPPINGS
;
440 * Take care not to create a writable alias for the
441 * read-only text and rodata sections of the kernel image.
442 * So temporarily mark them as NOMAP to skip mappings in
443 * the following for-loop
445 memblock_mark_nomap(kernel_start
, kernel_end
- kernel_start
);
446 #ifdef CONFIG_KEXEC_CORE
448 memblock_mark_nomap(crashk_res
.start
,
449 resource_size(&crashk_res
));
452 /* map all the memory banks */
453 for_each_memblock(memory
, reg
) {
454 phys_addr_t start
= reg
->base
;
455 phys_addr_t end
= start
+ reg
->size
;
459 if (memblock_is_nomap(reg
))
462 __map_memblock(pgdp
, start
, end
, PAGE_KERNEL
, flags
);
466 * Map the linear alias of the [_text, __init_begin) interval
467 * as non-executable now, and remove the write permission in
468 * mark_linear_text_alias_ro() below (which will be called after
469 * alternative patching has completed). This makes the contents
470 * of the region accessible to subsystems such as hibernate,
471 * but protects it from inadvertent modification or execution.
472 * Note that contiguous mappings cannot be remapped in this way,
473 * so we should avoid them here.
475 __map_memblock(pgdp
, kernel_start
, kernel_end
,
476 PAGE_KERNEL
, NO_CONT_MAPPINGS
);
477 memblock_clear_nomap(kernel_start
, kernel_end
- kernel_start
);
479 #ifdef CONFIG_KEXEC_CORE
481 * Use page-level mappings here so that we can shrink the region
482 * in page granularity and put back unused memory to buddy system
483 * through /sys/kernel/kexec_crash_size interface.
485 if (crashk_res
.end
) {
486 __map_memblock(pgdp
, crashk_res
.start
, crashk_res
.end
+ 1,
488 NO_BLOCK_MAPPINGS
| NO_CONT_MAPPINGS
);
489 memblock_clear_nomap(crashk_res
.start
,
490 resource_size(&crashk_res
));
495 void mark_rodata_ro(void)
497 unsigned long section_size
;
500 * mark .rodata as read only. Use __init_begin rather than __end_rodata
501 * to cover NOTES and EXCEPTION_TABLE.
503 section_size
= (unsigned long)__init_begin
- (unsigned long)__start_rodata
;
504 update_mapping_prot(__pa_symbol(__start_rodata
), (unsigned long)__start_rodata
,
505 section_size
, PAGE_KERNEL_RO
);
510 static void __init
map_kernel_segment(pgd_t
*pgdp
, void *va_start
, void *va_end
,
511 pgprot_t prot
, struct vm_struct
*vma
,
512 int flags
, unsigned long vm_flags
)
514 phys_addr_t pa_start
= __pa_symbol(va_start
);
515 unsigned long size
= va_end
- va_start
;
517 BUG_ON(!PAGE_ALIGNED(pa_start
));
518 BUG_ON(!PAGE_ALIGNED(size
));
520 __create_pgd_mapping(pgdp
, pa_start
, (unsigned long)va_start
, size
, prot
,
521 early_pgtable_alloc
, flags
);
523 if (!(vm_flags
& VM_NO_GUARD
))
526 vma
->addr
= va_start
;
527 vma
->phys_addr
= pa_start
;
529 vma
->flags
= VM_MAP
| vm_flags
;
530 vma
->caller
= __builtin_return_address(0);
532 vm_area_add_early(vma
);
535 static int __init
parse_rodata(char *arg
)
537 return strtobool(arg
, &rodata_enabled
);
539 early_param("rodata", parse_rodata
);
541 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
542 static int __init
map_entry_trampoline(void)
544 pgprot_t prot
= rodata_enabled
? PAGE_KERNEL_ROX
: PAGE_KERNEL_EXEC
;
545 phys_addr_t pa_start
= __pa_symbol(__entry_tramp_text_start
);
547 /* The trampoline is always mapped and can therefore be global */
548 pgprot_val(prot
) &= ~PTE_NG
;
550 /* Map only the text into the trampoline page table */
551 memset(tramp_pg_dir
, 0, PGD_SIZE
);
552 __create_pgd_mapping(tramp_pg_dir
, pa_start
, TRAMP_VALIAS
, PAGE_SIZE
,
553 prot
, pgd_pgtable_alloc
, 0);
555 /* Map both the text and data into the kernel page table */
556 __set_fixmap(FIX_ENTRY_TRAMP_TEXT
, pa_start
, prot
);
557 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE
)) {
558 extern char __entry_tramp_data_start
[];
560 __set_fixmap(FIX_ENTRY_TRAMP_DATA
,
561 __pa_symbol(__entry_tramp_data_start
),
567 core_initcall(map_entry_trampoline
);
571 * Create fine-grained mappings for the kernel.
573 static void __init
map_kernel(pgd_t
*pgdp
)
575 static struct vm_struct vmlinux_text
, vmlinux_rodata
, vmlinux_inittext
,
576 vmlinux_initdata
, vmlinux_data
;
579 * External debuggers may need to write directly to the text
580 * mapping to install SW breakpoints. Allow this (only) when
581 * explicitly requested with rodata=off.
583 pgprot_t text_prot
= rodata_enabled
? PAGE_KERNEL_ROX
: PAGE_KERNEL_EXEC
;
586 * Only rodata will be remapped with different permissions later on,
587 * all other segments are allowed to use contiguous mappings.
589 map_kernel_segment(pgdp
, _text
, _etext
, text_prot
, &vmlinux_text
, 0,
591 map_kernel_segment(pgdp
, __start_rodata
, __inittext_begin
, PAGE_KERNEL
,
592 &vmlinux_rodata
, NO_CONT_MAPPINGS
, VM_NO_GUARD
);
593 map_kernel_segment(pgdp
, __inittext_begin
, __inittext_end
, text_prot
,
594 &vmlinux_inittext
, 0, VM_NO_GUARD
);
595 map_kernel_segment(pgdp
, __initdata_begin
, __initdata_end
, PAGE_KERNEL
,
596 &vmlinux_initdata
, 0, VM_NO_GUARD
);
597 map_kernel_segment(pgdp
, _data
, _end
, PAGE_KERNEL
, &vmlinux_data
, 0, 0);
599 if (!READ_ONCE(pgd_val(*pgd_offset_raw(pgdp
, FIXADDR_START
)))) {
601 * The fixmap falls in a separate pgd to the kernel, and doesn't
602 * live in the carveout for the swapper_pg_dir. We can simply
603 * re-use the existing dir for the fixmap.
605 set_pgd(pgd_offset_raw(pgdp
, FIXADDR_START
),
606 READ_ONCE(*pgd_offset_k(FIXADDR_START
)));
607 } else if (CONFIG_PGTABLE_LEVELS
> 3) {
609 * The fixmap shares its top level pgd entry with the kernel
610 * mapping. This can really only occur when we are running
611 * with 16k/4 levels, so we can simply reuse the pud level
614 BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES
));
615 pud_populate(&init_mm
,
616 pud_set_fixmap_offset(pgdp
, FIXADDR_START
),
623 kasan_copy_shadow(pgdp
);
627 * paging_init() sets up the page tables, initialises the zone memory
628 * maps and sets up the zero page.
630 void __init
paging_init(void)
632 phys_addr_t pgd_phys
= early_pgtable_alloc();
633 pgd_t
*pgdp
= pgd_set_fixmap(pgd_phys
);
639 * We want to reuse the original swapper_pg_dir so we don't have to
640 * communicate the new address to non-coherent secondaries in
641 * secondary_entry, and so cpu_switch_mm can generate the address with
642 * adrp+add rather than a load from some global variable.
644 * To do this we need to go via a temporary pgd.
646 cpu_replace_ttbr1(__va(pgd_phys
));
647 memcpy(swapper_pg_dir
, pgdp
, PGD_SIZE
);
648 cpu_replace_ttbr1(lm_alias(swapper_pg_dir
));
651 memblock_free(pgd_phys
, PAGE_SIZE
);
654 * We only reuse the PGD from the swapper_pg_dir, not the pud + pmd
657 memblock_free(__pa_symbol(swapper_pg_dir
) + PAGE_SIZE
,
658 __pa_symbol(swapper_pg_end
) - __pa_symbol(swapper_pg_dir
)
663 * Check whether a kernel address is valid (derived from arch/x86/).
665 int kern_addr_valid(unsigned long addr
)
672 if ((((long)addr
) >> VA_BITS
) != -1UL)
675 pgdp
= pgd_offset_k(addr
);
676 if (pgd_none(READ_ONCE(*pgdp
)))
679 pudp
= pud_offset(pgdp
, addr
);
680 pud
= READ_ONCE(*pudp
);
685 return pfn_valid(pud_pfn(pud
));
687 pmdp
= pmd_offset(pudp
, addr
);
688 pmd
= READ_ONCE(*pmdp
);
693 return pfn_valid(pmd_pfn(pmd
));
695 ptep
= pte_offset_kernel(pmdp
, addr
);
696 pte
= READ_ONCE(*ptep
);
700 return pfn_valid(pte_pfn(pte
));
702 #ifdef CONFIG_SPARSEMEM_VMEMMAP
703 #if !ARM64_SWAPPER_USES_SECTION_MAPS
704 int __meminit
vmemmap_populate(unsigned long start
, unsigned long end
, int node
,
705 struct vmem_altmap
*altmap
)
707 return vmemmap_populate_basepages(start
, end
, node
);
709 #else /* !ARM64_SWAPPER_USES_SECTION_MAPS */
710 int __meminit
vmemmap_populate(unsigned long start
, unsigned long end
, int node
,
711 struct vmem_altmap
*altmap
)
713 unsigned long addr
= start
;
720 next
= pmd_addr_end(addr
, end
);
722 pgdp
= vmemmap_pgd_populate(addr
, node
);
726 pudp
= vmemmap_pud_populate(pgdp
, addr
, node
);
730 pmdp
= pmd_offset(pudp
, addr
);
731 if (pmd_none(READ_ONCE(*pmdp
))) {
734 p
= vmemmap_alloc_block_buf(PMD_SIZE
, node
);
738 pmd_set_huge(pmdp
, __pa(p
), __pgprot(PROT_SECT_NORMAL
));
740 vmemmap_verify((pte_t
*)pmdp
, node
, addr
, next
);
741 } while (addr
= next
, addr
!= end
);
745 #endif /* CONFIG_ARM64_64K_PAGES */
746 void vmemmap_free(unsigned long start
, unsigned long end
,
747 struct vmem_altmap
*altmap
)
750 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
752 static inline pud_t
* fixmap_pud(unsigned long addr
)
754 pgd_t
*pgdp
= pgd_offset_k(addr
);
755 pgd_t pgd
= READ_ONCE(*pgdp
);
757 BUG_ON(pgd_none(pgd
) || pgd_bad(pgd
));
759 return pud_offset_kimg(pgdp
, addr
);
762 static inline pmd_t
* fixmap_pmd(unsigned long addr
)
764 pud_t
*pudp
= fixmap_pud(addr
);
765 pud_t pud
= READ_ONCE(*pudp
);
767 BUG_ON(pud_none(pud
) || pud_bad(pud
));
769 return pmd_offset_kimg(pudp
, addr
);
772 static inline pte_t
* fixmap_pte(unsigned long addr
)
774 return &bm_pte
[pte_index(addr
)];
778 * The p*d_populate functions call virt_to_phys implicitly so they can't be used
779 * directly on kernel symbols (bm_p*d). This function is called too early to use
780 * lm_alias so __p*d_populate functions must be used to populate with the
781 * physical address from __pa_symbol.
783 void __init
early_fixmap_init(void)
788 unsigned long addr
= FIXADDR_START
;
790 pgdp
= pgd_offset_k(addr
);
791 pgd
= READ_ONCE(*pgdp
);
792 if (CONFIG_PGTABLE_LEVELS
> 3 &&
793 !(pgd_none(pgd
) || pgd_page_paddr(pgd
) == __pa_symbol(bm_pud
))) {
795 * We only end up here if the kernel mapping and the fixmap
796 * share the top level pgd entry, which should only happen on
797 * 16k/4 levels configurations.
799 BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES
));
800 pudp
= pud_offset_kimg(pgdp
, addr
);
803 __pgd_populate(pgdp
, __pa_symbol(bm_pud
), PUD_TYPE_TABLE
);
804 pudp
= fixmap_pud(addr
);
806 if (pud_none(READ_ONCE(*pudp
)))
807 __pud_populate(pudp
, __pa_symbol(bm_pmd
), PMD_TYPE_TABLE
);
808 pmdp
= fixmap_pmd(addr
);
809 __pmd_populate(pmdp
, __pa_symbol(bm_pte
), PMD_TYPE_TABLE
);
812 * The boot-ioremap range spans multiple pmds, for which
813 * we are not prepared:
815 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN
) >> PMD_SHIFT
)
816 != (__fix_to_virt(FIX_BTMAP_END
) >> PMD_SHIFT
));
818 if ((pmdp
!= fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN
)))
819 || pmdp
!= fixmap_pmd(fix_to_virt(FIX_BTMAP_END
))) {
821 pr_warn("pmdp %p != %p, %p\n",
822 pmdp
, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN
)),
823 fixmap_pmd(fix_to_virt(FIX_BTMAP_END
)));
824 pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
825 fix_to_virt(FIX_BTMAP_BEGIN
));
826 pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n",
827 fix_to_virt(FIX_BTMAP_END
));
829 pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END
);
830 pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN
);
835 * Unusually, this is also called in IRQ context (ghes_iounmap_irq) so if we
836 * ever need to use IPIs for TLB broadcasting, then we're in trouble here.
838 void __set_fixmap(enum fixed_addresses idx
,
839 phys_addr_t phys
, pgprot_t flags
)
841 unsigned long addr
= __fix_to_virt(idx
);
844 BUG_ON(idx
<= FIX_HOLE
|| idx
>= __end_of_fixed_addresses
);
846 ptep
= fixmap_pte(addr
);
848 if (pgprot_val(flags
)) {
849 set_pte(ptep
, pfn_pte(phys
>> PAGE_SHIFT
, flags
));
851 pte_clear(&init_mm
, addr
, ptep
);
852 flush_tlb_kernel_range(addr
, addr
+PAGE_SIZE
);
856 void *__init
__fixmap_remap_fdt(phys_addr_t dt_phys
, int *size
, pgprot_t prot
)
858 const u64 dt_virt_base
= __fix_to_virt(FIX_FDT
);
863 * Check whether the physical FDT address is set and meets the minimum
864 * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be
865 * at least 8 bytes so that we can always access the magic and size
866 * fields of the FDT header after mapping the first chunk, double check
867 * here if that is indeed the case.
869 BUILD_BUG_ON(MIN_FDT_ALIGN
< 8);
870 if (!dt_phys
|| dt_phys
% MIN_FDT_ALIGN
)
874 * Make sure that the FDT region can be mapped without the need to
875 * allocate additional translation table pages, so that it is safe
876 * to call create_mapping_noalloc() this early.
878 * On 64k pages, the FDT will be mapped using PTEs, so we need to
879 * be in the same PMD as the rest of the fixmap.
880 * On 4k pages, we'll use section mappings for the FDT so we only
881 * have to be in the same PUD.
883 BUILD_BUG_ON(dt_virt_base
% SZ_2M
);
885 BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END
) >> SWAPPER_TABLE_SHIFT
!=
886 __fix_to_virt(FIX_BTMAP_BEGIN
) >> SWAPPER_TABLE_SHIFT
);
888 offset
= dt_phys
% SWAPPER_BLOCK_SIZE
;
889 dt_virt
= (void *)dt_virt_base
+ offset
;
891 /* map the first chunk so we can read the size from the header */
892 create_mapping_noalloc(round_down(dt_phys
, SWAPPER_BLOCK_SIZE
),
893 dt_virt_base
, SWAPPER_BLOCK_SIZE
, prot
);
895 if (fdt_magic(dt_virt
) != FDT_MAGIC
)
898 *size
= fdt_totalsize(dt_virt
);
899 if (*size
> MAX_FDT_SIZE
)
902 if (offset
+ *size
> SWAPPER_BLOCK_SIZE
)
903 create_mapping_noalloc(round_down(dt_phys
, SWAPPER_BLOCK_SIZE
), dt_virt_base
,
904 round_up(offset
+ *size
, SWAPPER_BLOCK_SIZE
), prot
);
909 void *__init
fixmap_remap_fdt(phys_addr_t dt_phys
)
914 dt_virt
= __fixmap_remap_fdt(dt_phys
, &size
, PAGE_KERNEL_RO
);
918 memblock_reserve(dt_phys
, size
);
922 int __init
arch_ioremap_pud_supported(void)
924 /* only 4k granule supports level 1 block mappings */
925 return IS_ENABLED(CONFIG_ARM64_4K_PAGES
);
928 int __init
arch_ioremap_pmd_supported(void)
933 int pud_set_huge(pud_t
*pudp
, phys_addr_t phys
, pgprot_t prot
)
935 pgprot_t sect_prot
= __pgprot(PUD_TYPE_SECT
|
936 pgprot_val(mk_sect_prot(prot
)));
937 pud_t new_pud
= pfn_pud(__phys_to_pfn(phys
), sect_prot
);
939 /* Only allow permission changes for now */
940 if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp
)),
944 BUG_ON(phys
& ~PUD_MASK
);
945 set_pud(pudp
, new_pud
);
949 int pmd_set_huge(pmd_t
*pmdp
, phys_addr_t phys
, pgprot_t prot
)
951 pgprot_t sect_prot
= __pgprot(PMD_TYPE_SECT
|
952 pgprot_val(mk_sect_prot(prot
)));
953 pmd_t new_pmd
= pfn_pmd(__phys_to_pfn(phys
), sect_prot
);
955 /* Only allow permission changes for now */
956 if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp
)),
960 BUG_ON(phys
& ~PMD_MASK
);
961 set_pmd(pmdp
, new_pmd
);
965 int pud_clear_huge(pud_t
*pudp
)
967 if (!pud_sect(READ_ONCE(*pudp
)))
973 int pmd_clear_huge(pmd_t
*pmdp
)
975 if (!pmd_sect(READ_ONCE(*pmdp
)))
981 int pmd_free_pte_page(pmd_t
*pmdp
, unsigned long addr
)
986 pmd
= READ_ONCE(*pmdp
);
988 if (!pmd_present(pmd
))
990 if (!pmd_table(pmd
)) {
991 VM_WARN_ON(!pmd_table(pmd
));
995 table
= pte_offset_kernel(pmdp
, addr
);
997 __flush_tlb_kernel_pgtable(addr
);
998 pte_free_kernel(NULL
, table
);
1002 int pud_free_pmd_page(pud_t
*pudp
, unsigned long addr
)
1007 unsigned long next
, end
;
1009 pud
= READ_ONCE(*pudp
);
1011 if (!pud_present(pud
))
1013 if (!pud_table(pud
)) {
1014 VM_WARN_ON(!pud_table(pud
));
1018 table
= pmd_offset(pudp
, addr
);
1021 end
= addr
+ PUD_SIZE
;
1023 pmd_free_pte_page(pmdp
, next
);
1024 } while (pmdp
++, next
+= PMD_SIZE
, next
!= end
);
1027 __flush_tlb_kernel_pgtable(addr
);
1028 pmd_free(NULL
, table
);