2 * PowerPC64 port by Mike Corrigan and Dave Engebretsen
3 * {mikejc|engebret}@us.ibm.com
5 * Copyright (c) 2000 Mike Corrigan <mikejc@us.ibm.com>
7 * SMP scalability work:
8 * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
13 * PowerPC Hashed Page Table functions
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
24 #include <linux/spinlock.h>
25 #include <linux/errno.h>
26 #include <linux/sched.h>
27 #include <linux/proc_fs.h>
28 #include <linux/stat.h>
29 #include <linux/sysctl.h>
30 #include <linux/export.h>
31 #include <linux/ctype.h>
32 #include <linux/cache.h>
33 #include <linux/init.h>
34 #include <linux/signal.h>
35 #include <linux/memblock.h>
36 #include <linux/context_tracking.h>
37 #include <linux/libfdt.h>
39 #include <asm/processor.h>
40 #include <asm/pgtable.h>
42 #include <asm/mmu_context.h>
44 #include <asm/types.h>
45 #include <linux/uaccess.h>
46 #include <asm/machdep.h>
48 #include <asm/tlbflush.h>
52 #include <asm/cacheflush.h>
53 #include <asm/cputable.h>
54 #include <asm/sections.h>
55 #include <asm/copro.h>
57 #include <asm/code-patching.h>
58 #include <asm/fadump.h>
59 #include <asm/firmware.h>
61 #include <asm/trace.h>
65 #define DBG(fmt...) udbg_printf(fmt)
71 #define DBG_LOW(fmt...) udbg_printf(fmt)
73 #define DBG_LOW(fmt...)
81 * Note: pte --> Linux PTE
82 * HPTE --> PowerPC Hashed Page Table Entry
85 * htab_initialize is called with the MMU off (of course), but
86 * the kernel has been copied down to zero so it can directly
87 * reference global data. At this point it is very difficult
88 * to print debug info.
92 static unsigned long _SDR1
;
93 struct mmu_psize_def mmu_psize_defs
[MMU_PAGE_COUNT
];
94 EXPORT_SYMBOL_GPL(mmu_psize_defs
);
96 u8 hpte_page_sizes
[1 << LP_BITS
];
97 EXPORT_SYMBOL_GPL(hpte_page_sizes
);
99 struct hash_pte
*htab_address
;
100 unsigned long htab_size_bytes
;
101 unsigned long htab_hash_mask
;
102 EXPORT_SYMBOL_GPL(htab_hash_mask
);
103 int mmu_linear_psize
= MMU_PAGE_4K
;
104 EXPORT_SYMBOL_GPL(mmu_linear_psize
);
105 int mmu_virtual_psize
= MMU_PAGE_4K
;
106 int mmu_vmalloc_psize
= MMU_PAGE_4K
;
107 #ifdef CONFIG_SPARSEMEM_VMEMMAP
108 int mmu_vmemmap_psize
= MMU_PAGE_4K
;
110 int mmu_io_psize
= MMU_PAGE_4K
;
111 int mmu_kernel_ssize
= MMU_SEGSIZE_256M
;
112 EXPORT_SYMBOL_GPL(mmu_kernel_ssize
);
113 int mmu_highuser_ssize
= MMU_SEGSIZE_256M
;
114 u16 mmu_slb_size
= 64;
115 EXPORT_SYMBOL_GPL(mmu_slb_size
);
116 #ifdef CONFIG_PPC_64K_PAGES
117 int mmu_ci_restrictions
;
119 #ifdef CONFIG_DEBUG_PAGEALLOC
120 static u8
*linear_map_hash_slots
;
121 static unsigned long linear_map_hash_count
;
122 static DEFINE_SPINLOCK(linear_map_hash_lock
);
123 #endif /* CONFIG_DEBUG_PAGEALLOC */
124 struct mmu_hash_ops mmu_hash_ops
;
125 EXPORT_SYMBOL(mmu_hash_ops
);
127 /* There are definitions of page sizes arrays to be used when none
128 * is provided by the firmware.
131 /* Pre-POWER4 CPUs (4k pages only)
133 static struct mmu_psize_def mmu_psize_defaults_old
[] = {
137 .penc
= {[MMU_PAGE_4K
] = 0, [1 ... MMU_PAGE_COUNT
- 1] = -1},
143 /* POWER4, GPUL, POWER5
145 * Support for 16Mb large pages
147 static struct mmu_psize_def mmu_psize_defaults_gp
[] = {
151 .penc
= {[MMU_PAGE_4K
] = 0, [1 ... MMU_PAGE_COUNT
- 1] = -1},
158 .penc
= {[0 ... MMU_PAGE_16M
- 1] = -1, [MMU_PAGE_16M
] = 0,
159 [MMU_PAGE_16M
+ 1 ... MMU_PAGE_COUNT
- 1] = -1 },
166 * 'R' and 'C' update notes:
167 * - Under pHyp or KVM, the updatepp path will not set C, thus it *will*
168 * create writeable HPTEs without C set, because the hcall H_PROTECT
169 * that we use in that case will not update C
170 * - The above is however not a problem, because we also don't do that
171 * fancy "no flush" variant of eviction and we use H_REMOVE which will
172 * do the right thing and thus we don't have the race I described earlier
174 * - Under bare metal, we do have the race, so we need R and C set
175 * - We make sure R is always set and never lost
176 * - C is _PAGE_DIRTY, and *should* always be set for a writeable mapping
178 unsigned long htab_convert_pte_flags(unsigned long pteflags
)
180 unsigned long rflags
= 0;
182 /* _PAGE_EXEC -> NOEXEC */
183 if ((pteflags
& _PAGE_EXEC
) == 0)
187 * Linux uses slb key 0 for kernel and 1 for user.
188 * kernel RW areas are mapped with PPP=0b000
189 * User area is mapped with PPP=0b010 for read/write
190 * or PPP=0b011 for read-only (including writeable but clean pages).
192 if (pteflags
& _PAGE_PRIVILEGED
) {
194 * Kernel read only mapped with ppp bits 0b110
196 if (!(pteflags
& _PAGE_WRITE
)) {
197 if (mmu_has_feature(MMU_FTR_KERNEL_RO
))
198 rflags
|= (HPTE_R_PP0
| 0x2);
203 if (pteflags
& _PAGE_RWX
)
205 if (!((pteflags
& _PAGE_WRITE
) && (pteflags
& _PAGE_DIRTY
)))
209 * We can't allow hardware to update hpte bits. Hence always
210 * set 'R' bit and set 'C' if it is a write fault
214 if (pteflags
& _PAGE_DIRTY
)
220 if ((pteflags
& _PAGE_CACHE_CTL
) == _PAGE_TOLERANT
)
222 else if ((pteflags
& _PAGE_CACHE_CTL
) == _PAGE_NON_IDEMPOTENT
)
223 rflags
|= (HPTE_R_I
| HPTE_R_G
);
224 else if ((pteflags
& _PAGE_CACHE_CTL
) == _PAGE_SAO
)
225 rflags
|= (HPTE_R_W
| HPTE_R_I
| HPTE_R_M
);
228 * Add memory coherence if cache inhibited is not set
235 int htab_bolt_mapping(unsigned long vstart
, unsigned long vend
,
236 unsigned long pstart
, unsigned long prot
,
237 int psize
, int ssize
)
239 unsigned long vaddr
, paddr
;
240 unsigned int step
, shift
;
243 shift
= mmu_psize_defs
[psize
].shift
;
246 prot
= htab_convert_pte_flags(prot
);
248 DBG("htab_bolt_mapping(%lx..%lx -> %lx (%lx,%d,%d)\n",
249 vstart
, vend
, pstart
, prot
, psize
, ssize
);
251 for (vaddr
= vstart
, paddr
= pstart
; vaddr
< vend
;
252 vaddr
+= step
, paddr
+= step
) {
253 unsigned long hash
, hpteg
;
254 unsigned long vsid
= get_kernel_vsid(vaddr
, ssize
);
255 unsigned long vpn
= hpt_vpn(vaddr
, vsid
, ssize
);
256 unsigned long tprot
= prot
;
259 * If we hit a bad address return error.
263 /* Make kernel text executable */
264 if (overlaps_kernel_text(vaddr
, vaddr
+ step
))
267 /* Make kvm guest trampolines executable */
268 if (overlaps_kvm_tmp(vaddr
, vaddr
+ step
))
272 * If relocatable, check if it overlaps interrupt vectors that
273 * are copied down to real 0. For relocatable kernel
274 * (e.g. kdump case) we copy interrupt vectors down to real
275 * address 0. Mark that region as executable. This is
276 * because on p8 system with relocation on exception feature
277 * enabled, exceptions are raised with MMU (IR=DR=1) ON. Hence
278 * in order to execute the interrupt handlers in virtual
279 * mode the vector region need to be marked as executable.
281 if ((PHYSICAL_START
> MEMORY_START
) &&
282 overlaps_interrupt_vector_text(vaddr
, vaddr
+ step
))
285 hash
= hpt_hash(vpn
, shift
, ssize
);
286 hpteg
= ((hash
& htab_hash_mask
) * HPTES_PER_GROUP
);
288 BUG_ON(!mmu_hash_ops
.hpte_insert
);
289 ret
= mmu_hash_ops
.hpte_insert(hpteg
, vpn
, paddr
, tprot
,
290 HPTE_V_BOLTED
, psize
, psize
,
296 #ifdef CONFIG_DEBUG_PAGEALLOC
297 if (debug_pagealloc_enabled() &&
298 (paddr
>> PAGE_SHIFT
) < linear_map_hash_count
)
299 linear_map_hash_slots
[paddr
>> PAGE_SHIFT
] = ret
| 0x80;
300 #endif /* CONFIG_DEBUG_PAGEALLOC */
302 return ret
< 0 ? ret
: 0;
305 int htab_remove_mapping(unsigned long vstart
, unsigned long vend
,
306 int psize
, int ssize
)
309 unsigned int step
, shift
;
313 shift
= mmu_psize_defs
[psize
].shift
;
316 if (!mmu_hash_ops
.hpte_removebolted
)
319 for (vaddr
= vstart
; vaddr
< vend
; vaddr
+= step
) {
320 rc
= mmu_hash_ops
.hpte_removebolted(vaddr
, psize
, ssize
);
332 static bool disable_1tb_segments
= false;
334 static int __init
parse_disable_1tb_segments(char *p
)
336 disable_1tb_segments
= true;
339 early_param("disable_1tb_segments", parse_disable_1tb_segments
);
341 static int __init
htab_dt_scan_seg_sizes(unsigned long node
,
342 const char *uname
, int depth
,
345 const char *type
= of_get_flat_dt_prop(node
, "device_type", NULL
);
349 /* We are scanning "cpu" nodes only */
350 if (type
== NULL
|| strcmp(type
, "cpu") != 0)
353 prop
= of_get_flat_dt_prop(node
, "ibm,processor-segment-sizes", &size
);
356 for (; size
>= 4; size
-= 4, ++prop
) {
357 if (be32_to_cpu(prop
[0]) == 40) {
358 DBG("1T segment support detected\n");
360 if (disable_1tb_segments
) {
361 DBG("1T segments disabled by command line\n");
365 cur_cpu_spec
->mmu_features
|= MMU_FTR_1T_SEGMENT
;
369 cur_cpu_spec
->mmu_features
&= ~MMU_FTR_NO_SLBIE_B
;
373 static int __init
get_idx_from_shift(unsigned int shift
)
397 static int __init
htab_dt_scan_page_sizes(unsigned long node
,
398 const char *uname
, int depth
,
401 const char *type
= of_get_flat_dt_prop(node
, "device_type", NULL
);
405 /* We are scanning "cpu" nodes only */
406 if (type
== NULL
|| strcmp(type
, "cpu") != 0)
409 prop
= of_get_flat_dt_prop(node
, "ibm,segment-page-sizes", &size
);
413 pr_info("Page sizes from device-tree:\n");
415 cur_cpu_spec
->mmu_features
&= ~(MMU_FTR_16M_PAGE
);
417 unsigned int base_shift
= be32_to_cpu(prop
[0]);
418 unsigned int slbenc
= be32_to_cpu(prop
[1]);
419 unsigned int lpnum
= be32_to_cpu(prop
[2]);
420 struct mmu_psize_def
*def
;
423 size
-= 3; prop
+= 3;
424 base_idx
= get_idx_from_shift(base_shift
);
426 /* skip the pte encoding also */
427 prop
+= lpnum
* 2; size
-= lpnum
* 2;
430 def
= &mmu_psize_defs
[base_idx
];
431 if (base_idx
== MMU_PAGE_16M
)
432 cur_cpu_spec
->mmu_features
|= MMU_FTR_16M_PAGE
;
434 def
->shift
= base_shift
;
435 if (base_shift
<= 23)
438 def
->avpnm
= (1 << (base_shift
- 23)) - 1;
441 * We don't know for sure what's up with tlbiel, so
442 * for now we only set it for 4K and 64K pages
444 if (base_idx
== MMU_PAGE_4K
|| base_idx
== MMU_PAGE_64K
)
449 while (size
> 0 && lpnum
) {
450 unsigned int shift
= be32_to_cpu(prop
[0]);
451 int penc
= be32_to_cpu(prop
[1]);
453 prop
+= 2; size
-= 2;
456 idx
= get_idx_from_shift(shift
);
461 pr_err("Invalid penc for base_shift=%d "
462 "shift=%d\n", base_shift
, shift
);
464 def
->penc
[idx
] = penc
;
465 pr_info("base_shift=%d: shift=%d, sllp=0x%04lx,"
466 " avpnm=0x%08lx, tlbiel=%d, penc=%d\n",
467 base_shift
, shift
, def
->sllp
,
468 def
->avpnm
, def
->tlbiel
, def
->penc
[idx
]);
475 #ifdef CONFIG_HUGETLB_PAGE
476 /* Scan for 16G memory blocks that have been set aside for huge pages
477 * and reserve those blocks for 16G huge pages.
479 static int __init
htab_dt_scan_hugepage_blocks(unsigned long node
,
480 const char *uname
, int depth
,
482 const char *type
= of_get_flat_dt_prop(node
, "device_type", NULL
);
483 const __be64
*addr_prop
;
484 const __be32
*page_count_prop
;
485 unsigned int expected_pages
;
486 long unsigned int phys_addr
;
487 long unsigned int block_size
;
489 /* We are scanning "memory" nodes only */
490 if (type
== NULL
|| strcmp(type
, "memory") != 0)
493 /* This property is the log base 2 of the number of virtual pages that
494 * will represent this memory block. */
495 page_count_prop
= of_get_flat_dt_prop(node
, "ibm,expected#pages", NULL
);
496 if (page_count_prop
== NULL
)
498 expected_pages
= (1 << be32_to_cpu(page_count_prop
[0]));
499 addr_prop
= of_get_flat_dt_prop(node
, "reg", NULL
);
500 if (addr_prop
== NULL
)
502 phys_addr
= be64_to_cpu(addr_prop
[0]);
503 block_size
= be64_to_cpu(addr_prop
[1]);
504 if (block_size
!= (16 * GB
))
506 printk(KERN_INFO
"Huge page(16GB) memory: "
507 "addr = 0x%lX size = 0x%lX pages = %d\n",
508 phys_addr
, block_size
, expected_pages
);
509 if (phys_addr
+ (16 * GB
) <= memblock_end_of_DRAM()) {
510 memblock_reserve(phys_addr
, block_size
* expected_pages
);
511 add_gpage(phys_addr
, block_size
, expected_pages
);
515 #endif /* CONFIG_HUGETLB_PAGE */
517 static void mmu_psize_set_default_penc(void)
520 for (bpsize
= 0; bpsize
< MMU_PAGE_COUNT
; bpsize
++)
521 for (apsize
= 0; apsize
< MMU_PAGE_COUNT
; apsize
++)
522 mmu_psize_defs
[bpsize
].penc
[apsize
] = -1;
525 #ifdef CONFIG_PPC_64K_PAGES
527 static bool might_have_hea(void)
530 * The HEA ethernet adapter requires awareness of the
531 * GX bus. Without that awareness we can easily assume
532 * we will never see an HEA ethernet device.
534 #ifdef CONFIG_IBMEBUS
535 return !cpu_has_feature(CPU_FTR_ARCH_207S
) &&
536 firmware_has_feature(FW_FEATURE_SPLPAR
);
542 #endif /* #ifdef CONFIG_PPC_64K_PAGES */
544 static void __init
htab_scan_page_sizes(void)
548 /* se the invalid penc to -1 */
549 mmu_psize_set_default_penc();
551 /* Default to 4K pages only */
552 memcpy(mmu_psize_defs
, mmu_psize_defaults_old
,
553 sizeof(mmu_psize_defaults_old
));
556 * Try to find the available page sizes in the device-tree
558 rc
= of_scan_flat_dt(htab_dt_scan_page_sizes
, NULL
);
559 if (rc
== 0 && early_mmu_has_feature(MMU_FTR_16M_PAGE
)) {
561 * Nothing in the device-tree, but the CPU supports 16M pages,
562 * so let's fallback on a known size list for 16M capable CPUs.
564 memcpy(mmu_psize_defs
, mmu_psize_defaults_gp
,
565 sizeof(mmu_psize_defaults_gp
));
568 #ifdef CONFIG_HUGETLB_PAGE
569 /* Reserve 16G huge page memory sections for huge pages */
570 of_scan_flat_dt(htab_dt_scan_hugepage_blocks
, NULL
);
571 #endif /* CONFIG_HUGETLB_PAGE */
575 * Fill in the hpte_page_sizes[] array.
576 * We go through the mmu_psize_defs[] array looking for all the
577 * supported base/actual page size combinations. Each combination
578 * has a unique pagesize encoding (penc) value in the low bits of
579 * the LP field of the HPTE. For actual page sizes less than 1MB,
580 * some of the upper LP bits are used for RPN bits, meaning that
581 * we need to fill in several entries in hpte_page_sizes[].
583 * In diagrammatic form, with r = RPN bits and z = page size bits:
584 * PTE LP actual page size
591 * The zzzz bits are implementation-specific but are chosen so that
592 * no encoding for a larger page size uses the same value in its
593 * low-order N bits as the encoding for the 2^(12+N) byte page size
596 static void init_hpte_page_sizes(void)
599 long int shift
, penc
;
601 for (bp
= 0; bp
< MMU_PAGE_COUNT
; ++bp
) {
602 if (!mmu_psize_defs
[bp
].shift
)
603 continue; /* not a supported page size */
604 for (ap
= bp
; ap
< MMU_PAGE_COUNT
; ++ap
) {
605 penc
= mmu_psize_defs
[bp
].penc
[ap
];
608 shift
= mmu_psize_defs
[ap
].shift
- LP_SHIFT
;
610 continue; /* should never happen */
612 * For page sizes less than 1MB, this loop
613 * replicates the entry for all possible values
616 while (penc
< (1 << LP_BITS
)) {
617 hpte_page_sizes
[penc
] = (ap
<< 4) | bp
;
624 static void __init
htab_init_page_sizes(void)
626 init_hpte_page_sizes();
628 if (!debug_pagealloc_enabled()) {
630 * Pick a size for the linear mapping. Currently, we only
631 * support 16M, 1M and 4K which is the default
633 if (mmu_psize_defs
[MMU_PAGE_16M
].shift
)
634 mmu_linear_psize
= MMU_PAGE_16M
;
635 else if (mmu_psize_defs
[MMU_PAGE_1M
].shift
)
636 mmu_linear_psize
= MMU_PAGE_1M
;
639 #ifdef CONFIG_PPC_64K_PAGES
641 * Pick a size for the ordinary pages. Default is 4K, we support
642 * 64K for user mappings and vmalloc if supported by the processor.
643 * We only use 64k for ioremap if the processor
644 * (and firmware) support cache-inhibited large pages.
645 * If not, we use 4k and set mmu_ci_restrictions so that
646 * hash_page knows to switch processes that use cache-inhibited
647 * mappings to 4k pages.
649 if (mmu_psize_defs
[MMU_PAGE_64K
].shift
) {
650 mmu_virtual_psize
= MMU_PAGE_64K
;
651 mmu_vmalloc_psize
= MMU_PAGE_64K
;
652 if (mmu_linear_psize
== MMU_PAGE_4K
)
653 mmu_linear_psize
= MMU_PAGE_64K
;
654 if (mmu_has_feature(MMU_FTR_CI_LARGE_PAGE
)) {
656 * When running on pSeries using 64k pages for ioremap
657 * would stop us accessing the HEA ethernet. So if we
658 * have the chance of ever seeing one, stay at 4k.
660 if (!might_have_hea())
661 mmu_io_psize
= MMU_PAGE_64K
;
663 mmu_ci_restrictions
= 1;
665 #endif /* CONFIG_PPC_64K_PAGES */
667 #ifdef CONFIG_SPARSEMEM_VMEMMAP
668 /* We try to use 16M pages for vmemmap if that is supported
669 * and we have at least 1G of RAM at boot
671 if (mmu_psize_defs
[MMU_PAGE_16M
].shift
&&
672 memblock_phys_mem_size() >= 0x40000000)
673 mmu_vmemmap_psize
= MMU_PAGE_16M
;
674 else if (mmu_psize_defs
[MMU_PAGE_64K
].shift
)
675 mmu_vmemmap_psize
= MMU_PAGE_64K
;
677 mmu_vmemmap_psize
= MMU_PAGE_4K
;
678 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
680 printk(KERN_DEBUG
"Page orders: linear mapping = %d, "
681 "virtual = %d, io = %d"
682 #ifdef CONFIG_SPARSEMEM_VMEMMAP
686 mmu_psize_defs
[mmu_linear_psize
].shift
,
687 mmu_psize_defs
[mmu_virtual_psize
].shift
,
688 mmu_psize_defs
[mmu_io_psize
].shift
689 #ifdef CONFIG_SPARSEMEM_VMEMMAP
690 ,mmu_psize_defs
[mmu_vmemmap_psize
].shift
695 static int __init
htab_dt_scan_pftsize(unsigned long node
,
696 const char *uname
, int depth
,
699 const char *type
= of_get_flat_dt_prop(node
, "device_type", NULL
);
702 /* We are scanning "cpu" nodes only */
703 if (type
== NULL
|| strcmp(type
, "cpu") != 0)
706 prop
= of_get_flat_dt_prop(node
, "ibm,pft-size", NULL
);
708 /* pft_size[0] is the NUMA CEC cookie */
709 ppc64_pft_size
= be32_to_cpu(prop
[1]);
715 unsigned htab_shift_for_mem_size(unsigned long mem_size
)
717 unsigned memshift
= __ilog2(mem_size
);
718 unsigned pshift
= mmu_psize_defs
[mmu_virtual_psize
].shift
;
721 /* round mem_size up to next power of 2 */
722 if ((1UL << memshift
) < mem_size
)
725 /* aim for 2 pages / pteg */
726 pteg_shift
= memshift
- (pshift
+ 1);
729 * 2^11 PTEGS of 128 bytes each, ie. 2^18 bytes is the minimum htab
730 * size permitted by the architecture.
732 return max(pteg_shift
+ 7, 18U);
735 static unsigned long __init
htab_get_table_size(void)
737 /* If hash size isn't already provided by the platform, we try to
738 * retrieve it from the device-tree. If it's not there neither, we
739 * calculate it now based on the total RAM size
741 if (ppc64_pft_size
== 0)
742 of_scan_flat_dt(htab_dt_scan_pftsize
, NULL
);
744 return 1UL << ppc64_pft_size
;
746 return 1UL << htab_shift_for_mem_size(memblock_phys_mem_size());
749 #ifdef CONFIG_MEMORY_HOTPLUG
750 int create_section_mapping(unsigned long start
, unsigned long end
)
752 int rc
= htab_bolt_mapping(start
, end
, __pa(start
),
753 pgprot_val(PAGE_KERNEL
), mmu_linear_psize
,
757 int rc2
= htab_remove_mapping(start
, end
, mmu_linear_psize
,
759 BUG_ON(rc2
&& (rc2
!= -ENOENT
));
764 int remove_section_mapping(unsigned long start
, unsigned long end
)
766 int rc
= htab_remove_mapping(start
, end
, mmu_linear_psize
,
771 #endif /* CONFIG_MEMORY_HOTPLUG */
773 static void update_hid_for_hash(void)
776 unsigned long rb
= 3UL << PPC_BITLSHIFT(53); /* IS = 3 */
778 asm volatile("ptesync": : :"memory");
779 /* prs = 0, ric = 2, rs = 0, r = 1 is = 3 */
780 asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
781 : : "r"(rb
), "i"(0), "i"(0), "i"(2), "r"(0) : "memory");
782 asm volatile("eieio; tlbsync; ptesync; isync; slbia": : :"memory");
786 hid0
= mfspr(SPRN_HID0
);
787 hid0
&= ~HID0_POWER9_RADIX
;
788 mtspr(SPRN_HID0
, hid0
);
789 asm volatile("isync": : :"memory");
791 /* Wait for it to happen */
792 while ((mfspr(SPRN_HID0
) & HID0_POWER9_RADIX
))
796 static void __init
hash_init_partition_table(phys_addr_t hash_table
,
797 unsigned long htab_size
)
799 mmu_partition_table_init();
802 * PS field (VRMA page size) is not used for LPID 0, hence set to 0.
803 * For now, UPRT is 0 and we have no segment table.
805 htab_size
= __ilog2(htab_size
) - 18;
806 mmu_partition_table_set_entry(0, hash_table
| htab_size
, 0);
807 pr_info("Partition table %p\n", partition_tb
);
808 if (cpu_has_feature(CPU_FTR_POWER9_DD1
))
809 update_hid_for_hash();
812 static void __init
htab_initialize(void)
815 unsigned long pteg_count
;
817 unsigned long base
= 0, size
= 0;
818 struct memblock_region
*reg
;
820 DBG(" -> htab_initialize()\n");
822 if (mmu_has_feature(MMU_FTR_1T_SEGMENT
)) {
823 mmu_kernel_ssize
= MMU_SEGSIZE_1T
;
824 mmu_highuser_ssize
= MMU_SEGSIZE_1T
;
825 printk(KERN_INFO
"Using 1TB segments\n");
829 * Calculate the required size of the htab. We want the number of
830 * PTEGs to equal one half the number of real pages.
832 htab_size_bytes
= htab_get_table_size();
833 pteg_count
= htab_size_bytes
>> 7;
835 htab_hash_mask
= pteg_count
- 1;
837 if (firmware_has_feature(FW_FEATURE_LPAR
) ||
838 firmware_has_feature(FW_FEATURE_PS3_LV1
)) {
839 /* Using a hypervisor which owns the htab */
842 #ifdef CONFIG_FA_DUMP
844 * If firmware assisted dump is active firmware preserves
845 * the contents of htab along with entire partition memory.
846 * Clear the htab if firmware assisted dump is active so
847 * that we dont end up using old mappings.
849 if (is_fadump_active() && mmu_hash_ops
.hpte_clear_all
)
850 mmu_hash_ops
.hpte_clear_all();
853 unsigned long limit
= MEMBLOCK_ALLOC_ANYWHERE
;
855 #ifdef CONFIG_PPC_CELL
857 * Cell may require the hash table down low when using the
858 * Axon IOMMU in order to fit the dynamic region over it, see
859 * comments in cell/iommu.c
861 if (fdt_subnode_offset(initial_boot_params
, 0, "axon") > 0) {
863 pr_info("Hash table forced below 2G for Axon IOMMU\n");
865 #endif /* CONFIG_PPC_CELL */
867 table
= memblock_alloc_base(htab_size_bytes
, htab_size_bytes
,
870 DBG("Hash table allocated at %lx, size: %lx\n", table
,
873 htab_address
= __va(table
);
875 /* htab absolute addr + encoded htabsize */
876 _SDR1
= table
+ __ilog2(htab_size_bytes
) - 18;
878 /* Initialize the HPT with no entries */
879 memset((void *)table
, 0, htab_size_bytes
);
881 if (!cpu_has_feature(CPU_FTR_ARCH_300
))
883 mtspr(SPRN_SDR1
, _SDR1
);
885 hash_init_partition_table(table
, htab_size_bytes
);
888 prot
= pgprot_val(PAGE_KERNEL
);
890 #ifdef CONFIG_DEBUG_PAGEALLOC
891 if (debug_pagealloc_enabled()) {
892 linear_map_hash_count
= memblock_end_of_DRAM() >> PAGE_SHIFT
;
893 linear_map_hash_slots
= __va(memblock_alloc_base(
894 linear_map_hash_count
, 1, ppc64_rma_size
));
895 memset(linear_map_hash_slots
, 0, linear_map_hash_count
);
897 #endif /* CONFIG_DEBUG_PAGEALLOC */
899 /* On U3 based machines, we need to reserve the DART area and
900 * _NOT_ map it to avoid cache paradoxes as it's remapped non
904 /* create bolted the linear mapping in the hash table */
905 for_each_memblock(memory
, reg
) {
906 base
= (unsigned long)__va(reg
->base
);
909 DBG("creating mapping for region: %lx..%lx (prot: %lx)\n",
912 BUG_ON(htab_bolt_mapping(base
, base
+ size
, __pa(base
),
913 prot
, mmu_linear_psize
, mmu_kernel_ssize
));
915 memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE
);
918 * If we have a memory_limit and we've allocated TCEs then we need to
919 * explicitly map the TCE area at the top of RAM. We also cope with the
920 * case that the TCEs start below memory_limit.
921 * tce_alloc_start/end are 16MB aligned so the mapping should work
922 * for either 4K or 16MB pages.
924 if (tce_alloc_start
) {
925 tce_alloc_start
= (unsigned long)__va(tce_alloc_start
);
926 tce_alloc_end
= (unsigned long)__va(tce_alloc_end
);
928 if (base
+ size
>= tce_alloc_start
)
929 tce_alloc_start
= base
+ size
+ 1;
931 BUG_ON(htab_bolt_mapping(tce_alloc_start
, tce_alloc_end
,
932 __pa(tce_alloc_start
), prot
,
933 mmu_linear_psize
, mmu_kernel_ssize
));
937 DBG(" <- htab_initialize()\n");
942 void __init
hash__early_init_devtree(void)
944 /* Initialize segment sizes */
945 of_scan_flat_dt(htab_dt_scan_seg_sizes
, NULL
);
947 /* Initialize page sizes */
948 htab_scan_page_sizes();
951 void __init
hash__early_init_mmu(void)
953 htab_init_page_sizes();
956 * initialize page table size
958 __pte_frag_nr
= H_PTE_FRAG_NR
;
959 __pte_frag_size_shift
= H_PTE_FRAG_SIZE_SHIFT
;
961 __pte_index_size
= H_PTE_INDEX_SIZE
;
962 __pmd_index_size
= H_PMD_INDEX_SIZE
;
963 __pud_index_size
= H_PUD_INDEX_SIZE
;
964 __pgd_index_size
= H_PGD_INDEX_SIZE
;
965 __pmd_cache_index
= H_PMD_CACHE_INDEX
;
966 __pte_table_size
= H_PTE_TABLE_SIZE
;
967 __pmd_table_size
= H_PMD_TABLE_SIZE
;
968 __pud_table_size
= H_PUD_TABLE_SIZE
;
969 __pgd_table_size
= H_PGD_TABLE_SIZE
;
971 * 4k use hugepd format, so for hash set then to
978 __kernel_virt_start
= H_KERN_VIRT_START
;
979 __kernel_virt_size
= H_KERN_VIRT_SIZE
;
980 __vmalloc_start
= H_VMALLOC_START
;
981 __vmalloc_end
= H_VMALLOC_END
;
982 vmemmap
= (struct page
*)H_VMEMMAP_BASE
;
983 ioremap_bot
= IOREMAP_BASE
;
986 pci_io_base
= ISA_IO_BASE
;
989 /* Select appropriate backend */
990 if (firmware_has_feature(FW_FEATURE_PS3_LV1
))
992 else if (firmware_has_feature(FW_FEATURE_LPAR
))
994 else if (IS_ENABLED(CONFIG_PPC_NATIVE
))
997 if (!mmu_hash_ops
.hpte_insert
)
998 panic("hash__early_init_mmu: No MMU hash ops defined!\n");
1000 /* Initialize the MMU Hash table and create the linear mapping
1001 * of memory. Has to be done before SLB initialization as this is
1002 * currently where the page size encoding is obtained.
1006 pr_info("Initializing hash mmu with SLB\n");
1007 /* Initialize SLB management */
1012 void hash__early_init_mmu_secondary(void)
1014 /* Initialize hash table for that CPU */
1015 if (!firmware_has_feature(FW_FEATURE_LPAR
)) {
1017 if (cpu_has_feature(CPU_FTR_POWER9_DD1
))
1018 update_hid_for_hash();
1020 if (!cpu_has_feature(CPU_FTR_ARCH_300
))
1021 mtspr(SPRN_SDR1
, _SDR1
);
1024 __pa(partition_tb
) | (PATB_SIZE_SHIFT
- 12));
1026 /* Initialize SLB */
1029 #endif /* CONFIG_SMP */
1032 * Called by asm hashtable.S for doing lazy icache flush
1034 unsigned int hash_page_do_lazy_icache(unsigned int pp
, pte_t pte
, int trap
)
1038 if (!pfn_valid(pte_pfn(pte
)))
1041 page
= pte_page(pte
);
1044 if (!test_bit(PG_arch_1
, &page
->flags
) && !PageReserved(page
)) {
1045 if (trap
== 0x400) {
1046 flush_dcache_icache_page(page
);
1047 set_bit(PG_arch_1
, &page
->flags
);
1054 #ifdef CONFIG_PPC_MM_SLICES
1055 static unsigned int get_paca_psize(unsigned long addr
)
1058 unsigned char *hpsizes
;
1059 unsigned long index
, mask_index
;
1061 if (addr
< SLICE_LOW_TOP
) {
1062 lpsizes
= get_paca()->mm_ctx_low_slices_psize
;
1063 index
= GET_LOW_SLICE_INDEX(addr
);
1064 return (lpsizes
>> (index
* 4)) & 0xF;
1066 hpsizes
= get_paca()->mm_ctx_high_slices_psize
;
1067 index
= GET_HIGH_SLICE_INDEX(addr
);
1068 mask_index
= index
& 0x1;
1069 return (hpsizes
[index
>> 1] >> (mask_index
* 4)) & 0xF;
1073 unsigned int get_paca_psize(unsigned long addr
)
1075 return get_paca()->mm_ctx_user_psize
;
1080 * Demote a segment to using 4k pages.
1081 * For now this makes the whole process use 4k pages.
1083 #ifdef CONFIG_PPC_64K_PAGES
1084 void demote_segment_4k(struct mm_struct
*mm
, unsigned long addr
)
1086 if (get_slice_psize(mm
, addr
) == MMU_PAGE_4K
)
1088 slice_set_range_psize(mm
, addr
, 1, MMU_PAGE_4K
);
1089 copro_flush_all_slbs(mm
);
1090 if ((get_paca_psize(addr
) != MMU_PAGE_4K
) && (current
->mm
== mm
)) {
1092 copy_mm_to_paca(&mm
->context
);
1093 slb_flush_and_rebolt();
1096 #endif /* CONFIG_PPC_64K_PAGES */
1098 #ifdef CONFIG_PPC_SUBPAGE_PROT
1100 * This looks up a 2-bit protection code for a 4k subpage of a 64k page.
1101 * Userspace sets the subpage permissions using the subpage_prot system call.
1103 * Result is 0: full permissions, _PAGE_RW: read-only,
1104 * _PAGE_RWX: no access.
1106 static int subpage_protection(struct mm_struct
*mm
, unsigned long ea
)
1108 struct subpage_prot_table
*spt
= &mm
->context
.spt
;
1112 if (ea
>= spt
->maxaddr
)
1114 if (ea
< 0x100000000UL
) {
1115 /* addresses below 4GB use spt->low_prot */
1116 sbpm
= spt
->low_prot
;
1118 sbpm
= spt
->protptrs
[ea
>> SBP_L3_SHIFT
];
1122 sbpp
= sbpm
[(ea
>> SBP_L2_SHIFT
) & (SBP_L2_COUNT
- 1)];
1125 spp
= sbpp
[(ea
>> PAGE_SHIFT
) & (SBP_L1_COUNT
- 1)];
1127 /* extract 2-bit bitfield for this 4k subpage */
1128 spp
>>= 30 - 2 * ((ea
>> 12) & 0xf);
1131 * 0 -> full premission
1134 * We return the flag that need to be cleared.
1136 spp
= ((spp
& 2) ? _PAGE_RWX
: 0) | ((spp
& 1) ? _PAGE_WRITE
: 0);
1140 #else /* CONFIG_PPC_SUBPAGE_PROT */
1141 static inline int subpage_protection(struct mm_struct
*mm
, unsigned long ea
)
1147 void hash_failure_debug(unsigned long ea
, unsigned long access
,
1148 unsigned long vsid
, unsigned long trap
,
1149 int ssize
, int psize
, int lpsize
, unsigned long pte
)
1151 if (!printk_ratelimit())
1153 pr_info("mm: Hashing failure ! EA=0x%lx access=0x%lx current=%s\n",
1154 ea
, access
, current
->comm
);
1155 pr_info(" trap=0x%lx vsid=0x%lx ssize=%d base psize=%d psize %d pte=0x%lx\n",
1156 trap
, vsid
, ssize
, psize
, lpsize
, pte
);
1159 static void check_paca_psize(unsigned long ea
, struct mm_struct
*mm
,
1160 int psize
, bool user_region
)
1163 if (psize
!= get_paca_psize(ea
)) {
1164 copy_mm_to_paca(&mm
->context
);
1165 slb_flush_and_rebolt();
1167 } else if (get_paca()->vmalloc_sllp
!=
1168 mmu_psize_defs
[mmu_vmalloc_psize
].sllp
) {
1169 get_paca()->vmalloc_sllp
=
1170 mmu_psize_defs
[mmu_vmalloc_psize
].sllp
;
1171 slb_vmalloc_update();
1177 * 1 - normal page fault
1178 * -1 - critical hash insertion error
1179 * -2 - access not permitted by subpage protection mechanism
1181 int hash_page_mm(struct mm_struct
*mm
, unsigned long ea
,
1182 unsigned long access
, unsigned long trap
,
1183 unsigned long flags
)
1186 enum ctx_state prev_state
= exception_enter();
1191 const struct cpumask
*tmp
;
1192 int rc
, user_region
= 0;
1195 DBG_LOW("hash_page(ea=%016lx, access=%lx, trap=%lx\n",
1197 trace_hash_fault(ea
, access
, trap
);
1199 /* Get region & vsid */
1200 switch (REGION_ID(ea
)) {
1201 case USER_REGION_ID
:
1204 DBG_LOW(" user region with no mm !\n");
1208 psize
= get_slice_psize(mm
, ea
);
1209 ssize
= user_segment_size(ea
);
1210 vsid
= get_vsid(mm
->context
.id
, ea
, ssize
);
1212 case VMALLOC_REGION_ID
:
1213 vsid
= get_kernel_vsid(ea
, mmu_kernel_ssize
);
1214 if (ea
< VMALLOC_END
)
1215 psize
= mmu_vmalloc_psize
;
1217 psize
= mmu_io_psize
;
1218 ssize
= mmu_kernel_ssize
;
1221 /* Not a valid range
1222 * Send the problem up to do_page_fault
1227 DBG_LOW(" mm=%p, mm->pgdir=%p, vsid=%016lx\n", mm
, mm
->pgd
, vsid
);
1231 DBG_LOW("Bad address!\n");
1237 if (pgdir
== NULL
) {
1242 /* Check CPU locality */
1243 tmp
= cpumask_of(smp_processor_id());
1244 if (user_region
&& cpumask_equal(mm_cpumask(mm
), tmp
))
1245 flags
|= HPTE_LOCAL_UPDATE
;
1247 #ifndef CONFIG_PPC_64K_PAGES
1248 /* If we use 4K pages and our psize is not 4K, then we might
1249 * be hitting a special driver mapping, and need to align the
1250 * address before we fetch the PTE.
1252 * It could also be a hugepage mapping, in which case this is
1253 * not necessary, but it's not harmful, either.
1255 if (psize
!= MMU_PAGE_4K
)
1256 ea
&= ~((1ul << mmu_psize_defs
[psize
].shift
) - 1);
1257 #endif /* CONFIG_PPC_64K_PAGES */
1259 /* Get PTE and page size from page tables */
1260 ptep
= __find_linux_pte_or_hugepte(pgdir
, ea
, &is_thp
, &hugeshift
);
1261 if (ptep
== NULL
|| !pte_present(*ptep
)) {
1262 DBG_LOW(" no PTE !\n");
1267 /* Add _PAGE_PRESENT to the required access perm */
1268 access
|= _PAGE_PRESENT
;
1270 /* Pre-check access permissions (will be re-checked atomically
1271 * in __hash_page_XX but this pre-check is a fast path
1273 if (!check_pte_access(access
, pte_val(*ptep
))) {
1274 DBG_LOW(" no access !\n");
1281 rc
= __hash_page_thp(ea
, access
, vsid
, (pmd_t
*)ptep
,
1282 trap
, flags
, ssize
, psize
);
1283 #ifdef CONFIG_HUGETLB_PAGE
1285 rc
= __hash_page_huge(ea
, access
, vsid
, ptep
, trap
,
1286 flags
, ssize
, hugeshift
, psize
);
1290 * if we have hugeshift, and is not transhuge with
1291 * hugetlb disabled, something is really wrong.
1297 if (current
->mm
== mm
)
1298 check_paca_psize(ea
, mm
, psize
, user_region
);
1303 #ifndef CONFIG_PPC_64K_PAGES
1304 DBG_LOW(" i-pte: %016lx\n", pte_val(*ptep
));
1306 DBG_LOW(" i-pte: %016lx %016lx\n", pte_val(*ptep
),
1307 pte_val(*(ptep
+ PTRS_PER_PTE
)));
1309 /* Do actual hashing */
1310 #ifdef CONFIG_PPC_64K_PAGES
1311 /* If H_PAGE_4K_PFN is set, make sure this is a 4k segment */
1312 if ((pte_val(*ptep
) & H_PAGE_4K_PFN
) && psize
== MMU_PAGE_64K
) {
1313 demote_segment_4k(mm
, ea
);
1314 psize
= MMU_PAGE_4K
;
1317 /* If this PTE is non-cacheable and we have restrictions on
1318 * using non cacheable large pages, then we switch to 4k
1320 if (mmu_ci_restrictions
&& psize
== MMU_PAGE_64K
&& pte_ci(*ptep
)) {
1322 demote_segment_4k(mm
, ea
);
1323 psize
= MMU_PAGE_4K
;
1324 } else if (ea
< VMALLOC_END
) {
1326 * some driver did a non-cacheable mapping
1327 * in vmalloc space, so switch vmalloc
1330 printk(KERN_ALERT
"Reducing vmalloc segment "
1331 "to 4kB pages because of "
1332 "non-cacheable mapping\n");
1333 psize
= mmu_vmalloc_psize
= MMU_PAGE_4K
;
1334 copro_flush_all_slbs(mm
);
1338 #endif /* CONFIG_PPC_64K_PAGES */
1340 if (current
->mm
== mm
)
1341 check_paca_psize(ea
, mm
, psize
, user_region
);
1343 #ifdef CONFIG_PPC_64K_PAGES
1344 if (psize
== MMU_PAGE_64K
)
1345 rc
= __hash_page_64K(ea
, access
, vsid
, ptep
, trap
,
1348 #endif /* CONFIG_PPC_64K_PAGES */
1350 int spp
= subpage_protection(mm
, ea
);
1354 rc
= __hash_page_4K(ea
, access
, vsid
, ptep
, trap
,
1358 /* Dump some info in case of hash insertion failure, they should
1359 * never happen so it is really useful to know if/when they do
1362 hash_failure_debug(ea
, access
, vsid
, trap
, ssize
, psize
,
1363 psize
, pte_val(*ptep
));
1364 #ifndef CONFIG_PPC_64K_PAGES
1365 DBG_LOW(" o-pte: %016lx\n", pte_val(*ptep
));
1367 DBG_LOW(" o-pte: %016lx %016lx\n", pte_val(*ptep
),
1368 pte_val(*(ptep
+ PTRS_PER_PTE
)));
1370 DBG_LOW(" -> rc=%d\n", rc
);
1373 exception_exit(prev_state
);
1376 EXPORT_SYMBOL_GPL(hash_page_mm
);
1378 int hash_page(unsigned long ea
, unsigned long access
, unsigned long trap
,
1379 unsigned long dsisr
)
1381 unsigned long flags
= 0;
1382 struct mm_struct
*mm
= current
->mm
;
1384 if (REGION_ID(ea
) == VMALLOC_REGION_ID
)
1387 if (dsisr
& DSISR_NOHPTE
)
1388 flags
|= HPTE_NOHPTE_UPDATE
;
1390 return hash_page_mm(mm
, ea
, access
, trap
, flags
);
1392 EXPORT_SYMBOL_GPL(hash_page
);
1394 int __hash_page(unsigned long ea
, unsigned long msr
, unsigned long trap
,
1395 unsigned long dsisr
)
1397 unsigned long access
= _PAGE_PRESENT
| _PAGE_READ
;
1398 unsigned long flags
= 0;
1399 struct mm_struct
*mm
= current
->mm
;
1401 if (REGION_ID(ea
) == VMALLOC_REGION_ID
)
1404 if (dsisr
& DSISR_NOHPTE
)
1405 flags
|= HPTE_NOHPTE_UPDATE
;
1407 if (dsisr
& DSISR_ISSTORE
)
1408 access
|= _PAGE_WRITE
;
1410 * We set _PAGE_PRIVILEGED only when
1411 * kernel mode access kernel space.
1413 * _PAGE_PRIVILEGED is NOT set
1414 * 1) when kernel mode access user space
1415 * 2) user space access kernel space.
1417 access
|= _PAGE_PRIVILEGED
;
1418 if ((msr
& MSR_PR
) || (REGION_ID(ea
) == USER_REGION_ID
))
1419 access
&= ~_PAGE_PRIVILEGED
;
1422 access
|= _PAGE_EXEC
;
1424 return hash_page_mm(mm
, ea
, access
, trap
, flags
);
1427 #ifdef CONFIG_PPC_MM_SLICES
1428 static bool should_hash_preload(struct mm_struct
*mm
, unsigned long ea
)
1430 int psize
= get_slice_psize(mm
, ea
);
1432 /* We only prefault standard pages for now */
1433 if (unlikely(psize
!= mm
->context
.user_psize
))
1437 * Don't prefault if subpage protection is enabled for the EA.
1439 if (unlikely((psize
== MMU_PAGE_4K
) && subpage_protection(mm
, ea
)))
1445 static bool should_hash_preload(struct mm_struct
*mm
, unsigned long ea
)
1451 void hash_preload(struct mm_struct
*mm
, unsigned long ea
,
1452 unsigned long access
, unsigned long trap
)
1458 unsigned long flags
;
1459 int rc
, ssize
, update_flags
= 0;
1461 BUG_ON(REGION_ID(ea
) != USER_REGION_ID
);
1463 if (!should_hash_preload(mm
, ea
))
1466 DBG_LOW("hash_preload(mm=%p, mm->pgdir=%p, ea=%016lx, access=%lx,"
1467 " trap=%lx\n", mm
, mm
->pgd
, ea
, access
, trap
);
1469 /* Get Linux PTE if available */
1475 ssize
= user_segment_size(ea
);
1476 vsid
= get_vsid(mm
->context
.id
, ea
, ssize
);
1480 * Hash doesn't like irqs. Walking linux page table with irq disabled
1481 * saves us from holding multiple locks.
1483 local_irq_save(flags
);
1486 * THP pages use update_mmu_cache_pmd. We don't do
1487 * hash preload there. Hence can ignore THP here
1489 ptep
= find_linux_pte_or_hugepte(pgdir
, ea
, NULL
, &hugepage_shift
);
1493 WARN_ON(hugepage_shift
);
1494 #ifdef CONFIG_PPC_64K_PAGES
1495 /* If either H_PAGE_4K_PFN or cache inhibited is set (and we are on
1496 * a 64K kernel), then we don't preload, hash_page() will take
1497 * care of it once we actually try to access the page.
1498 * That way we don't have to duplicate all of the logic for segment
1499 * page size demotion here
1501 if ((pte_val(*ptep
) & H_PAGE_4K_PFN
) || pte_ci(*ptep
))
1503 #endif /* CONFIG_PPC_64K_PAGES */
1505 /* Is that local to this CPU ? */
1506 if (cpumask_equal(mm_cpumask(mm
), cpumask_of(smp_processor_id())))
1507 update_flags
|= HPTE_LOCAL_UPDATE
;
1510 #ifdef CONFIG_PPC_64K_PAGES
1511 if (mm
->context
.user_psize
== MMU_PAGE_64K
)
1512 rc
= __hash_page_64K(ea
, access
, vsid
, ptep
, trap
,
1513 update_flags
, ssize
);
1515 #endif /* CONFIG_PPC_64K_PAGES */
1516 rc
= __hash_page_4K(ea
, access
, vsid
, ptep
, trap
, update_flags
,
1517 ssize
, subpage_protection(mm
, ea
));
1519 /* Dump some info in case of hash insertion failure, they should
1520 * never happen so it is really useful to know if/when they do
1523 hash_failure_debug(ea
, access
, vsid
, trap
, ssize
,
1524 mm
->context
.user_psize
,
1525 mm
->context
.user_psize
,
1528 local_irq_restore(flags
);
1531 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1532 static inline void tm_flush_hash_page(int local
)
1535 * Transactions are not aborted by tlbiel, only tlbie. Without, syncing a
1536 * page back to a block device w/PIO could pick up transactional data
1537 * (bad!) so we force an abort here. Before the sync the page will be
1538 * made read-only, which will flush_hash_page. BIG ISSUE here: if the
1539 * kernel uses a page from userspace without unmapping it first, it may
1540 * see the speculated version.
1542 if (local
&& cpu_has_feature(CPU_FTR_TM
) && current
->thread
.regs
&&
1543 MSR_TM_ACTIVE(current
->thread
.regs
->msr
)) {
1545 tm_abort(TM_CAUSE_TLBI
);
1549 static inline void tm_flush_hash_page(int local
)
1554 /* WARNING: This is called from hash_low_64.S, if you change this prototype,
1555 * do not forget to update the assembly call site !
1557 void flush_hash_page(unsigned long vpn
, real_pte_t pte
, int psize
, int ssize
,
1558 unsigned long flags
)
1560 unsigned long hash
, index
, shift
, hidx
, slot
;
1561 int local
= flags
& HPTE_LOCAL_UPDATE
;
1563 DBG_LOW("flush_hash_page(vpn=%016lx)\n", vpn
);
1564 pte_iterate_hashed_subpages(pte
, psize
, vpn
, index
, shift
) {
1565 hash
= hpt_hash(vpn
, shift
, ssize
);
1566 hidx
= __rpte_to_hidx(pte
, index
);
1567 if (hidx
& _PTEIDX_SECONDARY
)
1569 slot
= (hash
& htab_hash_mask
) * HPTES_PER_GROUP
;
1570 slot
+= hidx
& _PTEIDX_GROUP_IX
;
1571 DBG_LOW(" sub %ld: hash=%lx, hidx=%lx\n", index
, slot
, hidx
);
1573 * We use same base page size and actual psize, because we don't
1574 * use these functions for hugepage
1576 mmu_hash_ops
.hpte_invalidate(slot
, vpn
, psize
, psize
,
1578 } pte_iterate_hashed_end();
1580 tm_flush_hash_page(local
);
1583 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1584 void flush_hash_hugepage(unsigned long vsid
, unsigned long addr
,
1585 pmd_t
*pmdp
, unsigned int psize
, int ssize
,
1586 unsigned long flags
)
1588 int i
, max_hpte_count
, valid
;
1589 unsigned long s_addr
;
1590 unsigned char *hpte_slot_array
;
1591 unsigned long hidx
, shift
, vpn
, hash
, slot
;
1592 int local
= flags
& HPTE_LOCAL_UPDATE
;
1594 s_addr
= addr
& HPAGE_PMD_MASK
;
1595 hpte_slot_array
= get_hpte_slot_array(pmdp
);
1597 * IF we try to do a HUGE PTE update after a withdraw is done.
1598 * we will find the below NULL. This happens when we do
1599 * split_huge_page_pmd
1601 if (!hpte_slot_array
)
1604 if (mmu_hash_ops
.hugepage_invalidate
) {
1605 mmu_hash_ops
.hugepage_invalidate(vsid
, s_addr
, hpte_slot_array
,
1606 psize
, ssize
, local
);
1610 * No bluk hpte removal support, invalidate each entry
1612 shift
= mmu_psize_defs
[psize
].shift
;
1613 max_hpte_count
= HPAGE_PMD_SIZE
>> shift
;
1614 for (i
= 0; i
< max_hpte_count
; i
++) {
1616 * 8 bits per each hpte entries
1617 * 000| [ secondary group (one bit) | hidx (3 bits) | valid bit]
1619 valid
= hpte_valid(hpte_slot_array
, i
);
1622 hidx
= hpte_hash_index(hpte_slot_array
, i
);
1625 addr
= s_addr
+ (i
* (1ul << shift
));
1626 vpn
= hpt_vpn(addr
, vsid
, ssize
);
1627 hash
= hpt_hash(vpn
, shift
, ssize
);
1628 if (hidx
& _PTEIDX_SECONDARY
)
1631 slot
= (hash
& htab_hash_mask
) * HPTES_PER_GROUP
;
1632 slot
+= hidx
& _PTEIDX_GROUP_IX
;
1633 mmu_hash_ops
.hpte_invalidate(slot
, vpn
, psize
,
1634 MMU_PAGE_16M
, ssize
, local
);
1637 tm_flush_hash_page(local
);
1639 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1641 void flush_hash_range(unsigned long number
, int local
)
1643 if (mmu_hash_ops
.flush_hash_range
)
1644 mmu_hash_ops
.flush_hash_range(number
, local
);
1647 struct ppc64_tlb_batch
*batch
=
1648 this_cpu_ptr(&ppc64_tlb_batch
);
1650 for (i
= 0; i
< number
; i
++)
1651 flush_hash_page(batch
->vpn
[i
], batch
->pte
[i
],
1652 batch
->psize
, batch
->ssize
, local
);
1657 * low_hash_fault is called when we the low level hash code failed
1658 * to instert a PTE due to an hypervisor error
1660 void low_hash_fault(struct pt_regs
*regs
, unsigned long address
, int rc
)
1662 enum ctx_state prev_state
= exception_enter();
1664 if (user_mode(regs
)) {
1665 #ifdef CONFIG_PPC_SUBPAGE_PROT
1667 _exception(SIGSEGV
, regs
, SEGV_ACCERR
, address
);
1670 _exception(SIGBUS
, regs
, BUS_ADRERR
, address
);
1672 bad_page_fault(regs
, address
, SIGBUS
);
1674 exception_exit(prev_state
);
1677 long hpte_insert_repeating(unsigned long hash
, unsigned long vpn
,
1678 unsigned long pa
, unsigned long rflags
,
1679 unsigned long vflags
, int psize
, int ssize
)
1681 unsigned long hpte_group
;
1685 hpte_group
= ((hash
& htab_hash_mask
) *
1686 HPTES_PER_GROUP
) & ~0x7UL
;
1688 /* Insert into the hash table, primary slot */
1689 slot
= mmu_hash_ops
.hpte_insert(hpte_group
, vpn
, pa
, rflags
, vflags
,
1690 psize
, psize
, ssize
);
1692 /* Primary is full, try the secondary */
1693 if (unlikely(slot
== -1)) {
1694 hpte_group
= ((~hash
& htab_hash_mask
) *
1695 HPTES_PER_GROUP
) & ~0x7UL
;
1696 slot
= mmu_hash_ops
.hpte_insert(hpte_group
, vpn
, pa
, rflags
,
1697 vflags
| HPTE_V_SECONDARY
,
1698 psize
, psize
, ssize
);
1701 hpte_group
= ((hash
& htab_hash_mask
) *
1702 HPTES_PER_GROUP
)&~0x7UL
;
1704 mmu_hash_ops
.hpte_remove(hpte_group
);
1712 #ifdef CONFIG_DEBUG_PAGEALLOC
1713 static void kernel_map_linear_page(unsigned long vaddr
, unsigned long lmi
)
1716 unsigned long vsid
= get_kernel_vsid(vaddr
, mmu_kernel_ssize
);
1717 unsigned long vpn
= hpt_vpn(vaddr
, vsid
, mmu_kernel_ssize
);
1718 unsigned long mode
= htab_convert_pte_flags(pgprot_val(PAGE_KERNEL
));
1721 hash
= hpt_hash(vpn
, PAGE_SHIFT
, mmu_kernel_ssize
);
1723 /* Don't create HPTE entries for bad address */
1727 ret
= hpte_insert_repeating(hash
, vpn
, __pa(vaddr
), mode
,
1729 mmu_linear_psize
, mmu_kernel_ssize
);
1732 spin_lock(&linear_map_hash_lock
);
1733 BUG_ON(linear_map_hash_slots
[lmi
] & 0x80);
1734 linear_map_hash_slots
[lmi
] = ret
| 0x80;
1735 spin_unlock(&linear_map_hash_lock
);
1738 static void kernel_unmap_linear_page(unsigned long vaddr
, unsigned long lmi
)
1740 unsigned long hash
, hidx
, slot
;
1741 unsigned long vsid
= get_kernel_vsid(vaddr
, mmu_kernel_ssize
);
1742 unsigned long vpn
= hpt_vpn(vaddr
, vsid
, mmu_kernel_ssize
);
1744 hash
= hpt_hash(vpn
, PAGE_SHIFT
, mmu_kernel_ssize
);
1745 spin_lock(&linear_map_hash_lock
);
1746 BUG_ON(!(linear_map_hash_slots
[lmi
] & 0x80));
1747 hidx
= linear_map_hash_slots
[lmi
] & 0x7f;
1748 linear_map_hash_slots
[lmi
] = 0;
1749 spin_unlock(&linear_map_hash_lock
);
1750 if (hidx
& _PTEIDX_SECONDARY
)
1752 slot
= (hash
& htab_hash_mask
) * HPTES_PER_GROUP
;
1753 slot
+= hidx
& _PTEIDX_GROUP_IX
;
1754 mmu_hash_ops
.hpte_invalidate(slot
, vpn
, mmu_linear_psize
,
1756 mmu_kernel_ssize
, 0);
1759 void __kernel_map_pages(struct page
*page
, int numpages
, int enable
)
1761 unsigned long flags
, vaddr
, lmi
;
1764 local_irq_save(flags
);
1765 for (i
= 0; i
< numpages
; i
++, page
++) {
1766 vaddr
= (unsigned long)page_address(page
);
1767 lmi
= __pa(vaddr
) >> PAGE_SHIFT
;
1768 if (lmi
>= linear_map_hash_count
)
1771 kernel_map_linear_page(vaddr
, lmi
);
1773 kernel_unmap_linear_page(vaddr
, lmi
);
1775 local_irq_restore(flags
);
1777 #endif /* CONFIG_DEBUG_PAGEALLOC */
1779 void hash__setup_initial_memory_limit(phys_addr_t first_memblock_base
,
1780 phys_addr_t first_memblock_size
)
1782 /* We don't currently support the first MEMBLOCK not mapping 0
1783 * physical on those processors
1785 BUG_ON(first_memblock_base
!= 0);
1787 /* On LPAR systems, the first entry is our RMA region,
1788 * non-LPAR 64-bit hash MMU systems don't have a limitation
1789 * on real mode access, but using the first entry works well
1790 * enough. We also clamp it to 1G to avoid some funky things
1791 * such as RTAS bugs etc...
1793 ppc64_rma_size
= min_t(u64
, first_memblock_size
, 0x40000000);
1795 /* Finally limit subsequent allocations */
1796 memblock_set_current_limit(ppc64_rma_size
);