4 * Privileged Space Mapping Buffer (PMB) Support.
6 * Copyright (C) 2005 - 2010 Paul Mundt
7 * Copyright (C) 2010 Matt Fleming
9 * This file is subject to the terms and conditions of the GNU General Public
10 * License. See the file "COPYING" in the main directory of this archive
13 #include <linux/init.h>
14 #include <linux/kernel.h>
15 #include <linux/sysdev.h>
16 #include <linux/cpu.h>
17 #include <linux/module.h>
18 #include <linux/bitops.h>
19 #include <linux/debugfs.h>
21 #include <linux/seq_file.h>
22 #include <linux/err.h>
24 #include <linux/spinlock.h>
25 #include <linux/vmalloc.h>
26 #include <asm/cacheflush.h>
27 #include <asm/sizes.h>
28 #include <asm/system.h>
29 #include <asm/uaccess.h>
30 #include <asm/pgtable.h>
33 #include <asm/mmu_context.h>
46 * 0 .. NR_PMB_ENTRIES for specific entry selection, or
47 * PMB_NO_ENTRY to search for a free one
51 /* Adjacent entry link for contiguous multi-entry mappings */
52 struct pmb_entry
*link
;
59 { .size
= SZ_512M
, .flag
= PMB_SZ_512M
, },
60 { .size
= SZ_128M
, .flag
= PMB_SZ_128M
, },
61 { .size
= SZ_64M
, .flag
= PMB_SZ_64M
, },
62 { .size
= SZ_16M
, .flag
= PMB_SZ_16M
, },
65 static void pmb_unmap_entry(struct pmb_entry
*, int depth
);
67 static DEFINE_RWLOCK(pmb_rwlock
);
68 static struct pmb_entry pmb_entry_list
[NR_PMB_ENTRIES
];
69 static DECLARE_BITMAP(pmb_map
, NR_PMB_ENTRIES
);
71 static unsigned int pmb_iomapping_enabled
;
73 static __always_inline
unsigned long mk_pmb_entry(unsigned int entry
)
75 return (entry
& PMB_E_MASK
) << PMB_E_SHIFT
;
78 static __always_inline
unsigned long mk_pmb_addr(unsigned int entry
)
80 return mk_pmb_entry(entry
) | PMB_ADDR
;
83 static __always_inline
unsigned long mk_pmb_data(unsigned int entry
)
85 return mk_pmb_entry(entry
) | PMB_DATA
;
88 static __always_inline
unsigned int pmb_ppn_in_range(unsigned long ppn
)
90 return ppn
>= __pa(memory_start
) && ppn
< __pa(memory_end
);
94 * Ensure that the PMB entries match our cache configuration.
96 * When we are in 32-bit address extended mode, CCR.CB becomes
97 * invalid, so care must be taken to manually adjust cacheable
100 static __always_inline
unsigned long pmb_cache_flags(void)
102 unsigned long flags
= 0;
104 #if defined(CONFIG_CACHE_OFF)
105 flags
|= PMB_WT
| PMB_UB
;
106 #elif defined(CONFIG_CACHE_WRITETHROUGH)
107 flags
|= PMB_C
| PMB_WT
| PMB_UB
;
108 #elif defined(CONFIG_CACHE_WRITEBACK)
116 * Convert typical pgprot value to the PMB equivalent
118 static inline unsigned long pgprot_to_pmb_flags(pgprot_t prot
)
120 unsigned long pmb_flags
= 0;
121 u64 flags
= pgprot_val(prot
);
123 if (flags
& _PAGE_CACHABLE
)
125 if (flags
& _PAGE_WT
)
126 pmb_flags
|= PMB_WT
| PMB_UB
;
131 static inline bool pmb_can_merge(struct pmb_entry
*a
, struct pmb_entry
*b
)
133 return (b
->vpn
== (a
->vpn
+ a
->size
)) &&
134 (b
->ppn
== (a
->ppn
+ a
->size
)) &&
135 (b
->flags
== a
->flags
);
138 static bool pmb_mapping_exists(unsigned long vaddr
, phys_addr_t phys
,
143 read_lock(&pmb_rwlock
);
145 for (i
= 0; i
< ARRAY_SIZE(pmb_entry_list
); i
++) {
146 struct pmb_entry
*pmbe
, *iter
;
149 if (!test_bit(i
, pmb_map
))
152 pmbe
= &pmb_entry_list
[i
];
155 * See if VPN and PPN are bounded by an existing mapping.
157 if ((vaddr
< pmbe
->vpn
) || (vaddr
>= (pmbe
->vpn
+ pmbe
->size
)))
159 if ((phys
< pmbe
->ppn
) || (phys
>= (pmbe
->ppn
+ pmbe
->size
)))
163 * Now see if we're in range of a simple mapping.
165 if (size
<= pmbe
->size
) {
166 read_unlock(&pmb_rwlock
);
173 * Finally for sizes that involve compound mappings, walk
176 for (iter
= pmbe
->link
; iter
; iter
= iter
->link
)
180 * Nothing else to do if the range requirements are met.
183 read_unlock(&pmb_rwlock
);
188 read_unlock(&pmb_rwlock
);
192 static bool pmb_size_valid(unsigned long size
)
196 for (i
= 0; i
< ARRAY_SIZE(pmb_sizes
); i
++)
197 if (pmb_sizes
[i
].size
== size
)
203 static inline bool pmb_addr_valid(unsigned long addr
, unsigned long size
)
205 return (addr
>= P1SEG
&& (addr
+ size
- 1) < P3SEG
);
208 static inline bool pmb_prot_valid(pgprot_t prot
)
210 return (pgprot_val(prot
) & _PAGE_USER
) == 0;
213 static int pmb_size_to_flags(unsigned long size
)
217 for (i
= 0; i
< ARRAY_SIZE(pmb_sizes
); i
++)
218 if (pmb_sizes
[i
].size
== size
)
219 return pmb_sizes
[i
].flag
;
224 static int pmb_alloc_entry(void)
228 pos
= find_first_zero_bit(pmb_map
, NR_PMB_ENTRIES
);
229 if (pos
>= 0 && pos
< NR_PMB_ENTRIES
)
230 __set_bit(pos
, pmb_map
);
237 static struct pmb_entry
*pmb_alloc(unsigned long vpn
, unsigned long ppn
,
238 unsigned long flags
, int entry
)
240 struct pmb_entry
*pmbe
;
241 unsigned long irqflags
;
245 write_lock_irqsave(&pmb_rwlock
, irqflags
);
247 if (entry
== PMB_NO_ENTRY
) {
248 pos
= pmb_alloc_entry();
249 if (unlikely(pos
< 0)) {
254 if (__test_and_set_bit(entry
, pmb_map
)) {
255 ret
= ERR_PTR(-ENOSPC
);
262 write_unlock_irqrestore(&pmb_rwlock
, irqflags
);
264 pmbe
= &pmb_entry_list
[pos
];
266 memset(pmbe
, 0, sizeof(struct pmb_entry
));
268 spin_lock_init(&pmbe
->lock
);
278 write_unlock_irqrestore(&pmb_rwlock
, irqflags
);
282 static void pmb_free(struct pmb_entry
*pmbe
)
284 __clear_bit(pmbe
->entry
, pmb_map
);
286 pmbe
->entry
= PMB_NO_ENTRY
;
291 * Must be run uncached.
293 static void __set_pmb_entry(struct pmb_entry
*pmbe
)
295 unsigned long addr
, data
;
297 addr
= mk_pmb_addr(pmbe
->entry
);
298 data
= mk_pmb_data(pmbe
->entry
);
303 __raw_writel(pmbe
->vpn
| PMB_V
, addr
);
304 __raw_writel(pmbe
->ppn
| pmbe
->flags
| PMB_V
, data
);
309 static void __clear_pmb_entry(struct pmb_entry
*pmbe
)
311 unsigned long addr
, data
;
312 unsigned long addr_val
, data_val
;
314 addr
= mk_pmb_addr(pmbe
->entry
);
315 data
= mk_pmb_data(pmbe
->entry
);
317 addr_val
= __raw_readl(addr
);
318 data_val
= __raw_readl(data
);
321 writel_uncached(addr_val
& ~PMB_V
, addr
);
322 writel_uncached(data_val
& ~PMB_V
, data
);
326 static void set_pmb_entry(struct pmb_entry
*pmbe
)
330 spin_lock_irqsave(&pmbe
->lock
, flags
);
331 __set_pmb_entry(pmbe
);
332 spin_unlock_irqrestore(&pmbe
->lock
, flags
);
334 #endif /* CONFIG_PM */
336 int pmb_bolt_mapping(unsigned long vaddr
, phys_addr_t phys
,
337 unsigned long size
, pgprot_t prot
)
339 struct pmb_entry
*pmbp
, *pmbe
;
340 unsigned long orig_addr
, orig_size
;
341 unsigned long flags
, pmb_flags
;
346 if (!pmb_addr_valid(vaddr
, size
))
348 if (pmb_mapping_exists(vaddr
, phys
, size
))
354 flush_tlb_kernel_range(vaddr
, vaddr
+ size
);
356 pmb_flags
= pgprot_to_pmb_flags(prot
);
360 for (i
= mapped
= 0; i
< ARRAY_SIZE(pmb_sizes
); i
++) {
361 if (size
< pmb_sizes
[i
].size
)
364 pmbe
= pmb_alloc(vaddr
, phys
, pmb_flags
|
365 pmb_sizes
[i
].flag
, PMB_NO_ENTRY
);
367 pmb_unmap_entry(pmbp
, mapped
);
368 return PTR_ERR(pmbe
);
371 spin_lock_irqsave(&pmbe
->lock
, flags
);
373 pmbe
->size
= pmb_sizes
[i
].size
;
375 __set_pmb_entry(pmbe
);
382 * Link adjacent entries that span multiple PMB
383 * entries for easier tear-down.
386 spin_lock(&pmbp
->lock
);
388 spin_unlock(&pmbp
->lock
);
394 * Instead of trying smaller sizes on every
395 * iteration (even if we succeed in allocating
396 * space), try using pmb_sizes[i].size again.
401 spin_unlock_irqrestore(&pmbe
->lock
, flags
);
403 } while (size
>= SZ_16M
);
405 flush_cache_vmap(orig_addr
, orig_addr
+ orig_size
);
410 void __iomem
*pmb_remap_caller(phys_addr_t phys
, unsigned long size
,
411 pgprot_t prot
, void *caller
)
414 phys_addr_t offset
, last_addr
;
415 phys_addr_t align_mask
;
416 unsigned long aligned
;
417 struct vm_struct
*area
;
420 if (!pmb_iomapping_enabled
)
424 * Small mappings need to go through the TLB.
427 return ERR_PTR(-EINVAL
);
428 if (!pmb_prot_valid(prot
))
429 return ERR_PTR(-EINVAL
);
431 for (i
= 0; i
< ARRAY_SIZE(pmb_sizes
); i
++)
432 if (size
>= pmb_sizes
[i
].size
)
435 last_addr
= phys
+ size
;
436 align_mask
= ~(pmb_sizes
[i
].size
- 1);
437 offset
= phys
& ~align_mask
;
439 aligned
= ALIGN(last_addr
, pmb_sizes
[i
].size
) - phys
;
442 * XXX: This should really start from uncached_end, but this
443 * causes the MMU to reset, so for now we restrict it to the
444 * 0xb000...0xc000 range.
446 area
= __get_vm_area_caller(aligned
, VM_IOREMAP
, 0xb0000000,
451 area
->phys_addr
= phys
;
452 vaddr
= (unsigned long)area
->addr
;
454 ret
= pmb_bolt_mapping(vaddr
, phys
, size
, prot
);
455 if (unlikely(ret
!= 0))
458 return (void __iomem
*)(offset
+ (char *)vaddr
);
461 int pmb_unmap(void __iomem
*addr
)
463 struct pmb_entry
*pmbe
= NULL
;
464 unsigned long vaddr
= (unsigned long __force
)addr
;
467 read_lock(&pmb_rwlock
);
469 for (i
= 0; i
< ARRAY_SIZE(pmb_entry_list
); i
++) {
470 if (test_bit(i
, pmb_map
)) {
471 pmbe
= &pmb_entry_list
[i
];
472 if (pmbe
->vpn
== vaddr
) {
479 read_unlock(&pmb_rwlock
);
482 pmb_unmap_entry(pmbe
, NR_PMB_ENTRIES
);
489 static void __pmb_unmap_entry(struct pmb_entry
*pmbe
, int depth
)
492 struct pmb_entry
*pmblink
= pmbe
;
495 * We may be called before this pmb_entry has been
496 * entered into the PMB table via set_pmb_entry(), but
497 * that's OK because we've allocated a unique slot for
498 * this entry in pmb_alloc() (even if we haven't filled
501 * Therefore, calling __clear_pmb_entry() is safe as no
502 * other mapping can be using that slot.
504 __clear_pmb_entry(pmbe
);
506 flush_cache_vunmap(pmbe
->vpn
, pmbe
->vpn
+ pmbe
->size
);
508 pmbe
= pmblink
->link
;
511 } while (pmbe
&& --depth
);
514 static void pmb_unmap_entry(struct pmb_entry
*pmbe
, int depth
)
521 write_lock_irqsave(&pmb_rwlock
, flags
);
522 __pmb_unmap_entry(pmbe
, depth
);
523 write_unlock_irqrestore(&pmb_rwlock
, flags
);
526 static void __init
pmb_notify(void)
530 pr_info("PMB: boot mappings:\n");
532 read_lock(&pmb_rwlock
);
534 for (i
= 0; i
< ARRAY_SIZE(pmb_entry_list
); i
++) {
535 struct pmb_entry
*pmbe
;
537 if (!test_bit(i
, pmb_map
))
540 pmbe
= &pmb_entry_list
[i
];
542 pr_info(" 0x%08lx -> 0x%08lx [ %4ldMB %2scached ]\n",
543 pmbe
->vpn
>> PAGE_SHIFT
, pmbe
->ppn
>> PAGE_SHIFT
,
544 pmbe
->size
>> 20, (pmbe
->flags
& PMB_C
) ? "" : "un");
547 read_unlock(&pmb_rwlock
);
551 * Sync our software copy of the PMB mappings with those in hardware. The
552 * mappings in the hardware PMB were either set up by the bootloader or
553 * very early on by the kernel.
555 static void __init
pmb_synchronize(void)
557 struct pmb_entry
*pmbp
= NULL
;
561 * Run through the initial boot mappings, log the established
562 * ones, and blow away anything that falls outside of the valid
563 * PPN range. Specifically, we only care about existing mappings
564 * that impact the cached/uncached sections.
566 * Note that touching these can be a bit of a minefield; the boot
567 * loader can establish multi-page mappings with the same caching
568 * attributes, so we need to ensure that we aren't modifying a
569 * mapping that we're presently executing from, or may execute
570 * from in the case of straddling page boundaries.
572 * In the future we will have to tidy up after the boot loader by
573 * jumping between the cached and uncached mappings and tearing
574 * down alternating mappings while executing from the other.
576 for (i
= 0; i
< NR_PMB_ENTRIES
; i
++) {
577 unsigned long addr
, data
;
578 unsigned long addr_val
, data_val
;
579 unsigned long ppn
, vpn
, flags
;
580 unsigned long irqflags
;
582 struct pmb_entry
*pmbe
;
584 addr
= mk_pmb_addr(i
);
585 data
= mk_pmb_data(i
);
587 addr_val
= __raw_readl(addr
);
588 data_val
= __raw_readl(data
);
591 * Skip over any bogus entries
593 if (!(data_val
& PMB_V
) || !(addr_val
& PMB_V
))
596 ppn
= data_val
& PMB_PFN_MASK
;
597 vpn
= addr_val
& PMB_PFN_MASK
;
600 * Only preserve in-range mappings.
602 if (!pmb_ppn_in_range(ppn
)) {
604 * Invalidate anything out of bounds.
606 writel_uncached(addr_val
& ~PMB_V
, addr
);
607 writel_uncached(data_val
& ~PMB_V
, data
);
612 * Update the caching attributes if necessary
614 if (data_val
& PMB_C
) {
615 data_val
&= ~PMB_CACHE_MASK
;
616 data_val
|= pmb_cache_flags();
618 writel_uncached(data_val
, data
);
621 size
= data_val
& PMB_SZ_MASK
;
622 flags
= size
| (data_val
& PMB_CACHE_MASK
);
624 pmbe
= pmb_alloc(vpn
, ppn
, flags
, i
);
630 spin_lock_irqsave(&pmbe
->lock
, irqflags
);
632 for (j
= 0; j
< ARRAY_SIZE(pmb_sizes
); j
++)
633 if (pmb_sizes
[j
].flag
== size
)
634 pmbe
->size
= pmb_sizes
[j
].size
;
637 spin_lock(&pmbp
->lock
);
640 * Compare the previous entry against the current one to
641 * see if the entries span a contiguous mapping. If so,
642 * setup the entry links accordingly. Compound mappings
643 * are later coalesced.
645 if (pmb_can_merge(pmbp
, pmbe
))
648 spin_unlock(&pmbp
->lock
);
653 spin_unlock_irqrestore(&pmbe
->lock
, irqflags
);
657 static void __init
pmb_merge(struct pmb_entry
*head
)
659 unsigned long span
, newsize
;
660 struct pmb_entry
*tail
;
661 int i
= 1, depth
= 0;
663 span
= newsize
= head
->size
;
669 if (pmb_size_valid(span
)) {
674 /* This is the end of the line.. */
683 * The merged page size must be valid.
685 if (!depth
|| !pmb_size_valid(newsize
))
688 head
->flags
&= ~PMB_SZ_MASK
;
689 head
->flags
|= pmb_size_to_flags(newsize
);
691 head
->size
= newsize
;
693 __pmb_unmap_entry(head
->link
, depth
);
694 __set_pmb_entry(head
);
697 static void __init
pmb_coalesce(void)
702 write_lock_irqsave(&pmb_rwlock
, flags
);
704 for (i
= 0; i
< ARRAY_SIZE(pmb_entry_list
); i
++) {
705 struct pmb_entry
*pmbe
;
707 if (!test_bit(i
, pmb_map
))
710 pmbe
= &pmb_entry_list
[i
];
713 * We're only interested in compound mappings
719 * Nothing to do if it already uses the largest possible
722 if (pmbe
->size
== SZ_512M
)
728 write_unlock_irqrestore(&pmb_rwlock
, flags
);
731 #ifdef CONFIG_UNCACHED_MAPPING
732 static void __init
pmb_resize(void)
737 * If the uncached mapping was constructed by the kernel, it will
738 * already be a reasonable size.
740 if (uncached_size
== SZ_16M
)
743 read_lock(&pmb_rwlock
);
745 for (i
= 0; i
< ARRAY_SIZE(pmb_entry_list
); i
++) {
746 struct pmb_entry
*pmbe
;
749 if (!test_bit(i
, pmb_map
))
752 pmbe
= &pmb_entry_list
[i
];
754 if (pmbe
->vpn
!= uncached_start
)
758 * Found it, now resize it.
760 spin_lock_irqsave(&pmbe
->lock
, flags
);
763 pmbe
->flags
&= ~PMB_SZ_MASK
;
764 pmbe
->flags
|= pmb_size_to_flags(pmbe
->size
);
766 uncached_resize(pmbe
->size
);
768 __set_pmb_entry(pmbe
);
770 spin_unlock_irqrestore(&pmbe
->lock
, flags
);
773 read_unlock(&pmb_rwlock
);
777 static int __init
early_pmb(char *p
)
782 if (strstr(p
, "iomap"))
783 pmb_iomapping_enabled
= 1;
787 early_param("pmb", early_pmb
);
789 void __init
pmb_init(void)
791 /* Synchronize software state */
794 /* Attempt to combine compound mappings */
797 #ifdef CONFIG_UNCACHED_MAPPING
798 /* Resize initial mappings, if necessary */
805 writel_uncached(0, PMB_IRMCR
);
807 /* Flush out the TLB */
808 local_flush_tlb_all();
812 bool __in_29bit_mode(void)
814 return (__raw_readl(PMB_PASCR
) & PASCR_SE
) == 0;
817 static int pmb_seq_show(struct seq_file
*file
, void *iter
)
821 seq_printf(file
, "V: Valid, C: Cacheable, WT: Write-Through\n"
822 "CB: Copy-Back, B: Buffered, UB: Unbuffered\n");
823 seq_printf(file
, "ety vpn ppn size flags\n");
825 for (i
= 0; i
< NR_PMB_ENTRIES
; i
++) {
826 unsigned long addr
, data
;
830 addr
= __raw_readl(mk_pmb_addr(i
));
831 data
= __raw_readl(mk_pmb_data(i
));
833 size
= data
& PMB_SZ_MASK
;
834 sz_str
= (size
== PMB_SZ_16M
) ? " 16MB":
835 (size
== PMB_SZ_64M
) ? " 64MB":
836 (size
== PMB_SZ_128M
) ? "128MB":
839 /* 02: V 0x88 0x08 128MB C CB B */
840 seq_printf(file
, "%02d: %c 0x%02lx 0x%02lx %s %c %s %s\n",
841 i
, ((addr
& PMB_V
) && (data
& PMB_V
)) ? 'V' : ' ',
842 (addr
>> 24) & 0xff, (data
>> 24) & 0xff,
843 sz_str
, (data
& PMB_C
) ? 'C' : ' ',
844 (data
& PMB_WT
) ? "WT" : "CB",
845 (data
& PMB_UB
) ? "UB" : " B");
851 static int pmb_debugfs_open(struct inode
*inode
, struct file
*file
)
853 return single_open(file
, pmb_seq_show
, NULL
);
856 static const struct file_operations pmb_debugfs_fops
= {
857 .owner
= THIS_MODULE
,
858 .open
= pmb_debugfs_open
,
861 .release
= single_release
,
864 static int __init
pmb_debugfs_init(void)
866 struct dentry
*dentry
;
868 dentry
= debugfs_create_file("pmb", S_IFREG
| S_IRUGO
,
869 sh_debugfs_root
, NULL
, &pmb_debugfs_fops
);
873 return PTR_ERR(dentry
);
877 subsys_initcall(pmb_debugfs_init
);
880 static int pmb_sysdev_suspend(struct sys_device
*dev
, pm_message_t state
)
882 static pm_message_t prev_state
;
885 /* Restore the PMB after a resume from hibernation */
886 if (state
.event
== PM_EVENT_ON
&&
887 prev_state
.event
== PM_EVENT_FREEZE
) {
888 struct pmb_entry
*pmbe
;
890 read_lock(&pmb_rwlock
);
892 for (i
= 0; i
< ARRAY_SIZE(pmb_entry_list
); i
++) {
893 if (test_bit(i
, pmb_map
)) {
894 pmbe
= &pmb_entry_list
[i
];
899 read_unlock(&pmb_rwlock
);
907 static int pmb_sysdev_resume(struct sys_device
*dev
)
909 return pmb_sysdev_suspend(dev
, PMSG_ON
);
912 static struct sysdev_driver pmb_sysdev_driver
= {
913 .suspend
= pmb_sysdev_suspend
,
914 .resume
= pmb_sysdev_resume
,
917 static int __init
pmb_sysdev_init(void)
919 return sysdev_driver_register(&cpu_sysdev_class
, &pmb_sysdev_driver
);
921 subsys_initcall(pmb_sysdev_init
);