1 /* $Id: init.c,v 1.209 2002/02/09 19:49:31 davem Exp $
2 * arch/sparc64/mm/init.c
4 * Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
8 #include <linux/module.h>
9 #include <linux/kernel.h>
10 #include <linux/sched.h>
11 #include <linux/string.h>
12 #include <linux/init.h>
13 #include <linux/bootmem.h>
15 #include <linux/hugetlb.h>
16 #include <linux/slab.h>
17 #include <linux/initrd.h>
18 #include <linux/swap.h>
19 #include <linux/pagemap.h>
20 #include <linux/poison.h>
22 #include <linux/seq_file.h>
23 #include <linux/kprobes.h>
24 #include <linux/cache.h>
25 #include <linux/sort.h>
28 #include <asm/system.h>
30 #include <asm/pgalloc.h>
31 #include <asm/pgtable.h>
32 #include <asm/oplib.h>
33 #include <asm/iommu.h>
35 #include <asm/uaccess.h>
36 #include <asm/mmu_context.h>
37 #include <asm/tlbflush.h>
39 #include <asm/starfire.h>
41 #include <asm/spitfire.h>
42 #include <asm/sections.h>
44 #include <asm/hypervisor.h>
47 extern void device_scan(void);
49 #define MAX_PHYS_ADDRESS (1UL << 42UL)
50 #define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL)
51 #define KPTE_BITMAP_BYTES \
52 ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 8)
54 unsigned long kern_linear_pte_xor
[2] __read_mostly
;
56 /* A bitmap, one bit for every 256MB of physical memory. If the bit
57 * is clear, we should use a 4MB page (via kern_linear_pte_xor[0]) else
58 * if set we should use a 256MB page (via kern_linear_pte_xor[1]).
60 unsigned long kpte_linear_bitmap
[KPTE_BITMAP_BYTES
/ sizeof(unsigned long)];
62 /* A special kernel TSB for 4MB and 256MB linear mappings. */
63 struct tsb swapper_4m_tsb
[KERNEL_TSB4M_NENTRIES
];
67 static struct linux_prom64_registers pavail
[MAX_BANKS
] __initdata
;
68 static struct linux_prom64_registers pavail_rescan
[MAX_BANKS
] __initdata
;
69 static int pavail_ents __initdata
;
70 static int pavail_rescan_ents __initdata
;
72 static int cmp_p64(const void *a
, const void *b
)
74 const struct linux_prom64_registers
*x
= a
, *y
= b
;
76 if (x
->phys_addr
> y
->phys_addr
)
78 if (x
->phys_addr
< y
->phys_addr
)
83 static void __init
read_obp_memory(const char *property
,
84 struct linux_prom64_registers
*regs
,
87 int node
= prom_finddevice("/memory");
88 int prop_size
= prom_getproplen(node
, property
);
91 ents
= prop_size
/ sizeof(struct linux_prom64_registers
);
92 if (ents
> MAX_BANKS
) {
93 prom_printf("The machine has more %s property entries than "
94 "this kernel can support (%d).\n",
99 ret
= prom_getproperty(node
, property
, (char *) regs
, prop_size
);
101 prom_printf("Couldn't get %s property from /memory.\n");
105 /* Sanitize what we got from the firmware, by page aligning
108 for (i
= 0; i
< ents
; i
++) {
109 unsigned long base
, size
;
111 base
= regs
[i
].phys_addr
;
112 size
= regs
[i
].reg_size
;
115 if (base
& ~PAGE_MASK
) {
116 unsigned long new_base
= PAGE_ALIGN(base
);
118 size
-= new_base
- base
;
119 if ((long) size
< 0L)
123 regs
[i
].phys_addr
= base
;
124 regs
[i
].reg_size
= size
;
127 for (i
= 0; i
< ents
; i
++) {
128 if (regs
[i
].reg_size
== 0UL) {
131 for (j
= i
; j
< ents
- 1; j
++) {
145 sort(regs
, ents
, sizeof(struct linux_prom64_registers
),
149 unsigned long *sparc64_valid_addr_bitmap __read_mostly
;
151 /* Kernel physical address base and size in bytes. */
152 unsigned long kern_base __read_mostly
;
153 unsigned long kern_size __read_mostly
;
155 /* get_new_mmu_context() uses "cache + 1". */
156 DEFINE_SPINLOCK(ctx_alloc_lock
);
157 unsigned long tlb_context_cache
= CTX_FIRST_VERSION
- 1;
158 #define CTX_BMAP_SLOTS (1UL << (CTX_NR_BITS - 6))
159 unsigned long mmu_context_bmap
[CTX_BMAP_SLOTS
];
161 /* References to special section boundaries */
162 extern char _start
[], _end
[];
164 /* Initial ramdisk setup */
165 extern unsigned long sparc_ramdisk_image64
;
166 extern unsigned int sparc_ramdisk_image
;
167 extern unsigned int sparc_ramdisk_size
;
169 struct page
*mem_map_zero __read_mostly
;
171 unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly
;
173 unsigned long sparc64_kern_pri_context __read_mostly
;
174 unsigned long sparc64_kern_pri_nuc_bits __read_mostly
;
175 unsigned long sparc64_kern_sec_context __read_mostly
;
179 kmem_cache_t
*pgtable_cache __read_mostly
;
181 static void zero_ctor(void *addr
, kmem_cache_t
*cache
, unsigned long flags
)
186 extern void tsb_cache_init(void);
188 void pgtable_cache_init(void)
190 pgtable_cache
= kmem_cache_create("pgtable_cache",
191 PAGE_SIZE
, PAGE_SIZE
,
193 SLAB_MUST_HWCACHE_ALIGN
,
196 if (!pgtable_cache
) {
197 prom_printf("Could not create pgtable_cache\n");
203 #ifdef CONFIG_DEBUG_DCFLUSH
204 atomic_t dcpage_flushes
= ATOMIC_INIT(0);
206 atomic_t dcpage_flushes_xcall
= ATOMIC_INIT(0);
210 inline void flush_dcache_page_impl(struct page
*page
)
212 BUG_ON(tlb_type
== hypervisor
);
213 #ifdef CONFIG_DEBUG_DCFLUSH
214 atomic_inc(&dcpage_flushes
);
217 #ifdef DCACHE_ALIASING_POSSIBLE
218 __flush_dcache_page(page_address(page
),
219 ((tlb_type
== spitfire
) &&
220 page_mapping(page
) != NULL
));
222 if (page_mapping(page
) != NULL
&&
223 tlb_type
== spitfire
)
224 __flush_icache_page(__pa(page_address(page
)));
228 #define PG_dcache_dirty PG_arch_1
229 #define PG_dcache_cpu_shift 24UL
230 #define PG_dcache_cpu_mask (256UL - 1UL)
233 #error D-cache dirty tracking and thread_info->cpu need fixing for > 256 cpus
236 #define dcache_dirty_cpu(page) \
237 (((page)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)
239 static __inline__
void set_dcache_dirty(struct page
*page
, int this_cpu
)
241 unsigned long mask
= this_cpu
;
242 unsigned long non_cpu_bits
;
244 non_cpu_bits
= ~(PG_dcache_cpu_mask
<< PG_dcache_cpu_shift
);
245 mask
= (mask
<< PG_dcache_cpu_shift
) | (1UL << PG_dcache_dirty
);
247 __asm__
__volatile__("1:\n\t"
249 "and %%g7, %1, %%g1\n\t"
250 "or %%g1, %0, %%g1\n\t"
251 "casx [%2], %%g7, %%g1\n\t"
253 "membar #StoreLoad | #StoreStore\n\t"
254 "bne,pn %%xcc, 1b\n\t"
257 : "r" (mask
), "r" (non_cpu_bits
), "r" (&page
->flags
)
261 static __inline__
void clear_dcache_dirty_cpu(struct page
*page
, unsigned long cpu
)
263 unsigned long mask
= (1UL << PG_dcache_dirty
);
265 __asm__
__volatile__("! test_and_clear_dcache_dirty\n"
268 "srlx %%g7, %4, %%g1\n\t"
269 "and %%g1, %3, %%g1\n\t"
271 "bne,pn %%icc, 2f\n\t"
272 " andn %%g7, %1, %%g1\n\t"
273 "casx [%2], %%g7, %%g1\n\t"
275 "membar #StoreLoad | #StoreStore\n\t"
276 "bne,pn %%xcc, 1b\n\t"
280 : "r" (cpu
), "r" (mask
), "r" (&page
->flags
),
281 "i" (PG_dcache_cpu_mask
),
282 "i" (PG_dcache_cpu_shift
)
286 static inline void tsb_insert(struct tsb
*ent
, unsigned long tag
, unsigned long pte
)
288 unsigned long tsb_addr
= (unsigned long) ent
;
290 if (tlb_type
== cheetah_plus
|| tlb_type
== hypervisor
)
291 tsb_addr
= __pa(tsb_addr
);
293 __tsb_insert(tsb_addr
, tag
, pte
);
296 unsigned long _PAGE_ALL_SZ_BITS __read_mostly
;
297 unsigned long _PAGE_SZBITS __read_mostly
;
299 void update_mmu_cache(struct vm_area_struct
*vma
, unsigned long address
, pte_t pte
)
301 struct mm_struct
*mm
;
303 unsigned long tag
, flags
;
304 unsigned long tsb_index
, tsb_hash_shift
;
306 if (tlb_type
!= hypervisor
) {
307 unsigned long pfn
= pte_pfn(pte
);
308 unsigned long pg_flags
;
311 if (pfn_valid(pfn
) &&
312 (page
= pfn_to_page(pfn
), page_mapping(page
)) &&
313 ((pg_flags
= page
->flags
) & (1UL << PG_dcache_dirty
))) {
314 int cpu
= ((pg_flags
>> PG_dcache_cpu_shift
) &
316 int this_cpu
= get_cpu();
318 /* This is just to optimize away some function calls
322 flush_dcache_page_impl(page
);
324 smp_flush_dcache_page_impl(page
, cpu
);
326 clear_dcache_dirty_cpu(page
, cpu
);
334 tsb_index
= MM_TSB_BASE
;
335 tsb_hash_shift
= PAGE_SHIFT
;
337 spin_lock_irqsave(&mm
->context
.lock
, flags
);
339 #ifdef CONFIG_HUGETLB_PAGE
340 if (mm
->context
.tsb_block
[MM_TSB_HUGE
].tsb
!= NULL
) {
341 if ((tlb_type
== hypervisor
&&
342 (pte_val(pte
) & _PAGE_SZALL_4V
) == _PAGE_SZHUGE_4V
) ||
343 (tlb_type
!= hypervisor
&&
344 (pte_val(pte
) & _PAGE_SZALL_4U
) == _PAGE_SZHUGE_4U
)) {
345 tsb_index
= MM_TSB_HUGE
;
346 tsb_hash_shift
= HPAGE_SHIFT
;
351 tsb
= mm
->context
.tsb_block
[tsb_index
].tsb
;
352 tsb
+= ((address
>> tsb_hash_shift
) &
353 (mm
->context
.tsb_block
[tsb_index
].tsb_nentries
- 1UL));
354 tag
= (address
>> 22UL);
355 tsb_insert(tsb
, tag
, pte_val(pte
));
357 spin_unlock_irqrestore(&mm
->context
.lock
, flags
);
360 void flush_dcache_page(struct page
*page
)
362 struct address_space
*mapping
;
365 if (tlb_type
== hypervisor
)
368 /* Do not bother with the expensive D-cache flush if it
369 * is merely the zero page. The 'bigcore' testcase in GDB
370 * causes this case to run millions of times.
372 if (page
== ZERO_PAGE(0))
375 this_cpu
= get_cpu();
377 mapping
= page_mapping(page
);
378 if (mapping
&& !mapping_mapped(mapping
)) {
379 int dirty
= test_bit(PG_dcache_dirty
, &page
->flags
);
381 int dirty_cpu
= dcache_dirty_cpu(page
);
383 if (dirty_cpu
== this_cpu
)
385 smp_flush_dcache_page_impl(page
, dirty_cpu
);
387 set_dcache_dirty(page
, this_cpu
);
389 /* We could delay the flush for the !page_mapping
390 * case too. But that case is for exec env/arg
391 * pages and those are %99 certainly going to get
392 * faulted into the tlb (and thus flushed) anyways.
394 flush_dcache_page_impl(page
);
401 void __kprobes
flush_icache_range(unsigned long start
, unsigned long end
)
403 /* Cheetah and Hypervisor platform cpus have coherent I-cache. */
404 if (tlb_type
== spitfire
) {
407 for (kaddr
= start
; kaddr
< end
; kaddr
+= PAGE_SIZE
)
408 __flush_icache_page(__get_phys(kaddr
));
414 printk("Mem-info:\n");
416 printk("Free swap: %6ldkB\n",
417 nr_swap_pages
<< (PAGE_SHIFT
-10));
418 printk("%ld pages of RAM\n", num_physpages
);
419 printk("%d free pages\n", nr_free_pages());
422 void mmu_info(struct seq_file
*m
)
424 if (tlb_type
== cheetah
)
425 seq_printf(m
, "MMU Type\t: Cheetah\n");
426 else if (tlb_type
== cheetah_plus
)
427 seq_printf(m
, "MMU Type\t: Cheetah+\n");
428 else if (tlb_type
== spitfire
)
429 seq_printf(m
, "MMU Type\t: Spitfire\n");
430 else if (tlb_type
== hypervisor
)
431 seq_printf(m
, "MMU Type\t: Hypervisor (sun4v)\n");
433 seq_printf(m
, "MMU Type\t: ???\n");
435 #ifdef CONFIG_DEBUG_DCFLUSH
436 seq_printf(m
, "DCPageFlushes\t: %d\n",
437 atomic_read(&dcpage_flushes
));
439 seq_printf(m
, "DCPageFlushesXC\t: %d\n",
440 atomic_read(&dcpage_flushes_xcall
));
441 #endif /* CONFIG_SMP */
442 #endif /* CONFIG_DEBUG_DCFLUSH */
445 struct linux_prom_translation
{
451 /* Exported for kernel TLB miss handling in ktlb.S */
452 struct linux_prom_translation prom_trans
[512] __read_mostly
;
453 unsigned int prom_trans_ents __read_mostly
;
455 /* Exported for SMP bootup purposes. */
456 unsigned long kern_locked_tte_data
;
458 /* The obp translations are saved based on 8k pagesize, since obp can
459 * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS ->
460 * HI_OBP_ADDRESS range are handled in ktlb.S.
462 static inline int in_obp_range(unsigned long vaddr
)
464 return (vaddr
>= LOW_OBP_ADDRESS
&&
465 vaddr
< HI_OBP_ADDRESS
);
468 static int cmp_ptrans(const void *a
, const void *b
)
470 const struct linux_prom_translation
*x
= a
, *y
= b
;
472 if (x
->virt
> y
->virt
)
474 if (x
->virt
< y
->virt
)
479 /* Read OBP translations property into 'prom_trans[]'. */
480 static void __init
read_obp_translations(void)
482 int n
, node
, ents
, first
, last
, i
;
484 node
= prom_finddevice("/virtual-memory");
485 n
= prom_getproplen(node
, "translations");
486 if (unlikely(n
== 0 || n
== -1)) {
487 prom_printf("prom_mappings: Couldn't get size.\n");
490 if (unlikely(n
> sizeof(prom_trans
))) {
491 prom_printf("prom_mappings: Size %Zd is too big.\n", n
);
495 if ((n
= prom_getproperty(node
, "translations",
496 (char *)&prom_trans
[0],
497 sizeof(prom_trans
))) == -1) {
498 prom_printf("prom_mappings: Couldn't get property.\n");
502 n
= n
/ sizeof(struct linux_prom_translation
);
506 sort(prom_trans
, ents
, sizeof(struct linux_prom_translation
),
509 /* Now kick out all the non-OBP entries. */
510 for (i
= 0; i
< ents
; i
++) {
511 if (in_obp_range(prom_trans
[i
].virt
))
515 for (; i
< ents
; i
++) {
516 if (!in_obp_range(prom_trans
[i
].virt
))
521 for (i
= 0; i
< (last
- first
); i
++) {
522 struct linux_prom_translation
*src
= &prom_trans
[i
+ first
];
523 struct linux_prom_translation
*dest
= &prom_trans
[i
];
527 for (; i
< ents
; i
++) {
528 struct linux_prom_translation
*dest
= &prom_trans
[i
];
529 dest
->virt
= dest
->size
= dest
->data
= 0x0UL
;
532 prom_trans_ents
= last
- first
;
534 if (tlb_type
== spitfire
) {
535 /* Clear diag TTE bits. */
536 for (i
= 0; i
< prom_trans_ents
; i
++)
537 prom_trans
[i
].data
&= ~0x0003fe0000000000UL
;
541 static void __init
hypervisor_tlb_lock(unsigned long vaddr
,
545 register unsigned long func
asm("%o5");
546 register unsigned long arg0
asm("%o0");
547 register unsigned long arg1
asm("%o1");
548 register unsigned long arg2
asm("%o2");
549 register unsigned long arg3
asm("%o3");
551 func
= HV_FAST_MMU_MAP_PERM_ADDR
;
556 __asm__
__volatile__("ta 0x80"
557 : "=&r" (func
), "=&r" (arg0
),
558 "=&r" (arg1
), "=&r" (arg2
),
560 : "0" (func
), "1" (arg0
), "2" (arg1
),
561 "3" (arg2
), "4" (arg3
));
563 prom_printf("hypervisor_tlb_lock[%lx:%lx:%lx:%lx]: "
564 "errors with %lx\n", vaddr
, 0, pte
, mmu
, arg0
);
569 static unsigned long kern_large_tte(unsigned long paddr
);
571 static void __init
remap_kernel(void)
573 unsigned long phys_page
, tte_vaddr
, tte_data
;
574 int tlb_ent
= sparc64_highest_locked_tlbent();
576 tte_vaddr
= (unsigned long) KERNBASE
;
577 phys_page
= (prom_boot_mapping_phys_low
>> 22UL) << 22UL;
578 tte_data
= kern_large_tte(phys_page
);
580 kern_locked_tte_data
= tte_data
;
582 /* Now lock us into the TLBs via Hypervisor or OBP. */
583 if (tlb_type
== hypervisor
) {
584 hypervisor_tlb_lock(tte_vaddr
, tte_data
, HV_MMU_DMMU
);
585 hypervisor_tlb_lock(tte_vaddr
, tte_data
, HV_MMU_IMMU
);
587 tte_vaddr
+= 0x400000;
588 tte_data
+= 0x400000;
589 hypervisor_tlb_lock(tte_vaddr
, tte_data
, HV_MMU_DMMU
);
590 hypervisor_tlb_lock(tte_vaddr
, tte_data
, HV_MMU_IMMU
);
593 prom_dtlb_load(tlb_ent
, tte_data
, tte_vaddr
);
594 prom_itlb_load(tlb_ent
, tte_data
, tte_vaddr
);
597 prom_dtlb_load(tlb_ent
,
599 tte_vaddr
+ 0x400000);
600 prom_itlb_load(tlb_ent
,
602 tte_vaddr
+ 0x400000);
604 sparc64_highest_unlocked_tlb_ent
= tlb_ent
- 1;
606 if (tlb_type
== cheetah_plus
) {
607 sparc64_kern_pri_context
= (CTX_CHEETAH_PLUS_CTX0
|
608 CTX_CHEETAH_PLUS_NUC
);
609 sparc64_kern_pri_nuc_bits
= CTX_CHEETAH_PLUS_NUC
;
610 sparc64_kern_sec_context
= CTX_CHEETAH_PLUS_CTX0
;
615 static void __init
inherit_prom_mappings(void)
617 read_obp_translations();
619 /* Now fixup OBP's idea about where we really are mapped. */
620 prom_printf("Remapping the kernel... ");
622 prom_printf("done.\n");
625 void prom_world(int enter
)
628 set_fs((mm_segment_t
) { get_thread_current_ds() });
630 __asm__
__volatile__("flushw");
633 #ifdef DCACHE_ALIASING_POSSIBLE
634 void __flush_dcache_range(unsigned long start
, unsigned long end
)
638 if (tlb_type
== spitfire
) {
641 for (va
= start
; va
< end
; va
+= 32) {
642 spitfire_put_dcache_tag(va
& 0x3fe0, 0x0);
646 } else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
) {
649 for (va
= start
; va
< end
; va
+= 32)
650 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
654 "i" (ASI_DCACHE_INVALIDATE
));
657 #endif /* DCACHE_ALIASING_POSSIBLE */
659 /* Caller does TLB context flushing on local CPU if necessary.
660 * The caller also ensures that CTX_VALID(mm->context) is false.
662 * We must be careful about boundary cases so that we never
663 * let the user have CTX 0 (nucleus) or we ever use a CTX
664 * version of zero (and thus NO_CONTEXT would not be caught
665 * by version mis-match tests in mmu_context.h).
667 * Always invoked with interrupts disabled.
669 void get_new_mmu_context(struct mm_struct
*mm
)
671 unsigned long ctx
, new_ctx
;
672 unsigned long orig_pgsz_bits
;
676 spin_lock_irqsave(&ctx_alloc_lock
, flags
);
677 orig_pgsz_bits
= (mm
->context
.sparc64_ctx_val
& CTX_PGSZ_MASK
);
678 ctx
= (tlb_context_cache
+ 1) & CTX_NR_MASK
;
679 new_ctx
= find_next_zero_bit(mmu_context_bmap
, 1 << CTX_NR_BITS
, ctx
);
681 if (new_ctx
>= (1 << CTX_NR_BITS
)) {
682 new_ctx
= find_next_zero_bit(mmu_context_bmap
, ctx
, 1);
683 if (new_ctx
>= ctx
) {
685 new_ctx
= (tlb_context_cache
& CTX_VERSION_MASK
) +
688 new_ctx
= CTX_FIRST_VERSION
;
690 /* Don't call memset, for 16 entries that's just
693 mmu_context_bmap
[0] = 3;
694 mmu_context_bmap
[1] = 0;
695 mmu_context_bmap
[2] = 0;
696 mmu_context_bmap
[3] = 0;
697 for (i
= 4; i
< CTX_BMAP_SLOTS
; i
+= 4) {
698 mmu_context_bmap
[i
+ 0] = 0;
699 mmu_context_bmap
[i
+ 1] = 0;
700 mmu_context_bmap
[i
+ 2] = 0;
701 mmu_context_bmap
[i
+ 3] = 0;
707 mmu_context_bmap
[new_ctx
>>6] |= (1UL << (new_ctx
& 63));
708 new_ctx
|= (tlb_context_cache
& CTX_VERSION_MASK
);
710 tlb_context_cache
= new_ctx
;
711 mm
->context
.sparc64_ctx_val
= new_ctx
| orig_pgsz_bits
;
712 spin_unlock_irqrestore(&ctx_alloc_lock
, flags
);
714 if (unlikely(new_version
))
715 smp_new_mmu_context_version();
718 void sparc_ultra_dump_itlb(void)
722 if (tlb_type
== spitfire
) {
723 printk ("Contents of itlb: ");
724 for (slot
= 0; slot
< 14; slot
++) printk (" ");
725 printk ("%2x:%016lx,%016lx\n",
727 spitfire_get_itlb_tag(0), spitfire_get_itlb_data(0));
728 for (slot
= 1; slot
< 64; slot
+=3) {
729 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
731 spitfire_get_itlb_tag(slot
), spitfire_get_itlb_data(slot
),
733 spitfire_get_itlb_tag(slot
+1), spitfire_get_itlb_data(slot
+1),
735 spitfire_get_itlb_tag(slot
+2), spitfire_get_itlb_data(slot
+2));
737 } else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
) {
738 printk ("Contents of itlb0:\n");
739 for (slot
= 0; slot
< 16; slot
+=2) {
740 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
742 cheetah_get_litlb_tag(slot
), cheetah_get_litlb_data(slot
),
744 cheetah_get_litlb_tag(slot
+1), cheetah_get_litlb_data(slot
+1));
746 printk ("Contents of itlb2:\n");
747 for (slot
= 0; slot
< 128; slot
+=2) {
748 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
750 cheetah_get_itlb_tag(slot
), cheetah_get_itlb_data(slot
),
752 cheetah_get_itlb_tag(slot
+1), cheetah_get_itlb_data(slot
+1));
757 void sparc_ultra_dump_dtlb(void)
761 if (tlb_type
== spitfire
) {
762 printk ("Contents of dtlb: ");
763 for (slot
= 0; slot
< 14; slot
++) printk (" ");
764 printk ("%2x:%016lx,%016lx\n", 0,
765 spitfire_get_dtlb_tag(0), spitfire_get_dtlb_data(0));
766 for (slot
= 1; slot
< 64; slot
+=3) {
767 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
769 spitfire_get_dtlb_tag(slot
), spitfire_get_dtlb_data(slot
),
771 spitfire_get_dtlb_tag(slot
+1), spitfire_get_dtlb_data(slot
+1),
773 spitfire_get_dtlb_tag(slot
+2), spitfire_get_dtlb_data(slot
+2));
775 } else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
) {
776 printk ("Contents of dtlb0:\n");
777 for (slot
= 0; slot
< 16; slot
+=2) {
778 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
780 cheetah_get_ldtlb_tag(slot
), cheetah_get_ldtlb_data(slot
),
782 cheetah_get_ldtlb_tag(slot
+1), cheetah_get_ldtlb_data(slot
+1));
784 printk ("Contents of dtlb2:\n");
785 for (slot
= 0; slot
< 512; slot
+=2) {
786 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
788 cheetah_get_dtlb_tag(slot
, 2), cheetah_get_dtlb_data(slot
, 2),
790 cheetah_get_dtlb_tag(slot
+1, 2), cheetah_get_dtlb_data(slot
+1, 2));
792 if (tlb_type
== cheetah_plus
) {
793 printk ("Contents of dtlb3:\n");
794 for (slot
= 0; slot
< 512; slot
+=2) {
795 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
797 cheetah_get_dtlb_tag(slot
, 3), cheetah_get_dtlb_data(slot
, 3),
799 cheetah_get_dtlb_tag(slot
+1, 3), cheetah_get_dtlb_data(slot
+1, 3));
805 extern unsigned long cmdline_memory_size
;
807 /* Find a free area for the bootmem map, avoiding the kernel image
808 * and the initial ramdisk.
810 static unsigned long __init
choose_bootmap_pfn(unsigned long start_pfn
,
811 unsigned long end_pfn
)
813 unsigned long avoid_start
, avoid_end
, bootmap_size
;
816 bootmap_size
= ((end_pfn
- start_pfn
) + 7) / 8;
817 bootmap_size
= ALIGN(bootmap_size
, sizeof(long));
819 avoid_start
= avoid_end
= 0;
820 #ifdef CONFIG_BLK_DEV_INITRD
821 avoid_start
= initrd_start
;
822 avoid_end
= PAGE_ALIGN(initrd_end
);
825 #ifdef CONFIG_DEBUG_BOOTMEM
826 prom_printf("choose_bootmap_pfn: kern[%lx:%lx] avoid[%lx:%lx]\n",
827 kern_base
, PAGE_ALIGN(kern_base
+ kern_size
),
828 avoid_start
, avoid_end
);
830 for (i
= 0; i
< pavail_ents
; i
++) {
831 unsigned long start
, end
;
833 start
= pavail
[i
].phys_addr
;
834 end
= start
+ pavail
[i
].reg_size
;
836 while (start
< end
) {
837 if (start
>= kern_base
&&
838 start
< PAGE_ALIGN(kern_base
+ kern_size
)) {
839 start
= PAGE_ALIGN(kern_base
+ kern_size
);
842 if (start
>= avoid_start
&& start
< avoid_end
) {
847 if ((end
- start
) < bootmap_size
)
850 if (start
< kern_base
&&
851 (start
+ bootmap_size
) > kern_base
) {
852 start
= PAGE_ALIGN(kern_base
+ kern_size
);
856 if (start
< avoid_start
&&
857 (start
+ bootmap_size
) > avoid_start
) {
862 /* OK, it doesn't overlap anything, use it. */
863 #ifdef CONFIG_DEBUG_BOOTMEM
864 prom_printf("choose_bootmap_pfn: Using %lx [%lx]\n",
865 start
>> PAGE_SHIFT
, start
);
867 return start
>> PAGE_SHIFT
;
871 prom_printf("Cannot find free area for bootmap, aborting.\n");
875 static void __init
trim_pavail(unsigned long *cur_size_p
,
876 unsigned long *end_of_phys_p
)
878 unsigned long to_trim
= *cur_size_p
- cmdline_memory_size
;
879 unsigned long avoid_start
, avoid_end
;
882 to_trim
= PAGE_ALIGN(to_trim
);
884 avoid_start
= avoid_end
= 0;
885 #ifdef CONFIG_BLK_DEV_INITRD
886 avoid_start
= initrd_start
;
887 avoid_end
= PAGE_ALIGN(initrd_end
);
890 /* Trim some pavail[] entries in order to satisfy the
891 * requested "mem=xxx" kernel command line specification.
893 * We must not trim off the kernel image area nor the
894 * initial ramdisk range (if any). Also, we must not trim
895 * any pavail[] entry down to zero in order to preserve
896 * the invariant that all pavail[] entries have a non-zero
897 * size which is assumed by all of the code in here.
899 for (i
= 0; i
< pavail_ents
; i
++) {
900 unsigned long start
, end
, kern_end
;
901 unsigned long trim_low
, trim_high
, n
;
903 kern_end
= PAGE_ALIGN(kern_base
+ kern_size
);
905 trim_low
= start
= pavail
[i
].phys_addr
;
906 trim_high
= end
= start
+ pavail
[i
].reg_size
;
908 if (kern_base
>= start
&&
910 trim_low
= kern_base
;
914 if (kern_end
>= start
&&
916 trim_high
= kern_end
;
919 avoid_start
>= start
&&
921 if (trim_low
> avoid_start
)
922 trim_low
= avoid_start
;
923 if (avoid_end
>= end
)
927 avoid_end
>= start
&&
929 if (trim_high
< avoid_end
)
930 trim_high
= avoid_end
;
933 if (trim_high
<= trim_low
)
936 if (trim_low
== start
&& trim_high
== end
) {
937 /* Whole chunk is available for trimming.
938 * Trim all except one page, in order to keep
941 n
= (end
- start
) - PAGE_SIZE
;
946 pavail
[i
].phys_addr
+= n
;
947 pavail
[i
].reg_size
-= n
;
951 n
= (trim_low
- start
);
956 pavail
[i
].phys_addr
+= n
;
957 pavail
[i
].reg_size
-= n
;
965 pavail
[i
].reg_size
-= n
;
977 for (i
= 0; i
< pavail_ents
; i
++) {
978 *end_of_phys_p
= pavail
[i
].phys_addr
+
980 *cur_size_p
+= pavail
[i
].reg_size
;
984 static unsigned long __init
bootmem_init(unsigned long *pages_avail
,
985 unsigned long phys_base
)
987 unsigned long bootmap_size
, end_pfn
;
988 unsigned long end_of_phys_memory
= 0UL;
989 unsigned long bootmap_pfn
, bytes_avail
, size
;
992 #ifdef CONFIG_DEBUG_BOOTMEM
993 prom_printf("bootmem_init: Scan pavail, ");
997 for (i
= 0; i
< pavail_ents
; i
++) {
998 end_of_phys_memory
= pavail
[i
].phys_addr
+
1000 bytes_avail
+= pavail
[i
].reg_size
;
1003 /* Determine the location of the initial ramdisk before trying
1004 * to honor the "mem=xxx" command line argument. We must know
1005 * where the kernel image and the ramdisk image are so that we
1006 * do not trim those two areas from the physical memory map.
1009 #ifdef CONFIG_BLK_DEV_INITRD
1010 /* Now have to check initial ramdisk, so that bootmap does not overwrite it */
1011 if (sparc_ramdisk_image
|| sparc_ramdisk_image64
) {
1012 unsigned long ramdisk_image
= sparc_ramdisk_image
?
1013 sparc_ramdisk_image
: sparc_ramdisk_image64
;
1014 ramdisk_image
-= KERNBASE
;
1015 initrd_start
= ramdisk_image
+ phys_base
;
1016 initrd_end
= initrd_start
+ sparc_ramdisk_size
;
1017 if (initrd_end
> end_of_phys_memory
) {
1018 printk(KERN_CRIT
"initrd extends beyond end of memory "
1019 "(0x%016lx > 0x%016lx)\ndisabling initrd\n",
1020 initrd_end
, end_of_phys_memory
);
1027 if (cmdline_memory_size
&&
1028 bytes_avail
> cmdline_memory_size
)
1029 trim_pavail(&bytes_avail
,
1030 &end_of_phys_memory
);
1032 *pages_avail
= bytes_avail
>> PAGE_SHIFT
;
1034 end_pfn
= end_of_phys_memory
>> PAGE_SHIFT
;
1036 /* Initialize the boot-time allocator. */
1037 max_pfn
= max_low_pfn
= end_pfn
;
1038 min_low_pfn
= (phys_base
>> PAGE_SHIFT
);
1040 bootmap_pfn
= choose_bootmap_pfn(min_low_pfn
, end_pfn
);
1042 #ifdef CONFIG_DEBUG_BOOTMEM
1043 prom_printf("init_bootmem(min[%lx], bootmap[%lx], max[%lx])\n",
1044 min_low_pfn
, bootmap_pfn
, max_low_pfn
);
1046 bootmap_size
= init_bootmem_node(NODE_DATA(0), bootmap_pfn
,
1047 min_low_pfn
, end_pfn
);
1049 /* Now register the available physical memory with the
1052 for (i
= 0; i
< pavail_ents
; i
++) {
1053 #ifdef CONFIG_DEBUG_BOOTMEM
1054 prom_printf("free_bootmem(pavail:%d): base[%lx] size[%lx]\n",
1055 i
, pavail
[i
].phys_addr
, pavail
[i
].reg_size
);
1057 free_bootmem(pavail
[i
].phys_addr
, pavail
[i
].reg_size
);
1060 #ifdef CONFIG_BLK_DEV_INITRD
1062 size
= initrd_end
- initrd_start
;
1064 /* Resert the initrd image area. */
1065 #ifdef CONFIG_DEBUG_BOOTMEM
1066 prom_printf("reserve_bootmem(initrd): base[%llx] size[%lx]\n",
1067 initrd_start
, initrd_end
);
1069 reserve_bootmem(initrd_start
, size
);
1070 *pages_avail
-= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
1072 initrd_start
+= PAGE_OFFSET
;
1073 initrd_end
+= PAGE_OFFSET
;
1076 /* Reserve the kernel text/data/bss. */
1077 #ifdef CONFIG_DEBUG_BOOTMEM
1078 prom_printf("reserve_bootmem(kernel): base[%lx] size[%lx]\n", kern_base
, kern_size
);
1080 reserve_bootmem(kern_base
, kern_size
);
1081 *pages_avail
-= PAGE_ALIGN(kern_size
) >> PAGE_SHIFT
;
1083 /* Reserve the bootmem map. We do not account for it
1084 * in pages_avail because we will release that memory
1085 * in free_all_bootmem.
1087 size
= bootmap_size
;
1088 #ifdef CONFIG_DEBUG_BOOTMEM
1089 prom_printf("reserve_bootmem(bootmap): base[%lx] size[%lx]\n",
1090 (bootmap_pfn
<< PAGE_SHIFT
), size
);
1092 reserve_bootmem((bootmap_pfn
<< PAGE_SHIFT
), size
);
1093 *pages_avail
-= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
1095 for (i
= 0; i
< pavail_ents
; i
++) {
1096 unsigned long start_pfn
, end_pfn
;
1098 start_pfn
= pavail
[i
].phys_addr
>> PAGE_SHIFT
;
1099 end_pfn
= (start_pfn
+ (pavail
[i
].reg_size
>> PAGE_SHIFT
));
1100 #ifdef CONFIG_DEBUG_BOOTMEM
1101 prom_printf("memory_present(0, %lx, %lx)\n",
1102 start_pfn
, end_pfn
);
1104 memory_present(0, start_pfn
, end_pfn
);
1112 static struct linux_prom64_registers pall
[MAX_BANKS
] __initdata
;
1113 static int pall_ents __initdata
;
1115 #ifdef CONFIG_DEBUG_PAGEALLOC
1116 static unsigned long kernel_map_range(unsigned long pstart
, unsigned long pend
, pgprot_t prot
)
1118 unsigned long vstart
= PAGE_OFFSET
+ pstart
;
1119 unsigned long vend
= PAGE_OFFSET
+ pend
;
1120 unsigned long alloc_bytes
= 0UL;
1122 if ((vstart
& ~PAGE_MASK
) || (vend
& ~PAGE_MASK
)) {
1123 prom_printf("kernel_map: Unaligned physmem[%lx:%lx]\n",
1128 while (vstart
< vend
) {
1129 unsigned long this_end
, paddr
= __pa(vstart
);
1130 pgd_t
*pgd
= pgd_offset_k(vstart
);
1135 pud
= pud_offset(pgd
, vstart
);
1136 if (pud_none(*pud
)) {
1139 new = __alloc_bootmem(PAGE_SIZE
, PAGE_SIZE
, PAGE_SIZE
);
1140 alloc_bytes
+= PAGE_SIZE
;
1141 pud_populate(&init_mm
, pud
, new);
1144 pmd
= pmd_offset(pud
, vstart
);
1145 if (!pmd_present(*pmd
)) {
1148 new = __alloc_bootmem(PAGE_SIZE
, PAGE_SIZE
, PAGE_SIZE
);
1149 alloc_bytes
+= PAGE_SIZE
;
1150 pmd_populate_kernel(&init_mm
, pmd
, new);
1153 pte
= pte_offset_kernel(pmd
, vstart
);
1154 this_end
= (vstart
+ PMD_SIZE
) & PMD_MASK
;
1155 if (this_end
> vend
)
1158 while (vstart
< this_end
) {
1159 pte_val(*pte
) = (paddr
| pgprot_val(prot
));
1161 vstart
+= PAGE_SIZE
;
1170 extern unsigned int kvmap_linear_patch
[1];
1171 #endif /* CONFIG_DEBUG_PAGEALLOC */
1173 static void __init
mark_kpte_bitmap(unsigned long start
, unsigned long end
)
1175 const unsigned long shift_256MB
= 28;
1176 const unsigned long mask_256MB
= ((1UL << shift_256MB
) - 1UL);
1177 const unsigned long size_256MB
= (1UL << shift_256MB
);
1179 while (start
< end
) {
1182 remains
= end
- start
;
1183 if (remains
< size_256MB
)
1186 if (start
& mask_256MB
) {
1187 start
= (start
+ size_256MB
) & ~mask_256MB
;
1191 while (remains
>= size_256MB
) {
1192 unsigned long index
= start
>> shift_256MB
;
1194 __set_bit(index
, kpte_linear_bitmap
);
1196 start
+= size_256MB
;
1197 remains
-= size_256MB
;
1202 static void __init
kernel_physical_mapping_init(void)
1205 #ifdef CONFIG_DEBUG_PAGEALLOC
1206 unsigned long mem_alloced
= 0UL;
1209 read_obp_memory("reg", &pall
[0], &pall_ents
);
1211 for (i
= 0; i
< pall_ents
; i
++) {
1212 unsigned long phys_start
, phys_end
;
1214 phys_start
= pall
[i
].phys_addr
;
1215 phys_end
= phys_start
+ pall
[i
].reg_size
;
1217 mark_kpte_bitmap(phys_start
, phys_end
);
1219 #ifdef CONFIG_DEBUG_PAGEALLOC
1220 mem_alloced
+= kernel_map_range(phys_start
, phys_end
,
1225 #ifdef CONFIG_DEBUG_PAGEALLOC
1226 printk("Allocated %ld bytes for kernel page tables.\n",
1229 kvmap_linear_patch
[0] = 0x01000000; /* nop */
1230 flushi(&kvmap_linear_patch
[0]);
1236 #ifdef CONFIG_DEBUG_PAGEALLOC
1237 void kernel_map_pages(struct page
*page
, int numpages
, int enable
)
1239 unsigned long phys_start
= page_to_pfn(page
) << PAGE_SHIFT
;
1240 unsigned long phys_end
= phys_start
+ (numpages
* PAGE_SIZE
);
1242 kernel_map_range(phys_start
, phys_end
,
1243 (enable
? PAGE_KERNEL
: __pgprot(0)));
1245 flush_tsb_kernel_range(PAGE_OFFSET
+ phys_start
,
1246 PAGE_OFFSET
+ phys_end
);
1248 /* we should perform an IPI and flush all tlbs,
1249 * but that can deadlock->flush only current cpu.
1251 __flush_tlb_kernel_range(PAGE_OFFSET
+ phys_start
,
1252 PAGE_OFFSET
+ phys_end
);
1256 unsigned long __init
find_ecache_flush_span(unsigned long size
)
1260 for (i
= 0; i
< pavail_ents
; i
++) {
1261 if (pavail
[i
].reg_size
>= size
)
1262 return pavail
[i
].phys_addr
;
1268 static void __init
tsb_phys_patch(void)
1270 struct tsb_ldquad_phys_patch_entry
*pquad
;
1271 struct tsb_phys_patch_entry
*p
;
1273 pquad
= &__tsb_ldquad_phys_patch
;
1274 while (pquad
< &__tsb_ldquad_phys_patch_end
) {
1275 unsigned long addr
= pquad
->addr
;
1277 if (tlb_type
== hypervisor
)
1278 *(unsigned int *) addr
= pquad
->sun4v_insn
;
1280 *(unsigned int *) addr
= pquad
->sun4u_insn
;
1282 __asm__
__volatile__("flush %0"
1289 p
= &__tsb_phys_patch
;
1290 while (p
< &__tsb_phys_patch_end
) {
1291 unsigned long addr
= p
->addr
;
1293 *(unsigned int *) addr
= p
->insn
;
1295 __asm__
__volatile__("flush %0"
1303 /* Don't mark as init, we give this to the Hypervisor. */
1304 static struct hv_tsb_descr ktsb_descr
[2];
1305 extern struct tsb swapper_tsb
[KERNEL_TSB_NENTRIES
];
1307 static void __init
sun4v_ktsb_init(void)
1309 unsigned long ktsb_pa
;
1311 /* First KTSB for PAGE_SIZE mappings. */
1312 ktsb_pa
= kern_base
+ ((unsigned long)&swapper_tsb
[0] - KERNBASE
);
1314 switch (PAGE_SIZE
) {
1317 ktsb_descr
[0].pgsz_idx
= HV_PGSZ_IDX_8K
;
1318 ktsb_descr
[0].pgsz_mask
= HV_PGSZ_MASK_8K
;
1322 ktsb_descr
[0].pgsz_idx
= HV_PGSZ_IDX_64K
;
1323 ktsb_descr
[0].pgsz_mask
= HV_PGSZ_MASK_64K
;
1327 ktsb_descr
[0].pgsz_idx
= HV_PGSZ_IDX_512K
;
1328 ktsb_descr
[0].pgsz_mask
= HV_PGSZ_MASK_512K
;
1331 case 4 * 1024 * 1024:
1332 ktsb_descr
[0].pgsz_idx
= HV_PGSZ_IDX_4MB
;
1333 ktsb_descr
[0].pgsz_mask
= HV_PGSZ_MASK_4MB
;
1337 ktsb_descr
[0].assoc
= 1;
1338 ktsb_descr
[0].num_ttes
= KERNEL_TSB_NENTRIES
;
1339 ktsb_descr
[0].ctx_idx
= 0;
1340 ktsb_descr
[0].tsb_base
= ktsb_pa
;
1341 ktsb_descr
[0].resv
= 0;
1343 /* Second KTSB for 4MB/256MB mappings. */
1344 ktsb_pa
= (kern_base
+
1345 ((unsigned long)&swapper_4m_tsb
[0] - KERNBASE
));
1347 ktsb_descr
[1].pgsz_idx
= HV_PGSZ_IDX_4MB
;
1348 ktsb_descr
[1].pgsz_mask
= (HV_PGSZ_MASK_4MB
|
1349 HV_PGSZ_MASK_256MB
);
1350 ktsb_descr
[1].assoc
= 1;
1351 ktsb_descr
[1].num_ttes
= KERNEL_TSB4M_NENTRIES
;
1352 ktsb_descr
[1].ctx_idx
= 0;
1353 ktsb_descr
[1].tsb_base
= ktsb_pa
;
1354 ktsb_descr
[1].resv
= 0;
1357 void __cpuinit
sun4v_ktsb_register(void)
1359 register unsigned long func
asm("%o5");
1360 register unsigned long arg0
asm("%o0");
1361 register unsigned long arg1
asm("%o1");
1364 pa
= kern_base
+ ((unsigned long)&ktsb_descr
[0] - KERNBASE
);
1366 func
= HV_FAST_MMU_TSB_CTX0
;
1369 __asm__
__volatile__("ta %6"
1370 : "=&r" (func
), "=&r" (arg0
), "=&r" (arg1
)
1371 : "0" (func
), "1" (arg0
), "2" (arg1
),
1372 "i" (HV_FAST_TRAP
));
1375 /* paging_init() sets up the page tables */
1377 extern void cheetah_ecache_flush_init(void);
1378 extern void sun4v_patch_tlb_handlers(void);
1380 static unsigned long last_valid_pfn
;
1381 pgd_t swapper_pg_dir
[2048];
1383 static void sun4u_pgprot_init(void);
1384 static void sun4v_pgprot_init(void);
1386 void __init
paging_init(void)
1388 unsigned long end_pfn
, pages_avail
, shift
, phys_base
;
1389 unsigned long real_end
, i
;
1391 kern_base
= (prom_boot_mapping_phys_low
>> 22UL) << 22UL;
1392 kern_size
= (unsigned long)&_end
- (unsigned long)KERNBASE
;
1394 /* Invalidate both kernel TSBs. */
1395 memset(swapper_tsb
, 0x40, sizeof(swapper_tsb
));
1396 memset(swapper_4m_tsb
, 0x40, sizeof(swapper_4m_tsb
));
1398 if (tlb_type
== hypervisor
)
1399 sun4v_pgprot_init();
1401 sun4u_pgprot_init();
1403 if (tlb_type
== cheetah_plus
||
1404 tlb_type
== hypervisor
)
1407 if (tlb_type
== hypervisor
) {
1408 sun4v_patch_tlb_handlers();
1412 /* Find available physical memory... */
1413 read_obp_memory("available", &pavail
[0], &pavail_ents
);
1415 phys_base
= 0xffffffffffffffffUL
;
1416 for (i
= 0; i
< pavail_ents
; i
++)
1417 phys_base
= min(phys_base
, pavail
[i
].phys_addr
);
1419 set_bit(0, mmu_context_bmap
);
1421 shift
= kern_base
+ PAGE_OFFSET
- ((unsigned long)KERNBASE
);
1423 real_end
= (unsigned long)_end
;
1424 if ((real_end
> ((unsigned long)KERNBASE
+ 0x400000)))
1426 if ((real_end
> ((unsigned long)KERNBASE
+ 0x800000))) {
1427 prom_printf("paging_init: Kernel > 8MB, too large.\n");
1431 /* Set kernel pgd to upper alias so physical page computations
1434 init_mm
.pgd
+= ((shift
) / (sizeof(pgd_t
)));
1436 memset(swapper_low_pmd_dir
, 0, sizeof(swapper_low_pmd_dir
));
1438 /* Now can init the kernel/bad page tables. */
1439 pud_set(pud_offset(&swapper_pg_dir
[0], 0),
1440 swapper_low_pmd_dir
+ (shift
/ sizeof(pgd_t
)));
1442 inherit_prom_mappings();
1444 /* Ok, we can use our TLB miss and window trap handlers safely. */
1449 if (tlb_type
== hypervisor
)
1450 sun4v_ktsb_register();
1452 /* Setup bootmem... */
1454 last_valid_pfn
= end_pfn
= bootmem_init(&pages_avail
, phys_base
);
1456 max_mapnr
= last_valid_pfn
;
1458 kernel_physical_mapping_init();
1460 prom_build_devicetree();
1463 unsigned long zones_size
[MAX_NR_ZONES
];
1464 unsigned long zholes_size
[MAX_NR_ZONES
];
1467 for (znum
= 0; znum
< MAX_NR_ZONES
; znum
++)
1468 zones_size
[znum
] = zholes_size
[znum
] = 0;
1470 zones_size
[ZONE_DMA
] = end_pfn
;
1471 zholes_size
[ZONE_DMA
] = end_pfn
- pages_avail
;
1473 free_area_init_node(0, &contig_page_data
, zones_size
,
1474 __pa(PAGE_OFFSET
) >> PAGE_SHIFT
,
1481 static void __init
taint_real_pages(void)
1485 read_obp_memory("available", &pavail_rescan
[0], &pavail_rescan_ents
);
1487 /* Find changes discovered in the physmem available rescan and
1488 * reserve the lost portions in the bootmem maps.
1490 for (i
= 0; i
< pavail_ents
; i
++) {
1491 unsigned long old_start
, old_end
;
1493 old_start
= pavail
[i
].phys_addr
;
1494 old_end
= old_start
+
1496 while (old_start
< old_end
) {
1499 for (n
= 0; n
< pavail_rescan_ents
; n
++) {
1500 unsigned long new_start
, new_end
;
1502 new_start
= pavail_rescan
[n
].phys_addr
;
1503 new_end
= new_start
+
1504 pavail_rescan
[n
].reg_size
;
1506 if (new_start
<= old_start
&&
1507 new_end
>= (old_start
+ PAGE_SIZE
)) {
1508 set_bit(old_start
>> 22,
1509 sparc64_valid_addr_bitmap
);
1513 reserve_bootmem(old_start
, PAGE_SIZE
);
1516 old_start
+= PAGE_SIZE
;
1521 int __init
page_in_phys_avail(unsigned long paddr
)
1527 for (i
= 0; i
< pavail_rescan_ents
; i
++) {
1528 unsigned long start
, end
;
1530 start
= pavail_rescan
[i
].phys_addr
;
1531 end
= start
+ pavail_rescan
[i
].reg_size
;
1533 if (paddr
>= start
&& paddr
< end
)
1536 if (paddr
>= kern_base
&& paddr
< (kern_base
+ kern_size
))
1538 #ifdef CONFIG_BLK_DEV_INITRD
1539 if (paddr
>= __pa(initrd_start
) &&
1540 paddr
< __pa(PAGE_ALIGN(initrd_end
)))
1547 void __init
mem_init(void)
1549 unsigned long codepages
, datapages
, initpages
;
1550 unsigned long addr
, last
;
1553 i
= last_valid_pfn
>> ((22 - PAGE_SHIFT
) + 6);
1555 sparc64_valid_addr_bitmap
= (unsigned long *) alloc_bootmem(i
<< 3);
1556 if (sparc64_valid_addr_bitmap
== NULL
) {
1557 prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n");
1560 memset(sparc64_valid_addr_bitmap
, 0, i
<< 3);
1562 addr
= PAGE_OFFSET
+ kern_base
;
1563 last
= PAGE_ALIGN(kern_size
) + addr
;
1564 while (addr
< last
) {
1565 set_bit(__pa(addr
) >> 22, sparc64_valid_addr_bitmap
);
1571 high_memory
= __va(last_valid_pfn
<< PAGE_SHIFT
);
1573 #ifdef CONFIG_DEBUG_BOOTMEM
1574 prom_printf("mem_init: Calling free_all_bootmem().\n");
1576 totalram_pages
= num_physpages
= free_all_bootmem() - 1;
1579 * Set up the zero page, mark it reserved, so that page count
1580 * is not manipulated when freeing the page from user ptes.
1582 mem_map_zero
= alloc_pages(GFP_KERNEL
|__GFP_ZERO
, 0);
1583 if (mem_map_zero
== NULL
) {
1584 prom_printf("paging_init: Cannot alloc zero page.\n");
1587 SetPageReserved(mem_map_zero
);
1589 codepages
= (((unsigned long) _etext
) - ((unsigned long) _start
));
1590 codepages
= PAGE_ALIGN(codepages
) >> PAGE_SHIFT
;
1591 datapages
= (((unsigned long) _edata
) - ((unsigned long) _etext
));
1592 datapages
= PAGE_ALIGN(datapages
) >> PAGE_SHIFT
;
1593 initpages
= (((unsigned long) __init_end
) - ((unsigned long) __init_begin
));
1594 initpages
= PAGE_ALIGN(initpages
) >> PAGE_SHIFT
;
1596 printk("Memory: %uk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n",
1597 nr_free_pages() << (PAGE_SHIFT
-10),
1598 codepages
<< (PAGE_SHIFT
-10),
1599 datapages
<< (PAGE_SHIFT
-10),
1600 initpages
<< (PAGE_SHIFT
-10),
1601 PAGE_OFFSET
, (last_valid_pfn
<< PAGE_SHIFT
));
1603 if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
)
1604 cheetah_ecache_flush_init();
1607 void free_initmem(void)
1609 unsigned long addr
, initend
;
1612 * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
1614 addr
= PAGE_ALIGN((unsigned long)(__init_begin
));
1615 initend
= (unsigned long)(__init_end
) & PAGE_MASK
;
1616 for (; addr
< initend
; addr
+= PAGE_SIZE
) {
1621 ((unsigned long) __va(kern_base
)) -
1622 ((unsigned long) KERNBASE
));
1623 memset((void *)addr
, POISON_FREE_INITMEM
, PAGE_SIZE
);
1624 p
= virt_to_page(page
);
1626 ClearPageReserved(p
);
1634 #ifdef CONFIG_BLK_DEV_INITRD
1635 void free_initrd_mem(unsigned long start
, unsigned long end
)
1638 printk ("Freeing initrd memory: %ldk freed\n", (end
- start
) >> 10);
1639 for (; start
< end
; start
+= PAGE_SIZE
) {
1640 struct page
*p
= virt_to_page(start
);
1642 ClearPageReserved(p
);
1651 #define _PAGE_CACHE_4U (_PAGE_CP_4U | _PAGE_CV_4U)
1652 #define _PAGE_CACHE_4V (_PAGE_CP_4V | _PAGE_CV_4V)
1653 #define __DIRTY_BITS_4U (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U)
1654 #define __DIRTY_BITS_4V (_PAGE_MODIFIED_4V | _PAGE_WRITE_4V | _PAGE_W_4V)
1655 #define __ACCESS_BITS_4U (_PAGE_ACCESSED_4U | _PAGE_READ_4U | _PAGE_R)
1656 #define __ACCESS_BITS_4V (_PAGE_ACCESSED_4V | _PAGE_READ_4V | _PAGE_R)
1658 pgprot_t PAGE_KERNEL __read_mostly
;
1659 EXPORT_SYMBOL(PAGE_KERNEL
);
1661 pgprot_t PAGE_KERNEL_LOCKED __read_mostly
;
1662 pgprot_t PAGE_COPY __read_mostly
;
1664 pgprot_t PAGE_SHARED __read_mostly
;
1665 EXPORT_SYMBOL(PAGE_SHARED
);
1667 pgprot_t PAGE_EXEC __read_mostly
;
1668 unsigned long pg_iobits __read_mostly
;
1670 unsigned long _PAGE_IE __read_mostly
;
1671 EXPORT_SYMBOL(_PAGE_IE
);
1673 unsigned long _PAGE_E __read_mostly
;
1674 EXPORT_SYMBOL(_PAGE_E
);
1676 unsigned long _PAGE_CACHE __read_mostly
;
1677 EXPORT_SYMBOL(_PAGE_CACHE
);
1679 static void prot_init_common(unsigned long page_none
,
1680 unsigned long page_shared
,
1681 unsigned long page_copy
,
1682 unsigned long page_readonly
,
1683 unsigned long page_exec_bit
)
1685 PAGE_COPY
= __pgprot(page_copy
);
1686 PAGE_SHARED
= __pgprot(page_shared
);
1688 protection_map
[0x0] = __pgprot(page_none
);
1689 protection_map
[0x1] = __pgprot(page_readonly
& ~page_exec_bit
);
1690 protection_map
[0x2] = __pgprot(page_copy
& ~page_exec_bit
);
1691 protection_map
[0x3] = __pgprot(page_copy
& ~page_exec_bit
);
1692 protection_map
[0x4] = __pgprot(page_readonly
);
1693 protection_map
[0x5] = __pgprot(page_readonly
);
1694 protection_map
[0x6] = __pgprot(page_copy
);
1695 protection_map
[0x7] = __pgprot(page_copy
);
1696 protection_map
[0x8] = __pgprot(page_none
);
1697 protection_map
[0x9] = __pgprot(page_readonly
& ~page_exec_bit
);
1698 protection_map
[0xa] = __pgprot(page_shared
& ~page_exec_bit
);
1699 protection_map
[0xb] = __pgprot(page_shared
& ~page_exec_bit
);
1700 protection_map
[0xc] = __pgprot(page_readonly
);
1701 protection_map
[0xd] = __pgprot(page_readonly
);
1702 protection_map
[0xe] = __pgprot(page_shared
);
1703 protection_map
[0xf] = __pgprot(page_shared
);
1706 static void __init
sun4u_pgprot_init(void)
1708 unsigned long page_none
, page_shared
, page_copy
, page_readonly
;
1709 unsigned long page_exec_bit
;
1711 PAGE_KERNEL
= __pgprot (_PAGE_PRESENT_4U
| _PAGE_VALID
|
1712 _PAGE_CACHE_4U
| _PAGE_P_4U
|
1713 __ACCESS_BITS_4U
| __DIRTY_BITS_4U
|
1715 PAGE_KERNEL_LOCKED
= __pgprot (_PAGE_PRESENT_4U
| _PAGE_VALID
|
1716 _PAGE_CACHE_4U
| _PAGE_P_4U
|
1717 __ACCESS_BITS_4U
| __DIRTY_BITS_4U
|
1718 _PAGE_EXEC_4U
| _PAGE_L_4U
);
1719 PAGE_EXEC
= __pgprot(_PAGE_EXEC_4U
);
1721 _PAGE_IE
= _PAGE_IE_4U
;
1722 _PAGE_E
= _PAGE_E_4U
;
1723 _PAGE_CACHE
= _PAGE_CACHE_4U
;
1725 pg_iobits
= (_PAGE_VALID
| _PAGE_PRESENT_4U
| __DIRTY_BITS_4U
|
1726 __ACCESS_BITS_4U
| _PAGE_E_4U
);
1728 kern_linear_pte_xor
[0] = (_PAGE_VALID
| _PAGE_SZ4MB_4U
) ^
1730 kern_linear_pte_xor
[0] |= (_PAGE_CP_4U
| _PAGE_CV_4U
|
1731 _PAGE_P_4U
| _PAGE_W_4U
);
1733 /* XXX Should use 256MB on Panther. XXX */
1734 kern_linear_pte_xor
[1] = kern_linear_pte_xor
[0];
1736 _PAGE_SZBITS
= _PAGE_SZBITS_4U
;
1737 _PAGE_ALL_SZ_BITS
= (_PAGE_SZ4MB_4U
| _PAGE_SZ512K_4U
|
1738 _PAGE_SZ64K_4U
| _PAGE_SZ8K_4U
|
1739 _PAGE_SZ32MB_4U
| _PAGE_SZ256MB_4U
);
1742 page_none
= _PAGE_PRESENT_4U
| _PAGE_ACCESSED_4U
| _PAGE_CACHE_4U
;
1743 page_shared
= (_PAGE_VALID
| _PAGE_PRESENT_4U
| _PAGE_CACHE_4U
|
1744 __ACCESS_BITS_4U
| _PAGE_WRITE_4U
| _PAGE_EXEC_4U
);
1745 page_copy
= (_PAGE_VALID
| _PAGE_PRESENT_4U
| _PAGE_CACHE_4U
|
1746 __ACCESS_BITS_4U
| _PAGE_EXEC_4U
);
1747 page_readonly
= (_PAGE_VALID
| _PAGE_PRESENT_4U
| _PAGE_CACHE_4U
|
1748 __ACCESS_BITS_4U
| _PAGE_EXEC_4U
);
1750 page_exec_bit
= _PAGE_EXEC_4U
;
1752 prot_init_common(page_none
, page_shared
, page_copy
, page_readonly
,
1756 static void __init
sun4v_pgprot_init(void)
1758 unsigned long page_none
, page_shared
, page_copy
, page_readonly
;
1759 unsigned long page_exec_bit
;
1761 PAGE_KERNEL
= __pgprot (_PAGE_PRESENT_4V
| _PAGE_VALID
|
1762 _PAGE_CACHE_4V
| _PAGE_P_4V
|
1763 __ACCESS_BITS_4V
| __DIRTY_BITS_4V
|
1765 PAGE_KERNEL_LOCKED
= PAGE_KERNEL
;
1766 PAGE_EXEC
= __pgprot(_PAGE_EXEC_4V
);
1768 _PAGE_IE
= _PAGE_IE_4V
;
1769 _PAGE_E
= _PAGE_E_4V
;
1770 _PAGE_CACHE
= _PAGE_CACHE_4V
;
1772 kern_linear_pte_xor
[0] = (_PAGE_VALID
| _PAGE_SZ4MB_4V
) ^
1774 kern_linear_pte_xor
[0] |= (_PAGE_CP_4V
| _PAGE_CV_4V
|
1775 _PAGE_P_4V
| _PAGE_W_4V
);
1777 kern_linear_pte_xor
[1] = (_PAGE_VALID
| _PAGE_SZ256MB_4V
) ^
1779 kern_linear_pte_xor
[1] |= (_PAGE_CP_4V
| _PAGE_CV_4V
|
1780 _PAGE_P_4V
| _PAGE_W_4V
);
1782 pg_iobits
= (_PAGE_VALID
| _PAGE_PRESENT_4V
| __DIRTY_BITS_4V
|
1783 __ACCESS_BITS_4V
| _PAGE_E_4V
);
1785 _PAGE_SZBITS
= _PAGE_SZBITS_4V
;
1786 _PAGE_ALL_SZ_BITS
= (_PAGE_SZ16GB_4V
| _PAGE_SZ2GB_4V
|
1787 _PAGE_SZ256MB_4V
| _PAGE_SZ32MB_4V
|
1788 _PAGE_SZ4MB_4V
| _PAGE_SZ512K_4V
|
1789 _PAGE_SZ64K_4V
| _PAGE_SZ8K_4V
);
1791 page_none
= _PAGE_PRESENT_4V
| _PAGE_ACCESSED_4V
| _PAGE_CACHE_4V
;
1792 page_shared
= (_PAGE_VALID
| _PAGE_PRESENT_4V
| _PAGE_CACHE_4V
|
1793 __ACCESS_BITS_4V
| _PAGE_WRITE_4V
| _PAGE_EXEC_4V
);
1794 page_copy
= (_PAGE_VALID
| _PAGE_PRESENT_4V
| _PAGE_CACHE_4V
|
1795 __ACCESS_BITS_4V
| _PAGE_EXEC_4V
);
1796 page_readonly
= (_PAGE_VALID
| _PAGE_PRESENT_4V
| _PAGE_CACHE_4V
|
1797 __ACCESS_BITS_4V
| _PAGE_EXEC_4V
);
1799 page_exec_bit
= _PAGE_EXEC_4V
;
1801 prot_init_common(page_none
, page_shared
, page_copy
, page_readonly
,
1805 unsigned long pte_sz_bits(unsigned long sz
)
1807 if (tlb_type
== hypervisor
) {
1811 return _PAGE_SZ8K_4V
;
1813 return _PAGE_SZ64K_4V
;
1815 return _PAGE_SZ512K_4V
;
1816 case 4 * 1024 * 1024:
1817 return _PAGE_SZ4MB_4V
;
1823 return _PAGE_SZ8K_4U
;
1825 return _PAGE_SZ64K_4U
;
1827 return _PAGE_SZ512K_4U
;
1828 case 4 * 1024 * 1024:
1829 return _PAGE_SZ4MB_4U
;
1834 pte_t
mk_pte_io(unsigned long page
, pgprot_t prot
, int space
, unsigned long page_size
)
1838 pte_val(pte
) = page
| pgprot_val(pgprot_noncached(prot
));
1839 pte_val(pte
) |= (((unsigned long)space
) << 32);
1840 pte_val(pte
) |= pte_sz_bits(page_size
);
1845 static unsigned long kern_large_tte(unsigned long paddr
)
1849 val
= (_PAGE_VALID
| _PAGE_SZ4MB_4U
|
1850 _PAGE_CP_4U
| _PAGE_CV_4U
| _PAGE_P_4U
|
1851 _PAGE_EXEC_4U
| _PAGE_L_4U
| _PAGE_W_4U
);
1852 if (tlb_type
== hypervisor
)
1853 val
= (_PAGE_VALID
| _PAGE_SZ4MB_4V
|
1854 _PAGE_CP_4V
| _PAGE_CV_4V
| _PAGE_P_4V
|
1855 _PAGE_EXEC_4V
| _PAGE_W_4V
);
1861 * Translate PROM's mapping we capture at boot time into physical address.
1862 * The second parameter is only set from prom_callback() invocations.
1864 unsigned long prom_virt_to_phys(unsigned long promva
, int *error
)
1869 mask
= _PAGE_PADDR_4U
;
1870 if (tlb_type
== hypervisor
)
1871 mask
= _PAGE_PADDR_4V
;
1873 for (i
= 0; i
< prom_trans_ents
; i
++) {
1874 struct linux_prom_translation
*p
= &prom_trans
[i
];
1876 if (promva
>= p
->virt
&&
1877 promva
< (p
->virt
+ p
->size
)) {
1878 unsigned long base
= p
->data
& mask
;
1882 return base
+ (promva
& (8192 - 1));
1890 /* XXX We should kill off this ugly thing at so me point. XXX */
1891 unsigned long sun4u_get_pte(unsigned long addr
)
1897 unsigned long mask
= _PAGE_PADDR_4U
;
1899 if (tlb_type
== hypervisor
)
1900 mask
= _PAGE_PADDR_4V
;
1902 if (addr
>= PAGE_OFFSET
)
1905 if ((addr
>= LOW_OBP_ADDRESS
) && (addr
< HI_OBP_ADDRESS
))
1906 return prom_virt_to_phys(addr
, NULL
);
1908 pgdp
= pgd_offset_k(addr
);
1909 pudp
= pud_offset(pgdp
, addr
);
1910 pmdp
= pmd_offset(pudp
, addr
);
1911 ptep
= pte_offset_kernel(pmdp
, addr
);
1913 return pte_val(*ptep
) & mask
;
1916 /* If not locked, zap it. */
1917 void __flush_tlb_all(void)
1919 unsigned long pstate
;
1922 __asm__
__volatile__("flushw\n\t"
1923 "rdpr %%pstate, %0\n\t"
1924 "wrpr %0, %1, %%pstate"
1927 if (tlb_type
== spitfire
) {
1928 for (i
= 0; i
< 64; i
++) {
1929 /* Spitfire Errata #32 workaround */
1930 /* NOTE: Always runs on spitfire, so no
1931 * cheetah+ page size encodings.
1933 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
1937 "r" (PRIMARY_CONTEXT
), "i" (ASI_DMMU
));
1939 if (!(spitfire_get_dtlb_data(i
) & _PAGE_L_4U
)) {
1940 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
1943 : "r" (TLB_TAG_ACCESS
), "i" (ASI_DMMU
));
1944 spitfire_put_dtlb_data(i
, 0x0UL
);
1947 /* Spitfire Errata #32 workaround */
1948 /* NOTE: Always runs on spitfire, so no
1949 * cheetah+ page size encodings.
1951 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
1955 "r" (PRIMARY_CONTEXT
), "i" (ASI_DMMU
));
1957 if (!(spitfire_get_itlb_data(i
) & _PAGE_L_4U
)) {
1958 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
1961 : "r" (TLB_TAG_ACCESS
), "i" (ASI_IMMU
));
1962 spitfire_put_itlb_data(i
, 0x0UL
);
1965 } else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
) {
1966 cheetah_flush_dtlb_all();
1967 cheetah_flush_itlb_all();
1969 __asm__
__volatile__("wrpr %0, 0, %%pstate"
1973 #ifdef CONFIG_MEMORY_HOTPLUG
1975 void online_page(struct page
*page
)
1977 ClearPageReserved(page
);
1978 init_page_count(page
);
1984 int remove_memory(u64 start
, u64 size
)
1989 #endif /* CONFIG_MEMORY_HOTPLUG */