1 /* $Id: init.c,v 1.209 2002/02/09 19:49:31 davem Exp $
2 * arch/sparc64/mm/init.c
4 * Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
8 #include <linux/config.h>
9 #include <linux/kernel.h>
10 #include <linux/sched.h>
11 #include <linux/string.h>
12 #include <linux/init.h>
13 #include <linux/bootmem.h>
15 #include <linux/hugetlb.h>
16 #include <linux/slab.h>
17 #include <linux/initrd.h>
18 #include <linux/swap.h>
19 #include <linux/pagemap.h>
21 #include <linux/seq_file.h>
24 #include <asm/system.h>
26 #include <asm/pgalloc.h>
27 #include <asm/pgtable.h>
28 #include <asm/oplib.h>
29 #include <asm/iommu.h>
31 #include <asm/uaccess.h>
32 #include <asm/mmu_context.h>
33 #include <asm/tlbflush.h>
35 #include <asm/starfire.h>
37 #include <asm/spitfire.h>
38 #include <asm/sections.h>
40 extern void device_scan(void);
42 struct sparc_phys_banks sp_banks
[SPARC_PHYS_BANKS
];
44 unsigned long *sparc64_valid_addr_bitmap
;
46 /* Ugly, but necessary... -DaveM */
47 unsigned long phys_base
;
48 unsigned long kern_base
;
49 unsigned long kern_size
;
50 unsigned long pfn_base
;
52 /* This is even uglier. We have a problem where the kernel may not be
53 * located at phys_base. However, initial __alloc_bootmem() calls need to
54 * be adjusted to be within the 4-8Megs that the kernel is mapped to, else
55 * those page mappings wont work. Things are ok after inherit_prom_mappings
56 * is called though. Dave says he'll clean this up some other time.
59 static unsigned long bootmap_base
;
61 /* get_new_mmu_context() uses "cache + 1". */
62 DEFINE_SPINLOCK(ctx_alloc_lock
);
63 unsigned long tlb_context_cache
= CTX_FIRST_VERSION
- 1;
64 #define CTX_BMAP_SLOTS (1UL << (CTX_NR_BITS - 6))
65 unsigned long mmu_context_bmap
[CTX_BMAP_SLOTS
];
67 /* References to special section boundaries */
68 extern char _start
[], _end
[];
70 /* Initial ramdisk setup */
71 extern unsigned long sparc_ramdisk_image64
;
72 extern unsigned int sparc_ramdisk_image
;
73 extern unsigned int sparc_ramdisk_size
;
75 struct page
*mem_map_zero
;
79 /* XXX Tune this... */
80 #define PGT_CACHE_LOW 25
81 #define PGT_CACHE_HIGH 50
83 void check_pgt_cache(void)
86 if (pgtable_cache_size
> PGT_CACHE_HIGH
) {
89 free_pgd_slow(get_pgd_fast());
91 free_pte_slow(pte_alloc_one_fast(NULL
, 0));
93 free_pte_slow(pte_alloc_one_fast(NULL
, 1 << (PAGE_SHIFT
+ 10)));
94 } while (pgtable_cache_size
> PGT_CACHE_LOW
);
99 #ifdef CONFIG_DEBUG_DCFLUSH
100 atomic_t dcpage_flushes
= ATOMIC_INIT(0);
102 atomic_t dcpage_flushes_xcall
= ATOMIC_INIT(0);
106 __inline__
void flush_dcache_page_impl(struct page
*page
)
108 #ifdef CONFIG_DEBUG_DCFLUSH
109 atomic_inc(&dcpage_flushes
);
112 #ifdef DCACHE_ALIASING_POSSIBLE
113 __flush_dcache_page(page_address(page
),
114 ((tlb_type
== spitfire
) &&
115 page_mapping(page
) != NULL
));
117 if (page_mapping(page
) != NULL
&&
118 tlb_type
== spitfire
)
119 __flush_icache_page(__pa(page_address(page
)));
123 #define PG_dcache_dirty PG_arch_1
125 #define dcache_dirty_cpu(page) \
126 (((page)->flags >> 24) & (NR_CPUS - 1UL))
128 static __inline__
void set_dcache_dirty(struct page
*page
, int this_cpu
)
130 unsigned long mask
= this_cpu
;
131 unsigned long non_cpu_bits
= ~((NR_CPUS
- 1UL) << 24UL);
132 mask
= (mask
<< 24) | (1UL << PG_dcache_dirty
);
133 __asm__
__volatile__("1:\n\t"
135 "and %%g7, %1, %%g1\n\t"
136 "or %%g1, %0, %%g1\n\t"
137 "casx [%2], %%g7, %%g1\n\t"
139 "bne,pn %%xcc, 1b\n\t"
140 " membar #StoreLoad | #StoreStore"
142 : "r" (mask
), "r" (non_cpu_bits
), "r" (&page
->flags
)
146 static __inline__
void clear_dcache_dirty_cpu(struct page
*page
, unsigned long cpu
)
148 unsigned long mask
= (1UL << PG_dcache_dirty
);
150 __asm__
__volatile__("! test_and_clear_dcache_dirty\n"
153 "srlx %%g7, 24, %%g1\n\t"
154 "and %%g1, %3, %%g1\n\t"
156 "bne,pn %%icc, 2f\n\t"
157 " andn %%g7, %1, %%g1\n\t"
158 "casx [%2], %%g7, %%g1\n\t"
160 "bne,pn %%xcc, 1b\n\t"
161 " membar #StoreLoad | #StoreStore\n"
164 : "r" (cpu
), "r" (mask
), "r" (&page
->flags
),
169 extern void __update_mmu_cache(unsigned long mmu_context_hw
, unsigned long address
, pte_t pte
, int code
);
171 void update_mmu_cache(struct vm_area_struct
*vma
, unsigned long address
, pte_t pte
)
175 unsigned long pg_flags
;
178 if (pfn_valid(pfn
) &&
179 (page
= pfn_to_page(pfn
), page_mapping(page
)) &&
180 ((pg_flags
= page
->flags
) & (1UL << PG_dcache_dirty
))) {
181 int cpu
= ((pg_flags
>> 24) & (NR_CPUS
- 1UL));
182 int this_cpu
= get_cpu();
184 /* This is just to optimize away some function calls
188 flush_dcache_page_impl(page
);
190 smp_flush_dcache_page_impl(page
, cpu
);
192 clear_dcache_dirty_cpu(page
, cpu
);
197 if (get_thread_fault_code())
198 __update_mmu_cache(CTX_NRBITS(vma
->vm_mm
->context
),
199 address
, pte
, get_thread_fault_code());
202 void flush_dcache_page(struct page
*page
)
204 struct address_space
*mapping
= page_mapping(page
);
205 int dirty
= test_bit(PG_dcache_dirty
, &page
->flags
);
206 int dirty_cpu
= dcache_dirty_cpu(page
);
207 int this_cpu
= get_cpu();
209 if (mapping
&& !mapping_mapped(mapping
)) {
211 if (dirty_cpu
== this_cpu
)
213 smp_flush_dcache_page_impl(page
, dirty_cpu
);
215 set_dcache_dirty(page
, this_cpu
);
217 /* We could delay the flush for the !page_mapping
218 * case too. But that case is for exec env/arg
219 * pages and those are %99 certainly going to get
220 * faulted into the tlb (and thus flushed) anyways.
222 flush_dcache_page_impl(page
);
229 void flush_icache_range(unsigned long start
, unsigned long end
)
231 /* Cheetah has coherent I-cache. */
232 if (tlb_type
== spitfire
) {
235 for (kaddr
= start
; kaddr
< end
; kaddr
+= PAGE_SIZE
)
236 __flush_icache_page(__get_phys(kaddr
));
240 unsigned long page_to_pfn(struct page
*page
)
242 return (unsigned long) ((page
- mem_map
) + pfn_base
);
245 struct page
*pfn_to_page(unsigned long pfn
)
247 return (mem_map
+ (pfn
- pfn_base
));
252 printk("Mem-info:\n");
254 printk("Free swap: %6ldkB\n",
255 nr_swap_pages
<< (PAGE_SHIFT
-10));
256 printk("%ld pages of RAM\n", num_physpages
);
257 printk("%d free pages\n", nr_free_pages());
258 printk("%d pages in page table cache\n",pgtable_cache_size
);
261 void mmu_info(struct seq_file
*m
)
263 if (tlb_type
== cheetah
)
264 seq_printf(m
, "MMU Type\t: Cheetah\n");
265 else if (tlb_type
== cheetah_plus
)
266 seq_printf(m
, "MMU Type\t: Cheetah+\n");
267 else if (tlb_type
== spitfire
)
268 seq_printf(m
, "MMU Type\t: Spitfire\n");
270 seq_printf(m
, "MMU Type\t: ???\n");
272 #ifdef CONFIG_DEBUG_DCFLUSH
273 seq_printf(m
, "DCPageFlushes\t: %d\n",
274 atomic_read(&dcpage_flushes
));
276 seq_printf(m
, "DCPageFlushesXC\t: %d\n",
277 atomic_read(&dcpage_flushes_xcall
));
278 #endif /* CONFIG_SMP */
279 #endif /* CONFIG_DEBUG_DCFLUSH */
282 struct linux_prom_translation
{
288 extern unsigned long prom_boot_page
;
289 extern void prom_remap(unsigned long physpage
, unsigned long virtpage
, int mmu_ihandle
);
290 extern int prom_get_mmu_ihandle(void);
291 extern void register_prom_callbacks(void);
293 /* Exported for SMP bootup purposes. */
294 unsigned long kern_locked_tte_data
;
296 void __init
early_pgtable_allocfail(char *type
)
298 prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type
);
302 #define BASE_PAGE_SIZE 8192
303 static pmd_t
*prompmd
;
306 * Translate PROM's mapping we capture at boot time into physical address.
307 * The second parameter is only set from prom_callback() invocations.
309 unsigned long prom_virt_to_phys(unsigned long promva
, int *error
)
311 pmd_t
*pmdp
= prompmd
+ ((promva
>> 23) & 0x7ff);
315 if (pmd_none(*pmdp
)) {
320 ptep
= (pte_t
*)__pmd_page(*pmdp
) + ((promva
>> 13) & 0x3ff);
321 if (!pte_present(*ptep
)) {
328 return(pte_val(*ptep
));
330 base
= pte_val(*ptep
) & _PAGE_PADDR
;
331 return(base
+ (promva
& (BASE_PAGE_SIZE
- 1)));
334 static void inherit_prom_mappings(void)
336 struct linux_prom_translation
*trans
;
337 unsigned long phys_page
, tte_vaddr
, tte_data
;
338 void (*remap_func
)(unsigned long, unsigned long, int);
342 extern unsigned int obp_iaddr_patch
[2], obp_daddr_patch
[2];
344 node
= prom_finddevice("/virtual-memory");
345 n
= prom_getproplen(node
, "translations");
346 if (n
== 0 || n
== -1) {
347 prom_printf("Couldn't get translation property\n");
350 n
+= 5 * sizeof(struct linux_prom_translation
);
351 for (tsz
= 1; tsz
< n
; tsz
<<= 1)
353 trans
= __alloc_bootmem(tsz
, SMP_CACHE_BYTES
, bootmap_base
);
355 prom_printf("inherit_prom_mappings: Cannot alloc translations.\n");
358 memset(trans
, 0, tsz
);
360 if ((n
= prom_getproperty(node
, "translations", (char *)trans
, tsz
)) == -1) {
361 prom_printf("Couldn't get translation property\n");
364 n
= n
/ sizeof(*trans
);
367 * The obp translations are saved based on 8k pagesize, since obp can
368 * use a mixture of pagesizes. Misses to the 0xf0000000 - 0x100000000,
369 * ie obp range, are handled in entry.S and do not use the vpte scheme
370 * (see rant in inherit_locked_prom_mappings()).
372 #define OBP_PMD_SIZE 2048
373 prompmd
= __alloc_bootmem(OBP_PMD_SIZE
, OBP_PMD_SIZE
, bootmap_base
);
375 early_pgtable_allocfail("pmd");
376 memset(prompmd
, 0, OBP_PMD_SIZE
);
377 for (i
= 0; i
< n
; i
++) {
380 if (trans
[i
].virt
>= LOW_OBP_ADDRESS
&& trans
[i
].virt
< HI_OBP_ADDRESS
) {
381 for (vaddr
= trans
[i
].virt
;
382 ((vaddr
< trans
[i
].virt
+ trans
[i
].size
) &&
383 (vaddr
< HI_OBP_ADDRESS
));
384 vaddr
+= BASE_PAGE_SIZE
) {
387 pmdp
= prompmd
+ ((vaddr
>> 23) & 0x7ff);
388 if (pmd_none(*pmdp
)) {
389 ptep
= __alloc_bootmem(BASE_PAGE_SIZE
,
393 early_pgtable_allocfail("pte");
394 memset(ptep
, 0, BASE_PAGE_SIZE
);
397 ptep
= (pte_t
*)__pmd_page(*pmdp
) +
398 ((vaddr
>> 13) & 0x3ff);
402 /* Clear diag TTE bits. */
403 if (tlb_type
== spitfire
)
404 val
&= ~0x0003fe0000000000UL
;
406 set_pte_at(&init_mm
, vaddr
,
407 ptep
, __pte(val
| _PAGE_MODIFIED
));
408 trans
[i
].data
+= BASE_PAGE_SIZE
;
412 phys_page
= __pa(prompmd
);
413 obp_iaddr_patch
[0] |= (phys_page
>> 10);
414 obp_iaddr_patch
[1] |= (phys_page
& 0x3ff);
415 flushi((long)&obp_iaddr_patch
[0]);
416 obp_daddr_patch
[0] |= (phys_page
>> 10);
417 obp_daddr_patch
[1] |= (phys_page
& 0x3ff);
418 flushi((long)&obp_daddr_patch
[0]);
420 /* Now fixup OBP's idea about where we really are mapped. */
421 prom_printf("Remapping the kernel... ");
423 /* Spitfire Errata #32 workaround */
424 /* NOTE: Using plain zero for the context value is
425 * correct here, we are not using the Linux trap
426 * tables yet so we should not use the special
427 * UltraSPARC-III+ page size encodings yet.
429 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
432 : "r" (0), "r" (PRIMARY_CONTEXT
), "i" (ASI_DMMU
));
437 phys_page
= spitfire_get_dtlb_data(sparc64_highest_locked_tlbent());
442 phys_page
= cheetah_get_litlb_data(sparc64_highest_locked_tlbent());
446 phys_page
&= _PAGE_PADDR
;
447 phys_page
+= ((unsigned long)&prom_boot_page
-
448 (unsigned long)KERNBASE
);
450 if (tlb_type
== spitfire
) {
451 /* Lock this into i/d tlb entry 59 */
452 __asm__
__volatile__(
453 "stxa %%g0, [%2] %3\n\t"
454 "stxa %0, [%1] %4\n\t"
457 "stxa %%g0, [%2] %5\n\t"
458 "stxa %0, [%1] %6\n\t"
461 : : "r" (phys_page
| _PAGE_VALID
| _PAGE_SZ8K
| _PAGE_CP
|
462 _PAGE_CV
| _PAGE_P
| _PAGE_L
| _PAGE_W
),
463 "r" (59 << 3), "r" (TLB_TAG_ACCESS
),
464 "i" (ASI_DMMU
), "i" (ASI_DTLB_DATA_ACCESS
),
465 "i" (ASI_IMMU
), "i" (ASI_ITLB_DATA_ACCESS
)
467 } else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
) {
468 /* Lock this into i/d tlb-0 entry 11 */
469 __asm__
__volatile__(
470 "stxa %%g0, [%2] %3\n\t"
471 "stxa %0, [%1] %4\n\t"
474 "stxa %%g0, [%2] %5\n\t"
475 "stxa %0, [%1] %6\n\t"
478 : : "r" (phys_page
| _PAGE_VALID
| _PAGE_SZ8K
| _PAGE_CP
|
479 _PAGE_CV
| _PAGE_P
| _PAGE_L
| _PAGE_W
),
480 "r" ((0 << 16) | (11 << 3)), "r" (TLB_TAG_ACCESS
),
481 "i" (ASI_DMMU
), "i" (ASI_DTLB_DATA_ACCESS
),
482 "i" (ASI_IMMU
), "i" (ASI_ITLB_DATA_ACCESS
)
485 /* Implement me :-) */
489 tte_vaddr
= (unsigned long) KERNBASE
;
491 /* Spitfire Errata #32 workaround */
492 /* NOTE: Using plain zero for the context value is
493 * correct here, we are not using the Linux trap
494 * tables yet so we should not use the special
495 * UltraSPARC-III+ page size encodings yet.
497 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
501 "r" (PRIMARY_CONTEXT
), "i" (ASI_DMMU
));
503 if (tlb_type
== spitfire
)
504 tte_data
= spitfire_get_dtlb_data(sparc64_highest_locked_tlbent());
506 tte_data
= cheetah_get_ldtlb_data(sparc64_highest_locked_tlbent());
508 kern_locked_tte_data
= tte_data
;
510 remap_func
= (void *) ((unsigned long) &prom_remap
-
511 (unsigned long) &prom_boot_page
);
514 /* Spitfire Errata #32 workaround */
515 /* NOTE: Using plain zero for the context value is
516 * correct here, we are not using the Linux trap
517 * tables yet so we should not use the special
518 * UltraSPARC-III+ page size encodings yet.
520 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
524 "r" (PRIMARY_CONTEXT
), "i" (ASI_DMMU
));
526 remap_func((tlb_type
== spitfire
?
527 (spitfire_get_dtlb_data(sparc64_highest_locked_tlbent()) & _PAGE_PADDR
) :
528 (cheetah_get_litlb_data(sparc64_highest_locked_tlbent()) & _PAGE_PADDR
)),
529 (unsigned long) KERNBASE
,
530 prom_get_mmu_ihandle());
533 remap_func(((tte_data
+ 0x400000) & _PAGE_PADDR
),
534 (unsigned long) KERNBASE
+ 0x400000, prom_get_mmu_ihandle());
536 /* Flush out that temporary mapping. */
537 spitfire_flush_dtlb_nucleus_page(0x0);
538 spitfire_flush_itlb_nucleus_page(0x0);
540 /* Now lock us back into the TLBs via OBP. */
541 prom_dtlb_load(sparc64_highest_locked_tlbent(), tte_data
, tte_vaddr
);
542 prom_itlb_load(sparc64_highest_locked_tlbent(), tte_data
, tte_vaddr
);
544 prom_dtlb_load(sparc64_highest_locked_tlbent()-1, tte_data
+ 0x400000,
545 tte_vaddr
+ 0x400000);
546 prom_itlb_load(sparc64_highest_locked_tlbent()-1, tte_data
+ 0x400000,
547 tte_vaddr
+ 0x400000);
550 /* Re-read translations property. */
551 if ((n
= prom_getproperty(node
, "translations", (char *)trans
, tsz
)) == -1) {
552 prom_printf("Couldn't get translation property\n");
555 n
= n
/ sizeof(*trans
);
557 for (i
= 0; i
< n
; i
++) {
558 unsigned long vaddr
= trans
[i
].virt
;
559 unsigned long size
= trans
[i
].size
;
561 if (vaddr
< 0xf0000000UL
) {
562 unsigned long avoid_start
= (unsigned long) KERNBASE
;
563 unsigned long avoid_end
= avoid_start
+ (4 * 1024 * 1024);
566 avoid_end
+= (4 * 1024 * 1024);
567 if (vaddr
< avoid_start
) {
568 unsigned long top
= vaddr
+ size
;
570 if (top
> avoid_start
)
572 prom_unmap(top
- vaddr
, vaddr
);
574 if ((vaddr
+ size
) > avoid_end
) {
575 unsigned long bottom
= vaddr
;
577 if (bottom
< avoid_end
)
579 prom_unmap((vaddr
+ size
) - bottom
, bottom
);
584 prom_printf("done.\n");
586 register_prom_callbacks();
589 /* The OBP specifications for sun4u mark 0xfffffffc00000000 and
590 * upwards as reserved for use by the firmware (I wonder if this
591 * will be the same on Cheetah...). We use this virtual address
592 * range for the VPTE table mappings of the nucleus so we need
593 * to zap them when we enter the PROM. -DaveM
595 static void __flush_nucleus_vptes(void)
597 unsigned long prom_reserved_base
= 0xfffffffc00000000UL
;
600 /* Only DTLB must be checked for VPTE entries. */
601 if (tlb_type
== spitfire
) {
602 for (i
= 0; i
< 63; i
++) {
605 /* Spitfire Errata #32 workaround */
606 /* NOTE: Always runs on spitfire, so no cheetah+
607 * page size encodings.
609 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
613 "r" (PRIMARY_CONTEXT
), "i" (ASI_DMMU
));
615 tag
= spitfire_get_dtlb_tag(i
);
616 if (((tag
& ~(PAGE_MASK
)) == 0) &&
617 ((tag
& (PAGE_MASK
)) >= prom_reserved_base
)) {
618 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
621 : "r" (TLB_TAG_ACCESS
), "i" (ASI_DMMU
));
622 spitfire_put_dtlb_data(i
, 0x0UL
);
625 } else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
) {
626 for (i
= 0; i
< 512; i
++) {
627 unsigned long tag
= cheetah_get_dtlb_tag(i
, 2);
629 if ((tag
& ~PAGE_MASK
) == 0 &&
630 (tag
& PAGE_MASK
) >= prom_reserved_base
) {
631 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
634 : "r" (TLB_TAG_ACCESS
), "i" (ASI_DMMU
));
635 cheetah_put_dtlb_data(i
, 0x0UL
, 2);
638 if (tlb_type
!= cheetah_plus
)
641 tag
= cheetah_get_dtlb_tag(i
, 3);
643 if ((tag
& ~PAGE_MASK
) == 0 &&
644 (tag
& PAGE_MASK
) >= prom_reserved_base
) {
645 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
648 : "r" (TLB_TAG_ACCESS
), "i" (ASI_DMMU
));
649 cheetah_put_dtlb_data(i
, 0x0UL
, 3);
653 /* Implement me :-) */
658 static int prom_ditlb_set
;
659 struct prom_tlb_entry
{
661 unsigned long tlb_tag
;
662 unsigned long tlb_data
;
664 struct prom_tlb_entry prom_itlb
[16], prom_dtlb
[16];
666 void prom_world(int enter
)
668 unsigned long pstate
;
672 set_fs((mm_segment_t
) { get_thread_current_ds() });
677 /* Make sure the following runs atomically. */
678 __asm__
__volatile__("flushw\n\t"
679 "rdpr %%pstate, %0\n\t"
680 "wrpr %0, %1, %%pstate"
685 /* Kick out nucleus VPTEs. */
686 __flush_nucleus_vptes();
688 /* Install PROM world. */
689 for (i
= 0; i
< 16; i
++) {
690 if (prom_dtlb
[i
].tlb_ent
!= -1) {
691 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
693 : : "r" (prom_dtlb
[i
].tlb_tag
), "r" (TLB_TAG_ACCESS
),
695 if (tlb_type
== spitfire
)
696 spitfire_put_dtlb_data(prom_dtlb
[i
].tlb_ent
,
697 prom_dtlb
[i
].tlb_data
);
698 else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
)
699 cheetah_put_ldtlb_data(prom_dtlb
[i
].tlb_ent
,
700 prom_dtlb
[i
].tlb_data
);
702 if (prom_itlb
[i
].tlb_ent
!= -1) {
703 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
705 : : "r" (prom_itlb
[i
].tlb_tag
),
706 "r" (TLB_TAG_ACCESS
),
708 if (tlb_type
== spitfire
)
709 spitfire_put_itlb_data(prom_itlb
[i
].tlb_ent
,
710 prom_itlb
[i
].tlb_data
);
711 else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
)
712 cheetah_put_litlb_data(prom_itlb
[i
].tlb_ent
,
713 prom_itlb
[i
].tlb_data
);
717 for (i
= 0; i
< 16; i
++) {
718 if (prom_dtlb
[i
].tlb_ent
!= -1) {
719 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
721 : : "r" (TLB_TAG_ACCESS
), "i" (ASI_DMMU
));
722 if (tlb_type
== spitfire
)
723 spitfire_put_dtlb_data(prom_dtlb
[i
].tlb_ent
, 0x0UL
);
725 cheetah_put_ldtlb_data(prom_dtlb
[i
].tlb_ent
, 0x0UL
);
727 if (prom_itlb
[i
].tlb_ent
!= -1) {
728 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
730 : : "r" (TLB_TAG_ACCESS
),
732 if (tlb_type
== spitfire
)
733 spitfire_put_itlb_data(prom_itlb
[i
].tlb_ent
, 0x0UL
);
735 cheetah_put_litlb_data(prom_itlb
[i
].tlb_ent
, 0x0UL
);
739 __asm__
__volatile__("wrpr %0, 0, %%pstate"
743 void inherit_locked_prom_mappings(int save_p
)
749 /* Fucking losing PROM has more mappings in the TLB, but
750 * it (conveniently) fails to mention any of these in the
751 * translations property. The only ones that matter are
752 * the locked PROM tlb entries, so we impose the following
753 * irrecovable rule on the PROM, it is allowed 8 locked
754 * entries in the ITLB and 8 in the DTLB.
756 * Supposedly the upper 16GB of the address space is
757 * reserved for OBP, BUT I WISH THIS WAS DOCUMENTED
758 * SOMEWHERE!!!!!!!!!!!!!!!!! Furthermore the entire interface
759 * used between the client program and the firmware on sun5
760 * systems to coordinate mmu mappings is also COMPLETELY
761 * UNDOCUMENTED!!!!!! Thanks S(t)un!
764 for (i
= 0; i
< 16; i
++) {
765 prom_itlb
[i
].tlb_ent
= -1;
766 prom_dtlb
[i
].tlb_ent
= -1;
769 if (tlb_type
== spitfire
) {
770 int high
= SPITFIRE_HIGHEST_LOCKED_TLBENT
- bigkernel
;
771 for (i
= 0; i
< high
; i
++) {
774 /* Spitfire Errata #32 workaround */
775 /* NOTE: Always runs on spitfire, so no cheetah+
776 * page size encodings.
778 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
782 "r" (PRIMARY_CONTEXT
), "i" (ASI_DMMU
));
784 data
= spitfire_get_dtlb_data(i
);
785 if ((data
& (_PAGE_L
|_PAGE_VALID
)) == (_PAGE_L
|_PAGE_VALID
)) {
788 /* Spitfire Errata #32 workaround */
789 /* NOTE: Always runs on spitfire, so no
790 * cheetah+ page size encodings.
792 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
796 "r" (PRIMARY_CONTEXT
), "i" (ASI_DMMU
));
798 tag
= spitfire_get_dtlb_tag(i
);
800 prom_dtlb
[dtlb_seen
].tlb_ent
= i
;
801 prom_dtlb
[dtlb_seen
].tlb_tag
= tag
;
802 prom_dtlb
[dtlb_seen
].tlb_data
= data
;
804 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
806 : : "r" (TLB_TAG_ACCESS
), "i" (ASI_DMMU
));
807 spitfire_put_dtlb_data(i
, 0x0UL
);
815 for (i
= 0; i
< high
; i
++) {
818 /* Spitfire Errata #32 workaround */
819 /* NOTE: Always runs on spitfire, so no
820 * cheetah+ page size encodings.
822 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
826 "r" (PRIMARY_CONTEXT
), "i" (ASI_DMMU
));
828 data
= spitfire_get_itlb_data(i
);
829 if ((data
& (_PAGE_L
|_PAGE_VALID
)) == (_PAGE_L
|_PAGE_VALID
)) {
832 /* Spitfire Errata #32 workaround */
833 /* NOTE: Always runs on spitfire, so no
834 * cheetah+ page size encodings.
836 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
840 "r" (PRIMARY_CONTEXT
), "i" (ASI_DMMU
));
842 tag
= spitfire_get_itlb_tag(i
);
844 prom_itlb
[itlb_seen
].tlb_ent
= i
;
845 prom_itlb
[itlb_seen
].tlb_tag
= tag
;
846 prom_itlb
[itlb_seen
].tlb_data
= data
;
848 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
850 : : "r" (TLB_TAG_ACCESS
), "i" (ASI_IMMU
));
851 spitfire_put_itlb_data(i
, 0x0UL
);
858 } else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
) {
859 int high
= CHEETAH_HIGHEST_LOCKED_TLBENT
- bigkernel
;
861 for (i
= 0; i
< high
; i
++) {
864 data
= cheetah_get_ldtlb_data(i
);
865 if ((data
& (_PAGE_L
|_PAGE_VALID
)) == (_PAGE_L
|_PAGE_VALID
)) {
868 tag
= cheetah_get_ldtlb_tag(i
);
870 prom_dtlb
[dtlb_seen
].tlb_ent
= i
;
871 prom_dtlb
[dtlb_seen
].tlb_tag
= tag
;
872 prom_dtlb
[dtlb_seen
].tlb_data
= data
;
874 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
876 : : "r" (TLB_TAG_ACCESS
), "i" (ASI_DMMU
));
877 cheetah_put_ldtlb_data(i
, 0x0UL
);
885 for (i
= 0; i
< high
; i
++) {
888 data
= cheetah_get_litlb_data(i
);
889 if ((data
& (_PAGE_L
|_PAGE_VALID
)) == (_PAGE_L
|_PAGE_VALID
)) {
892 tag
= cheetah_get_litlb_tag(i
);
894 prom_itlb
[itlb_seen
].tlb_ent
= i
;
895 prom_itlb
[itlb_seen
].tlb_tag
= tag
;
896 prom_itlb
[itlb_seen
].tlb_data
= data
;
898 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
900 : : "r" (TLB_TAG_ACCESS
), "i" (ASI_IMMU
));
901 cheetah_put_litlb_data(i
, 0x0UL
);
909 /* Implement me :-) */
916 /* Give PROM back his world, done during reboots... */
917 void prom_reload_locked(void)
921 for (i
= 0; i
< 16; i
++) {
922 if (prom_dtlb
[i
].tlb_ent
!= -1) {
923 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
925 : : "r" (prom_dtlb
[i
].tlb_tag
), "r" (TLB_TAG_ACCESS
),
927 if (tlb_type
== spitfire
)
928 spitfire_put_dtlb_data(prom_dtlb
[i
].tlb_ent
,
929 prom_dtlb
[i
].tlb_data
);
930 else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
)
931 cheetah_put_ldtlb_data(prom_dtlb
[i
].tlb_ent
,
932 prom_dtlb
[i
].tlb_data
);
935 if (prom_itlb
[i
].tlb_ent
!= -1) {
936 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
938 : : "r" (prom_itlb
[i
].tlb_tag
),
939 "r" (TLB_TAG_ACCESS
),
941 if (tlb_type
== spitfire
)
942 spitfire_put_itlb_data(prom_itlb
[i
].tlb_ent
,
943 prom_itlb
[i
].tlb_data
);
945 cheetah_put_litlb_data(prom_itlb
[i
].tlb_ent
,
946 prom_itlb
[i
].tlb_data
);
951 #ifdef DCACHE_ALIASING_POSSIBLE
952 void __flush_dcache_range(unsigned long start
, unsigned long end
)
956 if (tlb_type
== spitfire
) {
959 for (va
= start
; va
< end
; va
+= 32) {
960 spitfire_put_dcache_tag(va
& 0x3fe0, 0x0);
967 for (va
= start
; va
< end
; va
+= 32)
968 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
972 "i" (ASI_DCACHE_INVALIDATE
));
975 #endif /* DCACHE_ALIASING_POSSIBLE */
977 /* If not locked, zap it. */
978 void __flush_tlb_all(void)
980 unsigned long pstate
;
983 __asm__
__volatile__("flushw\n\t"
984 "rdpr %%pstate, %0\n\t"
985 "wrpr %0, %1, %%pstate"
988 if (tlb_type
== spitfire
) {
989 for (i
= 0; i
< 64; i
++) {
990 /* Spitfire Errata #32 workaround */
991 /* NOTE: Always runs on spitfire, so no
992 * cheetah+ page size encodings.
994 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
998 "r" (PRIMARY_CONTEXT
), "i" (ASI_DMMU
));
1000 if (!(spitfire_get_dtlb_data(i
) & _PAGE_L
)) {
1001 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
1004 : "r" (TLB_TAG_ACCESS
), "i" (ASI_DMMU
));
1005 spitfire_put_dtlb_data(i
, 0x0UL
);
1008 /* Spitfire Errata #32 workaround */
1009 /* NOTE: Always runs on spitfire, so no
1010 * cheetah+ page size encodings.
1012 __asm__
__volatile__("stxa %0, [%1] %2\n\t"
1016 "r" (PRIMARY_CONTEXT
), "i" (ASI_DMMU
));
1018 if (!(spitfire_get_itlb_data(i
) & _PAGE_L
)) {
1019 __asm__
__volatile__("stxa %%g0, [%0] %1\n\t"
1022 : "r" (TLB_TAG_ACCESS
), "i" (ASI_IMMU
));
1023 spitfire_put_itlb_data(i
, 0x0UL
);
1026 } else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
) {
1027 cheetah_flush_dtlb_all();
1028 cheetah_flush_itlb_all();
1030 __asm__
__volatile__("wrpr %0, 0, %%pstate"
1034 /* Caller does TLB context flushing on local CPU if necessary.
1035 * The caller also ensures that CTX_VALID(mm->context) is false.
1037 * We must be careful about boundary cases so that we never
1038 * let the user have CTX 0 (nucleus) or we ever use a CTX
1039 * version of zero (and thus NO_CONTEXT would not be caught
1040 * by version mis-match tests in mmu_context.h).
1042 void get_new_mmu_context(struct mm_struct
*mm
)
1044 unsigned long ctx
, new_ctx
;
1045 unsigned long orig_pgsz_bits
;
1048 spin_lock(&ctx_alloc_lock
);
1049 orig_pgsz_bits
= (mm
->context
.sparc64_ctx_val
& CTX_PGSZ_MASK
);
1050 ctx
= (tlb_context_cache
+ 1) & CTX_NR_MASK
;
1051 new_ctx
= find_next_zero_bit(mmu_context_bmap
, 1 << CTX_NR_BITS
, ctx
);
1052 if (new_ctx
>= (1 << CTX_NR_BITS
)) {
1053 new_ctx
= find_next_zero_bit(mmu_context_bmap
, ctx
, 1);
1054 if (new_ctx
>= ctx
) {
1056 new_ctx
= (tlb_context_cache
& CTX_VERSION_MASK
) +
1059 new_ctx
= CTX_FIRST_VERSION
;
1061 /* Don't call memset, for 16 entries that's just
1064 mmu_context_bmap
[0] = 3;
1065 mmu_context_bmap
[1] = 0;
1066 mmu_context_bmap
[2] = 0;
1067 mmu_context_bmap
[3] = 0;
1068 for (i
= 4; i
< CTX_BMAP_SLOTS
; i
+= 4) {
1069 mmu_context_bmap
[i
+ 0] = 0;
1070 mmu_context_bmap
[i
+ 1] = 0;
1071 mmu_context_bmap
[i
+ 2] = 0;
1072 mmu_context_bmap
[i
+ 3] = 0;
1077 mmu_context_bmap
[new_ctx
>>6] |= (1UL << (new_ctx
& 63));
1078 new_ctx
|= (tlb_context_cache
& CTX_VERSION_MASK
);
1080 tlb_context_cache
= new_ctx
;
1081 mm
->context
.sparc64_ctx_val
= new_ctx
| orig_pgsz_bits
;
1082 spin_unlock(&ctx_alloc_lock
);
1086 struct pgtable_cache_struct pgt_quicklists
;
1089 /* OK, we have to color these pages. The page tables are accessed
1090 * by non-Dcache enabled mapping in the VPTE area by the dtlb_backend.S
1091 * code, as well as by PAGE_OFFSET range direct-mapped addresses by
1092 * other parts of the kernel. By coloring, we make sure that the tlbmiss
1093 * fast handlers do not get data from old/garbage dcache lines that
1094 * correspond to an old/stale virtual address (user/kernel) that
1095 * previously mapped the pagetable page while accessing vpte range
1096 * addresses. The idea is that if the vpte color and PAGE_OFFSET range
1097 * color is the same, then when the kernel initializes the pagetable
1098 * using the later address range, accesses with the first address
1099 * range will see the newly initialized data rather than the garbage.
1101 #ifdef DCACHE_ALIASING_POSSIBLE
1102 #define DC_ALIAS_SHIFT 1
1104 #define DC_ALIAS_SHIFT 0
1106 pte_t
*__pte_alloc_one_kernel(struct mm_struct
*mm
, unsigned long address
)
1109 unsigned long color
;
1112 pte_t
*ptep
= pte_alloc_one_fast(mm
, address
);
1118 color
= VPTE_COLOR(address
);
1119 page
= alloc_pages(GFP_KERNEL
|__GFP_REPEAT
, DC_ALIAS_SHIFT
);
1121 unsigned long *to_free
;
1122 unsigned long paddr
;
1125 #ifdef DCACHE_ALIASING_POSSIBLE
1126 set_page_count(page
, 1);
1127 ClearPageCompound(page
);
1129 set_page_count((page
+ 1), 1);
1130 ClearPageCompound(page
+ 1);
1132 paddr
= (unsigned long) page_address(page
);
1133 memset((char *)paddr
, 0, (PAGE_SIZE
<< DC_ALIAS_SHIFT
));
1136 pte
= (pte_t
*) paddr
;
1137 to_free
= (unsigned long *) (paddr
+ PAGE_SIZE
);
1139 pte
= (pte_t
*) (paddr
+ PAGE_SIZE
);
1140 to_free
= (unsigned long *) paddr
;
1143 #ifdef DCACHE_ALIASING_POSSIBLE
1144 /* Now free the other one up, adjust cache size. */
1146 *to_free
= (unsigned long) pte_quicklist
[color
^ 0x1];
1147 pte_quicklist
[color
^ 0x1] = to_free
;
1148 pgtable_cache_size
++;
1157 void sparc_ultra_dump_itlb(void)
1161 if (tlb_type
== spitfire
) {
1162 printk ("Contents of itlb: ");
1163 for (slot
= 0; slot
< 14; slot
++) printk (" ");
1164 printk ("%2x:%016lx,%016lx\n",
1166 spitfire_get_itlb_tag(0), spitfire_get_itlb_data(0));
1167 for (slot
= 1; slot
< 64; slot
+=3) {
1168 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
1170 spitfire_get_itlb_tag(slot
), spitfire_get_itlb_data(slot
),
1172 spitfire_get_itlb_tag(slot
+1), spitfire_get_itlb_data(slot
+1),
1174 spitfire_get_itlb_tag(slot
+2), spitfire_get_itlb_data(slot
+2));
1176 } else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
) {
1177 printk ("Contents of itlb0:\n");
1178 for (slot
= 0; slot
< 16; slot
+=2) {
1179 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
1181 cheetah_get_litlb_tag(slot
), cheetah_get_litlb_data(slot
),
1183 cheetah_get_litlb_tag(slot
+1), cheetah_get_litlb_data(slot
+1));
1185 printk ("Contents of itlb2:\n");
1186 for (slot
= 0; slot
< 128; slot
+=2) {
1187 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
1189 cheetah_get_itlb_tag(slot
), cheetah_get_itlb_data(slot
),
1191 cheetah_get_itlb_tag(slot
+1), cheetah_get_itlb_data(slot
+1));
1196 void sparc_ultra_dump_dtlb(void)
1200 if (tlb_type
== spitfire
) {
1201 printk ("Contents of dtlb: ");
1202 for (slot
= 0; slot
< 14; slot
++) printk (" ");
1203 printk ("%2x:%016lx,%016lx\n", 0,
1204 spitfire_get_dtlb_tag(0), spitfire_get_dtlb_data(0));
1205 for (slot
= 1; slot
< 64; slot
+=3) {
1206 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
1208 spitfire_get_dtlb_tag(slot
), spitfire_get_dtlb_data(slot
),
1210 spitfire_get_dtlb_tag(slot
+1), spitfire_get_dtlb_data(slot
+1),
1212 spitfire_get_dtlb_tag(slot
+2), spitfire_get_dtlb_data(slot
+2));
1214 } else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
) {
1215 printk ("Contents of dtlb0:\n");
1216 for (slot
= 0; slot
< 16; slot
+=2) {
1217 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
1219 cheetah_get_ldtlb_tag(slot
), cheetah_get_ldtlb_data(slot
),
1221 cheetah_get_ldtlb_tag(slot
+1), cheetah_get_ldtlb_data(slot
+1));
1223 printk ("Contents of dtlb2:\n");
1224 for (slot
= 0; slot
< 512; slot
+=2) {
1225 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
1227 cheetah_get_dtlb_tag(slot
, 2), cheetah_get_dtlb_data(slot
, 2),
1229 cheetah_get_dtlb_tag(slot
+1, 2), cheetah_get_dtlb_data(slot
+1, 2));
1231 if (tlb_type
== cheetah_plus
) {
1232 printk ("Contents of dtlb3:\n");
1233 for (slot
= 0; slot
< 512; slot
+=2) {
1234 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
1236 cheetah_get_dtlb_tag(slot
, 3), cheetah_get_dtlb_data(slot
, 3),
1238 cheetah_get_dtlb_tag(slot
+1, 3), cheetah_get_dtlb_data(slot
+1, 3));
1244 extern unsigned long cmdline_memory_size
;
1246 unsigned long __init
bootmem_init(unsigned long *pages_avail
)
1248 unsigned long bootmap_size
, start_pfn
, end_pfn
;
1249 unsigned long end_of_phys_memory
= 0UL;
1250 unsigned long bootmap_pfn
, bytes_avail
, size
;
1253 #ifdef CONFIG_DEBUG_BOOTMEM
1254 prom_printf("bootmem_init: Scan sp_banks, ");
1258 for (i
= 0; sp_banks
[i
].num_bytes
!= 0; i
++) {
1259 end_of_phys_memory
= sp_banks
[i
].base_addr
+
1260 sp_banks
[i
].num_bytes
;
1261 bytes_avail
+= sp_banks
[i
].num_bytes
;
1262 if (cmdline_memory_size
) {
1263 if (bytes_avail
> cmdline_memory_size
) {
1264 unsigned long slack
= bytes_avail
- cmdline_memory_size
;
1266 bytes_avail
-= slack
;
1267 end_of_phys_memory
-= slack
;
1269 sp_banks
[i
].num_bytes
-= slack
;
1270 if (sp_banks
[i
].num_bytes
== 0) {
1271 sp_banks
[i
].base_addr
= 0xdeadbeef;
1273 sp_banks
[i
+1].num_bytes
= 0;
1274 sp_banks
[i
+1].base_addr
= 0xdeadbeef;
1281 *pages_avail
= bytes_avail
>> PAGE_SHIFT
;
1283 /* Start with page aligned address of last symbol in kernel
1284 * image. The kernel is hard mapped below PAGE_OFFSET in a
1285 * 4MB locked TLB translation.
1287 start_pfn
= PAGE_ALIGN(kern_base
+ kern_size
) >> PAGE_SHIFT
;
1289 bootmap_pfn
= start_pfn
;
1291 end_pfn
= end_of_phys_memory
>> PAGE_SHIFT
;
1293 #ifdef CONFIG_BLK_DEV_INITRD
1294 /* Now have to check initial ramdisk, so that bootmap does not overwrite it */
1295 if (sparc_ramdisk_image
|| sparc_ramdisk_image64
) {
1296 unsigned long ramdisk_image
= sparc_ramdisk_image
?
1297 sparc_ramdisk_image
: sparc_ramdisk_image64
;
1298 if (ramdisk_image
>= (unsigned long)_end
- 2 * PAGE_SIZE
)
1299 ramdisk_image
-= KERNBASE
;
1300 initrd_start
= ramdisk_image
+ phys_base
;
1301 initrd_end
= initrd_start
+ sparc_ramdisk_size
;
1302 if (initrd_end
> end_of_phys_memory
) {
1303 printk(KERN_CRIT
"initrd extends beyond end of memory "
1304 "(0x%016lx > 0x%016lx)\ndisabling initrd\n",
1305 initrd_end
, end_of_phys_memory
);
1309 if (initrd_start
>= (start_pfn
<< PAGE_SHIFT
) &&
1310 initrd_start
< (start_pfn
<< PAGE_SHIFT
) + 2 * PAGE_SIZE
)
1311 bootmap_pfn
= PAGE_ALIGN (initrd_end
) >> PAGE_SHIFT
;
1315 /* Initialize the boot-time allocator. */
1316 max_pfn
= max_low_pfn
= end_pfn
;
1317 min_low_pfn
= pfn_base
;
1319 #ifdef CONFIG_DEBUG_BOOTMEM
1320 prom_printf("init_bootmem(min[%lx], bootmap[%lx], max[%lx])\n",
1321 min_low_pfn
, bootmap_pfn
, max_low_pfn
);
1323 bootmap_size
= init_bootmem_node(NODE_DATA(0), bootmap_pfn
, pfn_base
, end_pfn
);
1325 bootmap_base
= bootmap_pfn
<< PAGE_SHIFT
;
1327 /* Now register the available physical memory with the
1330 for (i
= 0; sp_banks
[i
].num_bytes
!= 0; i
++) {
1331 #ifdef CONFIG_DEBUG_BOOTMEM
1332 prom_printf("free_bootmem(sp_banks:%d): base[%lx] size[%lx]\n",
1333 i
, sp_banks
[i
].base_addr
, sp_banks
[i
].num_bytes
);
1335 free_bootmem(sp_banks
[i
].base_addr
, sp_banks
[i
].num_bytes
);
1338 #ifdef CONFIG_BLK_DEV_INITRD
1340 size
= initrd_end
- initrd_start
;
1342 /* Resert the initrd image area. */
1343 #ifdef CONFIG_DEBUG_BOOTMEM
1344 prom_printf("reserve_bootmem(initrd): base[%llx] size[%lx]\n",
1345 initrd_start
, initrd_end
);
1347 reserve_bootmem(initrd_start
, size
);
1348 *pages_avail
-= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
1350 initrd_start
+= PAGE_OFFSET
;
1351 initrd_end
+= PAGE_OFFSET
;
1354 /* Reserve the kernel text/data/bss. */
1355 #ifdef CONFIG_DEBUG_BOOTMEM
1356 prom_printf("reserve_bootmem(kernel): base[%lx] size[%lx]\n", kern_base
, kern_size
);
1358 reserve_bootmem(kern_base
, kern_size
);
1359 *pages_avail
-= PAGE_ALIGN(kern_size
) >> PAGE_SHIFT
;
1361 /* Reserve the bootmem map. We do not account for it
1362 * in pages_avail because we will release that memory
1363 * in free_all_bootmem.
1365 size
= bootmap_size
;
1366 #ifdef CONFIG_DEBUG_BOOTMEM
1367 prom_printf("reserve_bootmem(bootmap): base[%lx] size[%lx]\n",
1368 (bootmap_pfn
<< PAGE_SHIFT
), size
);
1370 reserve_bootmem((bootmap_pfn
<< PAGE_SHIFT
), size
);
1371 *pages_avail
-= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
1376 /* paging_init() sets up the page tables */
1378 extern void cheetah_ecache_flush_init(void);
1380 static unsigned long last_valid_pfn
;
1382 void __init
paging_init(void)
1384 extern pmd_t swapper_pmd_dir
[1024];
1385 extern unsigned int sparc64_vpte_patchme1
[1];
1386 extern unsigned int sparc64_vpte_patchme2
[1];
1387 unsigned long alias_base
= kern_base
+ PAGE_OFFSET
;
1388 unsigned long second_alias_page
= 0;
1389 unsigned long pt
, flags
, end_pfn
, pages_avail
;
1390 unsigned long shift
= alias_base
- ((unsigned long)KERNBASE
);
1391 unsigned long real_end
;
1393 set_bit(0, mmu_context_bmap
);
1395 real_end
= (unsigned long)_end
;
1396 if ((real_end
> ((unsigned long)KERNBASE
+ 0x400000)))
1398 #ifdef CONFIG_BLK_DEV_INITRD
1399 if (sparc_ramdisk_image
|| sparc_ramdisk_image64
)
1400 real_end
= (PAGE_ALIGN(real_end
) + PAGE_ALIGN(sparc_ramdisk_size
));
1403 /* We assume physical memory starts at some 4mb multiple,
1404 * if this were not true we wouldn't boot up to this point
1407 pt
= kern_base
| _PAGE_VALID
| _PAGE_SZ4MB
;
1408 pt
|= _PAGE_CP
| _PAGE_CV
| _PAGE_P
| _PAGE_L
| _PAGE_W
;
1409 local_irq_save(flags
);
1410 if (tlb_type
== spitfire
) {
1411 __asm__
__volatile__(
1412 " stxa %1, [%0] %3\n"
1413 " stxa %2, [%5] %4\n"
1420 : "r" (TLB_TAG_ACCESS
), "r" (alias_base
), "r" (pt
),
1421 "i" (ASI_DMMU
), "i" (ASI_DTLB_DATA_ACCESS
), "r" (61 << 3)
1423 if (real_end
>= KERNBASE
+ 0x340000) {
1424 second_alias_page
= alias_base
+ 0x400000;
1425 __asm__
__volatile__(
1426 " stxa %1, [%0] %3\n"
1427 " stxa %2, [%5] %4\n"
1434 : "r" (TLB_TAG_ACCESS
), "r" (second_alias_page
), "r" (pt
+ 0x400000),
1435 "i" (ASI_DMMU
), "i" (ASI_DTLB_DATA_ACCESS
), "r" (60 << 3)
1438 } else if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
) {
1439 __asm__
__volatile__(
1440 " stxa %1, [%0] %3\n"
1441 " stxa %2, [%5] %4\n"
1448 : "r" (TLB_TAG_ACCESS
), "r" (alias_base
), "r" (pt
),
1449 "i" (ASI_DMMU
), "i" (ASI_DTLB_DATA_ACCESS
), "r" ((0<<16) | (13<<3))
1451 if (real_end
>= KERNBASE
+ 0x340000) {
1452 second_alias_page
= alias_base
+ 0x400000;
1453 __asm__
__volatile__(
1454 " stxa %1, [%0] %3\n"
1455 " stxa %2, [%5] %4\n"
1462 : "r" (TLB_TAG_ACCESS
), "r" (second_alias_page
), "r" (pt
+ 0x400000),
1463 "i" (ASI_DMMU
), "i" (ASI_DTLB_DATA_ACCESS
), "r" ((0<<16) | (12<<3))
1467 local_irq_restore(flags
);
1469 /* Now set kernel pgd to upper alias so physical page computations
1472 init_mm
.pgd
+= ((shift
) / (sizeof(pgd_t
)));
1474 memset(swapper_pmd_dir
, 0, sizeof(swapper_pmd_dir
));
1476 /* Now can init the kernel/bad page tables. */
1477 pud_set(pud_offset(&swapper_pg_dir
[0], 0),
1478 swapper_pmd_dir
+ (shift
/ sizeof(pgd_t
)));
1480 sparc64_vpte_patchme1
[0] |=
1481 (((unsigned long)pgd_val(init_mm
.pgd
[0])) >> 10);
1482 sparc64_vpte_patchme2
[0] |=
1483 (((unsigned long)pgd_val(init_mm
.pgd
[0])) & 0x3ff);
1484 flushi((long)&sparc64_vpte_patchme1
[0]);
1486 /* Setup bootmem... */
1488 last_valid_pfn
= end_pfn
= bootmem_init(&pages_avail
);
1490 /* Inherit non-locked OBP mappings. */
1491 inherit_prom_mappings();
1493 /* Ok, we can use our TLB miss and window trap handlers safely.
1494 * We need to do a quick peek here to see if we are on StarFire
1495 * or not, so setup_tba can setup the IRQ globals correctly (it
1496 * needs to get the hard smp processor id correctly).
1499 extern void setup_tba(int);
1500 setup_tba(this_is_starfire
);
1503 inherit_locked_prom_mappings(1);
1505 /* We only created DTLB mapping of this stuff. */
1506 spitfire_flush_dtlb_nucleus_page(alias_base
);
1507 if (second_alias_page
)
1508 spitfire_flush_dtlb_nucleus_page(second_alias_page
);
1513 unsigned long zones_size
[MAX_NR_ZONES
];
1514 unsigned long zholes_size
[MAX_NR_ZONES
];
1515 unsigned long npages
;
1518 for (znum
= 0; znum
< MAX_NR_ZONES
; znum
++)
1519 zones_size
[znum
] = zholes_size
[znum
] = 0;
1521 npages
= end_pfn
- pfn_base
;
1522 zones_size
[ZONE_DMA
] = npages
;
1523 zholes_size
[ZONE_DMA
] = npages
- pages_avail
;
1525 free_area_init_node(0, &contig_page_data
, zones_size
,
1526 phys_base
>> PAGE_SHIFT
, zholes_size
);
1532 /* Ok, it seems that the prom can allocate some more memory chunks
1533 * as a side effect of some prom calls we perform during the
1534 * boot sequence. My most likely theory is that it is from the
1535 * prom_set_traptable() call, and OBP is allocating a scratchpad
1536 * for saving client program register state etc.
1538 static void __init
sort_memlist(struct linux_mlist_p1275
*thislist
)
1542 unsigned long tmpaddr
, tmpsize
;
1543 unsigned long lowest
;
1545 for (i
= 0; thislist
[i
].theres_more
!= 0; i
++) {
1546 lowest
= thislist
[i
].start_adr
;
1547 for (mitr
= i
+1; thislist
[mitr
-1].theres_more
!= 0; mitr
++)
1548 if (thislist
[mitr
].start_adr
< lowest
) {
1549 lowest
= thislist
[mitr
].start_adr
;
1552 if (lowest
== thislist
[i
].start_adr
)
1554 tmpaddr
= thislist
[swapi
].start_adr
;
1555 tmpsize
= thislist
[swapi
].num_bytes
;
1556 for (mitr
= swapi
; mitr
> i
; mitr
--) {
1557 thislist
[mitr
].start_adr
= thislist
[mitr
-1].start_adr
;
1558 thislist
[mitr
].num_bytes
= thislist
[mitr
-1].num_bytes
;
1560 thislist
[i
].start_adr
= tmpaddr
;
1561 thislist
[i
].num_bytes
= tmpsize
;
1565 void __init
rescan_sp_banks(void)
1567 struct linux_prom64_registers memlist
[64];
1568 struct linux_mlist_p1275 avail
[64], *mlist
;
1569 unsigned long bytes
, base_paddr
;
1570 int num_regs
, node
= prom_finddevice("/memory");
1573 num_regs
= prom_getproperty(node
, "available",
1574 (char *) memlist
, sizeof(memlist
));
1575 num_regs
= (num_regs
/ sizeof(struct linux_prom64_registers
));
1576 for (i
= 0; i
< num_regs
; i
++) {
1577 avail
[i
].start_adr
= memlist
[i
].phys_addr
;
1578 avail
[i
].num_bytes
= memlist
[i
].reg_size
;
1579 avail
[i
].theres_more
= &avail
[i
+ 1];
1581 avail
[i
- 1].theres_more
= NULL
;
1582 sort_memlist(avail
);
1586 bytes
= mlist
->num_bytes
;
1587 base_paddr
= mlist
->start_adr
;
1589 sp_banks
[0].base_addr
= base_paddr
;
1590 sp_banks
[0].num_bytes
= bytes
;
1592 while (mlist
->theres_more
!= NULL
){
1594 mlist
= mlist
->theres_more
;
1595 bytes
= mlist
->num_bytes
;
1596 if (i
>= SPARC_PHYS_BANKS
-1) {
1597 printk ("The machine has more banks than "
1598 "this kernel can support\n"
1599 "Increase the SPARC_PHYS_BANKS "
1600 "setting (currently %d)\n",
1602 i
= SPARC_PHYS_BANKS
-1;
1606 sp_banks
[i
].base_addr
= mlist
->start_adr
;
1607 sp_banks
[i
].num_bytes
= mlist
->num_bytes
;
1611 sp_banks
[i
].base_addr
= 0xdeadbeefbeefdeadUL
;
1612 sp_banks
[i
].num_bytes
= 0;
1614 for (i
= 0; sp_banks
[i
].num_bytes
!= 0; i
++)
1615 sp_banks
[i
].num_bytes
&= PAGE_MASK
;
1618 static void __init
taint_real_pages(void)
1620 struct sparc_phys_banks saved_sp_banks
[SPARC_PHYS_BANKS
];
1623 for (i
= 0; i
< SPARC_PHYS_BANKS
; i
++) {
1624 saved_sp_banks
[i
].base_addr
=
1625 sp_banks
[i
].base_addr
;
1626 saved_sp_banks
[i
].num_bytes
=
1627 sp_banks
[i
].num_bytes
;
1632 /* Find changes discovered in the sp_bank rescan and
1633 * reserve the lost portions in the bootmem maps.
1635 for (i
= 0; saved_sp_banks
[i
].num_bytes
; i
++) {
1636 unsigned long old_start
, old_end
;
1638 old_start
= saved_sp_banks
[i
].base_addr
;
1639 old_end
= old_start
+
1640 saved_sp_banks
[i
].num_bytes
;
1641 while (old_start
< old_end
) {
1644 for (n
= 0; sp_banks
[n
].num_bytes
; n
++) {
1645 unsigned long new_start
, new_end
;
1647 new_start
= sp_banks
[n
].base_addr
;
1648 new_end
= new_start
+ sp_banks
[n
].num_bytes
;
1650 if (new_start
<= old_start
&&
1651 new_end
>= (old_start
+ PAGE_SIZE
)) {
1652 set_bit (old_start
>> 22,
1653 sparc64_valid_addr_bitmap
);
1657 reserve_bootmem(old_start
, PAGE_SIZE
);
1660 old_start
+= PAGE_SIZE
;
1665 void __init
mem_init(void)
1667 unsigned long codepages
, datapages
, initpages
;
1668 unsigned long addr
, last
;
1671 i
= last_valid_pfn
>> ((22 - PAGE_SHIFT
) + 6);
1673 sparc64_valid_addr_bitmap
= (unsigned long *)
1674 __alloc_bootmem(i
<< 3, SMP_CACHE_BYTES
, bootmap_base
);
1675 if (sparc64_valid_addr_bitmap
== NULL
) {
1676 prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n");
1679 memset(sparc64_valid_addr_bitmap
, 0, i
<< 3);
1681 addr
= PAGE_OFFSET
+ kern_base
;
1682 last
= PAGE_ALIGN(kern_size
) + addr
;
1683 while (addr
< last
) {
1684 set_bit(__pa(addr
) >> 22, sparc64_valid_addr_bitmap
);
1690 max_mapnr
= last_valid_pfn
- pfn_base
;
1691 high_memory
= __va(last_valid_pfn
<< PAGE_SHIFT
);
1693 #ifdef CONFIG_DEBUG_BOOTMEM
1694 prom_printf("mem_init: Calling free_all_bootmem().\n");
1696 totalram_pages
= num_physpages
= free_all_bootmem() - 1;
1699 * Set up the zero page, mark it reserved, so that page count
1700 * is not manipulated when freeing the page from user ptes.
1702 mem_map_zero
= alloc_pages(GFP_KERNEL
|__GFP_ZERO
, 0);
1703 if (mem_map_zero
== NULL
) {
1704 prom_printf("paging_init: Cannot alloc zero page.\n");
1707 SetPageReserved(mem_map_zero
);
1709 codepages
= (((unsigned long) _etext
) - ((unsigned long) _start
));
1710 codepages
= PAGE_ALIGN(codepages
) >> PAGE_SHIFT
;
1711 datapages
= (((unsigned long) _edata
) - ((unsigned long) _etext
));
1712 datapages
= PAGE_ALIGN(datapages
) >> PAGE_SHIFT
;
1713 initpages
= (((unsigned long) __init_end
) - ((unsigned long) __init_begin
));
1714 initpages
= PAGE_ALIGN(initpages
) >> PAGE_SHIFT
;
1716 printk("Memory: %uk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n",
1717 nr_free_pages() << (PAGE_SHIFT
-10),
1718 codepages
<< (PAGE_SHIFT
-10),
1719 datapages
<< (PAGE_SHIFT
-10),
1720 initpages
<< (PAGE_SHIFT
-10),
1721 PAGE_OFFSET
, (last_valid_pfn
<< PAGE_SHIFT
));
1723 if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
)
1724 cheetah_ecache_flush_init();
1727 void free_initmem (void)
1729 unsigned long addr
, initend
;
1732 * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
1734 addr
= PAGE_ALIGN((unsigned long)(__init_begin
));
1735 initend
= (unsigned long)(__init_end
) & PAGE_MASK
;
1736 for (; addr
< initend
; addr
+= PAGE_SIZE
) {
1741 ((unsigned long) __va(kern_base
)) -
1742 ((unsigned long) KERNBASE
));
1743 memset((void *)addr
, 0xcc, PAGE_SIZE
);
1744 p
= virt_to_page(page
);
1746 ClearPageReserved(p
);
1747 set_page_count(p
, 1);
1754 #ifdef CONFIG_BLK_DEV_INITRD
1755 void free_initrd_mem(unsigned long start
, unsigned long end
)
1758 printk ("Freeing initrd memory: %ldk freed\n", (end
- start
) >> 10);
1759 for (; start
< end
; start
+= PAGE_SIZE
) {
1760 struct page
*p
= virt_to_page(start
);
1762 ClearPageReserved(p
);
1763 set_page_count(p
, 1);