2 * linux/arch/arm/mm/mm-armv.c
4 * Copyright (C) 1998-2002 Russell King
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * Page table sludge for ARM v3 and v4 processor architectures.
12 #include <linux/config.h>
13 #include <linux/module.h>
15 #include <linux/init.h>
16 #include <linux/bootmem.h>
17 #include <linux/highmem.h>
18 #include <linux/nodemask.h>
20 #include <asm/pgalloc.h>
23 #include <asm/setup.h>
24 #include <asm/tlbflush.h>
26 #include <asm/mach/map.h>
28 #define CPOLICY_UNCACHED 0
29 #define CPOLICY_BUFFERED 1
30 #define CPOLICY_WRITETHROUGH 2
31 #define CPOLICY_WRITEBACK 3
32 #define CPOLICY_WRITEALLOC 4
34 static unsigned int cachepolicy __initdata
= CPOLICY_WRITEBACK
;
35 static unsigned int ecc_mask __initdata
= 0;
36 pgprot_t pgprot_kernel
;
38 EXPORT_SYMBOL(pgprot_kernel
);
43 const char policy
[16];
49 static struct cachepolicy cache_policies
[] __initdata
= {
53 .pmd
= PMD_SECT_UNCACHED
,
58 .pmd
= PMD_SECT_BUFFERED
,
59 .pte
= PTE_BUFFERABLE
,
61 .policy
= "writethrough",
66 .policy
= "writeback",
69 .pte
= PTE_BUFFERABLE
|PTE_CACHEABLE
,
71 .policy
= "writealloc",
74 .pte
= PTE_BUFFERABLE
|PTE_CACHEABLE
,
79 * These are useful for identifing cache coherency
80 * problems by allowing the cache or the cache and
81 * writebuffer to be turned off. (Note: the write
82 * buffer should not be on and the cache off).
84 static void __init
early_cachepolicy(char **p
)
88 for (i
= 0; i
< ARRAY_SIZE(cache_policies
); i
++) {
89 int len
= strlen(cache_policies
[i
].policy
);
91 if (memcmp(*p
, cache_policies
[i
].policy
, len
) == 0) {
93 cr_alignment
&= ~cache_policies
[i
].cr_mask
;
94 cr_no_alignment
&= ~cache_policies
[i
].cr_mask
;
99 if (i
== ARRAY_SIZE(cache_policies
))
100 printk(KERN_ERR
"ERROR: unknown or unsupported cache policy\n");
102 set_cr(cr_alignment
);
105 static void __init
early_nocache(char **__unused
)
107 char *p
= "buffered";
108 printk(KERN_WARNING
"nocache is deprecated; use cachepolicy=%s\n", p
);
109 early_cachepolicy(&p
);
112 static void __init
early_nowrite(char **__unused
)
114 char *p
= "uncached";
115 printk(KERN_WARNING
"nowb is deprecated; use cachepolicy=%s\n", p
);
116 early_cachepolicy(&p
);
119 static void __init
early_ecc(char **p
)
121 if (memcmp(*p
, "on", 2) == 0) {
122 ecc_mask
= PMD_PROTECTION
;
124 } else if (memcmp(*p
, "off", 3) == 0) {
130 __early_param("nocache", early_nocache
);
131 __early_param("nowb", early_nowrite
);
132 __early_param("cachepolicy=", early_cachepolicy
);
133 __early_param("ecc=", early_ecc
);
135 static int __init
noalign_setup(char *__unused
)
137 cr_alignment
&= ~CR_A
;
138 cr_no_alignment
&= ~CR_A
;
139 set_cr(cr_alignment
);
143 __setup("noalign", noalign_setup
);
145 #define FIRST_KERNEL_PGD_NR (FIRST_USER_PGD_NR + USER_PTRS_PER_PGD)
147 static inline pmd_t
*pmd_off(pgd_t
*pgd
, unsigned long virt
)
149 return pmd_offset(pgd
, virt
);
152 static inline pmd_t
*pmd_off_k(unsigned long virt
)
154 return pmd_off(pgd_offset_k(virt
), virt
);
158 * need to get a 16k page for level 1
160 pgd_t
*get_pgd_slow(struct mm_struct
*mm
)
162 pgd_t
*new_pgd
, *init_pgd
;
163 pmd_t
*new_pmd
, *init_pmd
;
164 pte_t
*new_pte
, *init_pte
;
166 new_pgd
= (pgd_t
*)__get_free_pages(GFP_KERNEL
, 2);
170 memzero(new_pgd
, FIRST_KERNEL_PGD_NR
* sizeof(pgd_t
));
173 * Copy over the kernel and IO PGD entries
175 init_pgd
= pgd_offset_k(0);
176 memcpy(new_pgd
+ FIRST_KERNEL_PGD_NR
, init_pgd
+ FIRST_KERNEL_PGD_NR
,
177 (PTRS_PER_PGD
- FIRST_KERNEL_PGD_NR
) * sizeof(pgd_t
));
179 clean_dcache_area(new_pgd
, PTRS_PER_PGD
* sizeof(pgd_t
));
181 if (!vectors_high()) {
183 * This lock is here just to satisfy pmd_alloc and pte_lock
185 spin_lock(&mm
->page_table_lock
);
188 * On ARM, first page must always be allocated since it
189 * contains the machine vectors.
191 new_pmd
= pmd_alloc(mm
, new_pgd
, 0);
195 new_pte
= pte_alloc_map(mm
, new_pmd
, 0);
199 init_pmd
= pmd_offset(init_pgd
, 0);
200 init_pte
= pte_offset_map_nested(init_pmd
, 0);
201 set_pte(new_pte
, *init_pte
);
202 pte_unmap_nested(init_pte
);
205 spin_unlock(&mm
->page_table_lock
);
211 spin_unlock(&mm
->page_table_lock
);
213 free_pages((unsigned long)new_pgd
, 2);
217 spin_unlock(&mm
->page_table_lock
);
218 free_pages((unsigned long)new_pgd
, 2);
225 void free_pgd_slow(pgd_t
*pgd
)
233 /* pgd is always present and good */
234 pmd
= pmd_off(pgd
, 0);
243 pte
= pmd_page(*pmd
);
245 dec_page_state(nr_page_table_pages
);
249 free_pages((unsigned long) pgd
, 2);
253 * Create a SECTION PGD between VIRT and PHYS in domain
254 * DOMAIN with protection PROT. This operates on half-
255 * pgdir entry increments.
258 alloc_init_section(unsigned long virt
, unsigned long phys
, int prot
)
260 pmd_t
*pmdp
= pmd_off_k(virt
);
262 if (virt
& (1 << 20))
265 *pmdp
= __pmd(phys
| prot
);
266 flush_pmd_entry(pmdp
);
270 * Create a SUPER SECTION PGD between VIRT and PHYS with protection PROT
273 alloc_init_supersection(unsigned long virt
, unsigned long phys
, int prot
)
277 for (i
= 0; i
< 16; i
+= 1) {
278 alloc_init_section(virt
, phys
, prot
| PMD_SECT_SUPER
);
280 virt
+= (PGDIR_SIZE
/ 2);
285 * Add a PAGE mapping between VIRT and PHYS in domain
286 * DOMAIN with protection PROT. Note that due to the
287 * way we map the PTEs, we must allocate two PTE_SIZE'd
288 * blocks - one for the Linux pte table, and one for
289 * the hardware pte table.
292 alloc_init_page(unsigned long virt
, unsigned long phys
, unsigned int prot_l1
, pgprot_t prot
)
294 pmd_t
*pmdp
= pmd_off_k(virt
);
297 if (pmd_none(*pmdp
)) {
298 unsigned long pmdval
;
299 ptep
= alloc_bootmem_low_pages(2 * PTRS_PER_PTE
*
302 pmdval
= __pa(ptep
) | prot_l1
;
303 pmdp
[0] = __pmd(pmdval
);
304 pmdp
[1] = __pmd(pmdval
+ 256 * sizeof(pte_t
));
305 flush_pmd_entry(pmdp
);
307 ptep
= pte_offset_kernel(pmdp
, virt
);
309 set_pte(ptep
, pfn_pte(phys
>> PAGE_SHIFT
, prot
));
313 * Clear any PGD mapping. On a two-level page table system,
314 * the clearance is done by the middle-level functions (pmd)
315 * rather than the top-level (pgd) functions.
317 static inline void clear_mapping(unsigned long virt
)
319 pmd_clear(pmd_off_k(virt
));
323 unsigned int prot_pte
;
324 unsigned int prot_l1
;
325 unsigned int prot_sect
;
329 static struct mem_types mem_types
[] __initdata
= {
331 .prot_pte
= L_PTE_PRESENT
| L_PTE_YOUNG
| L_PTE_DIRTY
|
333 .prot_l1
= PMD_TYPE_TABLE
,
334 .prot_sect
= PMD_TYPE_SECT
| PMD_SECT_UNCACHED
|
339 .prot_sect
= PMD_TYPE_SECT
,
340 .domain
= DOMAIN_KERNEL
,
343 .prot_sect
= PMD_TYPE_SECT
| PMD_SECT_MINICACHE
,
344 .domain
= DOMAIN_KERNEL
,
347 .prot_pte
= L_PTE_PRESENT
| L_PTE_YOUNG
| L_PTE_DIRTY
|
349 .prot_l1
= PMD_TYPE_TABLE
,
350 .domain
= DOMAIN_USER
,
352 [MT_HIGH_VECTORS
] = {
353 .prot_pte
= L_PTE_PRESENT
| L_PTE_YOUNG
| L_PTE_DIRTY
|
354 L_PTE_USER
| L_PTE_EXEC
,
355 .prot_l1
= PMD_TYPE_TABLE
,
356 .domain
= DOMAIN_USER
,
359 .prot_sect
= PMD_TYPE_SECT
| PMD_SECT_AP_WRITE
,
360 .domain
= DOMAIN_KERNEL
,
363 .prot_sect
= PMD_TYPE_SECT
,
364 .domain
= DOMAIN_KERNEL
,
366 [MT_IXP2000_DEVICE
] = { /* IXP2400 requires XCB=101 for on-chip I/O */
367 .prot_pte
= L_PTE_PRESENT
| L_PTE_YOUNG
| L_PTE_DIRTY
|
369 .prot_l1
= PMD_TYPE_TABLE
,
370 .prot_sect
= PMD_TYPE_SECT
| PMD_SECT_UNCACHED
|
371 PMD_SECT_AP_WRITE
| PMD_SECT_BUFFERABLE
|
378 * Adjust the PMD section entries according to the CPU in use.
380 static void __init
build_mem_type_table(void)
382 struct cachepolicy
*cp
;
383 unsigned int cr
= get_cr();
384 unsigned int user_pgprot
;
385 int cpu_arch
= cpu_architecture();
388 #if defined(CONFIG_CPU_DCACHE_DISABLE)
389 if (cachepolicy
> CPOLICY_BUFFERED
)
390 cachepolicy
= CPOLICY_BUFFERED
;
391 #elif defined(CONFIG_CPU_DCACHE_WRITETHROUGH)
392 if (cachepolicy
> CPOLICY_WRITETHROUGH
)
393 cachepolicy
= CPOLICY_WRITETHROUGH
;
395 if (cpu_arch
< CPU_ARCH_ARMv5
) {
396 if (cachepolicy
>= CPOLICY_WRITEALLOC
)
397 cachepolicy
= CPOLICY_WRITEBACK
;
401 if (cpu_arch
<= CPU_ARCH_ARMv5TEJ
) {
402 for (i
= 0; i
< ARRAY_SIZE(mem_types
); i
++) {
403 if (mem_types
[i
].prot_l1
)
404 mem_types
[i
].prot_l1
|= PMD_BIT4
;
405 if (mem_types
[i
].prot_sect
)
406 mem_types
[i
].prot_sect
|= PMD_BIT4
;
410 cp
= &cache_policies
[cachepolicy
];
411 user_pgprot
= cp
->pte
;
414 * ARMv6 and above have extended page tables.
416 if (cpu_arch
>= CPU_ARCH_ARMv6
&& (cr
& CR_XP
)) {
418 * bit 4 becomes XN which we must clear for the
419 * kernel memory mapping.
421 mem_types
[MT_MEMORY
].prot_sect
&= ~PMD_BIT4
;
422 mem_types
[MT_ROM
].prot_sect
&= ~PMD_BIT4
;
424 * Mark cache clean areas and XIP ROM read only
425 * from SVC mode and no access from userspace.
427 mem_types
[MT_ROM
].prot_sect
|= PMD_SECT_APX
|PMD_SECT_AP_WRITE
;
428 mem_types
[MT_MINICLEAN
].prot_sect
|= PMD_SECT_APX
|PMD_SECT_AP_WRITE
;
429 mem_types
[MT_CACHECLEAN
].prot_sect
|= PMD_SECT_APX
|PMD_SECT_AP_WRITE
;
432 * Mark the device area as "shared device"
434 mem_types
[MT_DEVICE
].prot_pte
|= L_PTE_BUFFERABLE
;
435 mem_types
[MT_DEVICE
].prot_sect
|= PMD_SECT_BUFFERED
;
438 * User pages need to be mapped with the ASID
441 user_pgprot
|= L_PTE_ASID
;
444 if (cpu_arch
>= CPU_ARCH_ARMv5
) {
445 mem_types
[MT_LOW_VECTORS
].prot_pte
|= cp
->pte
& PTE_CACHEABLE
;
446 mem_types
[MT_HIGH_VECTORS
].prot_pte
|= cp
->pte
& PTE_CACHEABLE
;
448 mem_types
[MT_LOW_VECTORS
].prot_pte
|= cp
->pte
;
449 mem_types
[MT_HIGH_VECTORS
].prot_pte
|= cp
->pte
;
450 mem_types
[MT_MINICLEAN
].prot_sect
&= ~PMD_SECT_TEX(1);
453 mem_types
[MT_LOW_VECTORS
].prot_l1
|= ecc_mask
;
454 mem_types
[MT_HIGH_VECTORS
].prot_l1
|= ecc_mask
;
455 mem_types
[MT_MEMORY
].prot_sect
|= ecc_mask
| cp
->pmd
;
456 mem_types
[MT_ROM
].prot_sect
|= cp
->pmd
;
458 for (i
= 0; i
< 16; i
++) {
459 unsigned long v
= pgprot_val(protection_map
[i
]);
460 v
&= (~(PTE_BUFFERABLE
|PTE_CACHEABLE
)) | user_pgprot
;
461 protection_map
[i
] = __pgprot(v
);
464 pgprot_kernel
= __pgprot(L_PTE_PRESENT
| L_PTE_YOUNG
|
465 L_PTE_DIRTY
| L_PTE_WRITE
|
466 L_PTE_EXEC
| cp
->pte
);
470 mem_types
[MT_CACHECLEAN
].prot_sect
|= PMD_SECT_WT
;
474 mem_types
[MT_CACHECLEAN
].prot_sect
|= PMD_SECT_WB
;
477 printk("Memory policy: ECC %sabled, Data cache %s\n",
478 ecc_mask
? "en" : "dis", cp
->policy
);
481 #define vectors_base() (vectors_high() ? 0xffff0000 : 0)
484 * Create the page directory entries and any necessary
485 * page tables for the mapping specified by `md'. We
486 * are able to cope here with varying sizes and address
487 * offsets, and we take full advantage of sections and
490 static void __init
create_mapping(struct map_desc
*md
)
492 unsigned long virt
, length
;
493 int prot_sect
, prot_l1
, domain
;
497 if (md
->virtual != vectors_base() && md
->virtual < TASK_SIZE
) {
498 printk(KERN_WARNING
"BUG: not creating mapping for "
499 "0x%08lx at 0x%08lx in user region\n",
500 md
->physical
, md
->virtual);
504 if ((md
->type
== MT_DEVICE
|| md
->type
== MT_ROM
) &&
505 md
->virtual >= PAGE_OFFSET
&& md
->virtual < VMALLOC_END
) {
506 printk(KERN_WARNING
"BUG: mapping for 0x%08lx at 0x%08lx "
507 "overlaps vmalloc space\n",
508 md
->physical
, md
->virtual);
511 domain
= mem_types
[md
->type
].domain
;
512 prot_pte
= __pgprot(mem_types
[md
->type
].prot_pte
);
513 prot_l1
= mem_types
[md
->type
].prot_l1
| PMD_DOMAIN(domain
);
514 prot_sect
= mem_types
[md
->type
].prot_sect
| PMD_DOMAIN(domain
);
517 off
= md
->physical
- virt
;
520 if (mem_types
[md
->type
].prot_l1
== 0 &&
521 (virt
& 0xfffff || (virt
+ off
) & 0xfffff || (virt
+ length
) & 0xfffff)) {
522 printk(KERN_WARNING
"BUG: map for 0x%08lx at 0x%08lx can not "
523 "be mapped using pages, ignoring.\n",
524 md
->physical
, md
->virtual);
528 while ((virt
& 0xfffff || (virt
+ off
) & 0xfffff) && length
>= PAGE_SIZE
) {
529 alloc_init_page(virt
, virt
+ off
, prot_l1
, prot_pte
);
535 /* N.B. ARMv6 supersections are only defined to work with domain 0.
536 * Since domain assignments can in fact be arbitrary, the
537 * 'domain == 0' check below is required to insure that ARMv6
538 * supersections are only allocated for domain 0 regardless
539 * of the actual domain assignments in use.
541 if (cpu_architecture() >= CPU_ARCH_ARMv6
&& domain
== 0) {
542 /* Align to supersection boundary */
543 while ((virt
& ~SUPERSECTION_MASK
|| (virt
+ off
) &
544 ~SUPERSECTION_MASK
) && length
>= (PGDIR_SIZE
/ 2)) {
545 alloc_init_section(virt
, virt
+ off
, prot_sect
);
547 virt
+= (PGDIR_SIZE
/ 2);
548 length
-= (PGDIR_SIZE
/ 2);
551 while (length
>= SUPERSECTION_SIZE
) {
552 alloc_init_supersection(virt
, virt
+ off
, prot_sect
);
554 virt
+= SUPERSECTION_SIZE
;
555 length
-= SUPERSECTION_SIZE
;
560 * A section mapping covers half a "pgdir" entry.
562 while (length
>= (PGDIR_SIZE
/ 2)) {
563 alloc_init_section(virt
, virt
+ off
, prot_sect
);
565 virt
+= (PGDIR_SIZE
/ 2);
566 length
-= (PGDIR_SIZE
/ 2);
569 while (length
>= PAGE_SIZE
) {
570 alloc_init_page(virt
, virt
+ off
, prot_l1
, prot_pte
);
578 * In order to soft-boot, we need to insert a 1:1 mapping in place of
579 * the user-mode pages. This will then ensure that we have predictable
580 * results when turning the mmu off
582 void setup_mm_for_reboot(char mode
)
584 unsigned long pmdval
;
588 int cpu_arch
= cpu_architecture();
590 if (current
->mm
&& current
->mm
->pgd
)
591 pgd
= current
->mm
->pgd
;
595 for (i
= 0; i
< FIRST_USER_PGD_NR
+ USER_PTRS_PER_PGD
; i
++) {
596 pmdval
= (i
<< PGDIR_SHIFT
) |
597 PMD_SECT_AP_WRITE
| PMD_SECT_AP_READ
|
599 if (cpu_arch
<= CPU_ARCH_ARMv5TEJ
)
601 pmd
= pmd_off(pgd
, i
<< PGDIR_SHIFT
);
602 pmd
[0] = __pmd(pmdval
);
603 pmd
[1] = __pmd(pmdval
+ (1 << (PGDIR_SHIFT
- 1)));
604 flush_pmd_entry(pmd
);
608 extern void _stext
, _etext
;
611 * Setup initial mappings. We use the page we allocated for zero page to hold
612 * the mappings, which will get overwritten by the vectors in traps_init().
613 * The mappings must be in virtual address order.
615 void __init
memtable_init(struct meminfo
*mi
)
617 struct map_desc
*init_maps
, *p
, *q
;
618 unsigned long address
= 0;
621 build_mem_type_table();
623 init_maps
= p
= alloc_bootmem_low_pages(PAGE_SIZE
);
625 #ifdef CONFIG_XIP_KERNEL
626 p
->physical
= CONFIG_XIP_PHYS_ADDR
& PMD_MASK
;
627 p
->virtual = (unsigned long)&_stext
& PMD_MASK
;
628 p
->length
= ((unsigned long)&_etext
- p
->virtual + ~PMD_MASK
) & PMD_MASK
;
633 for (i
= 0; i
< mi
->nr_banks
; i
++) {
634 if (mi
->bank
[i
].size
== 0)
637 p
->physical
= mi
->bank
[i
].start
;
638 p
->virtual = __phys_to_virt(p
->physical
);
639 p
->length
= mi
->bank
[i
].size
;
645 p
->physical
= FLUSH_BASE_PHYS
;
646 p
->virtual = FLUSH_BASE
;
647 p
->length
= PGDIR_SIZE
;
648 p
->type
= MT_CACHECLEAN
;
652 #ifdef FLUSH_BASE_MINICACHE
653 p
->physical
= FLUSH_BASE_PHYS
+ PGDIR_SIZE
;
654 p
->virtual = FLUSH_BASE_MINICACHE
;
655 p
->length
= PGDIR_SIZE
;
656 p
->type
= MT_MINICLEAN
;
661 * Go through the initial mappings, but clear out any
662 * pgdir entries that are not in the description.
666 if (address
< q
->virtual || q
== p
) {
667 clear_mapping(address
);
668 address
+= PGDIR_SIZE
;
672 address
= q
->virtual + q
->length
;
673 address
= (address
+ PGDIR_SIZE
- 1) & PGDIR_MASK
;
677 } while (address
!= 0);
680 * Create a mapping for the machine vectors at the high-vectors
681 * location (0xffff0000). If we aren't using high-vectors, also
682 * create a mapping at the low-vectors virtual address.
684 init_maps
->physical
= virt_to_phys(init_maps
);
685 init_maps
->virtual = 0xffff0000;
686 init_maps
->length
= PAGE_SIZE
;
687 init_maps
->type
= MT_HIGH_VECTORS
;
688 create_mapping(init_maps
);
690 if (!vectors_high()) {
691 init_maps
->virtual = 0;
692 init_maps
->type
= MT_LOW_VECTORS
;
693 create_mapping(init_maps
);
697 local_flush_tlb_all();
699 top_pmd
= pmd_off_k(0xffff0000);
703 * Create the architecture specific mappings
705 void __init
iotable_init(struct map_desc
*io_desc
, int nr
)
709 for (i
= 0; i
< nr
; i
++)
710 create_mapping(io_desc
+ i
);