?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ARM: OMAP: Remove smp.h
[linux-2.6.git] / arch / arm / mm / mmu.c
bloba7a9e41fa2c2d598f8c3c31bad3e0d26b2283c14
1 /*
2 * linux/arch/arm/mm/mmu.c
3 *
4 * Copyright (C) 1995-2005 Russell King
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10 #include <linux/module.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/init.h>
14 #include <linux/mman.h>
15 #include <linux/nodemask.h>
16 #include <linux/memblock.h>
17 #include <linux/fs.h>
18 #include <linux/vmalloc.h>
19 #include <linux/sizes.h>
20
21 #include <asm/cp15.h>
22 #include <asm/cputype.h>
23 #include <asm/sections.h>
24 #include <asm/cachetype.h>
25 #include <asm/setup.h>
26 #include <asm/smp_plat.h>
27 #include <asm/tlb.h>
28 #include <asm/highmem.h>
29 #include <asm/system_info.h>
30 #include <asm/traps.h>
31
32 #include <asm/mach/arch.h>
33 #include <asm/mach/map.h>
34 #include <asm/mach/pci.h>
35
36 #include "mm.h"
37
38 /*
39 * empty_zero_page is a special page that is used for
40 * zero-initialized data and COW.
41 */
42 struct page *empty_zero_page;
43 EXPORT_SYMBOL(empty_zero_page);
44
45 /*
46 * The pmd table for the upper-most set of pages.
47 */
48 pmd_t *top_pmd;
49
50 #define CPOLICY_UNCACHED 0
51 #define CPOLICY_BUFFERED 1
52 #define CPOLICY_WRITETHROUGH 2
53 #define CPOLICY_WRITEBACK 3
54 #define CPOLICY_WRITEALLOC 4
55
56 static unsigned int cachepolicy __initdata = CPOLICY_WRITEBACK;
57 static unsigned int ecc_mask __initdata = 0;
58 pgprot_t pgprot_user;
59 pgprot_t pgprot_kernel;
60
61 EXPORT_SYMBOL(pgprot_user);
62 EXPORT_SYMBOL(pgprot_kernel);
63
64 struct cachepolicy {
65 const char policy[16];
66 unsigned int cr_mask;
67 pmdval_t pmd;
68 pteval_t pte;
69 };
70
71 static struct cachepolicy cache_policies[] __initdata = {
72 {
73 .policy = "uncached",
74 .cr_mask = CR_W|CR_C,
75 .pmd = PMD_SECT_UNCACHED,
76 .pte = L_PTE_MT_UNCACHED,
77 }, {
78 .policy = "buffered",
79 .cr_mask = CR_C,
80 .pmd = PMD_SECT_BUFFERED,
81 .pte = L_PTE_MT_BUFFERABLE,
82 }, {
83 .policy = "writethrough",
84 .cr_mask = 0,
85 .pmd = PMD_SECT_WT,
86 .pte = L_PTE_MT_WRITETHROUGH,
87 }, {
88 .policy = "writeback",
89 .cr_mask = 0,
90 .pmd = PMD_SECT_WB,
91 .pte = L_PTE_MT_WRITEBACK,
92 }, {
93 .policy = "writealloc",
94 .cr_mask = 0,
95 .pmd = PMD_SECT_WBWA,
96 .pte = L_PTE_MT_WRITEALLOC,
97 }
98 };
99
100 /*
101 * These are useful for identifying cache coherency
102 * problems by allowing the cache or the cache and
103 * writebuffer to be turned off. (Note: the write
104 * buffer should not be on and the cache off).
105 */
106 static int __init early_cachepolicy(char *p)
107 {
108 int i;
109
110 for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
111 int len = strlen(cache_policies[i].policy);
112
113 if (memcmp(p, cache_policies[i].policy, len) == 0) {
114 cachepolicy = i;
115 cr_alignment &= ~cache_policies[i].cr_mask;
116 cr_no_alignment &= ~cache_policies[i].cr_mask;
117 break;
118 }
119 }
120 if (i == ARRAY_SIZE(cache_policies))
121 printk(KERN_ERR "ERROR: unknown or unsupported cache policy\n");
122 /*
123 * This restriction is partly to do with the way we boot; it is
124 * unpredictable to have memory mapped using two different sets of
125 * memory attributes (shared, type, and cache attribs). We can not
126 * change these attributes once the initial assembly has setup the
127 * page tables.
128 */
129 if (cpu_architecture() >= CPU_ARCH_ARMv6) {
130 printk(KERN_WARNING "Only cachepolicy=writeback supported on ARMv6 and later\n");
131 cachepolicy = CPOLICY_WRITEBACK;
132 }
133 flush_cache_all();
134 set_cr(cr_alignment);
135 return 0;
136 }
137 early_param("cachepolicy", early_cachepolicy);
138
139 static int __init early_nocache(char *__unused)
140 {
141 char *p = "buffered";
142 printk(KERN_WARNING "nocache is deprecated; use cachepolicy=%s\n", p);
143 early_cachepolicy(p);
144 return 0;
145 }
146 early_param("nocache", early_nocache);
147
148 static int __init early_nowrite(char *__unused)
149 {
150 char *p = "uncached";
151 printk(KERN_WARNING "nowb is deprecated; use cachepolicy=%s\n", p);
152 early_cachepolicy(p);
153 return 0;
154 }
155 early_param("nowb", early_nowrite);
156
157 #ifndef CONFIG_ARM_LPAE
158 static int __init early_ecc(char *p)
159 {
160 if (memcmp(p, "on", 2) == 0)
161 ecc_mask = PMD_PROTECTION;
162 else if (memcmp(p, "off", 3) == 0)
163 ecc_mask = 0;
164 return 0;
165 }
166 early_param("ecc", early_ecc);
167 #endif
168
169 static int __init noalign_setup(char *__unused)
170 {
171 cr_alignment &= ~CR_A;
172 cr_no_alignment &= ~CR_A;
173 set_cr(cr_alignment);
174 return 1;
175 }
176 __setup("noalign", noalign_setup);
177
178 #ifndef CONFIG_SMP
179 void adjust_cr(unsigned long mask, unsigned long set)
180 {
181 unsigned long flags;
182
183 mask &= ~CR_A;
184
185 set &= mask;
186
187 local_irq_save(flags);
188
189 cr_no_alignment = (cr_no_alignment & ~mask) | set;
190 cr_alignment = (cr_alignment & ~mask) | set;
191
192 set_cr((get_cr() & ~mask) | set);
193
194 local_irq_restore(flags);
195 }
196 #endif
197
198 #define PROT_PTE_DEVICE L_PTE_PRESENT|L_PTE_YOUNG|L_PTE_DIRTY|L_PTE_XN
199 #define PROT_SECT_DEVICE PMD_TYPE_SECT|PMD_SECT_AP_WRITE
200
201 static struct mem_type mem_types[] = {
202 [MT_DEVICE] = { /* Strongly ordered / ARMv6 shared device */
203 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_SHARED |
204 L_PTE_SHARED,
205 .prot_l1 = PMD_TYPE_TABLE,
206 .prot_sect = PROT_SECT_DEVICE | PMD_SECT_S,
207 .domain = DOMAIN_IO,
208 },
209 [MT_DEVICE_NONSHARED] = { /* ARMv6 non-shared device */
210 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_NONSHARED,
211 .prot_l1 = PMD_TYPE_TABLE,
212 .prot_sect = PROT_SECT_DEVICE,
213 .domain = DOMAIN_IO,
214 },
215 [MT_DEVICE_CACHED] = { /* ioremap_cached */
216 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_CACHED,
217 .prot_l1 = PMD_TYPE_TABLE,
218 .prot_sect = PROT_SECT_DEVICE | PMD_SECT_WB,
219 .domain = DOMAIN_IO,
220 },
221 [MT_DEVICE_WC] = { /* ioremap_wc */
222 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_WC,
223 .prot_l1 = PMD_TYPE_TABLE,
224 .prot_sect = PROT_SECT_DEVICE,
225 .domain = DOMAIN_IO,
226 },
227 [MT_UNCACHED] = {
228 .prot_pte = PROT_PTE_DEVICE,
229 .prot_l1 = PMD_TYPE_TABLE,
230 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
231 .domain = DOMAIN_IO,
232 },
233 [MT_CACHECLEAN] = {
234 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
235 .domain = DOMAIN_KERNEL,
236 },
237 #ifndef CONFIG_ARM_LPAE
238 [MT_MINICLEAN] = {
239 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE,
240 .domain = DOMAIN_KERNEL,
241 },
242 #endif
243 [MT_LOW_VECTORS] = {
244 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
245 L_PTE_RDONLY,
246 .prot_l1 = PMD_TYPE_TABLE,
247 .domain = DOMAIN_USER,
248 },
249 [MT_HIGH_VECTORS] = {
250 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
251 L_PTE_USER | L_PTE_RDONLY,
252 .prot_l1 = PMD_TYPE_TABLE,
253 .domain = DOMAIN_USER,
254 },
255 [MT_MEMORY] = {
256 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
257 .prot_l1 = PMD_TYPE_TABLE,
258 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
259 .domain = DOMAIN_KERNEL,
260 },
261 [MT_ROM] = {
262 .prot_sect = PMD_TYPE_SECT,
263 .domain = DOMAIN_KERNEL,
264 },
265 [MT_MEMORY_NONCACHED] = {
266 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
267 L_PTE_MT_BUFFERABLE,
268 .prot_l1 = PMD_TYPE_TABLE,
269 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
270 .domain = DOMAIN_KERNEL,
271 },
272 [MT_MEMORY_DTCM] = {
273 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
274 L_PTE_XN,
275 .prot_l1 = PMD_TYPE_TABLE,
276 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
277 .domain = DOMAIN_KERNEL,
278 },
279 [MT_MEMORY_ITCM] = {
280 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
281 .prot_l1 = PMD_TYPE_TABLE,
282 .domain = DOMAIN_KERNEL,
283 },
284 [MT_MEMORY_SO] = {
285 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
286 L_PTE_MT_UNCACHED,
287 .prot_l1 = PMD_TYPE_TABLE,
288 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_S |
289 PMD_SECT_UNCACHED | PMD_SECT_XN,
290 .domain = DOMAIN_KERNEL,
291 },
292 [MT_MEMORY_DMA_READY] = {
293 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
294 .prot_l1 = PMD_TYPE_TABLE,
295 .domain = DOMAIN_KERNEL,
296 },
297 };
298
299 const struct mem_type *get_mem_type(unsigned int type)
300 {
301 return type < ARRAY_SIZE(mem_types) ? &mem_types[type] : NULL;
302 }
303 EXPORT_SYMBOL(get_mem_type);
304
305 /*
306 * Adjust the PMD section entries according to the CPU in use.
307 */
308 static void __init build_mem_type_table(void)
309 {
310 struct cachepolicy *cp;
311 unsigned int cr = get_cr();
312 pteval_t user_pgprot, kern_pgprot, vecs_pgprot;
313 int cpu_arch = cpu_architecture();
314 int i;
315
316 if (cpu_arch < CPU_ARCH_ARMv6) {
317 #if defined(CONFIG_CPU_DCACHE_DISABLE)
318 if (cachepolicy > CPOLICY_BUFFERED)
319 cachepolicy = CPOLICY_BUFFERED;
320 #elif defined(CONFIG_CPU_DCACHE_WRITETHROUGH)
321 if (cachepolicy > CPOLICY_WRITETHROUGH)
322 cachepolicy = CPOLICY_WRITETHROUGH;
323 #endif
324 }
325 if (cpu_arch < CPU_ARCH_ARMv5) {
326 if (cachepolicy >= CPOLICY_WRITEALLOC)
327 cachepolicy = CPOLICY_WRITEBACK;
328 ecc_mask = 0;
329 }
330 if (is_smp())
331 cachepolicy = CPOLICY_WRITEALLOC;
332
333 /*
334 * Strip out features not present on earlier architectures.
335 * Pre-ARMv5 CPUs don't have TEX bits. Pre-ARMv6 CPUs or those
336 * without extended page tables don't have the 'Shared' bit.
337 */
338 if (cpu_arch < CPU_ARCH_ARMv5)
339 for (i = 0; i < ARRAY_SIZE(mem_types); i++)
340 mem_types[i].prot_sect &= ~PMD_SECT_TEX(7);
341 if ((cpu_arch < CPU_ARCH_ARMv6 || !(cr & CR_XP)) && !cpu_is_xsc3())
342 for (i = 0; i < ARRAY_SIZE(mem_types); i++)
343 mem_types[i].prot_sect &= ~PMD_SECT_S;
344
345 /*
346 * ARMv5 and lower, bit 4 must be set for page tables (was: cache
347 * "update-able on write" bit on ARM610). However, Xscale and
348 * Xscale3 require this bit to be cleared.
349 */
350 if (cpu_is_xscale() || cpu_is_xsc3()) {
351 for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
352 mem_types[i].prot_sect &= ~PMD_BIT4;
353 mem_types[i].prot_l1 &= ~PMD_BIT4;
354 }
355 } else if (cpu_arch < CPU_ARCH_ARMv6) {
356 for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
357 if (mem_types[i].prot_l1)
358 mem_types[i].prot_l1 |= PMD_BIT4;
359 if (mem_types[i].prot_sect)
360 mem_types[i].prot_sect |= PMD_BIT4;
361 }
362 }
363
364 /*
365 * Mark the device areas according to the CPU/architecture.
366 */
367 if (cpu_is_xsc3() || (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP))) {
368 if (!cpu_is_xsc3()) {
369 /*
370 * Mark device regions on ARMv6+ as execute-never
371 * to prevent speculative instruction fetches.
372 */
373 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_XN;
374 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_XN;
375 mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_XN;
376 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_XN;
377 }
378 if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
379 /*
380 * For ARMv7 with TEX remapping,
381 * - shared device is SXCB=1100
382 * - nonshared device is SXCB=0100
383 * - write combine device mem is SXCB=0001
384 * (Uncached Normal memory)
385 */
386 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1);
387 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(1);
388 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE;
389 } else if (cpu_is_xsc3()) {
390 /*
391 * For Xscale3,
392 * - shared device is TEXCB=00101
393 * - nonshared device is TEXCB=01000
394 * - write combine device mem is TEXCB=00100
395 * (Inner/Outer Uncacheable in xsc3 parlance)
396 */
397 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1) | PMD_SECT_BUFFERED;
398 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2);
399 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1);
400 } else {
401 /*
402 * For ARMv6 and ARMv7 without TEX remapping,
403 * - shared device is TEXCB=00001
404 * - nonshared device is TEXCB=01000
405 * - write combine device mem is TEXCB=00100
406 * (Uncached Normal in ARMv6 parlance).
407 */
408 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_BUFFERED;
409 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2);
410 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1);
411 }
412 } else {
413 /*
414 * On others, write combining is "Uncached/Buffered"
415 */
416 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE;
417 }
418
419 /*
420 * Now deal with the memory-type mappings
421 */
422 cp = &cache_policies[cachepolicy];
423 vecs_pgprot = kern_pgprot = user_pgprot = cp->pte;
424
425 /*
426 * Enable CPU-specific coherency if supported.
427 * (Only available on XSC3 at the moment.)
428 */
429 if (arch_is_coherent() && cpu_is_xsc3()) {
430 mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S;
431 mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED;
432 mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED;
433 mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S;
434 mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED;
435 }
436 /*
437 * ARMv6 and above have extended page tables.
438 */
439 if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) {
440 #ifndef CONFIG_ARM_LPAE
441 /*
442 * Mark cache clean areas and XIP ROM read only
443 * from SVC mode and no access from userspace.
444 */
445 mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
446 mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
447 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
448 #endif
449
450 if (is_smp()) {
451 /*
452 * Mark memory with the "shared" attribute
453 * for SMP systems
454 */
455 user_pgprot |= L_PTE_SHARED;
456 kern_pgprot |= L_PTE_SHARED;
457 vecs_pgprot |= L_PTE_SHARED;
458 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S;
459 mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED;
460 mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S;
461 mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED;
462 mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S;
463 mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED;
464 mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED;
465 mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S;
466 mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED;
467 }
468 }
469
470 /*
471 * Non-cacheable Normal - intended for memory areas that must
472 * not cause dirty cache line writebacks when used
473 */
474 if (cpu_arch >= CPU_ARCH_ARMv6) {
475 if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
476 /* Non-cacheable Normal is XCB = 001 */
477 mem_types[MT_MEMORY_NONCACHED].prot_sect |=
478 PMD_SECT_BUFFERED;
479 } else {
480 /* For both ARMv6 and non-TEX-remapping ARMv7 */
481 mem_types[MT_MEMORY_NONCACHED].prot_sect |=
482 PMD_SECT_TEX(1);
483 }
484 } else {
485 mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE;
486 }
487
488 #ifdef CONFIG_ARM_LPAE
489 /*
490 * Do not generate access flag faults for the kernel mappings.
491 */
492 for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
493 mem_types[i].prot_pte |= PTE_EXT_AF;
494 if (mem_types[i].prot_sect)
495 mem_types[i].prot_sect |= PMD_SECT_AF;
496 }
497 kern_pgprot |= PTE_EXT_AF;
498 vecs_pgprot |= PTE_EXT_AF;
499 #endif
500
501 for (i = 0; i < 16; i++) {
502 unsigned long v = pgprot_val(protection_map[i]);
503 protection_map[i] = __pgprot(v | user_pgprot);
504 }
505
506 mem_types[MT_LOW_VECTORS].prot_pte |= vecs_pgprot;
507 mem_types[MT_HIGH_VECTORS].prot_pte |= vecs_pgprot;
508
509 pgprot_user = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot);
510 pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG |
511 L_PTE_DIRTY | kern_pgprot);
512
513 mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask;
514 mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
515 mem_types[MT_MEMORY].prot_sect |= ecc_mask | cp->pmd;
516 mem_types[MT_MEMORY].prot_pte |= kern_pgprot;
517 mem_types[MT_MEMORY_DMA_READY].prot_pte |= kern_pgprot;
518 mem_types[MT_MEMORY_NONCACHED].prot_sect |= ecc_mask;
519 mem_types[MT_ROM].prot_sect |= cp->pmd;
520
521 switch (cp->pmd) {
522 case PMD_SECT_WT:
523 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WT;
524 break;
525 case PMD_SECT_WB:
526 case PMD_SECT_WBWA:
527 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WB;
528 break;
529 }
530 printk("Memory policy: ECC %sabled, Data cache %s\n",
531 ecc_mask ? "en" : "dis", cp->policy);
532
533 for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
534 struct mem_type *t = &mem_types[i];
535 if (t->prot_l1)
536 t->prot_l1 |= PMD_DOMAIN(t->domain);
537 if (t->prot_sect)
538 t->prot_sect |= PMD_DOMAIN(t->domain);
539 }
540 }
541
542 #ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE
543 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
544 unsigned long size, pgprot_t vma_prot)
545 {
546 if (!pfn_valid(pfn))
547 return pgprot_noncached(vma_prot);
548 else if (file->f_flags & O_SYNC)
549 return pgprot_writecombine(vma_prot);
550 return vma_prot;
551 }
552 EXPORT_SYMBOL(phys_mem_access_prot);
553 #endif
554
555 #define vectors_base() (vectors_high() ? 0xffff0000 : 0)
556
557 static void __init *early_alloc_aligned(unsigned long sz, unsigned long align)
558 {
559 void *ptr = __va(memblock_alloc(sz, align));
560 memset(ptr, 0, sz);
561 return ptr;
562 }
563
564 static void __init *early_alloc(unsigned long sz)
565 {
566 return early_alloc_aligned(sz, sz);
567 }
568
569 static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr, unsigned long prot)
570 {
571 if (pmd_none(*pmd)) {
572 pte_t *pte = early_alloc(PTE_HWTABLE_OFF + PTE_HWTABLE_SIZE);
573 __pmd_populate(pmd, __pa(pte), prot);
574 }
575 BUG_ON(pmd_bad(*pmd));
576 return pte_offset_kernel(pmd, addr);
577 }
578
579 static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
580 unsigned long end, unsigned long pfn,
581 const struct mem_type *type)
582 {
583 pte_t *pte = early_pte_alloc(pmd, addr, type->prot_l1);
584 do {
585 set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)), 0);
586 pfn++;
587 } while (pte++, addr += PAGE_SIZE, addr != end);
588 }
589
590 static void __init alloc_init_section(pud_t *pud, unsigned long addr,
591 unsigned long end, phys_addr_t phys,
592 const struct mem_type *type)
593 {
594 pmd_t *pmd = pmd_offset(pud, addr);
595
596 /*
597 * Try a section mapping - end, addr and phys must all be aligned
598 * to a section boundary. Note that PMDs refer to the individual
599 * L1 entries, whereas PGDs refer to a group of L1 entries making
600 * up one logical pointer to an L2 table.
601 */
602 if (type->prot_sect && ((addr | end | phys) & ~SECTION_MASK) == 0) {
603 pmd_t *p = pmd;
604
605 #ifndef CONFIG_ARM_LPAE
606 if (addr & SECTION_SIZE)
607 pmd++;
608 #endif
609
610 do {
611 *pmd = __pmd(phys | type->prot_sect);
612 phys += SECTION_SIZE;
613 } while (pmd++, addr += SECTION_SIZE, addr != end);
614
615 flush_pmd_entry(p);
616 } else {
617 /*
618 * No need to loop; pte's aren't interested in the
619 * individual L1 entries.
620 */
621 alloc_init_pte(pmd, addr, end, __phys_to_pfn(phys), type);
622 }
623 }
624
625 static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
626 unsigned long end, unsigned long phys, const struct mem_type *type)
627 {
628 pud_t *pud = pud_offset(pgd, addr);
629 unsigned long next;
630
631 do {
632 next = pud_addr_end(addr, end);
633 alloc_init_section(pud, addr, next, phys, type);
634 phys += next - addr;
635 } while (pud++, addr = next, addr != end);
636 }
637
638 #ifndef CONFIG_ARM_LPAE
639 static void __init create_36bit_mapping(struct map_desc *md,
640 const struct mem_type *type)
641 {
642 unsigned long addr, length, end;
643 phys_addr_t phys;
644 pgd_t *pgd;
645
646 addr = md->virtual;
647 phys = __pfn_to_phys(md->pfn);
648 length = PAGE_ALIGN(md->length);
649
650 if (!(cpu_architecture() >= CPU_ARCH_ARMv6 || cpu_is_xsc3())) {
651 printk(KERN_ERR "MM: CPU does not support supersection "
652 "mapping for 0x%08llx at 0x%08lx\n",
653 (long long)__pfn_to_phys((u64)md->pfn), addr);
654 return;
655 }
656
657 /* N.B. ARMv6 supersections are only defined to work with domain 0.
658 * Since domain assignments can in fact be arbitrary, the
659 * 'domain == 0' check below is required to insure that ARMv6
660 * supersections are only allocated for domain 0 regardless
661 * of the actual domain assignments in use.
662 */
663 if (type->domain) {
664 printk(KERN_ERR "MM: invalid domain in supersection "
665 "mapping for 0x%08llx at 0x%08lx\n",
666 (long long)__pfn_to_phys((u64)md->pfn), addr);
667 return;
668 }
669
670 if ((addr | length | __pfn_to_phys(md->pfn)) & ~SUPERSECTION_MASK) {
671 printk(KERN_ERR "MM: cannot create mapping for 0x%08llx"
672 " at 0x%08lx invalid alignment\n",
673 (long long)__pfn_to_phys((u64)md->pfn), addr);
674 return;
675 }
676
677 /*
678 * Shift bits [35:32] of address into bits [23:20] of PMD
679 * (See ARMv6 spec).
680 */
681 phys |= (((md->pfn >> (32 - PAGE_SHIFT)) & 0xF) << 20);
682
683 pgd = pgd_offset_k(addr);
684 end = addr + length;
685 do {
686 pud_t *pud = pud_offset(pgd, addr);
687 pmd_t *pmd = pmd_offset(pud, addr);
688 int i;
689
690 for (i = 0; i < 16; i++)
691 *pmd++ = __pmd(phys | type->prot_sect | PMD_SECT_SUPER);
692
693 addr += SUPERSECTION_SIZE;
694 phys += SUPERSECTION_SIZE;
695 pgd += SUPERSECTION_SIZE >> PGDIR_SHIFT;
696 } while (addr != end);
697 }
698 #endif /* !CONFIG_ARM_LPAE */
699
700 /*
701 * Create the page directory entries and any necessary
702 * page tables for the mapping specified by `md'. We
703 * are able to cope here with varying sizes and address
704 * offsets, and we take full advantage of sections and
705 * supersections.
706 */
707 static void __init create_mapping(struct map_desc *md)
708 {
709 unsigned long addr, length, end;
710 phys_addr_t phys;
711 const struct mem_type *type;
712 pgd_t *pgd;
713
714 if (md->virtual != vectors_base() && md->virtual < TASK_SIZE) {
715 printk(KERN_WARNING "BUG: not creating mapping for 0x%08llx"
716 " at 0x%08lx in user region\n",
717 (long long)__pfn_to_phys((u64)md->pfn), md->virtual);
718 return;
719 }
720
721 if ((md->type == MT_DEVICE || md->type == MT_ROM) &&
722 md->virtual >= PAGE_OFFSET &&
723 (md->virtual < VMALLOC_START || md->virtual >= VMALLOC_END)) {
724 printk(KERN_WARNING "BUG: mapping for 0x%08llx"
725 " at 0x%08lx out of vmalloc space\n",
726 (long long)__pfn_to_phys((u64)md->pfn), md->virtual);
727 }
728
729 type = &mem_types[md->type];
730
731 #ifndef CONFIG_ARM_LPAE
732 /*
733 * Catch 36-bit addresses
734 */
735 if (md->pfn >= 0x100000) {
736 create_36bit_mapping(md, type);
737 return;
738 }
739 #endif
740
741 addr = md->virtual & PAGE_MASK;
742 phys = __pfn_to_phys(md->pfn);
743 length = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));
744
745 if (type->prot_l1 == 0 && ((addr | phys | length) & ~SECTION_MASK)) {
746 printk(KERN_WARNING "BUG: map for 0x%08llx at 0x%08lx can not "
747 "be mapped using pages, ignoring.\n",
748 (long long)__pfn_to_phys(md->pfn), addr);
749 return;
750 }
751
752 pgd = pgd_offset_k(addr);
753 end = addr + length;
754 do {
755 unsigned long next = pgd_addr_end(addr, end);
756
757 alloc_init_pud(pgd, addr, next, phys, type);
758
759 phys += next - addr;
760 addr = next;
761 } while (pgd++, addr != end);
762 }
763
764 /*
765 * Create the architecture specific mappings
766 */
767 void __init iotable_init(struct map_desc *io_desc, int nr)
768 {
769 struct map_desc *md;
770 struct vm_struct *vm;
771
772 if (!nr)
773 return;
774
775 vm = early_alloc_aligned(sizeof(*vm) * nr, __alignof__(*vm));
776
777 for (md = io_desc; nr; md++, nr--) {
778 create_mapping(md);
779 vm->addr = (void *)(md->virtual & PAGE_MASK);
780 vm->size = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));
781 vm->phys_addr = __pfn_to_phys(md->pfn);
782 vm->flags = VM_IOREMAP | VM_ARM_STATIC_MAPPING;
783 vm->flags |= VM_ARM_MTYPE(md->type);
784 vm->caller = iotable_init;
785 vm_area_add_early(vm++);
786 }
787 }
788
789 void __init vm_reserve_area_early(unsigned long addr, unsigned long size,
790 void *caller)
791 {
792 struct vm_struct *vm;
793
794 vm = early_alloc_aligned(sizeof(*vm), __alignof__(*vm));
795 vm->addr = (void *)addr;
796 vm->size = size;
797 vm->flags = VM_IOREMAP | VM_ARM_EMPTY_MAPPING;
798 vm->caller = caller;
799 vm_area_add_early(vm);
800 }
801
802 #ifndef CONFIG_ARM_LPAE
803
804 /*
805 * The Linux PMD is made of two consecutive section entries covering 2MB
806 * (see definition in include/asm/pgtable-2level.h). However a call to
807 * create_mapping() may optimize static mappings by using individual
808 * 1MB section mappings. This leaves the actual PMD potentially half
809 * initialized if the top or bottom section entry isn't used, leaving it
810 * open to problems if a subsequent ioremap() or vmalloc() tries to use
811 * the virtual space left free by that unused section entry.
812 *
813 * Let's avoid the issue by inserting dummy vm entries covering the unused
814 * PMD halves once the static mappings are in place.
815 */
816
817 static void __init pmd_empty_section_gap(unsigned long addr)
818 {
819 vm_reserve_area_early(addr, SECTION_SIZE, pmd_empty_section_gap);
820 }
821
822 static void __init fill_pmd_gaps(void)
823 {
824 struct vm_struct *vm;
825 unsigned long addr, next = 0;
826 pmd_t *pmd;
827
828 /* we're still single threaded hence no lock needed here */
829 for (vm = vmlist; vm; vm = vm->next) {
830 if (!(vm->flags & (VM_ARM_STATIC_MAPPING | VM_ARM_EMPTY_MAPPING)))
831 continue;
832 addr = (unsigned long)vm->addr;
833 if (addr < next)
834 continue;
835
836 /*
837 * Check if this vm starts on an odd section boundary.
838 * If so and the first section entry for this PMD is free
839 * then we block the corresponding virtual address.
840 */
841 if ((addr & ~PMD_MASK) == SECTION_SIZE) {
842 pmd = pmd_off_k(addr);
843 if (pmd_none(*pmd))
844 pmd_empty_section_gap(addr & PMD_MASK);
845 }
846
847 /*
848 * Then check if this vm ends on an odd section boundary.
849 * If so and the second section entry for this PMD is empty
850 * then we block the corresponding virtual address.
851 */
852 addr += vm->size;
853 if ((addr & ~PMD_MASK) == SECTION_SIZE) {
854 pmd = pmd_off_k(addr) + 1;
855 if (pmd_none(*pmd))
856 pmd_empty_section_gap(addr);
857 }
858
859 /* no need to look at any vm entry until we hit the next PMD */
860 next = (addr + PMD_SIZE - 1) & PMD_MASK;
861 }
862 }
863
864 #else
865 #define fill_pmd_gaps() do { } while (0)
866 #endif
867
868 #if defined(CONFIG_PCI) && !defined(CONFIG_NEED_MACH_IO_H)
869 static void __init pci_reserve_io(void)
870 {
871 struct vm_struct *vm;
872 unsigned long addr;
873
874 /* we're still single threaded hence no lock needed here */
875 for (vm = vmlist; vm; vm = vm->next) {
876 if (!(vm->flags & VM_ARM_STATIC_MAPPING))
877 continue;
878 addr = (unsigned long)vm->addr;
879 addr &= ~(SZ_2M - 1);
880 if (addr == PCI_IO_VIRT_BASE)
881 return;
882
883 }
884 vm_reserve_area_early(PCI_IO_VIRT_BASE, SZ_2M, pci_reserve_io);
885 }
886 #else
887 #define pci_reserve_io() do { } while (0)
888 #endif
889
890 static void * __initdata vmalloc_min =
891 (void *)(VMALLOC_END - (240 << 20) - VMALLOC_OFFSET);
892
893 /*
894 * vmalloc=size forces the vmalloc area to be exactly 'size'
895 * bytes. This can be used to increase (or decrease) the vmalloc
896 * area - the default is 240m.
897 */
898 static int __init early_vmalloc(char *arg)
899 {
900 unsigned long vmalloc_reserve = memparse(arg, NULL);
901
902 if (vmalloc_reserve < SZ_16M) {
903 vmalloc_reserve = SZ_16M;
904 printk(KERN_WARNING
905 "vmalloc area too small, limiting to %luMB\n",
906 vmalloc_reserve >> 20);
907 }
908
909 if (vmalloc_reserve > VMALLOC_END - (PAGE_OFFSET + SZ_32M)) {
910 vmalloc_reserve = VMALLOC_END - (PAGE_OFFSET + SZ_32M);
911 printk(KERN_WARNING
912 "vmalloc area is too big, limiting to %luMB\n",
913 vmalloc_reserve >> 20);
914 }
915
916 vmalloc_min = (void *)(VMALLOC_END - vmalloc_reserve);
917 return 0;
918 }
919 early_param("vmalloc", early_vmalloc);
920
921 phys_addr_t arm_lowmem_limit __initdata = 0;
922
923 void __init sanity_check_meminfo(void)
924 {
925 int i, j, highmem = 0;
926
927 for (i = 0, j = 0; i < meminfo.nr_banks; i++) {
928 struct membank *bank = &meminfo.bank[j];
929 *bank = meminfo.bank[i];
930
931 if (bank->start > ULONG_MAX)
932 highmem = 1;
933
934 #ifdef CONFIG_HIGHMEM
935 if (__va(bank->start) >= vmalloc_min ||
936 __va(bank->start) < (void *)PAGE_OFFSET)
937 highmem = 1;
938
939 bank->highmem = highmem;
940
941 /*
942 * Split those memory banks which are partially overlapping
943 * the vmalloc area greatly simplifying things later.
944 */
945 if (!highmem && __va(bank->start) < vmalloc_min &&
946 bank->size > vmalloc_min - __va(bank->start)) {
947 if (meminfo.nr_banks >= NR_BANKS) {
948 printk(KERN_CRIT "NR_BANKS too low, "
949 "ignoring high memory\n");
950 } else {
951 memmove(bank + 1, bank,
952 (meminfo.nr_banks - i) * sizeof(*bank));
953 meminfo.nr_banks++;
954 i++;
955 bank[1].size -= vmalloc_min - __va(bank->start);
956 bank[1].start = __pa(vmalloc_min - 1) + 1;
957 bank[1].highmem = highmem = 1;
958 j++;
959 }
960 bank->size = vmalloc_min - __va(bank->start);
961 }
962 #else
963 bank->highmem = highmem;
964
965 /*
966 * Highmem banks not allowed with !CONFIG_HIGHMEM.
967 */
968 if (highmem) {
969 printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx "
970 "(!CONFIG_HIGHMEM).\n",
971 (unsigned long long)bank->start,
972 (unsigned long long)bank->start + bank->size - 1);
973 continue;
974 }
975
976 /*
977 * Check whether this memory bank would entirely overlap
978 * the vmalloc area.
979 */
980 if (__va(bank->start) >= vmalloc_min ||
981 __va(bank->start) < (void *)PAGE_OFFSET) {
982 printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx "
983 "(vmalloc region overlap).\n",
984 (unsigned long long)bank->start,
985 (unsigned long long)bank->start + bank->size - 1);
986 continue;
987 }
988
989 /*
990 * Check whether this memory bank would partially overlap
991 * the vmalloc area.
992 */
993 if (__va(bank->start + bank->size) > vmalloc_min ||
994 __va(bank->start + bank->size) < __va(bank->start)) {
995 unsigned long newsize = vmalloc_min - __va(bank->start);
996 printk(KERN_NOTICE "Truncating RAM at %.8llx-%.8llx "
997 "to -%.8llx (vmalloc region overlap).\n",
998 (unsigned long long)bank->start,
999 (unsigned long long)bank->start + bank->size - 1,
1000 (unsigned long long)bank->start + newsize - 1);
1001 bank->size = newsize;
1002 }
1003 #endif
1004 if (!bank->highmem && bank->start + bank->size > arm_lowmem_limit)
1005 arm_lowmem_limit = bank->start + bank->size;
1006
1007 j++;
1008 }
1009 #ifdef CONFIG_HIGHMEM
1010 if (highmem) {
1011 const char *reason = NULL;
1012
1013 if (cache_is_vipt_aliasing()) {
1014 /*
1015 * Interactions between kmap and other mappings
1016 * make highmem support with aliasing VIPT caches
1017 * rather difficult.
1018 */
1019 reason = "with VIPT aliasing cache";
1020 }
1021 if (reason) {
1022 printk(KERN_CRIT "HIGHMEM is not supported %s, ignoring high memory\n",
1023 reason);
1024 while (j > 0 && meminfo.bank[j - 1].highmem)
1025 j--;
1026 }
1027 }
1028 #endif
1029 meminfo.nr_banks = j;
1030 high_memory = __va(arm_lowmem_limit - 1) + 1;
1031 memblock_set_current_limit(arm_lowmem_limit);
1032 }
1033
1034 static inline void prepare_page_table(void)
1035 {
1036 unsigned long addr;
1037 phys_addr_t end;
1038
1039 /*
1040 * Clear out all the mappings below the kernel image.
1041 */
1042 for (addr = 0; addr < MODULES_VADDR; addr += PMD_SIZE)
1043 pmd_clear(pmd_off_k(addr));
1044
1045 #ifdef CONFIG_XIP_KERNEL
1046 /* The XIP kernel is mapped in the module area -- skip over it */
1047 addr = ((unsigned long)_etext + PMD_SIZE - 1) & PMD_MASK;
1048 #endif
1049 for ( ; addr < PAGE_OFFSET; addr += PMD_SIZE)
1050 pmd_clear(pmd_off_k(addr));
1051
1052 /*
1053 * Find the end of the first block of lowmem.
1054 */
1055 end = memblock.memory.regions[0].base + memblock.memory.regions[0].size;
1056 if (end >= arm_lowmem_limit)
1057 end = arm_lowmem_limit;
1058
1059 /*
1060 * Clear out all the kernel space mappings, except for the first
1061 * memory bank, up to the vmalloc region.
1062 */
1063 for (addr = __phys_to_virt(end);
1064 addr < VMALLOC_START; addr += PMD_SIZE)
1065 pmd_clear(pmd_off_k(addr));
1066 }
1067
1068 #ifdef CONFIG_ARM_LPAE
1069 /* the first page is reserved for pgd */
1070 #define SWAPPER_PG_DIR_SIZE (PAGE_SIZE + \
1071 PTRS_PER_PGD * PTRS_PER_PMD * sizeof(pmd_t))
1072 #else
1073 #define SWAPPER_PG_DIR_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
1074 #endif
1075
1076 /*
1077 * Reserve the special regions of memory
1078 */
1079 void __init arm_mm_memblock_reserve(void)
1080 {
1081 /*
1082 * Reserve the page tables. These are already in use,
1083 * and can only be in node 0.
1084 */
1085 memblock_reserve(__pa(swapper_pg_dir), SWAPPER_PG_DIR_SIZE);
1086
1087 #ifdef CONFIG_SA1111
1088 /*
1089 * Because of the SA1111 DMA bug, we want to preserve our
1090 * precious DMA-able memory...
1091 */
1092 memblock_reserve(PHYS_OFFSET, __pa(swapper_pg_dir) - PHYS_OFFSET);
1093 #endif
1094 }
1095
1096 /*
1097 * Set up the device mappings. Since we clear out the page tables for all
1098 * mappings above VMALLOC_START, we will remove any debug device mappings.
1099 * This means you have to be careful how you debug this function, or any
1100 * called function. This means you can't use any function or debugging
1101 * method which may touch any device, otherwise the kernel _will_ crash.
1102 */
1103 static void __init devicemaps_init(struct machine_desc *mdesc)
1104 {
1105 struct map_desc map;
1106 unsigned long addr;
1107 void *vectors;
1108
1109 /*
1110 * Allocate the vector page early.
1111 */
1112 vectors = early_alloc(PAGE_SIZE);
1113
1114 early_trap_init(vectors);
1115
1116 for (addr = VMALLOC_START; addr; addr += PMD_SIZE)
1117 pmd_clear(pmd_off_k(addr));
1118
1119 /*
1120 * Map the kernel if it is XIP.
1121 * It is always first in the modulearea.
1122 */
1123 #ifdef CONFIG_XIP_KERNEL
1124 map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & SECTION_MASK);
1125 map.virtual = MODULES_VADDR;
1126 map.length = ((unsigned long)_etext - map.virtual + ~SECTION_MASK) & SECTION_MASK;
1127 map.type = MT_ROM;
1128 create_mapping(&map);
1129 #endif
1130
1131 /*
1132 * Map the cache flushing regions.
1133 */
1134 #ifdef FLUSH_BASE
1135 map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS);
1136 map.virtual = FLUSH_BASE;
1137 map.length = SZ_1M;
1138 map.type = MT_CACHECLEAN;
1139 create_mapping(&map);
1140 #endif
1141 #ifdef FLUSH_BASE_MINICACHE
1142 map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS + SZ_1M);
1143 map.virtual = FLUSH_BASE_MINICACHE;
1144 map.length = SZ_1M;
1145 map.type = MT_MINICLEAN;
1146 create_mapping(&map);
1147 #endif
1148
1149 /*
1150 * Create a mapping for the machine vectors at the high-vectors
1151 * location (0xffff0000). If we aren't using high-vectors, also
1152 * create a mapping at the low-vectors virtual address.
1153 */
1154 map.pfn = __phys_to_pfn(virt_to_phys(vectors));
1155 map.virtual = 0xffff0000;
1156 map.length = PAGE_SIZE;
1157 map.type = MT_HIGH_VECTORS;
1158 create_mapping(&map);
1159
1160 if (!vectors_high()) {
1161 map.virtual = 0;
1162 map.type = MT_LOW_VECTORS;
1163 create_mapping(&map);
1164 }
1165
1166 /*
1167 * Ask the machine support to map in the statically mapped devices.
1168 */
1169 if (mdesc->map_io)
1170 mdesc->map_io();
1171 fill_pmd_gaps();
1172
1173 /* Reserve fixed i/o space in VMALLOC region */
1174 pci_reserve_io();
1175
1176 /*
1177 * Finally flush the caches and tlb to ensure that we're in a
1178 * consistent state wrt the writebuffer. This also ensures that
1179 * any write-allocated cache lines in the vector page are written
1180 * back. After this point, we can start to touch devices again.
1181 */
1182 local_flush_tlb_all();
1183 flush_cache_all();
1184 }
1185
1186 static void __init kmap_init(void)
1187 {
1188 #ifdef CONFIG_HIGHMEM
1189 pkmap_page_table = early_pte_alloc(pmd_off_k(PKMAP_BASE),
1190 PKMAP_BASE, _PAGE_KERNEL_TABLE);
1191 #endif
1192 }
1193
1194 static void __init map_lowmem(void)
1195 {
1196 struct memblock_region *reg;
1197
1198 /* Map all the lowmem memory banks. */
1199 for_each_memblock(memory, reg) {
1200 phys_addr_t start = reg->base;
1201 phys_addr_t end = start + reg->size;
1202 struct map_desc map;
1203
1204 if (end > arm_lowmem_limit)
1205 end = arm_lowmem_limit;
1206 if (start >= end)
1207 break;
1208
1209 map.pfn = __phys_to_pfn(start);
1210 map.virtual = __phys_to_virt(start);
1211 map.length = end - start;
1212 map.type = MT_MEMORY;
1213
1214 create_mapping(&map);
1215 }
1216 }
1217
1218 /*
1219 * paging_init() sets up the page tables, initialises the zone memory
1220 * maps, and sets up the zero page, bad page and bad page tables.
1221 */
1222 void __init paging_init(struct machine_desc *mdesc)
1223 {
1224 void *zero_page;
1225
1226 memblock_set_current_limit(arm_lowmem_limit);
1227
1228 build_mem_type_table();
1229 prepare_page_table();
1230 map_lowmem();
1231 dma_contiguous_remap();
1232 devicemaps_init(mdesc);
1233 kmap_init();
1234
1235 top_pmd = pmd_off_k(0xffff0000);
1236
1237 /* allocate the zero page. */
1238 zero_page = early_alloc(PAGE_SIZE);
1239
1240 bootmem_init();
1241
1242 empty_zero_page = virt_to_page(zero_page);
1243 __flush_dcache_page(NULL, empty_zero_page);
1244 }
Linus' kernel tree