2 * linux/arch/x86_64/mm/init.c
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
6 * Copyright (C) 2002,2003 Andi Kleen <ak@suse.de>
9 #include <linux/signal.h>
10 #include <linux/sched.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/string.h>
14 #include <linux/types.h>
15 #include <linux/ptrace.h>
16 #include <linux/mman.h>
18 #include <linux/swap.h>
19 #include <linux/smp.h>
20 #include <linux/init.h>
21 #include <linux/pagemap.h>
22 #include <linux/bootmem.h>
23 #include <linux/proc_fs.h>
24 #include <linux/pci.h>
25 #include <linux/pfn.h>
26 #include <linux/poison.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/module.h>
29 #include <linux/memory_hotplug.h>
30 #include <linux/nmi.h>
32 #include <asm/processor.h>
33 #include <asm/system.h>
34 #include <asm/uaccess.h>
35 #include <asm/pgtable.h>
36 #include <asm/pgalloc.h>
38 #include <asm/fixmap.h>
42 #include <asm/mmu_context.h>
43 #include <asm/proto.h>
45 #include <asm/sections.h>
46 #include <asm/kdebug.h>
48 #include <asm/cacheflush.h>
50 static unsigned long dma_reserve __initdata
;
52 DEFINE_PER_CPU(struct mmu_gather
, mmu_gathers
);
54 int direct_gbpages __meminitdata
55 #ifdef CONFIG_DIRECT_GBPAGES
60 static int __init
parse_direct_gbpages_off(char *arg
)
65 early_param("nogbpages", parse_direct_gbpages_off
);
67 static int __init
parse_direct_gbpages_on(char *arg
)
72 early_param("gbpages", parse_direct_gbpages_on
);
75 * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
76 * physical space so we can cache the place of the first one and move
77 * around without checking the pgd every time.
82 long i
, total
= 0, reserved
= 0;
83 long shared
= 0, cached
= 0;
87 printk(KERN_INFO
"Mem-info:\n");
89 for_each_online_pgdat(pgdat
) {
90 for (i
= 0; i
< pgdat
->node_spanned_pages
; ++i
) {
92 * This loop can take a while with 256 GB and
93 * 4k pages so defer the NMI watchdog:
95 if (unlikely(i
% MAX_ORDER_NR_PAGES
== 0))
98 if (!pfn_valid(pgdat
->node_start_pfn
+ i
))
101 page
= pfn_to_page(pgdat
->node_start_pfn
+ i
);
103 if (PageReserved(page
))
105 else if (PageSwapCache(page
))
107 else if (page_count(page
))
108 shared
+= page_count(page
) - 1;
111 printk(KERN_INFO
"%lu pages of RAM\n", total
);
112 printk(KERN_INFO
"%lu reserved pages\n", reserved
);
113 printk(KERN_INFO
"%lu pages shared\n", shared
);
114 printk(KERN_INFO
"%lu pages swap cached\n", cached
);
119 static __init
void *spp_getpage(void)
124 ptr
= (void *) get_zeroed_page(GFP_ATOMIC
);
126 ptr
= alloc_bootmem_pages(PAGE_SIZE
);
128 if (!ptr
|| ((unsigned long)ptr
& ~PAGE_MASK
)) {
129 panic("set_pte_phys: cannot allocate page data %s\n",
130 after_bootmem
? "after bootmem" : "");
133 pr_debug("spp_getpage %p\n", ptr
);
139 set_pte_phys(unsigned long vaddr
, unsigned long phys
, pgprot_t prot
)
146 pr_debug("set_pte_phys %lx to %lx\n", vaddr
, phys
);
148 pgd
= pgd_offset_k(vaddr
);
149 if (pgd_none(*pgd
)) {
151 "PGD FIXMAP MISSING, it should be setup in head.S!\n");
154 pud
= pud_offset(pgd
, vaddr
);
155 if (pud_none(*pud
)) {
156 pmd
= (pmd_t
*) spp_getpage();
157 set_pud(pud
, __pud(__pa(pmd
) | _KERNPG_TABLE
| _PAGE_USER
));
158 if (pmd
!= pmd_offset(pud
, 0)) {
159 printk(KERN_ERR
"PAGETABLE BUG #01! %p <-> %p\n",
160 pmd
, pmd_offset(pud
, 0));
164 pmd
= pmd_offset(pud
, vaddr
);
165 if (pmd_none(*pmd
)) {
166 pte
= (pte_t
*) spp_getpage();
167 set_pmd(pmd
, __pmd(__pa(pte
) | _KERNPG_TABLE
| _PAGE_USER
));
168 if (pte
!= pte_offset_kernel(pmd
, 0)) {
169 printk(KERN_ERR
"PAGETABLE BUG #02!\n");
173 new_pte
= pfn_pte(phys
>> PAGE_SHIFT
, prot
);
175 pte
= pte_offset_kernel(pmd
, vaddr
);
176 if (!pte_none(*pte
) && pte_val(new_pte
) &&
177 pte_val(*pte
) != (pte_val(new_pte
) & __supported_pte_mask
))
179 set_pte(pte
, new_pte
);
182 * It's enough to flush this one mapping.
183 * (PGE mappings get flushed as well)
185 __flush_tlb_one(vaddr
);
189 * The head.S code sets up the kernel high mapping:
191 * from __START_KERNEL_map to __START_KERNEL_map + size (== _end-_text)
193 * phys_addr holds the negative offset to the kernel, which is added
194 * to the compile time generated pmds. This results in invalid pmds up
195 * to the point where we hit the physaddr 0 mapping.
197 * We limit the mappings to the region from _text to _end. _end is
198 * rounded up to the 2MB boundary. This catches the invalid pmds as
199 * well, as they are located before _text:
201 void __init
cleanup_highmap(void)
203 unsigned long vaddr
= __START_KERNEL_map
;
204 unsigned long end
= round_up((unsigned long)_end
, PMD_SIZE
) - 1;
205 pmd_t
*pmd
= level2_kernel_pgt
;
206 pmd_t
*last_pmd
= pmd
+ PTRS_PER_PMD
;
208 for (; pmd
< last_pmd
; pmd
++, vaddr
+= PMD_SIZE
) {
209 if (!pmd_present(*pmd
))
211 if (vaddr
< (unsigned long) _text
|| vaddr
> end
)
212 set_pmd(pmd
, __pmd(0));
216 /* NOTE: this is meant to be run only at boot */
217 void __set_fixmap(enum fixed_addresses idx
, unsigned long phys
, pgprot_t prot
)
219 unsigned long address
= __fix_to_virt(idx
);
221 if (idx
>= __end_of_fixed_addresses
) {
222 printk(KERN_ERR
"Invalid __set_fixmap\n");
225 set_pte_phys(address
, phys
, prot
);
228 static unsigned long __initdata table_start
;
229 static unsigned long __meminitdata table_end
;
231 static __meminit
void *alloc_low_page(unsigned long *phys
)
233 unsigned long pfn
= table_end
++;
237 adr
= (void *)get_zeroed_page(GFP_ATOMIC
);
244 panic("alloc_low_page: ran out of memory");
246 adr
= early_ioremap(pfn
* PAGE_SIZE
, PAGE_SIZE
);
247 memset(adr
, 0, PAGE_SIZE
);
248 *phys
= pfn
* PAGE_SIZE
;
252 static __meminit
void unmap_low_page(void *adr
)
257 early_iounmap(adr
, PAGE_SIZE
);
260 /* Must run before zap_low_mappings */
261 __meminit
void *early_ioremap(unsigned long addr
, unsigned long size
)
263 pmd_t
*pmd
, *last_pmd
;
267 pmds
= ((addr
& ~PMD_MASK
) + size
+ ~PMD_MASK
) / PMD_SIZE
;
268 vaddr
= __START_KERNEL_map
;
269 pmd
= level2_kernel_pgt
;
270 last_pmd
= level2_kernel_pgt
+ PTRS_PER_PMD
- 1;
272 for (; pmd
<= last_pmd
; pmd
++, vaddr
+= PMD_SIZE
) {
273 for (i
= 0; i
< pmds
; i
++) {
274 if (pmd_present(pmd
[i
]))
275 goto continue_outer_loop
;
277 vaddr
+= addr
& ~PMD_MASK
;
280 for (i
= 0; i
< pmds
; i
++, addr
+= PMD_SIZE
)
281 set_pmd(pmd
+i
, __pmd(addr
| __PAGE_KERNEL_LARGE_EXEC
));
284 return (void *)vaddr
;
288 printk(KERN_ERR
"early_ioremap(0x%lx, %lu) failed\n", addr
, size
);
294 * To avoid virtual aliases later:
296 __meminit
void early_iounmap(void *addr
, unsigned long size
)
302 vaddr
= (unsigned long)addr
;
303 pmds
= ((vaddr
& ~PMD_MASK
) + size
+ ~PMD_MASK
) / PMD_SIZE
;
304 pmd
= level2_kernel_pgt
+ pmd_index(vaddr
);
306 for (i
= 0; i
< pmds
; i
++)
312 static unsigned long __meminit
313 phys_pmd_init(pmd_t
*pmd_page
, unsigned long address
, unsigned long end
)
315 int i
= pmd_index(address
);
317 for (; i
< PTRS_PER_PMD
; i
++, address
+= PMD_SIZE
) {
318 pmd_t
*pmd
= pmd_page
+ pmd_index(address
);
320 if (address
>= end
) {
321 if (!after_bootmem
) {
322 for (; i
< PTRS_PER_PMD
; i
++, pmd
++)
323 set_pmd(pmd
, __pmd(0));
331 set_pte((pte_t
*)pmd
,
332 pfn_pte(address
>> PAGE_SHIFT
, PAGE_KERNEL_LARGE
));
337 static unsigned long __meminit
338 phys_pmd_update(pud_t
*pud
, unsigned long address
, unsigned long end
)
340 pmd_t
*pmd
= pmd_offset(pud
, 0);
341 unsigned long last_map_addr
;
343 spin_lock(&init_mm
.page_table_lock
);
344 last_map_addr
= phys_pmd_init(pmd
, address
, end
);
345 spin_unlock(&init_mm
.page_table_lock
);
347 return last_map_addr
;
350 static unsigned long __meminit
351 phys_pud_init(pud_t
*pud_page
, unsigned long addr
, unsigned long end
)
353 unsigned long last_map_addr
= end
;
354 int i
= pud_index(addr
);
356 for (; i
< PTRS_PER_PUD
; i
++, addr
= (addr
& PUD_MASK
) + PUD_SIZE
) {
357 unsigned long pmd_phys
;
358 pud_t
*pud
= pud_page
+ pud_index(addr
);
364 if (!after_bootmem
&&
365 !e820_any_mapped(addr
, addr
+PUD_SIZE
, 0)) {
366 set_pud(pud
, __pud(0));
371 if (!pud_large(*pud
))
372 last_map_addr
= phys_pmd_update(pud
, addr
, end
);
376 if (direct_gbpages
) {
377 set_pte((pte_t
*)pud
,
378 pfn_pte(addr
>> PAGE_SHIFT
, PAGE_KERNEL_LARGE
));
379 last_map_addr
= (addr
& PUD_MASK
) + PUD_SIZE
;
383 pmd
= alloc_low_page(&pmd_phys
);
385 spin_lock(&init_mm
.page_table_lock
);
386 set_pud(pud
, __pud(pmd_phys
| _KERNPG_TABLE
));
387 last_map_addr
= phys_pmd_init(pmd
, addr
, end
);
388 spin_unlock(&init_mm
.page_table_lock
);
394 return last_map_addr
>> PAGE_SHIFT
;
397 static void __init
find_early_table_space(unsigned long end
)
399 unsigned long puds
, pmds
, tables
, start
;
401 puds
= (end
+ PUD_SIZE
- 1) >> PUD_SHIFT
;
402 tables
= round_up(puds
* sizeof(pud_t
), PAGE_SIZE
);
403 if (!direct_gbpages
) {
404 pmds
= (end
+ PMD_SIZE
- 1) >> PMD_SHIFT
;
405 tables
+= round_up(pmds
* sizeof(pmd_t
), PAGE_SIZE
);
409 * RED-PEN putting page tables only on node 0 could
410 * cause a hotspot and fill up ZONE_DMA. The page tables
411 * need roughly 0.5KB per GB.
414 table_start
= find_e820_area(start
, end
, tables
, PAGE_SIZE
);
415 if (table_start
== -1UL)
416 panic("Cannot find space for the kernel page tables");
418 table_start
>>= PAGE_SHIFT
;
419 table_end
= table_start
;
421 early_printk("kernel direct mapping tables up to %lx @ %lx-%lx\n",
422 end
, table_start
<< PAGE_SHIFT
,
423 (table_start
<< PAGE_SHIFT
) + tables
);
426 static void __init
init_gbpages(void)
428 if (direct_gbpages
&& cpu_has_gbpages
)
429 printk(KERN_INFO
"Using GB pages for direct mapping\n");
434 #ifdef CONFIG_MEMTEST_BOOTPARAM
436 static void __init
memtest(unsigned long start_phys
, unsigned long size
,
440 unsigned long *start
;
441 unsigned long start_bad
;
442 unsigned long last_bad
;
444 unsigned long start_phys_aligned
;
456 val
= 0x5555555555555555UL
;
459 val
= 0xaaaaaaaaaaaaaaaaUL
;
465 incr
= sizeof(unsigned long);
466 start_phys_aligned
= ALIGN(start_phys
, incr
);
467 count
= (size
- (start_phys_aligned
- start_phys
))/incr
;
468 start
= __va(start_phys_aligned
);
472 for (i
= 0; i
< count
; i
++)
474 for (i
= 0; i
< count
; i
++, start
++, start_phys_aligned
+= incr
) {
476 if (start_phys_aligned
== last_bad
+ incr
) {
480 printk(KERN_CONT
"\n %016lx bad mem addr %016lx - %016lx reserved",
481 val
, start_bad
, last_bad
+ incr
);
482 reserve_early(start_bad
, last_bad
- start_bad
, "BAD RAM");
484 start_bad
= last_bad
= start_phys_aligned
;
489 printk(KERN_CONT
"\n %016lx bad mem addr %016lx - %016lx reserved",
490 val
, start_bad
, last_bad
+ incr
);
491 reserve_early(start_bad
, last_bad
- start_bad
, "BAD RAM");
496 static int memtest_pattern __initdata
= CONFIG_MEMTEST_BOOTPARAM_VALUE
;
498 static int __init
parse_memtest(char *arg
)
501 memtest_pattern
= simple_strtoul(arg
, NULL
, 0);
505 early_param("memtest", parse_memtest
);
507 static void __init
early_memtest(unsigned long start
, unsigned long end
)
509 unsigned long t_start
, t_size
;
512 if (!memtest_pattern
)
515 printk(KERN_INFO
"early_memtest: pattern num %d", memtest_pattern
);
516 for (pattern
= 0; pattern
< memtest_pattern
; pattern
++) {
519 while (t_start
< end
) {
520 t_start
= find_e820_area_size(t_start
, &t_size
, 1);
525 if (t_start
+ t_size
> end
)
526 t_size
= end
- t_start
;
528 printk(KERN_CONT
"\n %016lx - %016lx pattern %d",
529 t_start
, t_start
+ t_size
, pattern
);
531 memtest(t_start
, t_size
, pattern
);
536 printk(KERN_CONT
"\n");
539 static void __init
early_memtest(unsigned long start
, unsigned long end
)
545 * Setup the direct mapping of the physical memory at PAGE_OFFSET.
546 * This runs before bootmem is initialized and gets pages directly from
547 * the physical memory. To access them they are temporarily mapped.
549 unsigned long __init_refok
init_memory_mapping(unsigned long start
, unsigned long end
)
551 unsigned long next
, last_map_addr
= end
;
552 unsigned long start_phys
= start
, end_phys
= end
;
554 printk(KERN_INFO
"init_memory_mapping\n");
557 * Find space for the kernel direct mapping tables.
559 * Later we should allocate these tables in the local node of the
560 * memory mapped. Unfortunately this is done currently before the
561 * nodes are discovered.
563 if (!after_bootmem
) {
565 find_early_table_space(end
);
568 start
= (unsigned long)__va(start
);
569 end
= (unsigned long)__va(end
);
571 for (; start
< end
; start
= next
) {
572 pgd_t
*pgd
= pgd_offset_k(start
);
573 unsigned long pud_phys
;
577 pud
= pud_offset(pgd
, start
& PGDIR_MASK
);
579 pud
= alloc_low_page(&pud_phys
);
581 next
= start
+ PGDIR_SIZE
;
584 last_map_addr
= phys_pud_init(pud
, __pa(start
), __pa(next
));
586 set_pgd(pgd_offset_k(start
), mk_kernel_pgd(pud_phys
));
591 mmu_cr4_features
= read_cr4();
595 reserve_early(table_start
<< PAGE_SHIFT
,
596 table_end
<< PAGE_SHIFT
, "PGTABLE");
599 early_memtest(start_phys
, end_phys
);
601 return last_map_addr
;
605 void __init
paging_init(void)
607 unsigned long max_zone_pfns
[MAX_NR_ZONES
];
609 memset(max_zone_pfns
, 0, sizeof(max_zone_pfns
));
610 max_zone_pfns
[ZONE_DMA
] = MAX_DMA_PFN
;
611 max_zone_pfns
[ZONE_DMA32
] = MAX_DMA32_PFN
;
612 max_zone_pfns
[ZONE_NORMAL
] = end_pfn
;
614 memory_present(0, 0, end_pfn
);
616 free_area_init_nodes(max_zone_pfns
);
621 * Memory hotplug specific functions
623 void online_page(struct page
*page
)
625 ClearPageReserved(page
);
626 init_page_count(page
);
632 #ifdef CONFIG_MEMORY_HOTPLUG
634 * Memory is added always to NORMAL zone. This means you will never get
635 * additional DMA/DMA32 memory.
637 int arch_add_memory(int nid
, u64 start
, u64 size
)
639 struct pglist_data
*pgdat
= NODE_DATA(nid
);
640 struct zone
*zone
= pgdat
->node_zones
+ ZONE_NORMAL
;
641 unsigned long last_mapped_pfn
, start_pfn
= start
>> PAGE_SHIFT
;
642 unsigned long nr_pages
= size
>> PAGE_SHIFT
;
645 last_mapped_pfn
= init_memory_mapping(start
, start
+ size
-1);
646 if (last_mapped_pfn
> max_pfn_mapped
)
647 max_pfn_mapped
= last_mapped_pfn
;
649 ret
= __add_pages(zone
, start_pfn
, nr_pages
);
654 EXPORT_SYMBOL_GPL(arch_add_memory
);
656 #if !defined(CONFIG_ACPI_NUMA) && defined(CONFIG_NUMA)
657 int memory_add_physaddr_to_nid(u64 start
)
661 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid
);
664 #endif /* CONFIG_MEMORY_HOTPLUG */
667 * devmem_is_allowed() checks to see if /dev/mem access to a certain address
668 * is valid. The argument is a physical page number.
671 * On x86, access has to be given to the first megabyte of ram because that area
672 * contains bios code and data regions used by X and dosemu and similar apps.
673 * Access has to be given to non-kernel-ram areas as well, these contain the PCI
674 * mmio resources as well as potential bios/acpi data regions.
676 int devmem_is_allowed(unsigned long pagenr
)
680 if (!page_is_ram(pagenr
))
686 static struct kcore_list kcore_mem
, kcore_vmalloc
, kcore_kernel
,
687 kcore_modules
, kcore_vsyscall
;
689 void __init
mem_init(void)
691 long codesize
, reservedpages
, datasize
, initsize
;
695 /* clear_bss() already clear the empty_zero_page */
699 /* this will put all low memory onto the freelists */
701 totalram_pages
= numa_free_all_bootmem();
703 totalram_pages
= free_all_bootmem();
705 reservedpages
= end_pfn
- totalram_pages
-
706 absent_pages_in_range(0, end_pfn
);
709 codesize
= (unsigned long) &_etext
- (unsigned long) &_text
;
710 datasize
= (unsigned long) &_edata
- (unsigned long) &_etext
;
711 initsize
= (unsigned long) &__init_end
- (unsigned long) &__init_begin
;
713 /* Register memory areas for /proc/kcore */
714 kclist_add(&kcore_mem
, __va(0), max_low_pfn
<< PAGE_SHIFT
);
715 kclist_add(&kcore_vmalloc
, (void *)VMALLOC_START
,
716 VMALLOC_END
-VMALLOC_START
);
717 kclist_add(&kcore_kernel
, &_stext
, _end
- _stext
);
718 kclist_add(&kcore_modules
, (void *)MODULES_VADDR
, MODULES_LEN
);
719 kclist_add(&kcore_vsyscall
, (void *)VSYSCALL_START
,
720 VSYSCALL_END
- VSYSCALL_START
);
722 printk(KERN_INFO
"Memory: %luk/%luk available (%ldk kernel code, "
723 "%ldk reserved, %ldk data, %ldk init)\n",
724 (unsigned long) nr_free_pages() << (PAGE_SHIFT
-10),
725 end_pfn
<< (PAGE_SHIFT
-10),
727 reservedpages
<< (PAGE_SHIFT
-10),
734 void free_init_pages(char *what
, unsigned long begin
, unsigned long end
)
736 unsigned long addr
= begin
;
742 * If debugging page accesses then do not free this memory but
743 * mark them not present - any buggy init-section access will
744 * create a kernel page fault:
746 #ifdef CONFIG_DEBUG_PAGEALLOC
747 printk(KERN_INFO
"debug: unmapping init memory %08lx..%08lx\n",
748 begin
, PAGE_ALIGN(end
));
749 set_memory_np(begin
, (end
- begin
) >> PAGE_SHIFT
);
751 printk(KERN_INFO
"Freeing %s: %luk freed\n", what
, (end
- begin
) >> 10);
753 for (; addr
< end
; addr
+= PAGE_SIZE
) {
754 ClearPageReserved(virt_to_page(addr
));
755 init_page_count(virt_to_page(addr
));
756 memset((void *)(addr
& ~(PAGE_SIZE
-1)),
757 POISON_FREE_INITMEM
, PAGE_SIZE
);
764 void free_initmem(void)
766 free_init_pages("unused kernel memory",
767 (unsigned long)(&__init_begin
),
768 (unsigned long)(&__init_end
));
771 #ifdef CONFIG_DEBUG_RODATA
772 const int rodata_test_data
= 0xC3;
773 EXPORT_SYMBOL_GPL(rodata_test_data
);
775 void mark_rodata_ro(void)
777 unsigned long start
= PFN_ALIGN(_stext
), end
= PFN_ALIGN(__end_rodata
);
779 printk(KERN_INFO
"Write protecting the kernel read-only data: %luk\n",
780 (end
- start
) >> 10);
781 set_memory_ro(start
, (end
- start
) >> PAGE_SHIFT
);
784 * The rodata section (but not the kernel text!) should also be
787 start
= ((unsigned long)__start_rodata
+ PAGE_SIZE
- 1) & PAGE_MASK
;
788 set_memory_nx(start
, (end
- start
) >> PAGE_SHIFT
);
792 #ifdef CONFIG_CPA_DEBUG
793 printk(KERN_INFO
"Testing CPA: undo %lx-%lx\n", start
, end
);
794 set_memory_rw(start
, (end
-start
) >> PAGE_SHIFT
);
796 printk(KERN_INFO
"Testing CPA: again\n");
797 set_memory_ro(start
, (end
-start
) >> PAGE_SHIFT
);
803 #ifdef CONFIG_BLK_DEV_INITRD
804 void free_initrd_mem(unsigned long start
, unsigned long end
)
806 free_init_pages("initrd memory", start
, end
);
810 void __init
reserve_bootmem_generic(unsigned long phys
, unsigned len
)
813 int nid
= phys_to_nid(phys
);
815 unsigned long pfn
= phys
>> PAGE_SHIFT
;
817 if (pfn
>= end_pfn
) {
819 * This can happen with kdump kernels when accessing
822 if (pfn
< max_pfn_mapped
)
825 printk(KERN_ERR
"reserve_bootmem: illegal reserve %lx %u\n",
830 /* Should check here against the e820 map to avoid double free */
832 reserve_bootmem_node(NODE_DATA(nid
), phys
, len
, BOOTMEM_DEFAULT
);
834 reserve_bootmem(phys
, len
, BOOTMEM_DEFAULT
);
836 if (phys
+len
<= MAX_DMA_PFN
*PAGE_SIZE
) {
837 dma_reserve
+= len
/ PAGE_SIZE
;
838 set_dma_reserve(dma_reserve
);
842 int kern_addr_valid(unsigned long addr
)
844 unsigned long above
= ((long)addr
) >> __VIRTUAL_MASK_SHIFT
;
850 if (above
!= 0 && above
!= -1UL)
853 pgd
= pgd_offset_k(addr
);
857 pud
= pud_offset(pgd
, addr
);
861 pmd
= pmd_offset(pud
, addr
);
866 return pfn_valid(pmd_pfn(*pmd
));
868 pte
= pte_offset_kernel(pmd
, addr
);
872 return pfn_valid(pte_pfn(*pte
));
876 * A pseudo VMA to allow ptrace access for the vsyscall page. This only
877 * covers the 64bit vsyscall page now. 32bit has a real VMA now and does
878 * not need special handling anymore:
880 static struct vm_area_struct gate_vma
= {
881 .vm_start
= VSYSCALL_START
,
882 .vm_end
= VSYSCALL_START
+ (VSYSCALL_MAPPED_PAGES
* PAGE_SIZE
),
883 .vm_page_prot
= PAGE_READONLY_EXEC
,
884 .vm_flags
= VM_READ
| VM_EXEC
887 struct vm_area_struct
*get_gate_vma(struct task_struct
*tsk
)
889 #ifdef CONFIG_IA32_EMULATION
890 if (test_tsk_thread_flag(tsk
, TIF_IA32
))
896 int in_gate_area(struct task_struct
*task
, unsigned long addr
)
898 struct vm_area_struct
*vma
= get_gate_vma(task
);
903 return (addr
>= vma
->vm_start
) && (addr
< vma
->vm_end
);
907 * Use this when you have no reliable task/vma, typically from interrupt
908 * context. It is less reliable than using the task's vma and may give
911 int in_gate_area_no_task(unsigned long addr
)
913 return (addr
>= VSYSCALL_START
) && (addr
< VSYSCALL_END
);
916 const char *arch_vma_name(struct vm_area_struct
*vma
)
918 if (vma
->vm_mm
&& vma
->vm_start
== (long)vma
->vm_mm
->context
.vdso
)
920 if (vma
== &gate_vma
)
925 #ifdef CONFIG_SPARSEMEM_VMEMMAP
927 * Initialise the sparsemem vmemmap using huge-pages at the PMD level.
930 vmemmap_populate(struct page
*start_page
, unsigned long size
, int node
)
932 unsigned long addr
= (unsigned long)start_page
;
933 unsigned long end
= (unsigned long)(start_page
+ size
);
939 for (; addr
< end
; addr
= next
) {
940 next
= pmd_addr_end(addr
, end
);
942 pgd
= vmemmap_pgd_populate(addr
, node
);
946 pud
= vmemmap_pud_populate(pgd
, addr
, node
);
950 pmd
= pmd_offset(pud
, addr
);
951 if (pmd_none(*pmd
)) {
955 p
= vmemmap_alloc_block(PMD_SIZE
, node
);
959 entry
= pfn_pte(__pa(p
) >> PAGE_SHIFT
,
961 set_pmd(pmd
, __pmd(pte_val(entry
)));
963 printk(KERN_DEBUG
" [%lx-%lx] PMD ->%p on node %d\n",
964 addr
, addr
+ PMD_SIZE
- 1, p
, node
);
966 vmemmap_verify((pte_t
*)pmd
, node
, addr
, next
);