2 * linux/arch/x86_64/mm/init.c
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
6 * Copyright (C) 2002,2003 Andi Kleen <ak@suse.de>
9 #include <linux/signal.h>
10 #include <linux/sched.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/string.h>
14 #include <linux/types.h>
15 #include <linux/ptrace.h>
16 #include <linux/mman.h>
18 #include <linux/swap.h>
19 #include <linux/smp.h>
20 #include <linux/init.h>
21 #include <linux/pagemap.h>
22 #include <linux/bootmem.h>
23 #include <linux/proc_fs.h>
24 #include <linux/pci.h>
25 #include <linux/pfn.h>
26 #include <linux/poison.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/module.h>
29 #include <linux/memory_hotplug.h>
30 #include <linux/nmi.h>
32 #include <asm/processor.h>
33 #include <asm/system.h>
34 #include <asm/uaccess.h>
35 #include <asm/pgtable.h>
36 #include <asm/pgalloc.h>
38 #include <asm/fixmap.h>
42 #include <asm/mmu_context.h>
43 #include <asm/proto.h>
45 #include <asm/sections.h>
46 #include <asm/kdebug.h>
48 #include <asm/cacheflush.h>
50 const struct dma_mapping_ops
*dma_ops
;
51 EXPORT_SYMBOL(dma_ops
);
53 static unsigned long dma_reserve __initdata
;
55 DEFINE_PER_CPU(struct mmu_gather
, mmu_gathers
);
57 int direct_gbpages __meminitdata
58 #ifdef CONFIG_DIRECT_GBPAGES
63 static int __init
parse_direct_gbpages_off(char *arg
)
68 early_param("nogbpages", parse_direct_gbpages_off
);
70 static int __init
parse_direct_gbpages_on(char *arg
)
75 early_param("gbpages", parse_direct_gbpages_on
);
78 * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
79 * physical space so we can cache the place of the first one and move
80 * around without checking the pgd every time.
85 long i
, total
= 0, reserved
= 0;
86 long shared
= 0, cached
= 0;
90 printk(KERN_INFO
"Mem-info:\n");
92 for_each_online_pgdat(pgdat
) {
93 for (i
= 0; i
< pgdat
->node_spanned_pages
; ++i
) {
95 * This loop can take a while with 256 GB and
96 * 4k pages so defer the NMI watchdog:
98 if (unlikely(i
% MAX_ORDER_NR_PAGES
== 0))
101 if (!pfn_valid(pgdat
->node_start_pfn
+ i
))
104 page
= pfn_to_page(pgdat
->node_start_pfn
+ i
);
106 if (PageReserved(page
))
108 else if (PageSwapCache(page
))
110 else if (page_count(page
))
111 shared
+= page_count(page
) - 1;
114 printk(KERN_INFO
"%lu pages of RAM\n", total
);
115 printk(KERN_INFO
"%lu reserved pages\n", reserved
);
116 printk(KERN_INFO
"%lu pages shared\n", shared
);
117 printk(KERN_INFO
"%lu pages swap cached\n", cached
);
122 static __init
void *spp_getpage(void)
127 ptr
= (void *) get_zeroed_page(GFP_ATOMIC
);
129 ptr
= alloc_bootmem_pages(PAGE_SIZE
);
131 if (!ptr
|| ((unsigned long)ptr
& ~PAGE_MASK
)) {
132 panic("set_pte_phys: cannot allocate page data %s\n",
133 after_bootmem
? "after bootmem" : "");
136 pr_debug("spp_getpage %p\n", ptr
);
142 set_pte_phys(unsigned long vaddr
, unsigned long phys
, pgprot_t prot
)
149 pr_debug("set_pte_phys %lx to %lx\n", vaddr
, phys
);
151 pgd
= pgd_offset_k(vaddr
);
152 if (pgd_none(*pgd
)) {
154 "PGD FIXMAP MISSING, it should be setup in head.S!\n");
157 pud
= pud_offset(pgd
, vaddr
);
158 if (pud_none(*pud
)) {
159 pmd
= (pmd_t
*) spp_getpage();
160 set_pud(pud
, __pud(__pa(pmd
) | _KERNPG_TABLE
| _PAGE_USER
));
161 if (pmd
!= pmd_offset(pud
, 0)) {
162 printk(KERN_ERR
"PAGETABLE BUG #01! %p <-> %p\n",
163 pmd
, pmd_offset(pud
, 0));
167 pmd
= pmd_offset(pud
, vaddr
);
168 if (pmd_none(*pmd
)) {
169 pte
= (pte_t
*) spp_getpage();
170 set_pmd(pmd
, __pmd(__pa(pte
) | _KERNPG_TABLE
| _PAGE_USER
));
171 if (pte
!= pte_offset_kernel(pmd
, 0)) {
172 printk(KERN_ERR
"PAGETABLE BUG #02!\n");
176 new_pte
= pfn_pte(phys
>> PAGE_SHIFT
, prot
);
178 pte
= pte_offset_kernel(pmd
, vaddr
);
179 if (!pte_none(*pte
) &&
180 pte_val(*pte
) != (pte_val(new_pte
) & __supported_pte_mask
))
182 set_pte(pte
, new_pte
);
185 * It's enough to flush this one mapping.
186 * (PGE mappings get flushed as well)
188 __flush_tlb_one(vaddr
);
192 * The head.S code sets up the kernel high mapping:
194 * from __START_KERNEL_map to __START_KERNEL_map + size (== _end-_text)
196 * phys_addr holds the negative offset to the kernel, which is added
197 * to the compile time generated pmds. This results in invalid pmds up
198 * to the point where we hit the physaddr 0 mapping.
200 * We limit the mappings to the region from _text to _end. _end is
201 * rounded up to the 2MB boundary. This catches the invalid pmds as
202 * well, as they are located before _text:
204 void __init
cleanup_highmap(void)
206 unsigned long vaddr
= __START_KERNEL_map
;
207 unsigned long end
= round_up((unsigned long)_end
, PMD_SIZE
) - 1;
208 pmd_t
*pmd
= level2_kernel_pgt
;
209 pmd_t
*last_pmd
= pmd
+ PTRS_PER_PMD
;
211 for (; pmd
< last_pmd
; pmd
++, vaddr
+= PMD_SIZE
) {
212 if (!pmd_present(*pmd
))
214 if (vaddr
< (unsigned long) _text
|| vaddr
> end
)
215 set_pmd(pmd
, __pmd(0));
219 /* NOTE: this is meant to be run only at boot */
221 __set_fixmap(enum fixed_addresses idx
, unsigned long phys
, pgprot_t prot
)
223 unsigned long address
= __fix_to_virt(idx
);
225 if (idx
>= __end_of_fixed_addresses
) {
226 printk(KERN_ERR
"Invalid __set_fixmap\n");
229 set_pte_phys(address
, phys
, prot
);
232 static unsigned long __initdata table_start
;
233 static unsigned long __meminitdata table_end
;
235 static __meminit
void *alloc_low_page(unsigned long *phys
)
237 unsigned long pfn
= table_end
++;
241 adr
= (void *)get_zeroed_page(GFP_ATOMIC
);
248 panic("alloc_low_page: ran out of memory");
250 adr
= early_ioremap(pfn
* PAGE_SIZE
, PAGE_SIZE
);
251 memset(adr
, 0, PAGE_SIZE
);
252 *phys
= pfn
* PAGE_SIZE
;
256 static __meminit
void unmap_low_page(void *adr
)
261 early_iounmap(adr
, PAGE_SIZE
);
264 /* Must run before zap_low_mappings */
265 __meminit
void *early_ioremap(unsigned long addr
, unsigned long size
)
267 pmd_t
*pmd
, *last_pmd
;
271 pmds
= ((addr
& ~PMD_MASK
) + size
+ ~PMD_MASK
) / PMD_SIZE
;
272 vaddr
= __START_KERNEL_map
;
273 pmd
= level2_kernel_pgt
;
274 last_pmd
= level2_kernel_pgt
+ PTRS_PER_PMD
- 1;
276 for (; pmd
<= last_pmd
; pmd
++, vaddr
+= PMD_SIZE
) {
277 for (i
= 0; i
< pmds
; i
++) {
278 if (pmd_present(pmd
[i
]))
279 goto continue_outer_loop
;
281 vaddr
+= addr
& ~PMD_MASK
;
284 for (i
= 0; i
< pmds
; i
++, addr
+= PMD_SIZE
)
285 set_pmd(pmd
+i
, __pmd(addr
| __PAGE_KERNEL_LARGE_EXEC
));
288 return (void *)vaddr
;
292 printk(KERN_ERR
"early_ioremap(0x%lx, %lu) failed\n", addr
, size
);
298 * To avoid virtual aliases later:
300 __meminit
void early_iounmap(void *addr
, unsigned long size
)
306 vaddr
= (unsigned long)addr
;
307 pmds
= ((vaddr
& ~PMD_MASK
) + size
+ ~PMD_MASK
) / PMD_SIZE
;
308 pmd
= level2_kernel_pgt
+ pmd_index(vaddr
);
310 for (i
= 0; i
< pmds
; i
++)
316 static void __meminit
317 phys_pmd_init(pmd_t
*pmd_page
, unsigned long address
, unsigned long end
)
319 int i
= pmd_index(address
);
321 for (; i
< PTRS_PER_PMD
; i
++, address
+= PMD_SIZE
) {
322 pmd_t
*pmd
= pmd_page
+ pmd_index(address
);
324 if (address
>= end
) {
325 if (!after_bootmem
) {
326 for (; i
< PTRS_PER_PMD
; i
++, pmd
++)
327 set_pmd(pmd
, __pmd(0));
335 set_pte((pte_t
*)pmd
,
336 pfn_pte(address
>> PAGE_SHIFT
, PAGE_KERNEL_LARGE
));
340 static void __meminit
341 phys_pmd_update(pud_t
*pud
, unsigned long address
, unsigned long end
)
343 pmd_t
*pmd
= pmd_offset(pud
, 0);
344 spin_lock(&init_mm
.page_table_lock
);
345 phys_pmd_init(pmd
, address
, end
);
346 spin_unlock(&init_mm
.page_table_lock
);
350 static void __meminit
351 phys_pud_init(pud_t
*pud_page
, unsigned long addr
, unsigned long end
)
353 int i
= pud_index(addr
);
355 for (; i
< PTRS_PER_PUD
; i
++, addr
= (addr
& PUD_MASK
) + PUD_SIZE
) {
356 unsigned long pmd_phys
;
357 pud_t
*pud
= pud_page
+ pud_index(addr
);
363 if (!after_bootmem
&&
364 !e820_any_mapped(addr
, addr
+PUD_SIZE
, 0)) {
365 set_pud(pud
, __pud(0));
370 if (!pud_large(*pud
))
371 phys_pmd_update(pud
, addr
, end
);
375 if (direct_gbpages
) {
376 set_pte((pte_t
*)pud
,
377 pfn_pte(addr
>> PAGE_SHIFT
, PAGE_KERNEL_LARGE
));
381 pmd
= alloc_low_page(&pmd_phys
);
383 spin_lock(&init_mm
.page_table_lock
);
384 set_pud(pud
, __pud(pmd_phys
| _KERNPG_TABLE
));
385 phys_pmd_init(pmd
, addr
, end
);
386 spin_unlock(&init_mm
.page_table_lock
);
393 static void __init
find_early_table_space(unsigned long end
)
395 unsigned long puds
, pmds
, tables
, start
;
397 puds
= (end
+ PUD_SIZE
- 1) >> PUD_SHIFT
;
398 tables
= round_up(puds
* sizeof(pud_t
), PAGE_SIZE
);
399 if (!direct_gbpages
) {
400 pmds
= (end
+ PMD_SIZE
- 1) >> PMD_SHIFT
;
401 tables
+= round_up(pmds
* sizeof(pmd_t
), PAGE_SIZE
);
405 * RED-PEN putting page tables only on node 0 could
406 * cause a hotspot and fill up ZONE_DMA. The page tables
407 * need roughly 0.5KB per GB.
410 table_start
= find_e820_area(start
, end
, tables
, PAGE_SIZE
);
411 if (table_start
== -1UL)
412 panic("Cannot find space for the kernel page tables");
414 table_start
>>= PAGE_SHIFT
;
415 table_end
= table_start
;
417 early_printk("kernel direct mapping tables up to %lx @ %lx-%lx\n",
418 end
, table_start
<< PAGE_SHIFT
,
419 (table_start
<< PAGE_SHIFT
) + tables
);
422 static void __init
init_gbpages(void)
424 if (direct_gbpages
&& cpu_has_gbpages
)
425 printk(KERN_INFO
"Using GB pages for direct mapping\n");
430 #ifdef CONFIG_MEMTEST_BOOTPARAM
432 static void __init
memtest(unsigned long start_phys
, unsigned long size
,
436 unsigned long *start
;
437 unsigned long start_bad
;
438 unsigned long last_bad
;
440 unsigned long start_phys_aligned
;
452 val
= 0x5555555555555555UL
;
455 val
= 0xaaaaaaaaaaaaaaaaUL
;
461 incr
= sizeof(unsigned long);
462 start_phys_aligned
= ALIGN(start_phys
, incr
);
463 count
= (size
- (start_phys_aligned
- start_phys
))/incr
;
464 start
= __va(start_phys_aligned
);
468 for (i
= 0; i
< count
; i
++)
470 for (i
= 0; i
< count
; i
++, start
++, start_phys_aligned
+= incr
) {
472 if (start_phys_aligned
== last_bad
+ incr
) {
476 printk(KERN_CONT
"\n %016lx bad mem addr %016lx - %016lx reserved",
477 val
, start_bad
, last_bad
+ incr
);
478 reserve_early(start_bad
, last_bad
- start_bad
, "BAD RAM");
480 start_bad
= last_bad
= start_phys_aligned
;
485 printk(KERN_CONT
"\n %016lx bad mem addr %016lx - %016lx reserved",
486 val
, start_bad
, last_bad
+ incr
);
487 reserve_early(start_bad
, last_bad
- start_bad
, "BAD RAM");
492 static int memtest_pattern __initdata
= CONFIG_MEMTEST_BOOTPARAM_VALUE
;
494 static int __init
parse_memtest(char *arg
)
497 memtest_pattern
= simple_strtoul(arg
, NULL
, 0);
501 early_param("memtest", parse_memtest
);
503 static void __init
early_memtest(unsigned long start
, unsigned long end
)
505 unsigned long t_start
, t_size
;
508 if (!memtest_pattern
)
511 printk(KERN_INFO
"early_memtest: pattern num %d", memtest_pattern
);
512 for (pattern
= 0; pattern
< memtest_pattern
; pattern
++) {
515 while (t_start
< end
) {
516 t_start
= find_e820_area_size(t_start
, &t_size
, 1);
521 if (t_start
+ t_size
> end
)
522 t_size
= end
- t_start
;
524 printk(KERN_CONT
"\n %016lx - %016lx pattern %d",
525 t_start
, t_start
+ t_size
, pattern
);
527 memtest(t_start
, t_size
, pattern
);
532 printk(KERN_CONT
"\n");
535 static void __init
early_memtest(unsigned long start
, unsigned long end
)
541 * Setup the direct mapping of the physical memory at PAGE_OFFSET.
542 * This runs before bootmem is initialized and gets pages directly from
543 * the physical memory. To access them they are temporarily mapped.
545 void __init_refok
init_memory_mapping(unsigned long start
, unsigned long end
)
548 unsigned long start_phys
= start
, end_phys
= end
;
550 printk(KERN_INFO
"init_memory_mapping\n");
553 * Find space for the kernel direct mapping tables.
555 * Later we should allocate these tables in the local node of the
556 * memory mapped. Unfortunately this is done currently before the
557 * nodes are discovered.
559 if (!after_bootmem
) {
561 find_early_table_space(end
);
564 start
= (unsigned long)__va(start
);
565 end
= (unsigned long)__va(end
);
567 for (; start
< end
; start
= next
) {
568 pgd_t
*pgd
= pgd_offset_k(start
);
569 unsigned long pud_phys
;
573 pud
= pud_offset(pgd
, start
& PGDIR_MASK
);
575 pud
= alloc_low_page(&pud_phys
);
577 next
= start
+ PGDIR_SIZE
;
580 phys_pud_init(pud
, __pa(start
), __pa(next
));
582 set_pgd(pgd_offset_k(start
), mk_kernel_pgd(pud_phys
));
587 mmu_cr4_features
= read_cr4();
591 reserve_early(table_start
<< PAGE_SHIFT
,
592 table_end
<< PAGE_SHIFT
, "PGTABLE");
595 early_memtest(start_phys
, end_phys
);
599 void __init
paging_init(void)
601 unsigned long max_zone_pfns
[MAX_NR_ZONES
];
603 memset(max_zone_pfns
, 0, sizeof(max_zone_pfns
));
604 max_zone_pfns
[ZONE_DMA
] = MAX_DMA_PFN
;
605 max_zone_pfns
[ZONE_DMA32
] = MAX_DMA32_PFN
;
606 max_zone_pfns
[ZONE_NORMAL
] = end_pfn
;
608 memory_present(0, 0, end_pfn
);
610 free_area_init_nodes(max_zone_pfns
);
615 * Memory hotplug specific functions
617 void online_page(struct page
*page
)
619 ClearPageReserved(page
);
620 init_page_count(page
);
626 #ifdef CONFIG_MEMORY_HOTPLUG
628 * Memory is added always to NORMAL zone. This means you will never get
629 * additional DMA/DMA32 memory.
631 int arch_add_memory(int nid
, u64 start
, u64 size
)
633 struct pglist_data
*pgdat
= NODE_DATA(nid
);
634 struct zone
*zone
= pgdat
->node_zones
+ ZONE_NORMAL
;
635 unsigned long start_pfn
= start
>> PAGE_SHIFT
;
636 unsigned long nr_pages
= size
>> PAGE_SHIFT
;
639 init_memory_mapping(start
, start
+ size
-1);
641 ret
= __add_pages(zone
, start_pfn
, nr_pages
);
646 EXPORT_SYMBOL_GPL(arch_add_memory
);
648 #if !defined(CONFIG_ACPI_NUMA) && defined(CONFIG_NUMA)
649 int memory_add_physaddr_to_nid(u64 start
)
653 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid
);
656 #endif /* CONFIG_MEMORY_HOTPLUG */
658 static struct kcore_list kcore_mem
, kcore_vmalloc
, kcore_kernel
,
659 kcore_modules
, kcore_vsyscall
;
661 void __init
mem_init(void)
663 long codesize
, reservedpages
, datasize
, initsize
;
667 /* clear_bss() already clear the empty_zero_page */
671 /* this will put all low memory onto the freelists */
673 totalram_pages
= numa_free_all_bootmem();
675 totalram_pages
= free_all_bootmem();
677 reservedpages
= end_pfn
- totalram_pages
-
678 absent_pages_in_range(0, end_pfn
);
681 codesize
= (unsigned long) &_etext
- (unsigned long) &_text
;
682 datasize
= (unsigned long) &_edata
- (unsigned long) &_etext
;
683 initsize
= (unsigned long) &__init_end
- (unsigned long) &__init_begin
;
685 /* Register memory areas for /proc/kcore */
686 kclist_add(&kcore_mem
, __va(0), max_low_pfn
<< PAGE_SHIFT
);
687 kclist_add(&kcore_vmalloc
, (void *)VMALLOC_START
,
688 VMALLOC_END
-VMALLOC_START
);
689 kclist_add(&kcore_kernel
, &_stext
, _end
- _stext
);
690 kclist_add(&kcore_modules
, (void *)MODULES_VADDR
, MODULES_LEN
);
691 kclist_add(&kcore_vsyscall
, (void *)VSYSCALL_START
,
692 VSYSCALL_END
- VSYSCALL_START
);
694 printk(KERN_INFO
"Memory: %luk/%luk available (%ldk kernel code, "
695 "%ldk reserved, %ldk data, %ldk init)\n",
696 (unsigned long) nr_free_pages() << (PAGE_SHIFT
-10),
697 end_pfn
<< (PAGE_SHIFT
-10),
699 reservedpages
<< (PAGE_SHIFT
-10),
706 void free_init_pages(char *what
, unsigned long begin
, unsigned long end
)
708 unsigned long addr
= begin
;
714 * If debugging page accesses then do not free this memory but
715 * mark them not present - any buggy init-section access will
716 * create a kernel page fault:
718 #ifdef CONFIG_DEBUG_PAGEALLOC
719 printk(KERN_INFO
"debug: unmapping init memory %08lx..%08lx\n",
720 begin
, PAGE_ALIGN(end
));
721 set_memory_np(begin
, (end
- begin
) >> PAGE_SHIFT
);
723 printk(KERN_INFO
"Freeing %s: %luk freed\n", what
, (end
- begin
) >> 10);
725 for (; addr
< end
; addr
+= PAGE_SIZE
) {
726 ClearPageReserved(virt_to_page(addr
));
727 init_page_count(virt_to_page(addr
));
728 memset((void *)(addr
& ~(PAGE_SIZE
-1)),
729 POISON_FREE_INITMEM
, PAGE_SIZE
);
736 void free_initmem(void)
738 free_init_pages("unused kernel memory",
739 (unsigned long)(&__init_begin
),
740 (unsigned long)(&__init_end
));
743 #ifdef CONFIG_DEBUG_RODATA
744 const int rodata_test_data
= 0xC3;
745 EXPORT_SYMBOL_GPL(rodata_test_data
);
747 void mark_rodata_ro(void)
749 unsigned long start
= PFN_ALIGN(_stext
), end
= PFN_ALIGN(__end_rodata
);
751 printk(KERN_INFO
"Write protecting the kernel read-only data: %luk\n",
752 (end
- start
) >> 10);
753 set_memory_ro(start
, (end
- start
) >> PAGE_SHIFT
);
756 * The rodata section (but not the kernel text!) should also be
759 start
= ((unsigned long)__start_rodata
+ PAGE_SIZE
- 1) & PAGE_MASK
;
760 set_memory_nx(start
, (end
- start
) >> PAGE_SHIFT
);
764 #ifdef CONFIG_CPA_DEBUG
765 printk(KERN_INFO
"Testing CPA: undo %lx-%lx\n", start
, end
);
766 set_memory_rw(start
, (end
-start
) >> PAGE_SHIFT
);
768 printk(KERN_INFO
"Testing CPA: again\n");
769 set_memory_ro(start
, (end
-start
) >> PAGE_SHIFT
);
775 #ifdef CONFIG_BLK_DEV_INITRD
776 void free_initrd_mem(unsigned long start
, unsigned long end
)
778 free_init_pages("initrd memory", start
, end
);
782 void __init
reserve_bootmem_generic(unsigned long phys
, unsigned len
)
785 int nid
= phys_to_nid(phys
);
787 unsigned long pfn
= phys
>> PAGE_SHIFT
;
789 if (pfn
>= end_pfn
) {
791 * This can happen with kdump kernels when accessing
794 if (pfn
< max_pfn_mapped
)
797 printk(KERN_ERR
"reserve_bootmem: illegal reserve %lx %u\n",
802 /* Should check here against the e820 map to avoid double free */
804 reserve_bootmem_node(NODE_DATA(nid
), phys
, len
, BOOTMEM_DEFAULT
);
806 reserve_bootmem(phys
, len
, BOOTMEM_DEFAULT
);
808 if (phys
+len
<= MAX_DMA_PFN
*PAGE_SIZE
) {
809 dma_reserve
+= len
/ PAGE_SIZE
;
810 set_dma_reserve(dma_reserve
);
814 int kern_addr_valid(unsigned long addr
)
816 unsigned long above
= ((long)addr
) >> __VIRTUAL_MASK_SHIFT
;
822 if (above
!= 0 && above
!= -1UL)
825 pgd
= pgd_offset_k(addr
);
829 pud
= pud_offset(pgd
, addr
);
833 pmd
= pmd_offset(pud
, addr
);
838 return pfn_valid(pmd_pfn(*pmd
));
840 pte
= pte_offset_kernel(pmd
, addr
);
844 return pfn_valid(pte_pfn(*pte
));
848 * A pseudo VMA to allow ptrace access for the vsyscall page. This only
849 * covers the 64bit vsyscall page now. 32bit has a real VMA now and does
850 * not need special handling anymore:
852 static struct vm_area_struct gate_vma
= {
853 .vm_start
= VSYSCALL_START
,
854 .vm_end
= VSYSCALL_START
+ (VSYSCALL_MAPPED_PAGES
* PAGE_SIZE
),
855 .vm_page_prot
= PAGE_READONLY_EXEC
,
856 .vm_flags
= VM_READ
| VM_EXEC
859 struct vm_area_struct
*get_gate_vma(struct task_struct
*tsk
)
861 #ifdef CONFIG_IA32_EMULATION
862 if (test_tsk_thread_flag(tsk
, TIF_IA32
))
868 int in_gate_area(struct task_struct
*task
, unsigned long addr
)
870 struct vm_area_struct
*vma
= get_gate_vma(task
);
875 return (addr
>= vma
->vm_start
) && (addr
< vma
->vm_end
);
879 * Use this when you have no reliable task/vma, typically from interrupt
880 * context. It is less reliable than using the task's vma and may give
883 int in_gate_area_no_task(unsigned long addr
)
885 return (addr
>= VSYSCALL_START
) && (addr
< VSYSCALL_END
);
888 const char *arch_vma_name(struct vm_area_struct
*vma
)
890 if (vma
->vm_mm
&& vma
->vm_start
== (long)vma
->vm_mm
->context
.vdso
)
892 if (vma
== &gate_vma
)
897 #ifdef CONFIG_SPARSEMEM_VMEMMAP
899 * Initialise the sparsemem vmemmap using huge-pages at the PMD level.
902 vmemmap_populate(struct page
*start_page
, unsigned long size
, int node
)
904 unsigned long addr
= (unsigned long)start_page
;
905 unsigned long end
= (unsigned long)(start_page
+ size
);
911 for (; addr
< end
; addr
= next
) {
912 next
= pmd_addr_end(addr
, end
);
914 pgd
= vmemmap_pgd_populate(addr
, node
);
918 pud
= vmemmap_pud_populate(pgd
, addr
, node
);
922 pmd
= pmd_offset(pud
, addr
);
923 if (pmd_none(*pmd
)) {
927 p
= vmemmap_alloc_block(PMD_SIZE
, node
);
931 entry
= pfn_pte(__pa(p
) >> PAGE_SHIFT
,
933 set_pmd(pmd
, __pmd(pte_val(entry
)));
935 printk(KERN_DEBUG
" [%lx-%lx] PMD ->%p on node %d\n",
936 addr
, addr
+ PMD_SIZE
- 1, p
, node
);
938 vmemmap_verify((pte_t
*)pmd
, node
, addr
, next
);