2 * linux/arch/i386/mm/pgtable.c
5 #include <linux/sched.h>
6 #include <linux/kernel.h>
7 #include <linux/errno.h>
9 #include <linux/swap.h>
10 #include <linux/smp.h>
11 #include <linux/highmem.h>
12 #include <linux/slab.h>
13 #include <linux/pagemap.h>
14 #include <linux/spinlock.h>
15 #include <linux/module.h>
17 #include <asm/system.h>
18 #include <asm/pgtable.h>
19 #include <asm/pgalloc.h>
20 #include <asm/fixmap.h>
23 #include <asm/tlbflush.h>
27 int total
= 0, reserved
= 0;
28 int shared
= 0, cached
= 0;
35 printk(KERN_INFO
"Mem-info:\n");
37 printk(KERN_INFO
"Free swap: %6ldkB\n", nr_swap_pages
<<(PAGE_SHIFT
-10));
38 for_each_online_pgdat(pgdat
) {
39 pgdat_resize_lock(pgdat
, &flags
);
40 for (i
= 0; i
< pgdat
->node_spanned_pages
; ++i
) {
41 page
= pgdat_page_nr(pgdat
, i
);
43 if (PageHighMem(page
))
45 if (PageReserved(page
))
47 else if (PageSwapCache(page
))
49 else if (page_count(page
))
50 shared
+= page_count(page
) - 1;
52 pgdat_resize_unlock(pgdat
, &flags
);
54 printk(KERN_INFO
"%d pages of RAM\n", total
);
55 printk(KERN_INFO
"%d pages of HIGHMEM\n", highmem
);
56 printk(KERN_INFO
"%d reserved pages\n", reserved
);
57 printk(KERN_INFO
"%d pages shared\n", shared
);
58 printk(KERN_INFO
"%d pages swap cached\n", cached
);
60 printk(KERN_INFO
"%lu pages dirty\n", global_page_state(NR_FILE_DIRTY
));
61 printk(KERN_INFO
"%lu pages writeback\n",
62 global_page_state(NR_WRITEBACK
));
63 printk(KERN_INFO
"%lu pages mapped\n", global_page_state(NR_FILE_MAPPED
));
64 printk(KERN_INFO
"%lu pages slab\n",
65 global_page_state(NR_SLAB_RECLAIMABLE
) +
66 global_page_state(NR_SLAB_UNRECLAIMABLE
));
67 printk(KERN_INFO
"%lu pages pagetables\n",
68 global_page_state(NR_PAGETABLE
));
72 * Associate a virtual page frame with a given physical page frame
73 * and protection flags for that frame.
75 static void set_pte_pfn(unsigned long vaddr
, unsigned long pfn
, pgprot_t flags
)
82 pgd
= swapper_pg_dir
+ pgd_index(vaddr
);
87 pud
= pud_offset(pgd
, vaddr
);
92 pmd
= pmd_offset(pud
, vaddr
);
97 pte
= pte_offset_kernel(pmd
, vaddr
);
98 if (pgprot_val(flags
))
99 /* <pfn,flags> stored as-is, to permit clearing entries */
100 set_pte(pte
, pfn_pte(pfn
, flags
));
102 pte_clear(&init_mm
, vaddr
, pte
);
105 * It's enough to flush this one mapping.
106 * (PGE mappings get flushed as well)
108 __flush_tlb_one(vaddr
);
112 * Associate a large virtual page frame with a given physical page frame
113 * and protection flags for that frame. pfn is for the base of the page,
114 * vaddr is what the page gets mapped to - both must be properly aligned.
115 * The pmd must already be instantiated. Assumes PAE mode.
117 void set_pmd_pfn(unsigned long vaddr
, unsigned long pfn
, pgprot_t flags
)
123 if (vaddr
& (PMD_SIZE
-1)) { /* vaddr is misaligned */
124 printk(KERN_WARNING
"set_pmd_pfn: vaddr misaligned\n");
127 if (pfn
& (PTRS_PER_PTE
-1)) { /* pfn is misaligned */
128 printk(KERN_WARNING
"set_pmd_pfn: pfn misaligned\n");
131 pgd
= swapper_pg_dir
+ pgd_index(vaddr
);
132 if (pgd_none(*pgd
)) {
133 printk(KERN_WARNING
"set_pmd_pfn: pgd_none\n");
136 pud
= pud_offset(pgd
, vaddr
);
137 pmd
= pmd_offset(pud
, vaddr
);
138 set_pmd(pmd
, pfn_pmd(pfn
, flags
));
140 * It's enough to flush this one mapping.
141 * (PGE mappings get flushed as well)
143 __flush_tlb_one(vaddr
);
147 #ifndef CONFIG_COMPAT_VDSO
148 unsigned long __FIXADDR_TOP
= 0xfffff000;
149 EXPORT_SYMBOL(__FIXADDR_TOP
);
152 void __set_fixmap (enum fixed_addresses idx
, unsigned long phys
, pgprot_t flags
)
154 unsigned long address
= __fix_to_virt(idx
);
156 if (idx
>= __end_of_fixed_addresses
) {
160 set_pte_pfn(address
, phys
>> PAGE_SHIFT
, flags
);
165 * reserve_top_address - reserves a hole in the top of kernel address space
166 * @reserve - size of hole to reserve
168 * Can be used to relocate the fixmap area and poke a hole in the top
169 * of kernel address space to make room for a hypervisor.
171 void reserve_top_address(unsigned long reserve
)
174 printk(KERN_INFO
"Reserving virtual address space above 0x%08x\n",
176 #ifdef CONFIG_COMPAT_VDSO
177 BUG_ON(reserve
!= 0);
179 __FIXADDR_TOP
= -reserve
- PAGE_SIZE
;
180 __VMALLOC_RESERVE
+= reserve
;
184 pte_t
*pte_alloc_one_kernel(struct mm_struct
*mm
, unsigned long address
)
186 return (pte_t
*)__get_free_page(GFP_KERNEL
|__GFP_REPEAT
|__GFP_ZERO
);
189 struct page
*pte_alloc_one(struct mm_struct
*mm
, unsigned long address
)
193 #ifdef CONFIG_HIGHPTE
194 pte
= alloc_pages(GFP_KERNEL
|__GFP_HIGHMEM
|__GFP_REPEAT
|__GFP_ZERO
, 0);
196 pte
= alloc_pages(GFP_KERNEL
|__GFP_REPEAT
|__GFP_ZERO
, 0);
201 void pmd_ctor(void *pmd
, struct kmem_cache
*cache
, unsigned long flags
)
203 memset(pmd
, 0, PTRS_PER_PMD
*sizeof(pmd_t
));
207 * List of all pgd's needed for non-PAE so it can invalidate entries
208 * in both cached and uncached pgd's; not needed for PAE since the
209 * kernel pmd is shared. If PAE were not to share the pmd a similar
210 * tactic would be needed. This is essentially codepath-based locking
211 * against pageattr.c; it is the unique case in which a valid change
212 * of kernel pagetables can't be lazily synchronized by vmalloc faults.
213 * vmalloc faults work because attached pagetables are never freed.
214 * The locking scheme was chosen on the basis of manfred's
215 * recommendations and having no core impact whatsoever.
218 DEFINE_SPINLOCK(pgd_lock
);
219 struct page
*pgd_list
;
221 static inline void pgd_list_add(pgd_t
*pgd
)
223 struct page
*page
= virt_to_page(pgd
);
224 page
->index
= (unsigned long)pgd_list
;
226 set_page_private(pgd_list
, (unsigned long)&page
->index
);
228 set_page_private(page
, (unsigned long)&pgd_list
);
231 static inline void pgd_list_del(pgd_t
*pgd
)
233 struct page
*next
, **pprev
, *page
= virt_to_page(pgd
);
234 next
= (struct page
*)page
->index
;
235 pprev
= (struct page
**)page_private(page
);
238 set_page_private(next
, (unsigned long)pprev
);
241 void pgd_ctor(void *pgd
, struct kmem_cache
*cache
, unsigned long unused
)
245 if (PTRS_PER_PMD
== 1) {
246 memset(pgd
, 0, USER_PTRS_PER_PGD
*sizeof(pgd_t
));
247 spin_lock_irqsave(&pgd_lock
, flags
);
250 clone_pgd_range((pgd_t
*)pgd
+ USER_PTRS_PER_PGD
,
251 swapper_pg_dir
+ USER_PTRS_PER_PGD
,
254 if (PTRS_PER_PMD
> 1)
257 /* must happen under lock */
258 paravirt_alloc_pd_clone(__pa(pgd
) >> PAGE_SHIFT
,
259 __pa(swapper_pg_dir
) >> PAGE_SHIFT
,
260 USER_PTRS_PER_PGD
, PTRS_PER_PGD
- USER_PTRS_PER_PGD
);
263 spin_unlock_irqrestore(&pgd_lock
, flags
);
266 /* never called when PTRS_PER_PMD > 1 */
267 void pgd_dtor(void *pgd
, struct kmem_cache
*cache
, unsigned long unused
)
269 unsigned long flags
; /* can be called from interrupt context */
271 paravirt_release_pd(__pa(pgd
) >> PAGE_SHIFT
);
272 spin_lock_irqsave(&pgd_lock
, flags
);
274 spin_unlock_irqrestore(&pgd_lock
, flags
);
277 pgd_t
*pgd_alloc(struct mm_struct
*mm
)
280 pgd_t
*pgd
= kmem_cache_alloc(pgd_cache
, GFP_KERNEL
);
282 if (PTRS_PER_PMD
== 1 || !pgd
)
285 for (i
= 0; i
< USER_PTRS_PER_PGD
; ++i
) {
286 pmd_t
*pmd
= kmem_cache_alloc(pmd_cache
, GFP_KERNEL
);
289 paravirt_alloc_pd(__pa(pmd
) >> PAGE_SHIFT
);
290 set_pgd(&pgd
[i
], __pgd(1 + __pa(pmd
)));
295 for (i
--; i
>= 0; i
--) {
296 pgd_t pgdent
= pgd
[i
];
297 void* pmd
= (void *)__va(pgd_val(pgdent
)-1);
298 paravirt_release_pd(__pa(pmd
) >> PAGE_SHIFT
);
299 kmem_cache_free(pmd_cache
, pmd
);
301 kmem_cache_free(pgd_cache
, pgd
);
305 void pgd_free(pgd_t
*pgd
)
309 /* in the PAE case user pgd entries are overwritten before usage */
310 if (PTRS_PER_PMD
> 1)
311 for (i
= 0; i
< USER_PTRS_PER_PGD
; ++i
) {
312 pgd_t pgdent
= pgd
[i
];
313 void* pmd
= (void *)__va(pgd_val(pgdent
)-1);
314 paravirt_release_pd(__pa(pmd
) >> PAGE_SHIFT
);
315 kmem_cache_free(pmd_cache
, pmd
);
317 /* in the non-PAE case, free_pgtables() clears user pgd entries */
318 kmem_cache_free(pgd_cache
, pgd
);